diff --git a/Cargo.lock b/Cargo.lock index 063da9b90..2611563e3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -314,29 +314,38 @@ dependencies = [ ] [[package]] -name = "asap-aware-mapping" +name = "asap-devtools" version = "0.1.0" dependencies = [ "asap-frontend-promql", + "asap-frontend-sql", + "asap-logical-asap-optimization", + "asap-physical-asap-optimization", + "asap-plan-selection", "asap-types", - "asap_sketchlib", "serde", "serde_json", - "thiserror 2.0.18", + "serde_yaml", + "tokio", ] [[package]] -name = "asap-devtools" +name = "asap-executor" version = "0.1.0" dependencies = [ - "asap-aware-mapping", "asap-frontend-promql", - "asap-frontend-sql", + "asap-logical-asap-optimization", + "asap-plan-selection", "asap-types", + "asap_sketchlib", + "chrono", + "futures", + "regex", + "rmp-serde", "serde", "serde_json", - "serde_yaml", - "tokio", + "thiserror 2.0.18", + "tracing", ] [[package]] @@ -362,8 +371,9 @@ dependencies = [ name = "asap-frontend-promql" version = "0.1.0" dependencies = [ - "asap-aware-mapping", "asap-frontend-common", + "asap-logical-asap-optimization", + "asap-plan-selection", "asap-types", "promql-parser", ] @@ -372,8 +382,9 @@ dependencies = [ name = "asap-frontend-sql" version = "0.1.0" dependencies = [ - "asap-aware-mapping", "asap-frontend-common", + "asap-logical-asap-optimization", + "asap-plan-selection", "asap-sql-function-catalog", "asap-types", "datafusion", @@ -387,11 +398,13 @@ dependencies = [ name = "asap-integration-tests" version = "0.1.0" dependencies = [ - "asap-aware-mapping", "asap-devtools", + "asap-executor", "asap-frontend-promql", "asap-frontend-sql", - "asap-physical-operators", + "asap-logical-asap-optimization", + "asap-physical-asap-optimization", + "asap-plan-selection", "asap-planner", "asap-types", "asap_sketchlib", @@ -401,31 +414,49 @@ dependencies = [ ] [[package]] -name = "asap-physical-operators" +name = "asap-logical-asap-optimization" version = "0.1.0" dependencies = [ - "asap-aware-mapping", "asap-frontend-promql", "asap-types", "asap_sketchlib", - "chrono", - "futures", - "regex", - "rmp-serde", "serde", "serde_json", "thiserror 2.0.18", - "tracing", +] + +[[package]] +name = "asap-physical-asap-optimization" +version = "0.1.0" +dependencies = [ + "asap-logical-asap-optimization", + "asap-types", + "thiserror 2.0.18", +] + +[[package]] +name = "asap-plan-selection" +version = "0.1.0" +dependencies = [ + "asap-frontend-promql", + "asap-logical-asap-optimization", + "asap-physical-asap-optimization", + "asap-types", + "asap_sketchlib", + "serde", + "serde_json", + "thiserror 2.0.18", ] [[package]] name = "asap-planner" version = "0.1.0" dependencies = [ - "asap-aware-mapping", "asap-frontend-metricsql", "asap-frontend-promql", "asap-frontend-sql", + "asap-logical-asap-optimization", + "asap-plan-selection", "asap-types", "thiserror 2.0.18", "tokio", diff --git a/Cargo.toml b/Cargo.toml index a0d811d05..4d43df1d3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,10 +1,12 @@ [workspace] members = [ - "crates/asap-physical-operators", + "crates/executor", "crates/types", "crates/frontend-common", "crates/sql-function-catalog", - "crates/asap-aware-mapping", + "crates/logical-asap-optimization", + "crates/physical-asap-optimization", + "crates/plan-selection", "crates/frontend-promql", "crates/frontend-metricsql", "crates/metricsql-common-parser-support", diff --git a/crates/asap-aware-mapping/Cargo.toml b/crates/asap-aware-mapping/Cargo.toml deleted file mode 100644 index b2c03ed12..000000000 --- a/crates/asap-aware-mapping/Cargo.toml +++ /dev/null @@ -1,18 +0,0 @@ -[package] -name = "asap-aware-mapping" -version = "0.1.0" -edition = "2021" - -# The cost-aware optimizer layer (L4 decisions) over the L3 intent algebra: -# selects and sizes the summary family for each intent. Consumes and produces -# asap-types (L3 intent algebra + L4 sketch-bound IR) — never depends on a -# front end. -[dependencies] -asap_sketchlib = { workspace = true } -asap-types = { path = "../types" } -thiserror = "2" -serde = { version = "1", features = ["derive"] } -serde_json = "1" - -[dev-dependencies] -asap-frontend-promql = { path = "../frontend-promql" } diff --git a/crates/asap-aware-mapping/src/lib.rs b/crates/asap-aware-mapping/src/lib.rs deleted file mode 100644 index 8b8adefbd..000000000 --- a/crates/asap-aware-mapping/src/lib.rs +++ /dev/null @@ -1,104 +0,0 @@ -//! `asap-plan` — the cost-aware optimizer layer over the pre-ASAP intent algebra. -//! -//! This crate sits between the language-agnostic IR ([`asap_ir`]) and -//! any runtime: it consumes pre-ASAP [`OperatorNode`](asap_types::ir::OperatorNode) -//! DAGs and makes the cost-aware decisions the pre-ASAP IR deliberately -//! leaves open — which sketch (if any) realises each approximate intent. -//! -//! Identical-sub-DAG sharing over the pre-ASAP IR (`asap_types::ir::cse`) -//! runs before the stages; the stages add the ASAP-aware sharing that is -//! invisible at the pre-ASAP level ([`asap_aware_cse`], Pass 2), e.g. -//! `Quantile(x, 0.99)` and `Quantile(x, 0.95)` reading one built sketch. -//! -//! It depends only on the IR crate, never on a front end — the layering -//! invariant (arrows point up) holds here too. -//! -//! Post-lowering **canonicalization** is *not* here: it lives in -//! `asap_types::ir::canonicalize`, run inside `asap_frontend_common`'s shared -//! `resolve_root` so every front end normalizes before the IR leaves resolution -//! (issue #34, closed). -//! -//! ## Planning workflows -//! -//! The planner runs Stage 1 to Stage 3 in order: Stage 1, logical ASAP-aware -//! optimization ([`per_computation_candidates`]: Pass 1 and composition; -//! [`asap_aware_cse`]: Pass 2); Stage 2, physical ASAP-aware optimization -//! ([`stage2_physical_asap_optimization`]); Stage 3, plan selection -//! ([`stage3_plan_selection`]). -//! -//! - Run them for a parsed workload through [`optimize`] with -//! [`Stage1ToStage3Pass`], the default pass. -//! - Call [`run_stage1_to_stage3`] over the frontends' roots to read every stage's -//! candidates as well as the selected plan. -//! -//! Models and evidence determine which choices the helpers can justify. -//! Physical operator binding, placement, storage, deployment, and execution -//! remain downstream responsibilities. Neither taking the first candidate nor -//! assembling a logical DAG creates an executable deployment plan. -//! -//! ## Supporting components -//! -//! - [`realization`] — the summary families that can serve an aggregate -//! intent, the intent's accuracy target and `(ε, δ)` budget, and default -//! sketch sizing, shared by Pass 1 and Pass 2. -//! - [`accuracy`] — the [`AccuracyModel`](accuracy::AccuracyModel) -//! extension point (issue #172): the planning-time algebra that derives -//! a machine-readable [`AccuracyGuarantee`](asap_types::post_asap::AccuracyGuarantee) -//! for every finalized post-ASAP value and propagates it through -//! approximate-over-approximate compositions under conservative rules -//! (no independence assumptions, unknown statistics stay unknown). Plan -//! selection checks each candidate's guarantee against its query's -//! `AccuracyTarget` before pricing it. -//! - [`analytical_cost`] and [`physical_operator_statistics`] — the -//! analytical per-operator resource model Stage 3 prices with. -//! - [`maintained_population`] — builds a maintained-population sub-DAG for -//! an instant top-k/quantile/sum query; Pass 1 does not offer it yet (#681). - -pub mod accuracy; -pub mod analytical_cost; -/// The deployment's capabilities: supported summaries and what it retains. -pub mod deployment_capabilities; -mod function_rules; -pub mod pane_sharing; -pub mod pass; -pub mod physical_handoff_cost; -pub mod physical_operator_statistics; -pub mod query_physical_lowering; -pub mod realization; -pub mod storage_io; -#[cfg(test)] -mod test_support; - -pub use accuracy::{ - AccuracyEvidenceProvider, AccuracyModel, DefaultAccuracyModel, NoAccuracyEvidence, - PropagationStats, -}; -pub use deployment_capabilities::{ - DeploymentCapabilities, DeploymentCapabilitiesError, SupportedSummary, -}; -pub use pass::{ - optimize, run_stage1_to_stage3, OptimizationInput, OptimizationInputError, OptimizationPass, - OptimizeError, PassNameConflict, PassRegistry, PlanOutput, PlanningModels, QueryPlan, - Stage1ToStage3Pass, Stage1ToStage3Run, -}; -pub use realization::summary_candidates; - -pub mod maintained_population; - -/// Stage 1, Pass 1: per-computation candidate generation. No execution timing is -/// assigned: that is a Stage 2 materialization decision. -pub mod per_computation_candidates; - -/// Stage 1, Pass 2: ASAP-aware common-subexpression elimination (the -/// identical-expression rule). -pub mod asap_aware_cse; - -/// #509 Stage 2, physical ASAP-aware optimization (MVP): materialization -/// (every sub-DAG `NotMaterialized`) and physical operator implementation, -/// producing the CandidatePhysicalASAPDAGs. -pub mod stage2_physical_asap_optimization; - -/// #509 Stage 3, plan selection (MVP): accuracy check, analytical pricing -/// (a `NotMaterialized` node once per query that reads it, a materialized -/// one once), cheapest valid plan. -pub mod stage3_plan_selection; diff --git a/crates/asap-aware-mapping/tests/cost_independence.rs b/crates/asap-aware-mapping/tests/cost_independence.rs deleted file mode 100644 index 05915ac3b..000000000 --- a/crates/asap-aware-mapping/tests/cost_independence.rs +++ /dev/null @@ -1,152 +0,0 @@ -//! Only plan selection prices plans (#509): logical ASAP-aware optimization -//! and physical ASAP-aware optimization import no cost model. - -use std::path::Path; - -/// The logical and physical ASAP-aware optimization modules, relative to -/// `src/`. -const OPTIMIZATION_MODULES: &[&str] = &[ - "per_computation_candidates.rs", - "stage2_physical_asap_optimization.rs", -]; - -/// The modules that price plans. -const COST_MODULES: &[&str] = &[ - "analytical_cost", - "physical_handoff_cost", - "physical_operator_statistics", - "stage3_plan_selection", - "storage_io", -]; - -/// The source lines outside `#[cfg(test)]` items and comments, numbered. -fn production_lines(source: &str) -> Vec<(usize, &str)> { - let mut lines = Vec::new(); - let mut skip_next_item = false; - let mut depth = 0i64; - for (index, line) in source.lines().enumerate() { - let trimmed = line.trim(); - if depth > 0 { - depth += brace_balance(line); - continue; - } - if trimmed == "#[cfg(test)]" { - skip_next_item = true; - continue; - } - if skip_next_item { - if trimmed.starts_with("#[") || trimmed.is_empty() { - continue; - } - skip_next_item = false; - depth = brace_balance(line); - continue; - } - if !trimmed.starts_with("//") { - lines.push((index + 1, line)); - } - } - lines -} - -fn brace_balance(line: &str) -> i64 { - line.chars() - .map(|c| match c { - '{' => 1, - '}' => -1, - _ => 0, - }) - .sum() -} - -/// Names `lib.rs` re-exports from `module` (`pub use module::{...};`). -fn reexports<'a>(lib: &'a str, module: &str) -> Vec<&'a str> { - let start = format!("pub use {module}::{{"); - let Some(begin) = lib.find(&start) else { - return Vec::new(); - }; - let rest = &lib[begin + start.len()..]; - rest[..rest.find('}').expect("re-export list is closed")] - .split(',') - .map(str::trim) - .filter(|name| !name.is_empty()) - .collect() -} - -/// The production lines of `source` that import or name a cost module. -fn cost_imports(source: &str, forbidden: &[&str]) -> Vec<(usize, String)> { - let mut offenders = Vec::new(); - let mut in_use = false; - for (number, line) in production_lines(source) { - in_use |= line.trim_start().starts_with("use ") || line.contains(" use "); - let path_use = COST_MODULES - .iter() - .any(|module| line.contains(&format!("{module}::"))); - let imported = in_use - && line - .split(|c: char| !(c.is_alphanumeric() || c == '_')) - .any(|token| forbidden.contains(&token)); - if path_use || imported { - offenders.push((number, line.trim().to_string())); - } - if line.contains(';') { - in_use = false; - } - } - offenders -} - -#[test] -fn logical_and_physical_asap_optimization_import_no_cost_model() { - let src = Path::new(env!("CARGO_MANIFEST_DIR")).join("src"); - let lib = std::fs::read_to_string(src.join("lib.rs")).unwrap(); - for module in COST_MODULES { - assert!(src.join(format!("{module}.rs")).exists(), "{module}"); - } - let forbidden: Vec<&str> = COST_MODULES - .iter() - .copied() - .chain( - COST_MODULES - .iter() - .flat_map(|module| reexports(&lib, module)), - ) - .collect(); - let mut offenders = Vec::new(); - for module in OPTIMIZATION_MODULES { - let source = std::fs::read_to_string(src.join(module)).unwrap(); - for (number, line) in cost_imports(&source, &forbidden) { - offenders.push(format!("{module}:{number}: {line}")); - } - } - assert!( - offenders.is_empty(), - "only plan selection may price plans:\n{}", - offenders.join("\n") - ); -} - -/// The check finds a cost import in production code, through a path or a -/// re-exported name, and ignores tests and comments. -#[test] -fn the_check_finds_cost_imports_outside_tests() { - let source = "\ -// use crate::stage3_plan_selection::CostModel; -use crate::{ - accuracy::AccuracyModel, - ResourceUnitCost, -}; -fn f() -> f64 { - crate::analytical_cost::price() -} -#[cfg(test)] -mod tests { - use crate::stage3_plan_selection::CostModel; -} -"; - let lines: Vec<_> = cost_imports(source, &["ResourceUnitCost"]) - .into_iter() - .map(|(number, _)| number) - .collect(); - assert_eq!(lines, vec![4, 7]); -} diff --git a/crates/devtools/Cargo.toml b/crates/devtools/Cargo.toml index 5e4590954..0504f09f3 100644 --- a/crates/devtools/Cargo.toml +++ b/crates/devtools/Cargo.toml @@ -10,7 +10,9 @@ edition = "2021" [dependencies] asap-frontend-promql = { path = "../frontend-promql" } asap-frontend-sql = { path = "../frontend-sql" } -asap-aware-mapping = { path = "../asap-aware-mapping" } +asap-logical-asap-optimization = { path = "../logical-asap-optimization" } +asap-physical-asap-optimization = { path = "../physical-asap-optimization" } +asap-plan-selection = { path = "../plan-selection" } # Used by the show_ir / stage_pipeline / variant_coverage bins (catalog schemas, # async SQL path, JSON output) and by the topk_ir / canonical_examples diff --git a/crates/devtools/examples/canonical_examples.rs b/crates/devtools/examples/canonical_examples.rs index 4cf4db474..cad4fcbd5 100644 --- a/crates/devtools/examples/canonical_examples.rs +++ b/crates/devtools/examples/canonical_examples.rs @@ -5,8 +5,8 @@ use asap_devtools::lower_promql_with_sample_interval; use asap_frontend_sql::{lower_sql_dialect, SqlCatalog}; -use asap_types::pre_asap::schema::{DataType, Field, Schema}; -use asap_types::types::AccuracyTarget; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::workload::SqlDialect; fn col(name: &str, dtype: DataType) -> Field { diff --git a/crates/devtools/examples/topk_ir.rs b/crates/devtools/examples/topk_ir.rs index af21b1e1e..5464fcae4 100644 --- a/crates/devtools/examples/topk_ir.rs +++ b/crates/devtools/examples/topk_ir.rs @@ -4,8 +4,8 @@ // resulting pre-ASAP IR. Used for interactive exploration; not a test. use asap_devtools::{lower_promql_with_sample_interval, lower_sql, SqlCatalog}; -use asap_types::pre_asap::schema::{DataType, Field, Schema}; -use asap_types::types::AccuracyTarget; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::schema::{DataType, Field, Schema}; fn col(name: &str, dtype: DataType) -> Field { Field::plain(name, dtype, false) diff --git a/crates/devtools/src/bin/analyze_corpora.rs b/crates/devtools/src/bin/analyze_corpora.rs index 2d2f34e7b..dbd12e425 100644 --- a/crates/devtools/src/bin/analyze_corpora.rs +++ b/crates/devtools/src/bin/analyze_corpora.rs @@ -6,8 +6,8 @@ use asap_devtools::{lower_promql_with_sample_interval, SqlCatalog}; use asap_frontend_sql::lower_sql_dialect; -use asap_types::pre_asap::schema::{DataType, Field, Schema}; -use asap_types::types::AccuracyTarget; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::workload::SqlDialect; use serde::Serialize; use serde_json::Value; diff --git a/crates/devtools/src/bin/show_logical_dag.rs b/crates/devtools/src/bin/show_logical_dag.rs index 9b2ab90c1..4b3812a5f 100644 --- a/crates/devtools/src/bin/show_logical_dag.rs +++ b/crates/devtools/src/bin/show_logical_dag.rs @@ -18,8 +18,8 @@ // bytes)` catalog — the same table used in cross_language.rs and topk_ir.rs. use asap_devtools::{lower_promql_with_sample_interval, lower_sql, SqlCatalog}; -use asap_types::pre_asap::schema::{DataType, Field, Schema}; -use asap_types::types::AccuracyTarget; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::schema::{DataType, Field, Schema}; use std::io::Read; fn col(name: &str, dtype: DataType) -> Field { diff --git a/crates/devtools/src/bin/sketch_coverage.rs b/crates/devtools/src/bin/sketch_coverage.rs index 362ca9931..e0e7c7b53 100644 --- a/crates/devtools/src/bin/sketch_coverage.rs +++ b/crates/devtools/src/bin/sketch_coverage.rs @@ -13,14 +13,14 @@ // every corpus lowers with. Without an approximate target, Pass 1 offers no // summary candidate. -use asap_aware_mapping::per_computation_candidates::{ - generate_per_computation_candidates, ComputationCandidate, -}; use asap_devtools::lower_promql_with_sample_interval; use asap_frontend_sql::{lower_sql_dialect, SqlCatalog}; +use asap_logical_asap_optimization::per_computation_candidates::{ + generate_per_computation_candidates, ComputationCandidate, +}; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::{OperatorNode, QueryRoot}; -use asap_types::pre_asap::schema::{DataType, Field, Schema}; -use asap_types::types::AccuracyTarget; use asap_types::workload::SqlDialect; use std::collections::BTreeSet; use std::rc::Rc; diff --git a/crates/devtools/src/bin/stage_pipeline.rs b/crates/devtools/src/bin/stage_pipeline.rs index f6fb63f9a..09b969991 100644 --- a/crates/devtools/src/bin/stage_pipeline.rs +++ b/crates/devtools/src/bin/stage_pipeline.rs @@ -106,14 +106,6 @@ use std::collections::{BTreeSet, HashSet}; -use asap_aware_mapping::per_computation_candidates::{ - ComputationCandidate, PerComputationCandidates, PruneReason, -}; -use asap_aware_mapping::stage3_plan_selection::{ - ClusterLimits, ClusterObjective, CostModel, MeasuredOperationCost, PlanSelectionSettings, - Rejection, ResourceUnitCost, ResourceUsage, Selection, SelectionError, -}; -use asap_aware_mapping::DeploymentCapabilities; use asap_devtools::candidate_labels::{candidate_labels, computation_label, query_of_each_target}; use asap_devtools::planner_layering_examples::{ example1, example2, pattern_a, pattern_b, PatternARecurrence, SHARED_ROWS_PER_SECOND, @@ -124,10 +116,18 @@ use asap_devtools::planning_stages::{ }; use asap_devtools::SqlCatalog; use asap_frontend_promql::promql_metric_names; +use asap_logical_asap_optimization::per_computation_candidates::{ + ComputationCandidate, PerComputationCandidates, PruneReason, +}; +use asap_logical_asap_optimization::DeploymentCapabilities; +use asap_plan_selection::{ + ClusterLimits, ClusterObjective, CostModel, MeasuredOperationCost, PlanSelectionSettings, + Rejection, ResourceUnitCost, ResourceUsage, Selection, SelectionError, +}; use asap_types::ir::flat::{flatten, FlatDag}; +use asap_types::ir::properties::{default_metric_for_root, AccuracyMetric, AccuracyTarget}; +use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::QueryRoot; -use asap_types::pre_asap::schema::{DataType, Field, Schema}; -use asap_types::types::{default_metric_for_root, AccuracyMetric, AccuracyTarget}; use asap_types::workload::{ BatchEntry, DataArrival, DataWorkload, DurationMs, PlanningWorkload, Query, QueryLanguage, QueryRecurrence, QueryRequirements, QueryWorkload, RepeatedDemand, SqlDialect, @@ -817,7 +817,7 @@ mod tests { fn failed_selection_lists_rejections_without_a_selection() { let rejected = vec![Rejection { id: "P1".into(), - cause: asap_aware_mapping::stage3_plan_selection::RejectionCause::AccuracyTargetMissed, + cause: asap_plan_selection::RejectionCause::AccuracyTargetMissed, reason: "q1: misses target".into(), }]; let document = diff --git a/crates/devtools/src/bin/variant_coverage.rs b/crates/devtools/src/bin/variant_coverage.rs index 56d9596bb..f2a592405 100644 --- a/crates/devtools/src/bin/variant_coverage.rs +++ b/crates/devtools/src/bin/variant_coverage.rs @@ -8,9 +8,9 @@ use asap_devtools::lower_promql_with_sample_interval; use asap_frontend_sql::{lower_sql_dialect, SqlCatalog}; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::{OperatorNode, ScalarExpr}; -use asap_types::pre_asap::schema::{DataType, Field, Schema}; -use asap_types::types::AccuracyTarget; use asap_types::workload::SqlDialect; use std::collections::BTreeSet; use std::rc::Rc; diff --git a/crates/devtools/src/candidate_labels.rs b/crates/devtools/src/candidate_labels.rs index 9e2ec271d..9af353160 100644 --- a/crates/devtools/src/candidate_labels.rs +++ b/crates/devtools/src/candidate_labels.rs @@ -6,15 +6,16 @@ use std::collections::BTreeMap; use std::rc::Rc; -use asap_aware_mapping::asap_aware_cse::PaneCandidate; -use asap_aware_mapping::pass::Pass2Candidate; -use asap_aware_mapping::per_computation_candidates::{ +use asap_logical_asap_optimization::asap_aware_cse::PaneCandidate; +use asap_logical_asap_optimization::per_computation_candidates::{ ComputationCandidate, PerComputationCandidates, }; -use asap_aware_mapping::stage2_physical_asap_optimization::PhysicalCandidate; +use asap_logical_asap_optimization::workload_candidates::Pass2Candidate; +use asap_physical_asap_optimization::PhysicalCandidate; +use asap_types::ir::schema::{FieldDataType, SketchAlgorithm}; use asap_types::ir::NonASAPOp; use asap_types::ir::{ASAPOp, Operator, OperatorNode, QueryRoot}; -use asap_types::post_asap::{FieldDataType, Materialization, SketchAlgorithm}; +use asap_types::physical::Materialization; use crate::planning_stages::PlanningStagesRun; @@ -471,7 +472,7 @@ mod tests { #[test] fn identical_pane_chains_in_two_queries_are_listed_once() { use crate::planner_layering_examples::{shared_data_workload, SHARED_ROWS_PER_SECOND}; - use asap_types::types::AccuracyTarget; + use asap_types::ir::properties::AccuracyTarget; use asap_types::workload::{ DataArrival, PlanningWorkload, Predictability, Query, QueryLanguage, QueryRequirements, QueryWorkload, RepeatedDemand, RepeatingEntry, RepetitionInterval, @@ -532,7 +533,7 @@ mod tests { #[test] fn fused_computations_and_avg_states_have_their_own_labels() { use crate::planner_layering_examples::{shared_data_workload, SHARED_ROWS_PER_SECOND}; - use asap_types::types::AccuracyTarget; + use asap_types::ir::properties::AccuracyTarget; use asap_types::workload::{ DataArrival, PlanningWorkload, Predictability, Query, QueryLanguage, QueryRequirements, QueryWorkload, RepeatedDemand, RepeatingEntry, RepetitionInterval, diff --git a/crates/devtools/src/lib.rs b/crates/devtools/src/lib.rs index 4dcd229a4..85f206532 100644 --- a/crates/devtools/src/lib.rs +++ b/crates/devtools/src/lib.rs @@ -35,7 +35,7 @@ pub use asap_frontend_sql::{lower_sql, lower_sql_batch, SqlCatalog, SqlError, Sq /// default. pub fn lower_promql_with_sample_interval( query: &str, - accuracy: asap_types::types::AccuracyTarget, + accuracy: asap_types::ir::properties::AccuracyTarget, interval_ms: u64, ) -> Result, PromqlError> { use asap_types::workload::{ diff --git a/crates/devtools/src/planner_layering_examples.rs b/crates/devtools/src/planner_layering_examples.rs index f1160b364..a584dcabd 100644 --- a/crates/devtools/src/planner_layering_examples.rs +++ b/crates/devtools/src/planner_layering_examples.rs @@ -2,7 +2,7 @@ //! (`docs/design_docs/proposals/planner-layering.md`), shared by //! `stage_pipeline` and the planner-layering acceptance tests. -use asap_types::types::{AccuracyMetric, AccuracyTarget}; +use asap_types::ir::properties::{AccuracyMetric, AccuracyTarget}; use asap_types::workload::{ BatchEntry, DataArrival, DataWorkload, DurationMs, MetricType, PlanningWorkload, Predictability, Query, QueryLanguage, QueryRequirements, QueryWorkload, RepeatedDemand, @@ -92,7 +92,7 @@ pub fn example1() -> PlanningWorkload { /// design's Int64 `c * c` can overflow, so the SQL frontend recognizes /// only the Float64 product. pub fn example2() -> (PlanningWorkload, asap_frontend_sql::SqlCatalog) { - use asap_types::pre_asap::schema::{DataType, Field, Schema}; + use asap_types::ir::schema::{DataType, Field, Schema}; // (query, ε, δ) per panel. let panels = [ ( diff --git a/crates/devtools/src/planning_stages.rs b/crates/devtools/src/planning_stages.rs index 6242530c2..b94136dfc 100644 --- a/crates/devtools/src/planning_stages.rs +++ b/crates/devtools/src/planning_stages.rs @@ -2,16 +2,17 @@ //! one pipeline behind `stage_pipeline` and the planner-layering acceptance //! tests, so both see the same candidates, ids and selection. -use asap_aware_mapping::pass::{run_stage2_and_stage3, stage1_logical_asap_optimization}; -pub use asap_aware_mapping::pass::{LogicalCandidate, Pass2Candidate, PrunedCandidate}; -use asap_aware_mapping::per_computation_candidates::generate_per_computation_candidates; -use asap_aware_mapping::per_computation_candidates::PerComputationCandidates; -use asap_aware_mapping::stage2_physical_asap_optimization::{ - physical_candidate_count, PhysicalCandidate, -}; -use asap_aware_mapping::stage3_plan_selection::{PlanSelectionSettings, Selection, SelectionError}; -use asap_aware_mapping::{DeploymentCapabilities, PlanningModels}; -use asap_types::ir::schema_support::with_promql_series_identity; +use asap_logical_asap_optimization::per_computation_candidates::generate_per_computation_candidates; +use asap_logical_asap_optimization::per_computation_candidates::PerComputationCandidates; +use asap_logical_asap_optimization::workload_candidates::stage1_logical_asap_optimization; +pub use asap_logical_asap_optimization::workload_candidates::Pass2Candidate; +use asap_logical_asap_optimization::DeploymentCapabilities; +use asap_physical_asap_optimization::{physical_candidate_count, PhysicalCandidate}; +use asap_plan_selection::pass::run_stage2_and_stage3; +pub use asap_plan_selection::pass::{LogicalCandidate, PrunedCandidate}; +use asap_plan_selection::PlanningModels; +use asap_plan_selection::{PlanSelectionSettings, Selection, SelectionError}; +use asap_types::ir::schema::schema_support::with_promql_series_identity; use asap_types::ir::QueryRoot; use asap_types::workload::{DataWorkload, PlanningWorkload, QueryWorkloadEntry}; diff --git a/crates/devtools/src/rqe_atomic_costs.rs b/crates/devtools/src/rqe_atomic_costs.rs index ce809597e..b5ff3a244 100644 --- a/crates/devtools/src/rqe_atomic_costs.rs +++ b/crates/devtools/src/rqe_atomic_costs.rs @@ -22,10 +22,10 @@ //! the reason: exact accumulators, Hydra structures (shared across groups, //! measured at one label width) and sketch names with no planner sketch. //! -//! [`PlanSelectionSettings::measured_operation_costs`]: asap_aware_mapping::stage3_plan_selection::PlanSelectionSettings::measured_operation_costs +//! [`PlanSelectionSettings::measured_operation_costs`]: asap_plan_selection::PlanSelectionSettings::measured_operation_costs -use asap_aware_mapping::stage3_plan_selection::{MeasuredOperationCost, SummaryOperation}; -use asap_types::post_asap::SketchParams; +use asap_plan_selection::{MeasuredOperationCost, SummaryOperation}; +use asap_types::ir::schema::SketchParams; use serde_json::Value; /// A sketch-bench row that gave no entries, and why. diff --git a/crates/devtools/tests/cross_language.rs b/crates/devtools/tests/cross_language.rs index 0d589ed21..106d92bb8 100644 --- a/crates/devtools/tests/cross_language.rs +++ b/crates/devtools/tests/cross_language.rs @@ -14,10 +14,10 @@ //! explicit inner `Aggregate([Count])`. use asap_devtools::{lower_promql_with_sample_interval, lower_sql, SqlCatalog}; +use asap_types::ir::operator::{AggIntent, GroupKeys}; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::{NonASAPOp, OperatorNode}; -use asap_types::pre_asap::schema::{DataType, Field, Schema}; -use asap_types::pre_asap::{AggIntent, GroupKeys}; -use asap_types::types::AccuracyTarget; use std::rc::Rc; fn col(name: &str, dtype: DataType) -> Field { diff --git a/crates/devtools/tests/viewer_contract.rs b/crates/devtools/tests/viewer_contract.rs index 09f2b9862..a19729df4 100644 --- a/crates/devtools/tests/viewer_contract.rs +++ b/crates/devtools/tests/viewer_contract.rs @@ -7,17 +7,18 @@ use std::collections::{BTreeMap, BTreeSet}; use std::rc::Rc; use std::time::Duration; -use asap_types::ir::{ - ASAPOp, BinaryOperator, NonASAPOp, Operator, OperatorNode, Predicate, ScalarExpr, TimeRangeKind, -}; -use asap_types::post_asap::maintained_population::{ +use asap_types::ir::operator::maintained_population::{ MaintainedPopulation, PopulationInput, PopulationStatistic, }; -use asap_types::post_asap::sketch::{GroupingStrategy, SketchStatistic, SummaryUpdate}; -use asap_types::pre_asap::{ - ArithmeticOpKind, BinaryOpKind, ColumnRef, DataType, FieldDataType, GroupKeys, JoinKind, - Reduction, RelationalSetOpKind, SampleKind, ScalarValue, Schema, Source, TimeShift, - WindowFuncKind, +use asap_types::ir::operator::{ + BinaryOpKind, GroupKeys, JoinKind, Reduction, RelationalSetOpKind, SampleKind, Source, + TimeShift, WindowFuncKind, +}; +use asap_types::ir::scalar::{ArithmeticOpKind, ColumnRef, ScalarValue}; +use asap_types::ir::schema::sketch::{GroupingStrategy, SketchStatistic, SummaryUpdate}; +use asap_types::ir::schema::{DataType, FieldDataType, Schema}; +use asap_types::ir::{ + ASAPOp, BinaryOperator, NonASAPOp, Operator, OperatorNode, Predicate, ScalarExpr, TimeRangeKind, }; /// One instance of every `NonASAPOp` variant, children elided. The diff --git a/crates/asap-physical-operators/Cargo.toml b/crates/executor/Cargo.toml similarity index 76% rename from crates/asap-physical-operators/Cargo.toml rename to crates/executor/Cargo.toml index 2a159ef31..cf4c2304a 100644 --- a/crates/asap-physical-operators/Cargo.toml +++ b/crates/executor/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "asap-physical-operators" +name = "asap-executor" version = "0.1.0" edition = "2021" @@ -17,5 +17,6 @@ regex = "1" [dev-dependencies] rmp-serde = "1" -asap-aware-mapping = { path = "../asap-aware-mapping" } +asap-logical-asap-optimization = { path = "../logical-asap-optimization" } +asap-plan-selection = { path = "../plan-selection" } asap-frontend-promql = { path = "../frontend-promql" } diff --git a/crates/asap-physical-operators/README.md b/crates/executor/README.md similarity index 97% rename from crates/asap-physical-operators/README.md rename to crates/executor/README.md index 35b5c1dde..19c7d05a5 100644 --- a/crates/asap-physical-operators/README.md +++ b/crates/executor/README.md @@ -1,4 +1,4 @@ -# ASAP physical operators +# ASAP executor An independent Rust physical operator DAG runtime shared by ingestion time and query time execution. The library requires neither backend engine, a server, @@ -20,14 +20,14 @@ operator is needed. Summary construction updates state batch by batch. End of input means the supplied query range or ingestion window is complete. ```rust -use asap_physical_operators::{ +use asap_executor::{ expressions::Expression, operators::Operator, values::Value, plan::PhysicalDAG, runtime::{Limits, RunContext, Scope}, }; -use asap_physical_operators::planner::pre_asap::DataType; +use asap_executor::planner::ir::schema::DataType; use futures::{executor::block_on, StreamExt}; let source = Operator::scalar(Value::Int64(7), DataType::Int64)?; @@ -44,7 +44,7 @@ let run = RunContext::new( let mut output = plan.execute(&[1], run)?.remove(0); let batch = block_on(output.next()).unwrap()?; assert!(matches!(batch.rows()[0][0], Value::Int64(-7))); -# Ok::<(), asap_physical_operators::dag::Error>(()) +# Ok::<(), asap_executor::dag::Error>(()) ``` `physical_planner::compile` accepts a Planner `PhysicalASAPDAG` and typed input contracts. diff --git a/crates/asap-physical-operators/src/capability.rs b/crates/executor/src/capability.rs similarity index 93% rename from crates/asap-physical-operators/src/capability.rs rename to crates/executor/src/capability.rs index c02310bbf..9d535dd8e 100644 --- a/crates/asap-physical-operators/src/capability.rs +++ b/crates/executor/src/capability.rs @@ -9,7 +9,7 @@ //! Stored-state encodings belong to deployments. Full plan acceptance is //! owned by `binding`, which also validates schemas, expressions and inputs. use crate::Error; -use planner_types::post_asap::{ +use planner_types::ir::schema::{ ExactKind, ExactParams, FieldDataType as SummaryFamilyType, GroupingStrategy, SketchAlgorithm, SketchParams, SketchStatistic, SummaryUpdate, }; @@ -50,7 +50,7 @@ pub fn validate_summary_kernel( match (kind.algorithm(), kind.params()) { // `asap_sketchlib` silently caps a larger `k`. (A::Kll, P::Kll { k, .. }) - if (8..=planner_types::post_asap::sketch::KLL_MAX_K).contains(k) => + if (8..=planner_types::ir::schema::sketch::KLL_MAX_K).contains(k) => { false } @@ -133,7 +133,7 @@ fn validate_hydra_cms( input: &SummaryUpdate, grouping: &GroupingStrategy, ) -> Result<(), String> { - use planner_types::post_asap::{NonNegativeWeightProof, SummaryInputExpr, WeightDomain}; + use planner_types::ir::schema::{NonNegativeWeightProof, SummaryInputExpr, WeightDomain}; if !matches!(family, SummaryFamilyType::Sketch(_, layout) if layout == grouping) { return Err("Planner family and operator grouping disagree".into()); } @@ -157,7 +157,7 @@ fn validate_hydra_cms( pub(crate) fn hydra_cms_shape( family: &SummaryFamilyType, ) -> Result<(usize, usize, usize, usize), String> { - use planner_types::post_asap::{HydraKind, HydraParams}; + use planner_types::ir::schema::{HydraKind, HydraParams}; let SummaryFamilyType::Sketch( kind, GroupingStrategy::SharedMultiSubpopulation { @@ -227,9 +227,9 @@ fn valid_matrix(width: u32, depth: u32) -> bool { /// occurrence), the update an unkeyed frequency family (HLL, UnivMon) builds /// from that column's values. pub(crate) fn unit_frequency_column( - update: &planner_types::post_asap::SummaryUpdate, -) -> Option<&planner_types::pre_asap::ColumnRef> { - use planner_types::post_asap::{NonNegativeWeightProof, SummaryInputExpr, WeightDomain}; + update: &planner_types::ir::schema::SummaryUpdate, +) -> Option<&planner_types::ir::scalar::ColumnRef> { + use planner_types::ir::schema::{NonNegativeWeightProof, SummaryInputExpr, WeightDomain}; match (&update.item, &update.weight, &update.weight_domain) { ( Some(SummaryInputExpr::Column(column)), @@ -242,12 +242,12 @@ pub(crate) fn unit_frequency_column( } } -pub(crate) fn is_unit_sample_frequency(update: &planner_types::post_asap::SummaryUpdate) -> bool { - use planner_types::post_asap::{NonNegativeWeightProof, SummaryInputExpr, WeightDomain}; +pub(crate) fn is_unit_sample_frequency(update: &planner_types::ir::schema::SummaryUpdate) -> bool { + use planner_types::ir::schema::{NonNegativeWeightProof, SummaryInputExpr, WeightDomain}; matches!( update.item, Some(SummaryInputExpr::Column( - planner_types::pre_asap::ColumnRef::SampleValue + planner_types::ir::scalar::ColumnRef::SampleValue )) ) && matches!(update.weight, SummaryInputExpr::Constant(1.0)) && matches!( @@ -259,7 +259,7 @@ pub(crate) fn is_unit_sample_frequency(update: &planner_types::post_asap::Summar } pub fn validate_native_family(family: &SummaryFamilyType) -> Result<(), Error> { - use planner_types::post_asap::SketchAlgorithm as A; + use planner_types::ir::schema::SketchAlgorithm as A; if let SummaryFamilyType::Sketch(kind, grouping) = family { // Plain Count-Min is native as stored state only: it merges and reads // its bare count, but the DAG does not build it from rows. HydraCms @@ -303,8 +303,8 @@ pub fn validate_native_family(family: &SummaryFamilyType) -> Result<(), Error> { } crate::capability::validate_summary_kernel( family, - &planner_types::post_asap::SummaryUpdate::column( - planner_types::pre_asap::ColumnRef::SampleValue, + &planner_types::ir::schema::SummaryUpdate::column( + planner_types::ir::scalar::ColumnRef::SampleValue, ), &Default::default(), ) @@ -317,7 +317,7 @@ pub fn validate_sketch_evaluation( query: &SketchStatistic, ) -> Result<(), Error> { validate_native_family(family)?; - use planner_types::post_asap::SketchAlgorithm as A; + use planner_types::ir::schema::SketchAlgorithm as A; // A point count without an item value reads the total count. let bare_count = matches!(query, SketchStatistic::PointCount { value: None, .. }); let supported = match family { @@ -382,7 +382,7 @@ pub fn validate_exact_evaluation( ) -> Result<(), Error> { validate_native_family(family)?; use crate::Statistic as S; - use planner_types::post_asap::ExactKind as E; + use planner_types::ir::schema::ExactKind as E; let supported = matches!( (family, evaluation.statistic), (SummaryFamilyType::ExactAggregate(E::Sum, _), S::Sum) @@ -416,7 +416,7 @@ mod tests { #[test] fn filtered_summary_runs_only_at_query_time() { let filter = planner_types::ir::Predicate::(planner_types::ir::ScalarExpr::Literal( - planner_types::pre_asap::ScalarValue::Boolean(true), + planner_types::ir::scalar::ScalarValue::Boolean(true), )); assert!(validate_ingestion_time_summary(Some(&filter)).is_err()); assert!(validate_ingestion_time_summary::(None).is_ok()); diff --git a/crates/asap-physical-operators/src/dag/mod.rs b/crates/executor/src/dag/mod.rs similarity index 100% rename from crates/asap-physical-operators/src/dag/mod.rs rename to crates/executor/src/dag/mod.rs diff --git a/crates/asap-physical-operators/src/error.rs b/crates/executor/src/error.rs similarity index 100% rename from crates/asap-physical-operators/src/error.rs rename to crates/executor/src/error.rs diff --git a/crates/asap-physical-operators/src/evaluation.rs b/crates/executor/src/evaluation.rs similarity index 97% rename from crates/asap-physical-operators/src/evaluation.rs rename to crates/executor/src/evaluation.rs index 843145ce2..c0898e4f1 100644 --- a/crates/asap-physical-operators/src/evaluation.rs +++ b/crates/executor/src/evaluation.rs @@ -54,7 +54,7 @@ pub fn exact_evaluation( #[cfg(test)] mod counter_tests { use super::*; - use planner_types::post_asap::{ExactKind, ExactParams, FieldDataType as SummaryFamilyType}; + use planner_types::ir::schema::{ExactKind, ExactParams, FieldDataType as SummaryFamilyType}; fn counter(kind: ExactKind, params: ExactParams, keyed: bool) -> ExactAccumulator { ExactAccumulator::new(SummaryFamilyType::ExactAggregate(kind, params), keyed).unwrap() diff --git a/crates/asap-physical-operators/src/expressions/arithmetic.rs b/crates/executor/src/expressions/arithmetic.rs similarity index 88% rename from crates/asap-physical-operators/src/expressions/arithmetic.rs rename to crates/executor/src/expressions/arithmetic.rs index 64dbb8301..99a912012 100644 --- a/crates/asap-physical-operators/src/expressions/arithmetic.rs +++ b/crates/executor/src/expressions/arithmetic.rs @@ -2,11 +2,11 @@ //! Preserve IEEE non-finite results; callers own their output policies. pub fn evaluate_float64_arithmetic( - operator: &planner_types::pre_asap::ArithmeticOpKind, + operator: &planner_types::ir::scalar::ArithmeticOpKind, left: f64, right: f64, ) -> f64 { - use planner_types::pre_asap::ArithmeticOpKind::*; + use planner_types::ir::scalar::ArithmeticOpKind::*; match operator { Add => left + right, Sub => left - right, @@ -26,7 +26,7 @@ pub fn evaluate_binary( ) -> Result { use crate::expressions::binary::BinaryOpKind; use crate::{values::Value, Error}; - use planner_types::pre_asap::ArithmeticOpKind; + use planner_types::ir::scalar::ArithmeticOpKind; let invalid = || Error::Invalid("unsupported binary operation or invalid checked-division domain".into()); if operator.vector_match.is_some() { @@ -63,8 +63,8 @@ pub fn evaluate_binary( } /// IEEE comparison, as Go's: NaN is unequal to everything, itself included. -fn compare(op: &planner_types::pre_asap::CompareOpKind, left: f64, right: f64) -> Option { - use planner_types::pre_asap::CompareOpKind; +fn compare(op: &planner_types::ir::scalar::CompareOpKind, left: f64, right: f64) -> Option { + use planner_types::ir::scalar::CompareOpKind; Some(match op { CompareOpKind::Eq => left == right, CompareOpKind::Ne => left != right, diff --git a/crates/asap-physical-operators/src/expressions/binary.rs b/crates/executor/src/expressions/binary.rs similarity index 87% rename from crates/asap-physical-operators/src/expressions/binary.rs rename to crates/executor/src/expressions/binary.rs index 81c1c2656..5b027505e 100644 --- a/crates/asap-physical-operators/src/expressions/binary.rs +++ b/crates/executor/src/expressions/binary.rs @@ -1,7 +1,6 @@ //! Execution configuration for a binary kernel, including comparison evaluation mode. -use planner_types::pre_asap::{ - ArithmeticOpKind, CompareOpKind, PromQLVectorSetOpKind, VectorMatch, -}; +use planner_types::ir::operator::{PromQLVectorSetOpKind, VectorMatch}; +use planner_types::ir::scalar::{ArithmeticOpKind, CompareOpKind}; #[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] pub enum BinaryOpKind { Arithmetic(ArithmeticOpKind), @@ -18,7 +17,7 @@ pub struct BinaryOperator { } impl BinaryOperator { pub fn from_logical(operator: &planner_types::ir::BinaryOperator, return_bool: bool) -> Self { - use planner_types::pre_asap::BinaryOpKind as L; + use planner_types::ir::operator::BinaryOpKind as L; Self { kind: match &operator.kind { L::Arithmetic(op) => BinaryOpKind::Arithmetic(op.clone()), diff --git a/crates/asap-physical-operators/src/expressions/mod.rs b/crates/executor/src/expressions/mod.rs similarity index 99% rename from crates/asap-physical-operators/src/expressions/mod.rs rename to crates/executor/src/expressions/mod.rs index cb7c00f8a..5ea3a86c7 100644 --- a/crates/asap-physical-operators/src/expressions/mod.rs +++ b/crates/executor/src/expressions/mod.rs @@ -3,7 +3,8 @@ use crate::{ values::{plain, SchemaRef, Value}, Error, }; -use planner_types::pre_asap::{ArithmeticOpKind, DataType}; +use planner_types::ir::scalar::ArithmeticOpKind; +use planner_types::ir::schema::DataType; pub mod arithmetic; pub mod binary; mod planner; @@ -66,7 +67,7 @@ impl Expression { right, } => { use crate::expressions::binary::BinaryOpKind; - use planner_types::pre_asap::CompareOpKind; + use planner_types::ir::scalar::CompareOpKind; let (a, n) = left.dtype(input)?; let (b, m) = right.dtype(input)?; if a != DataType::Float64 || b != a || operator.vector_match.is_some() { diff --git a/crates/asap-physical-operators/src/expressions/planner.rs b/crates/executor/src/expressions/planner.rs similarity index 97% rename from crates/asap-physical-operators/src/expressions/planner.rs rename to crates/executor/src/expressions/planner.rs index 99f4c2b89..f877ff890 100644 --- a/crates/asap-physical-operators/src/expressions/planner.rs +++ b/crates/executor/src/expressions/planner.rs @@ -3,7 +3,8 @@ use crate::{ values::{SchemaRef, Value}, Error, }; -use planner_types::pre_asap::{ArithmeticOpKind, CompareOpKind, DataType, ScalarValue}; +use planner_types::ir::scalar::{ArithmeticOpKind, CompareOpKind, ScalarValue}; +use planner_types::ir::schema::DataType; use planner_types::ir::ScalarExpr; use std::{cmp::Ordering, sync::Arc}; @@ -11,7 +12,7 @@ use std::{cmp::Ordering, sync::Arc}; pub(super) fn evaluate( expr: &ScalarExpr, row: &[Value], - schema: &planner_types::pre_asap::Schema, + schema: &planner_types::ir::schema::Schema, ) -> Result { match expr { ScalarExpr::Column(index) => row.get(*index).cloned().ok_or(Error::Invalid(format!( @@ -115,8 +116,8 @@ pub(super) fn evaluate( Value::Null ))), ScalarExpr::FunctionCall { name, args } => { - use planner_types::pre_asap::scalar_type_rules::MapScalarFunction; - if planner_types::pre_asap::scalar_type_rules::promql_function_arity(name).is_some() { + use planner_types::ir::scalar::scalar_type_rules::MapScalarFunction; + if planner_types::ir::scalar::scalar_type_rules::promql_function_arity(name).is_some() { let values = args .iter() .map(|arg| match evaluate(arg, row, schema)? { @@ -497,7 +498,7 @@ fn promql_function(name: &str, args: &[f64]) -> Result { #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] pub struct CompiledExpression { expression: ScalarExpr, - schema: planner_types::pre_asap::Schema, + schema: planner_types::ir::schema::Schema, output: (DataType, bool), } impl CompiledExpression { @@ -563,7 +564,7 @@ impl CompiledExpression { evaluate(&self.expression, row, &self.schema) } } -fn validate(expr: &ScalarExpr, schema: &planner_types::pre_asap::Schema) -> Result<(), Error> { +fn validate(expr: &ScalarExpr, schema: &planner_types::ir::schema::Schema) -> Result<(), Error> { let invalid = || Error::Invalid(format!("unsupported scalar expression: {expr:?}")); expr.scalar_type(schema) .map_err(|e| Error::Invalid(e.to_string()))?; @@ -646,10 +647,11 @@ fn validate(expr: &ScalarExpr, schema: &planner_types::pre_asap::Schema) -> Resu } ScalarExpr::FunctionCall { name, args } => { if name != "promql_drop_metric_name" - && planner_types::pre_asap::scalar_type_rules::promql_function_arity(name).is_none() + && planner_types::ir::scalar::scalar_type_rules::promql_function_arity(name) + .is_none() && name != "asap_struct_field" && name != "asap_element_access" - && planner_types::pre_asap::scalar_type_rules::MapScalarFunction::from_name(name) + && planner_types::ir::scalar::scalar_type_rules::MapScalarFunction::from_name(name) .is_none() { return Err(invalid()); diff --git a/crates/asap-physical-operators/src/key_by_label_values.rs b/crates/executor/src/key_by_label_values.rs similarity index 100% rename from crates/asap-physical-operators/src/key_by_label_values.rs rename to crates/executor/src/key_by_label_values.rs diff --git a/crates/asap-physical-operators/src/lib.rs b/crates/executor/src/lib.rs similarity index 97% rename from crates/asap-physical-operators/src/lib.rs rename to crates/executor/src/lib.rs index ea6b73eaf..b75bd71e2 100644 --- a/crates/asap-physical-operators/src/lib.rs +++ b/crates/executor/src/lib.rs @@ -28,6 +28,7 @@ pub mod expressions; pub mod operators; pub mod physical_planner; pub mod plan; +pub mod query_time; pub mod runtime; pub mod sources; pub mod values; diff --git a/crates/asap-physical-operators/src/measurement.rs b/crates/executor/src/measurement.rs similarity index 100% rename from crates/asap-physical-operators/src/measurement.rs rename to crates/executor/src/measurement.rs diff --git a/crates/asap-physical-operators/src/operators/aggregate/mod.rs b/crates/executor/src/operators/aggregate/mod.rs similarity index 99% rename from crates/asap-physical-operators/src/operators/aggregate/mod.rs rename to crates/executor/src/operators/aggregate/mod.rs index 8f69dba57..9d85eb8af 100644 --- a/crates/asap-physical-operators/src/operators/aggregate/mod.rs +++ b/crates/executor/src/operators/aggregate/mod.rs @@ -122,13 +122,13 @@ impl Operator { } pub fn window( input: SchemaRef, - intent: planner_types::pre_asap::AggIntent, + intent: planner_types::ir::operator::AggIntent, coordinate: usize, value: usize, groups: Vec, window: Option<(i64, i64)>, ) -> Result { - use planner_types::pre_asap::AggIntent; + use planner_types::ir::operator::AggIntent; validate_groups(&input, &groups)?; let histogram = matches!(intent, AggIntent::HistogramQuantile { .. }); if !matches!( diff --git a/crates/asap-physical-operators/src/operators/aggregate/temporal.rs b/crates/executor/src/operators/aggregate/temporal.rs similarity index 98% rename from crates/asap-physical-operators/src/operators/aggregate/temporal.rs rename to crates/executor/src/operators/aggregate/temporal.rs index 9836534e7..77d49d365 100644 --- a/crates/asap-physical-operators/src/operators/aggregate/temporal.rs +++ b/crates/executor/src/operators/aggregate/temporal.rs @@ -10,7 +10,8 @@ use crate::{ values::{group_key, Value}, Error, }; -use planner_types::pre_asap::{AggIntent, ColumnRef}; +use planner_types::ir::operator::AggIntent; +use planner_types::ir::scalar::ColumnRef; use std::collections::BTreeMap; pub(in crate::operators) async fn reduce( @@ -307,10 +308,9 @@ mod tests { runtime::{batch_execution::evaluate_batch, Limits, RunContext, Scope}, values::Batch, }; - use planner_types::{ - post_asap::{Field as SummaryField, FieldDataType as SummaryFamilyType, Schema}, - pre_asap::DataType, - types::AccuracyTarget, + use planner_types::ir::properties::AccuracyTarget; + use planner_types::ir::schema::{ + DataType, Field as SummaryField, FieldDataType as SummaryFamilyType, Schema, }; use std::sync::Arc; diff --git a/crates/asap-physical-operators/src/operators/aligned_binary.rs b/crates/executor/src/operators/aligned_binary.rs similarity index 100% rename from crates/asap-physical-operators/src/operators/aligned_binary.rs rename to crates/executor/src/operators/aligned_binary.rs diff --git a/crates/asap-physical-operators/src/operators/common.rs b/crates/executor/src/operators/common.rs similarity index 100% rename from crates/asap-physical-operators/src/operators/common.rs rename to crates/executor/src/operators/common.rs diff --git a/crates/asap-physical-operators/src/operators/current_series.rs b/crates/executor/src/operators/current_series.rs similarity index 100% rename from crates/asap-physical-operators/src/operators/current_series.rs rename to crates/executor/src/operators/current_series.rs diff --git a/crates/asap-physical-operators/src/operators/filter.rs b/crates/executor/src/operators/filter.rs similarity index 100% rename from crates/asap-physical-operators/src/operators/filter.rs rename to crates/executor/src/operators/filter.rs diff --git a/crates/asap-physical-operators/src/operators/joins/mod.rs b/crates/executor/src/operators/joins/mod.rs similarity index 98% rename from crates/asap-physical-operators/src/operators/joins/mod.rs rename to crates/executor/src/operators/joins/mod.rs index 652fbcdcc..60e6d8c93 100644 --- a/crates/asap-physical-operators/src/operators/joins/mod.rs +++ b/crates/executor/src/operators/joins/mod.rs @@ -53,11 +53,11 @@ impl Operator { pub fn relational_join( left: SchemaRef, right: SchemaRef, - kind: planner_types::pre_asap::JoinKind, + kind: planner_types::ir::operator::JoinKind, predicate: &planner_types::ir::Predicate, output: SchemaRef, ) -> Result { - use planner_types::pre_asap::JoinKind; + use planner_types::ir::operator::JoinKind; let mut joined = left.fields.clone(); joined.extend(right.fields.clone()); let predicate = @@ -101,7 +101,7 @@ pub(super) fn execute<'a>( let right = inputs.pop().ok_or_else(|| invalid("right input missing"))?; let left = inputs.pop().ok_or_else(|| invalid("left input missing"))?; return Ok(futures::stream::once(async move { - use planner_types::pre_asap::JoinKind; + use planner_types::ir::operator::JoinKind; let ((left, _left_memory), (right, _right_memory)) = futures::try_join!(collect_rows(left, &context), collect_rows(right, &context))?; let mut workspace = Workspace::new(&context)?; diff --git a/crates/asap-physical-operators/src/operators/limit.rs b/crates/executor/src/operators/limit.rs similarity index 100% rename from crates/asap-physical-operators/src/operators/limit.rs rename to crates/executor/src/operators/limit.rs diff --git a/crates/asap-physical-operators/src/operators/mod.rs b/crates/executor/src/operators/mod.rs similarity index 95% rename from crates/asap-physical-operators/src/operators/mod.rs rename to crates/executor/src/operators/mod.rs index c4a0036bf..147475c16 100644 --- a/crates/asap-physical-operators/src/operators/mod.rs +++ b/crates/executor/src/operators/mod.rs @@ -7,9 +7,9 @@ use crate::{ Error, }; use futures::StreamExt; -use planner_types::{ - post_asap::{Field as SummaryField, FieldDataType as SummaryFamilyType, Schema, SummaryUpdate}, - pre_asap::{ColumnRef, DataType}, +use planner_types::ir::scalar::ColumnRef; +use planner_types::ir::schema::{ + DataType, Field as SummaryField, FieldDataType as SummaryFamilyType, Schema, SummaryUpdate, }; use std::{collections::BTreeMap, sync::Arc}; pub(crate) mod common; @@ -67,11 +67,11 @@ enum Kind { operator: crate::expressions::binary::BinaryOperator, }, RangeWindow { - intent: Box>, + intent: Box>, }, HistogramQuantile, SeriesWindow { - function: Option>>, + function: Option>>, coordinate: usize, value: usize, range_ms: i64, @@ -79,12 +79,12 @@ enum Kind { at_ms: Option, steps: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - range_at: Option, + range_at: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - steps_range_at: Option, + steps_range_at: Option, }, SeriesLabels { - kind: planner_types::pre_asap::VectorMatchKind, + kind: planner_types::ir::operator::VectorMatchKind, labels: Vec, unique: bool, }, @@ -114,7 +114,7 @@ enum Kind { groups: Vec, }, Window { - intent: Box>, + intent: Box>, coordinate: usize, value: usize, groups: Vec, @@ -132,7 +132,7 @@ enum Kind { require_complete_right: bool, }, Join { - kind: planner_types::pre_asap::JoinKind, + kind: planner_types::ir::operator::JoinKind, predicate: Box, }, SummaryBuild { diff --git a/crates/asap-physical-operators/src/operators/projection.rs b/crates/executor/src/operators/projection.rs similarity index 100% rename from crates/asap-physical-operators/src/operators/projection.rs rename to crates/executor/src/operators/projection.rs diff --git a/crates/asap-physical-operators/src/operators/scope_timestamp.rs b/crates/executor/src/operators/scope_timestamp.rs similarity index 100% rename from crates/asap-physical-operators/src/operators/scope_timestamp.rs rename to crates/executor/src/operators/scope_timestamp.rs diff --git a/crates/asap-physical-operators/src/operators/series_labels.rs b/crates/executor/src/operators/series_labels.rs similarity index 98% rename from crates/asap-physical-operators/src/operators/series_labels.rs rename to crates/executor/src/operators/series_labels.rs index 3987c5142..a7d5259fd 100644 --- a/crates/asap-physical-operators/src/operators/series_labels.rs +++ b/crates/executor/src/operators/series_labels.rs @@ -3,7 +3,8 @@ use super::*; use crate::expressions::binary::BinaryOpKind; use crate::expressions::binary::BinaryOperator; -use planner_types::pre_asap::{schema::PROMQL_SERIES_IDENTITY, VectorMatchKind}; +use planner_types::ir::operator::VectorMatchKind; +use planner_types::ir::schema::PROMQL_SERIES_IDENTITY; type Labels = BTreeMap; @@ -194,7 +195,8 @@ impl Operator { operator: BinaryOperator, scalars: [bool; 2], ) -> Result { - use planner_types::pre_asap::{CompareOpKind::*, GroupSide, PromQLVectorSetOpKind}; + use planner_types::ir::operator::{GroupSide, PromQLVectorSetOpKind}; + use planner_types::ir::scalar::CompareOpKind::*; let vectors = scalars == [false, false]; let valid = match &operator.kind { BinaryOpKind::Arithmetic(_) => true, @@ -317,7 +319,7 @@ async fn series_binary( work: &mut Cooperative, workspace: &mut Workspace, ) -> Result>, Error> { - use planner_types::pre_asap::{GroupSide, PromQLVectorSetOpKind}; + use planner_types::ir::operator::{GroupSide, PromQLVectorSetOpKind}; let drops_name = matches!( binary.kind, BinaryOpKind::Arithmetic(_) | BinaryOpKind::CompareBool(_) diff --git a/crates/asap-physical-operators/src/operators/series_window.rs b/crates/executor/src/operators/series_window.rs similarity index 99% rename from crates/asap-physical-operators/src/operators/series_window.rs rename to crates/executor/src/operators/series_window.rs index b22c8e0ac..7b18f78b0 100644 --- a/crates/asap-physical-operators/src/operators/series_window.rs +++ b/crates/executor/src/operators/series_window.rs @@ -1,6 +1,6 @@ //! PromQL per-series evaluation over the samples before an evaluation instant. use super::*; -use planner_types::pre_asap::{AggIntent, AtModifier}; +use planner_types::ir::operator::{AggIntent, AtModifier}; /// A PromQL subquery grid: every multiple of `step_ms` in /// `(T - offset_ms - range_ms, T - offset_ms]`. `T` is `at_ms` (the subquery's diff --git a/crates/asap-physical-operators/src/operators/sort.rs b/crates/executor/src/operators/sort.rs similarity index 100% rename from crates/asap-physical-operators/src/operators/sort.rs rename to crates/executor/src/operators/sort.rs diff --git a/crates/asap-physical-operators/src/operators/source.rs b/crates/executor/src/operators/source.rs similarity index 100% rename from crates/asap-physical-operators/src/operators/source.rs rename to crates/executor/src/operators/source.rs diff --git a/crates/asap-physical-operators/src/operators/summary/mod.rs b/crates/executor/src/operators/summary/mod.rs similarity index 97% rename from crates/asap-physical-operators/src/operators/summary/mod.rs rename to crates/executor/src/operators/summary/mod.rs index 178f3d494..9ee33f2c7 100644 --- a/crates/asap-physical-operators/src/operators/summary/mod.rs +++ b/crates/executor/src/operators/summary/mod.rs @@ -1,9 +1,9 @@ use super::*; -use planner_types::post_asap::{FamilyCapability, MergeInputRule}; +use planner_types::ir::schema::{FamilyCapability, MergeInputRule}; /// A summary evaluation: a sketch query, or an exact evaluation with typed parameters. #[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)] pub enum SummaryEvaluation { - Sketch(planner_types::post_asap::SketchStatistic), + Sketch(planner_types::ir::schema::SketchStatistic), Exact(crate::summary_kernels::exact::ExactEvaluation), } @@ -169,7 +169,7 @@ impl Operator { validate_groups(&input, &groups)?; let dtype = plain(&input, value)?.0; let typed_frequency = matches!(&family, SummaryFamilyType::Sketch(kind, _) - if kind.algorithm() == &planner_types::post_asap::SketchAlgorithm::UnivMon); + if kind.algorithm() == &planner_types::ir::schema::SketchAlgorithm::UnivMon); if *dtype != DataType::Float64 && !(typed_frequency && matches!(dtype, DataType::Utf8 | DataType::Int64 | DataType::Bool)) @@ -185,8 +185,8 @@ impl Operator { && matches!( family, SummaryFamilyType::ExactAggregate( - planner_types::post_asap::ExactKind::Rate - | planner_types::post_asap::ExactKind::Increase, + planner_types::ir::schema::ExactKind::Rate + | planner_types::ir::schema::ExactKind::Increase, _ ) ) @@ -260,7 +260,7 @@ impl Operator { let mut fields = input.fields.clone(); let result_type = if matches!( fields[state].dtype, - SummaryFamilyType::ExactAggregate(planner_types::post_asap::ExactKind::Count, _) + SummaryFamilyType::ExactAggregate(planner_types::ir::schema::ExactKind::Count, _) ) || integral_count(family, &query) { DataType::Int64 @@ -273,8 +273,8 @@ impl Operator { && matches!( fields[state].dtype, SummaryFamilyType::ExactAggregate( - planner_types::post_asap::ExactKind::Min - | planner_types::post_asap::ExactKind::Max, + planner_types::ir::schema::ExactKind::Min + | planner_types::ir::schema::ExactKind::Max, _ ) ); @@ -291,10 +291,10 @@ impl Operator { /// non-integral total rather than rounding it. fn integral_count(family: &SummaryFamilyType, query: &SummaryEvaluation) -> bool { matches!(family, SummaryFamilyType::Sketch(kind, _) - if kind.algorithm() == &planner_types::post_asap::SketchAlgorithm::Cms) + if kind.algorithm() == &planner_types::ir::schema::SketchAlgorithm::Cms) && matches!( query, - SummaryEvaluation::Sketch(planner_types::post_asap::SketchStatistic::PointCount { + SummaryEvaluation::Sketch(planner_types::ir::schema::SketchStatistic::PointCount { value: None, .. }) @@ -511,8 +511,8 @@ async fn build_summary( let ordered_time = matches!( family, SummaryFamilyType::ExactAggregate( - planner_types::post_asap::ExactKind::Rate - | planner_types::post_asap::ExactKind::Increase, + planner_types::ir::schema::ExactKind::Rate + | planner_types::ir::schema::ExactKind::Increase, _ ) ); diff --git a/crates/asap-physical-operators/src/operators/unchecked.rs b/crates/executor/src/operators/unchecked.rs similarity index 100% rename from crates/asap-physical-operators/src/operators/unchecked.rs rename to crates/executor/src/operators/unchecked.rs diff --git a/crates/asap-physical-operators/src/operators/vector_binary.rs b/crates/executor/src/operators/vector_binary.rs similarity index 100% rename from crates/asap-physical-operators/src/operators/vector_binary.rs rename to crates/executor/src/operators/vector_binary.rs diff --git a/crates/asap-physical-operators/src/operators/vector_window.rs b/crates/executor/src/operators/vector_window.rs similarity index 99% rename from crates/asap-physical-operators/src/operators/vector_window.rs rename to crates/executor/src/operators/vector_window.rs index 6b21ff6a4..afd99c49c 100644 --- a/crates/asap-physical-operators/src/operators/vector_window.rs +++ b/crates/executor/src/operators/vector_window.rs @@ -1,6 +1,6 @@ //! Window bounds are typed input data; aggregation and histogram semantics stay native. use super::*; -use planner_types::pre_asap::AggIntent; +use planner_types::ir::operator::AggIntent; pub(crate) fn matrix_schema() -> SchemaRef { let mut fields = vector_binary::value_schema(false).fields.clone(); diff --git a/crates/asap-physical-operators/src/physical_planner/candidates.rs b/crates/executor/src/physical_planner/candidates.rs similarity index 96% rename from crates/asap-physical-operators/src/physical_planner/candidates.rs rename to crates/executor/src/physical_planner/candidates.rs index b20c31c4b..141cb8503 100644 --- a/crates/asap-physical-operators/src/physical_planner/candidates.rs +++ b/crates/executor/src/physical_planner/candidates.rs @@ -97,7 +97,7 @@ pub fn cut_candidate( /// Only ingestion timing is read: a `MatAtQuery` output is not stored by /// this frontier yet (choosing and storing it is Stage 2's job). pub fn frontier_from_timing(dag: &PhysicalASAPDAG) -> Result, Error> { - use planner_types::post_asap::ExecutionTiming::IngestionTime; + use planner_types::ir::properties::ExecutionTiming::IngestionTime; let timing = dag .nodes .iter() @@ -350,7 +350,7 @@ mod tests { query_batch: Some(vec![BatchEntry { query: Query("sum by(job)(rate(m[1m]))".into()), requirements: QueryRequirements { - accuracy: planner_types::types::AccuracyTarget::exact(), + accuracy: planner_types::ir::properties::AccuracyTarget::exact(), ..Default::default() }, predictability: Predictability::Unknown, @@ -370,11 +370,11 @@ mod tests { let root = promql_rows::with_series_identity(&root).unwrap(); // The logical plan the planner pipeline selects, materialized at // ingestion time. - let run = asap_aware_mapping::run_stage1_to_stage3( + let run = asap_plan_selection::run_stage1_to_stage3( vec![(0, planner_types::ir::QueryRoot::Operator(root))], &workload.query_workload.entries().collect::>(), workload.data_workload.as_ref().unwrap(), - asap_aware_mapping::PlanningModels::builtin(), + asap_plan_selection::PlanningModels::builtin(), ) .unwrap(); let [planner_types::ir::QueryRoot::Operator(selected)] = run.selected_logical().unwrap() @@ -384,8 +384,8 @@ mod tests { let selected = planner_types::ir::apply_materialization_timings( selected, &planner_types::ir::MaterializationAssignment::with_default( - planner_types::ir::Materialization::MatAtIngestion( - planner_types::ir::Medium::Memory, + planner_types::physical::Materialization::MatAtIngestion( + planner_types::physical::Medium::Memory, ), ), &mut Default::default(), @@ -423,9 +423,10 @@ mod tests { fn with_timing( dag: &PhysicalASAPDAG, - timing: impl Fn(&PhysicalASAPDAGNode) -> planner_types::post_asap::ExecutionTiming, + timing: impl Fn(&PhysicalASAPDAGNode) -> planner_types::ir::properties::ExecutionTiming, ) -> PhysicalASAPDAG { - use planner_types::post_asap::{ExecutionTiming, Materialization, Medium}; + use planner_types::ir::properties::ExecutionTiming; + use planner_types::physical::{Materialization, Medium}; let mut timed = dag.clone(); for node in &mut timed.nodes { node.output_state.timing = timing(node); @@ -470,7 +471,7 @@ mod tests { /// lowers each Planner node once and matches `compile_candidate`. #[test] fn timing_cuts_share_one_lowering() { - use planner_types::post_asap::ExecutionTiming::QueryTime; + use planner_types::ir::properties::ExecutionTiming::QueryTime; let (mat_at_ingestion, _, root) = grouped_rate(); let not_materialized = with_timing(&mat_at_ingestion, |_| QueryTime); let inputs = raw_input(&mat_at_ingestion); @@ -500,7 +501,7 @@ mod tests { /// ingestion-time root is itself the frontier. #[test] fn frontier_from_timing_includes_ingestion_root() { - use planner_types::post_asap::ExecutionTiming::IngestionTime; + use planner_types::ir::properties::ExecutionTiming::IngestionTime; let (dag, _, root) = grouped_rate(); let timed = with_timing(&dag, |_| IngestionTime); assert_eq!(frontier_from_timing(&timed).unwrap(), [root]); @@ -510,8 +511,8 @@ mod tests { /// ingestion-time one is on the frontier like an operator root. #[test] fn frontier_from_timing_includes_ingestion_scalar_reference() { + use planner_types::ir::properties::ExecutionTiming::IngestionTime; use planner_types::ir::ScalarExpr; - use planner_types::post_asap::ExecutionTiming::IngestionTime; let (dag, _, root) = grouped_rate(); let mut timed = with_timing(&dag, |_| IngestionTime); let read = timed.roots.remove(0); @@ -524,7 +525,7 @@ mod tests { /// A query-time node feeding an ingestion-time node is rejected. #[test] fn frontier_from_timing_rejects_query_time_input_to_ingestion() { - use planner_types::post_asap::ExecutionTiming::{IngestionTime, QueryTime}; + use planner_types::ir::properties::ExecutionTiming::{IngestionTime, QueryTime}; let (dag, _, _) = grouped_rate(); let timed = with_timing(&dag, |node| { if node.id == dag.roots[0] { diff --git a/crates/asap-physical-operators/src/physical_planner/compiled.rs b/crates/executor/src/physical_planner/compiled.rs similarity index 100% rename from crates/asap-physical-operators/src/physical_planner/compiled.rs rename to crates/executor/src/physical_planner/compiled.rs diff --git a/crates/asap-physical-operators/src/physical_planner/logical.rs b/crates/executor/src/physical_planner/logical.rs similarity index 100% rename from crates/asap-physical-operators/src/physical_planner/logical.rs rename to crates/executor/src/physical_planner/logical.rs diff --git a/crates/asap-physical-operators/src/physical_planner/mod.rs b/crates/executor/src/physical_planner/mod.rs similarity index 96% rename from crates/asap-physical-operators/src/physical_planner/mod.rs rename to crates/executor/src/physical_planner/mod.rs index 9501007b0..0d0ce4dea 100644 --- a/crates/asap-physical-operators/src/physical_planner/mod.rs +++ b/crates/executor/src/physical_planner/mod.rs @@ -10,17 +10,16 @@ use crate::{ values::{Batch, SchemaRef}, Error, }; +use planner_types::ir::operator::{AggIntent, GroupKeys, Reduction as PlannerReduction}; use planner_types::ir::physical_export::{ PhysicalASAPDAG, PhysicalASAPDAGNode, PhysicalASAPNodeId, PhysicalASAPOperatorPayload as Payload, }; -use planner_types::ir::{ASAPOp, NonASAPOp, Operator as LogicalOperator, OperatorNode, ScalarExpr}; -use planner_types::{ - post_asap::{FamilyCapability, FieldDataType, SketchStatistic, SummaryInputExpr}, - pre_asap::{ - AggIntent, ColumnRef, CompareOpKind, DataType, GroupKeys, Reduction as PlannerReduction, - }, +use planner_types::ir::scalar::{ColumnRef, CompareOpKind}; +use planner_types::ir::schema::{ + DataType, FamilyCapability, FieldDataType, SketchStatistic, SummaryInputExpr, }; +use planner_types::ir::{ASAPOp, NonASAPOp, Operator as LogicalOperator, OperatorNode, ScalarExpr}; mod logical; use std::{ collections::{BTreeMap, BTreeSet}, @@ -141,7 +140,7 @@ fn compile_internal( { return Err(invalid(format!( "node {}: a PaneChain runs as its panes; expand it with \ - `asap_types::ir::pane_chain::expand_pane_chains` before compiling", + `asap_types::ir::operator::pane_chain::expand_pane_chains` before compiling", node.id ))); } @@ -234,7 +233,7 @@ fn compile_internal( schemas.truncate(1); } if let Payload::ASAP(ASAPOp::MaintainPopulation { population, .. }) = &node.payload { - use planner_types::post_asap::maintained_population::PopulationInput; + use planner_types::ir::operator::maintained_population::PopulationInput; let PopulationInput::CurrentSeries(spec) = &population.input else { return Err(invalid( "native maintained population requires a current-series input", @@ -267,7 +266,7 @@ fn compile_internal( continue; } if let Payload::ASAP(ASAPOp::EvaluatePopulation { evaluation, .. }) = &node.payload { - use planner_types::post_asap::maintained_population::{ + use planner_types::ir::operator::maintained_population::{ PopulationInput, PopulationStatistic, }; let [producer] = inputs.as_slice() else { @@ -495,7 +494,7 @@ fn compile_internal( *return_bool, ); let query_time = node.output_state.timing - == planner_types::post_asap::ExecutionTiming::QueryTime; + == planner_types::ir::properties::ExecutionTiming::QueryTime; let label_map = |schema: &SchemaRef| { schema .fields @@ -711,7 +710,8 @@ fn bind_operation(node: &PhysicalASAPDAGNode, inputs: &[SchemaRef]) -> Result Result { let columns = schema .fields @@ -719,7 +719,7 @@ fn bind_operation(node: &PhysicalASAPDAGNode, inputs: &[SchemaRef]) -> Result>(); @@ -763,7 +763,7 @@ fn bind_operation(node: &PhysicalASAPDAGNode, inputs: &[SchemaRef]) -> Result Result Result { let state = summary_column(input)?; use crate::Statistic as S; - use planner_types::post_asap::ExactKind as E; + use planner_types::ir::schema::ExactKind as E; let statistic = match &input.fields[state].dtype { FieldDataType::ExactAggregate(kind, _) => match kind { E::Sum => S::Sum, @@ -953,7 +953,8 @@ fn bind_operation(node: &PhysicalASAPDAGNode, inputs: &[SchemaRef]) -> Result { - if node.output_state.timing == planner_types::post_asap::ExecutionTiming::IngestionTime + if node.output_state.timing + == planner_types::ir::properties::ExecutionTiming::IngestionTime { crate::capability::validate_ingestion_time_summary(filter.as_ref()) .map_err(Error::Invalid)?; @@ -999,11 +1000,11 @@ fn bind_operation(node: &PhysicalASAPDAGNode, inputs: &[SchemaRef]) -> Result Result { - if grouping != &planner_types::post_asap::GroupingStrategy::PerSubpopulationInstance { + if grouping != &planner_types::ir::schema::GroupingStrategy::PerSubpopulationInstance { crate::capability::validate_summary_kernel(family, update, grouping) .map_err(Error::Invalid)?; let PlannerReduction::Reduce(keys) = reduction else { @@ -1055,10 +1056,10 @@ fn summary_build( "keyed summary weight must be a finalized value column", )); }; - if matches!(family, FieldDataType::Sketch(kind, _) if kind.algorithm() == &planner_types::post_asap::SketchAlgorithm::CmsWithHeap) + if matches!(family, FieldDataType::Sketch(kind, _) if kind.algorithm() == &planner_types::ir::schema::SketchAlgorithm::CmsWithHeap) && !matches!( update.weight_domain, - planner_types::post_asap::WeightDomain::NonNegative { .. } + planner_types::ir::schema::WeightDomain::NonNegative { .. } ) { return Err(invalid("CMS requires a nonnegative weight contract")); @@ -1113,8 +1114,8 @@ fn summary_build( fn unkeyed_frequency_family(family: &FieldDataType) -> bool { matches!(family, FieldDataType::Sketch(kind, _) if matches!( kind.algorithm(), - planner_types::post_asap::SketchAlgorithm::UnivMon - | planner_types::post_asap::SketchAlgorithm::Hll + planner_types::ir::schema::SketchAlgorithm::UnivMon + | planner_types::ir::schema::SketchAlgorithm::Hll )) } fn summary_column(input: &SchemaRef) -> Result { @@ -1303,8 +1304,8 @@ fn semi_join_keys( /// Deployments may use these positions to bind their source columns. pub fn equijoin_keys( pred: &planner_types::ir::Predicate, - left: &planner_types::post_asap::Schema, - right: &planner_types::post_asap::Schema, + left: &planner_types::ir::schema::Schema, + right: &planner_types::ir::schema::Schema, ) -> Result, Error> { let mut keys = Vec::new(); semi_join_keys(&pred.0, left.fields.len(), right.fields.len(), &mut keys)?; diff --git a/crates/asap-physical-operators/src/physical_planner/precompute.rs b/crates/executor/src/physical_planner/precompute.rs similarity index 94% rename from crates/asap-physical-operators/src/physical_planner/precompute.rs rename to crates/executor/src/physical_planner/precompute.rs index 1f4c4b22d..7bcb3b207 100644 --- a/crates/asap-physical-operators/src/physical_planner/precompute.rs +++ b/crates/executor/src/physical_planner/precompute.rs @@ -1,20 +1,18 @@ //! Compile immutable summary-input computation with explicit population and pane identity. use super::promql_rows::SERIES_IDENTITY_COLUMN as SERIES_IDENTITY; use super::*; +use planner_types::ir::properties::ExecutionTiming; +use planner_types::ir::schema::FieldDataType as SummaryFamilyType; +use planner_types::ir::schema::{DataType, GroupingStrategy, Schema}; use planner_types::ir::ASAPOp; use planner_types::ir::NonASAPOp; -use planner_types::post_asap::FieldDataType as SummaryFamilyType; -use planner_types::{ - post_asap::{ExecutionTiming, GroupingStrategy, Schema}, - pre_asap::DataType, -}; /// Physical rows carry the population and pane coordinate alongside the logical value. /// These fields preserve identities which are implicit in a stored summary instance. pub fn population_schema(family: SummaryFamilyType) -> SchemaRef { Arc::new(Schema { fields: vec![ - planner_types::post_asap::Field { + planner_types::ir::schema::Field { name: "$population".into(), dtype: SummaryFamilyType::Plain(DataType::Map { key: Box::new(DataType::Utf8), @@ -24,13 +22,13 @@ pub fn population_schema(family: SummaryFamilyType) -> SchemaRef { nullable: false, table: None, }, - planner_types::post_asap::Field { + planner_types::ir::schema::Field { name: "$window_end".into(), dtype: SummaryFamilyType::Plain(DataType::Timestamp), nullable: false, table: None, }, - planner_types::post_asap::Field { + planner_types::ir::schema::Field { name: "value".into(), dtype: family, nullable: false, @@ -104,7 +102,7 @@ pub fn boundary_schema( if !matches!( &scan.payload, Payload::NonASAP(NonASAPOp::Scan { - source: planner_types::pre_asap::Source::TimeSeries { .. }, + source: planner_types::ir::operator::Source::TimeSeries { .. }, .. }) ) { @@ -277,7 +275,7 @@ fn validate_value_output(node: &PhysicalASAPDAGNode) -> Result<(), Error> { let schema = &node.output_schema; // Physical population rows already carry the complete identity in `$population`. // Typed logical plans may expose its opaque series-identity column as metadata. - let identity = planner_types::pre_asap::schema::PROMQL_SERIES_IDENTITY; + let identity = planner_types::ir::schema::PROMQL_SERIES_IDENTITY; let identities = schema .fields .iter() @@ -366,11 +364,11 @@ fn fragment( }; validate_value_output(node)?; let statistic = match &input.fields[2].dtype { - SummaryFamilyType::ExactAggregate(planner_types::post_asap::ExactKind::Sum, _) => { + SummaryFamilyType::ExactAggregate(planner_types::ir::schema::ExactKind::Sum, _) => { crate::Statistic::Sum } SummaryFamilyType::ExactAggregate( - planner_types::post_asap::ExactKind::Count, + planner_types::ir::schema::ExactKind::Count, _, ) => crate::Statistic::Count, _ => { @@ -427,10 +425,10 @@ fn fragment( && crate::capability::is_unit_sample_frequency(update) && matches!(family, SummaryFamilyType::Sketch(kind, _) if !matches!( kind.algorithm(), - planner_types::post_asap::SketchAlgorithm::Cms - | planner_types::post_asap::SketchAlgorithm::CountSketch - | planner_types::post_asap::SketchAlgorithm::CmsWithHeap - | planner_types::post_asap::SketchAlgorithm::CountSketchWithHeap + planner_types::ir::schema::SketchAlgorithm::Cms + | planner_types::ir::schema::SketchAlgorithm::CountSketch + | planner_types::ir::schema::SketchAlgorithm::CmsWithHeap + | planner_types::ir::schema::SketchAlgorithm::CountSketchWithHeap )); let keyed = update.item.is_some() && !unit_frequency; if (keyed && !raw) || !matches!(grouping, GroupingStrategy::PerSubpopulationInstance) { @@ -443,8 +441,8 @@ fn fragment( if raw && matches!( update.weight_domain, - planner_types::post_asap::WeightDomain::NonNegative { - proof: planner_types::post_asap::NonNegativeWeightProof::ResetAwareCounterDerivative + planner_types::ir::schema::WeightDomain::NonNegative { + proof: planner_types::ir::schema::NonNegativeWeightProof::ResetAwareCounterDerivative } ) { @@ -453,10 +451,10 @@ fn fragment( )); } if keyed - && matches!(family, SummaryFamilyType::Sketch(kind, _) if kind.algorithm() == &planner_types::post_asap::SketchAlgorithm::CmsWithHeap) + && matches!(family, SummaryFamilyType::Sketch(kind, _) if kind.algorithm() == &planner_types::ir::schema::SketchAlgorithm::CmsWithHeap) && !matches!( update.weight_domain, - planner_types::post_asap::WeightDomain::NonNegative { .. } + planner_types::ir::schema::WeightDomain::NonNegative { .. } ) { return Err(invalid("CMS requires a nonnegative weight contract")); @@ -526,7 +524,7 @@ fn fragment( )?; for (index, (expression, dtype)) in items.into_iter().enumerate() { let name = format!("$item{index}"); - fields.push(planner_types::post_asap::Field { + fields.push(planner_types::ir::schema::Field { name: name.clone(), dtype: SummaryFamilyType::Plain(dtype), nullable: false, @@ -639,7 +637,7 @@ fn raw_items( DataType::Utf8, )), SummaryInputExpr::EntityIdentity( - planner_types::post_asap::EntityIdentity::PromqlLabelSet { excluding }, + planner_types::ir::schema::EntityIdentity::PromqlLabelSet { excluding }, ) => items.push(identity( excluding .iter() diff --git a/crates/asap-physical-operators/src/physical_planner/promql_rows.rs b/crates/executor/src/physical_planner/promql_rows.rs similarity index 89% rename from crates/asap-physical-operators/src/physical_planner/promql_rows.rs rename to crates/executor/src/physical_planner/promql_rows.rs index 004ec3c8e..88eeef055 100644 --- a/crates/asap-physical-operators/src/physical_planner/promql_rows.rs +++ b/crates/executor/src/physical_planner/promql_rows.rs @@ -4,14 +4,14 @@ use super::*; use planner_types::ir::physical_export::{ compile_physical_asap_dag, compile_physical_asap_dag_with_node_ids, }; +use planner_types::ir::schema::DataType; +use planner_types::ir::schema::FieldDataType as SummaryFamilyType; use planner_types::ir::ASAPOp; use planner_types::ir::NonASAPOp; -use planner_types::post_asap::FieldDataType as SummaryFamilyType; -use planner_types::pre_asap::DataType; use std::rc::Rc; /// Not a legal PromQL label name, so it cannot shadow a user label. -pub use planner_types::pre_asap::schema::PROMQL_SERIES_IDENTITY as SERIES_IDENTITY_COLUMN; +pub use planner_types::ir::schema::PROMQL_SERIES_IDENTITY as SERIES_IDENTITY_COLUMN; /// Canonical, reversible identity. JSON object encoding preserves label names, /// empty values and escaping; sorting makes ingestion order irrelevant. @@ -29,9 +29,9 @@ pub fn decode_series_identity(encoded: &str) -> Result, } /// Resolve the row representation before candidate search; see -/// [`planner_types::ir::schema_support::with_promql_series_identity`]. +/// [`planner_types::ir::schema::schema_support::with_promql_series_identity`]. pub fn with_series_identity(root: &Rc) -> Result, Error> { - planner_types::ir::schema_support::with_promql_series_identity(root).map_err(invalid) + planner_types::ir::schema::schema_support::with_promql_series_identity(root).map_err(invalid) } /// Construct source rows only from full identities. The named label columns @@ -87,14 +87,15 @@ pub fn series_row( pub fn compile_current_series_evaluation( selected: &Rc, ) -> Result { - use planner_types::post_asap::{ - maintained_population::PopulationStatistic, Field as SummaryField, - }; + use planner_types::ir::operator::maintained_population::PopulationStatistic; + use planner_types::ir::schema::Field as SummaryField; // This compiler emits maintained precompute: every sub-DAG `MatAtIngestion`. let selected = planner_types::ir::apply_materialization_timings( selected, &planner_types::ir::MaterializationAssignment::with_default( - planner_types::ir::Materialization::MatAtIngestion(planner_types::ir::Medium::Memory), + planner_types::physical::Materialization::MatAtIngestion( + planner_types::physical::Medium::Memory, + ), ), &mut planner_types::ir::TimingMemo::new(), ) @@ -105,7 +106,7 @@ pub fn compile_current_series_evaluation( // Cut at the population output, preserving all selected heap/evaluation nodes. let populations = dag.nodes.iter().filter(|node| matches!(&node.payload, Payload::ASAP(ASAPOp::MaintainPopulation { population, .. }) - if matches!(population.input, planner_types::post_asap::maintained_population::PopulationInput::CurrentSeries(_)) + if matches!(population.input, planner_types::ir::operator::maintained_population::PopulationInput::CurrentSeries(_)) )).collect::>(); if let [population] = populations.as_slice() { if population @@ -195,12 +196,12 @@ pub fn compile_current_series_evaluation( pub fn compile_rate_ranking( selected: &Rc, ) -> Result<(Rc, CompiledPhysicalDAG), Error> { - use planner_types::post_asap::ExactKind; + use planner_types::ir::schema::ExactKind; fn frontier(node: &Rc) -> Option> { if matches!(&node.operator, LogicalOperator::ASAP(ASAPOp::FinalizeExactAccumulator { child }) if matches!(&child.operator, LogicalOperator::ASAP(ASAPOp::SummaryAgg { family: FieldDataType::ExactAggregate(ExactKind::Rate, _), - reduction: planner_types::pre_asap::Reduction::PerEntity, child: raw, .. + reduction: planner_types::ir::operator::Reduction::PerEntity, child: raw, .. }) if matches!(raw.non_asap(), Some(NonASAPOp::TimeRange { .. })))) { return Some(Rc::clone(node)); @@ -211,7 +212,9 @@ pub fn compile_rate_ranking( let selected = planner_types::ir::apply_materialization_timings( selected, &planner_types::ir::MaterializationAssignment::with_default( - planner_types::ir::Materialization::MatAtIngestion(planner_types::ir::Medium::Memory), + planner_types::physical::Materialization::MatAtIngestion( + planner_types::physical::Medium::Memory, + ), ), &mut planner_types::ir::TimingMemo::new(), ) @@ -251,7 +254,8 @@ pub fn compile_rate_ranking( pub fn compile_fixed_window_rate_aggregation( dag: &planner_types::ir::physical_export::PhysicalASAPDAG, ) -> Result { - use planner_types::post_asap::{ExactKind, ExecutionTiming, SketchAlgorithm}; + use planner_types::ir::properties::ExecutionTiming; + use planner_types::ir::schema::{ExactKind, SketchAlgorithm}; let sources = dag .nodes .iter() @@ -260,7 +264,7 @@ pub fn compile_fixed_window_rate_aggregation( &n.payload, Payload::ASAP(ASAPOp::SummaryAgg { family: SummaryFamilyType::ExactAggregate(ExactKind::Rate, _), - reduction: planner_types::pre_asap::Reduction::PerEntity, + reduction: planner_types::ir::operator::Reduction::PerEntity, .. }) ) diff --git a/crates/asap-physical-operators/src/physical_planner/promql_values.rs b/crates/executor/src/physical_planner/promql_values.rs similarity index 98% rename from crates/asap-physical-operators/src/physical_planner/promql_values.rs rename to crates/executor/src/physical_planner/promql_values.rs index b55221168..a92db3d28 100644 --- a/crates/asap-physical-operators/src/physical_planner/promql_values.rs +++ b/crates/executor/src/physical_planner/promql_values.rs @@ -1,6 +1,6 @@ //! Physical scalar/vector contracts preserve complete label sets across native computation. use super::*; -use planner_types::post_asap::FieldDataType as SummaryFamilyType; +use planner_types::ir::schema::FieldDataType as SummaryFamilyType; pub fn scalar_schema() -> SchemaRef { crate::operators::vector_binary::value_schema(true) @@ -16,7 +16,7 @@ pub fn matrix_schema() -> SchemaRef { pub fn compile_scalar(value: f64) -> Result { let operator = Operator::scalar( crate::values::Value::Float64(value), - planner_types::pre_asap::DataType::Float64, + planner_types::ir::schema::DataType::Float64, )? .with_output_schema(scalar_schema())?; CompiledPhysicalDAG::from_operators( @@ -229,7 +229,7 @@ pub fn compile_exact_evaluation( lookback_ms: u64, preserve_metric_name: bool, ) -> Result { - use planner_types::post_asap::ExactKind; + use planner_types::ir::schema::ExactKind; let statistic = match &family { SummaryFamilyType::ExactAggregate(kind, _) => match kind { ExactKind::Sum => crate::Statistic::Sum, diff --git a/crates/asap-physical-operators/src/physical_planner/row_values.rs b/crates/executor/src/physical_planner/row_values.rs similarity index 94% rename from crates/asap-physical-operators/src/physical_planner/row_values.rs rename to crates/executor/src/physical_planner/row_values.rs index 14f3d4813..4c67064a2 100644 --- a/crates/asap-physical-operators/src/physical_planner/row_values.rs +++ b/crates/executor/src/physical_planner/row_values.rs @@ -1,7 +1,7 @@ //! Query-time PromQL value computation over logical row schemas. use super::*; -use planner_types::post_asap::maintained_population::PopulationStatistic; -use planner_types::pre_asap::DataType; +use planner_types::ir::operator::maintained_population::PopulationStatistic; +use planner_types::ir::schema::DataType; /// Aggregate evaluations of a maintained current-series population, as a chain. pub(super) fn population_aggregate( diff --git a/crates/asap-physical-operators/src/plan/mod.rs b/crates/executor/src/plan/mod.rs similarity index 100% rename from crates/asap-physical-operators/src/plan/mod.rs rename to crates/executor/src/plan/mod.rs diff --git a/crates/asap-physical-operators/src/plan/properties.rs b/crates/executor/src/plan/properties.rs similarity index 100% rename from crates/asap-physical-operators/src/plan/properties.rs rename to crates/executor/src/plan/properties.rs diff --git a/crates/types/src/post_asap/query_time/error_estimation.rs b/crates/executor/src/query_time/error_estimation.rs similarity index 98% rename from crates/types/src/post_asap/query_time/error_estimation.rs rename to crates/executor/src/query_time/error_estimation.rs index 9191325af..939299b76 100644 --- a/crates/types/src/post_asap/query_time/error_estimation.rs +++ b/crates/executor/src/query_time/error_estimation.rs @@ -8,7 +8,8 @@ //! The traditional CMS/Count-Sketch/CU-Sketch guarantee is an *a priori*, //! worst-case bound derived before any data is seen: e.g. classic CMS sizing //! (`w = ⌈e/ε⌉` counters/row, `r = ⌈ln(1/δ)⌉` rows — exactly -//! `crates/asap-aware-mapping/src/implementation.rs`'s `cms_width`/`cms_depth`) +//! `crates/logical-asap-optimization/src/accuracy/estimators/cms.rs`'s +//! `cms_width`/`cms_depth`) //! guarantees `Pr[error > ε·|F|₁] < δ` obliviously to the real data //! distribution, by construction assuming the adversarial worst case (§3, //! §3.3 of the paper). @@ -46,9 +47,10 @@ //! plan-time sizing. As of this module landing, **this repository has no //! vendored CMS/CountSketch/CU-Sketch runtime and no counter-array data //! structure anywhere** — confirmed by inspecting -//! `crates/types/src/post_asap/sketch.rs` (`SketchAlgorithm`, `SketchParams`, -//! this module's neighbors) and `crates/asap-aware-mapping/src/implementation.rs` -//! (`default_size_params`, `cms_width`, `cms_depth`): both are purely +//! `crates/types/src/ir/schema/sketch.rs` (`SketchAlgorithm`, `SketchParams`, +//! this module's former neighbors) and +//! `crates/logical-asap-optimization/src/accuracy/estimators/cms.rs` +//! (`cms_width`, `cms_depth`): both are purely //! planning-time sizing metadata. There is no `A[row][col]` counter matrix //! anywhere in the workspace for these functions to be handed at query //! time. So integration point (1) — reporting an *actual* query's posterior @@ -203,10 +205,9 @@ fn saturating_ceil(x: f64, lo: u32, hi: u32) -> u32 { /// The classic CMS `(ε, δ)` sizing formula (§3.3, restated just before /// Eq. 6; identical — formula *and* clamping — to -/// `crates/asap-aware-mapping/src/implementation.rs`'s private +/// `crates/logical-asap-optimization/src/accuracy/estimators/cms.rs`'s /// `cms_width`/`cms_depth`, reimplemented here — deliberately, not by -/// accident — because `asap-types` sits below `asap-aware-mapping` in the -/// workspace's dependency layering and cannot import from it): `w = ⌈e/ε⌉` +/// accident — because the executor depends on no planning stage crate): `w = ⌈e/ε⌉` /// counters/row, clamped to `[2, 2²⁶]`; `r = ⌈ln(1/δ)⌉` rows, clamped to /// `[1, 32]`. /// @@ -475,7 +476,7 @@ mod tests { #[test] fn classic_cms_sizing_matches_implementation_rs_formula() { // Same worked example as - // `asap-aware-mapping::accuracy::estimators::cms` sizing + // `asap_logical_asap_optimization::accuracy::estimators::cms` sizing // (eps=0.001, delta=0.001 ⇒ width=2719, depth=7), pinned here too so // the two independent (layering-forced) reimplementations can't // silently drift apart undetected. diff --git a/crates/executor/src/query_time/mod.rs b/crates/executor/src/query_time/mod.rs new file mode 100644 index 000000000..44bbbee94 --- /dev/null +++ b/crates/executor/src/query_time/mod.rs @@ -0,0 +1,22 @@ +//! Code meant to run at **query execution time**, not planning time: pure, +//! sketch-object-agnostic posterior error-bound math (given a sketch's real +//! counter values, compute a tighter posterior error bound) that only makes +//! sense to invoke *after* a real sketch has ingested data and is being +//! evaluated. +//! +//! It lives in the executor, not in `asap-types`, because `asap-types` +//! holds no execution logic (issue #190): everything there is planning-time +//! IR, describing *what* summary will be built, not *computing over* one +//! that has run. Nothing in the planner or the executor calls into this +//! module today; it is a library waiting for a sketch evaluation path that +//! uses it (see [`error_estimation`]'s own docs for exactly what's blocked +//! and why). See issue #250 for the still-unimplemented idea of recording +//! this module's query-time observations over time and feeding them into a +//! future replan. + +pub mod error_estimation; + +// Glob, not a named list: this module exists only to hold +// `error_estimation` today, so re-exporting everything it makes public +// keeps this hop in sync automatically. +pub use error_estimation::*; diff --git a/crates/asap-physical-operators/src/runtime/batch_execution.rs b/crates/executor/src/runtime/batch_execution.rs similarity index 98% rename from crates/asap-physical-operators/src/runtime/batch_execution.rs rename to crates/executor/src/runtime/batch_execution.rs index 733741022..50132f462 100644 --- a/crates/asap-physical-operators/src/runtime/batch_execution.rs +++ b/crates/executor/src/runtime/batch_execution.rs @@ -87,9 +87,8 @@ mod tests { runtime::{Limits, Scope}, values::Value, }; - use planner_types::{ - post_asap::{Field as SummaryField, FieldDataType as SummaryFamilyType, Schema}, - pre_asap::DataType, + use planner_types::ir::schema::{ + DataType, Field as SummaryField, FieldDataType as SummaryFamilyType, Schema, }; use std::sync::Arc; diff --git a/crates/asap-physical-operators/src/runtime/context.rs b/crates/executor/src/runtime/context.rs similarity index 100% rename from crates/asap-physical-operators/src/runtime/context.rs rename to crates/executor/src/runtime/context.rs diff --git a/crates/asap-physical-operators/src/runtime/cooperative.rs b/crates/executor/src/runtime/cooperative.rs similarity index 100% rename from crates/asap-physical-operators/src/runtime/cooperative.rs rename to crates/executor/src/runtime/cooperative.rs diff --git a/crates/asap-physical-operators/src/runtime/mod.rs b/crates/executor/src/runtime/mod.rs similarity index 100% rename from crates/asap-physical-operators/src/runtime/mod.rs rename to crates/executor/src/runtime/mod.rs diff --git a/crates/asap-physical-operators/src/runtime/tests.rs b/crates/executor/src/runtime/tests.rs similarity index 100% rename from crates/asap-physical-operators/src/runtime/tests.rs rename to crates/executor/src/runtime/tests.rs diff --git a/crates/asap-physical-operators/src/sources/memory.rs b/crates/executor/src/sources/memory.rs similarity index 100% rename from crates/asap-physical-operators/src/sources/memory.rs rename to crates/executor/src/sources/memory.rs diff --git a/crates/asap-physical-operators/src/sources/mod.rs b/crates/executor/src/sources/mod.rs similarity index 98% rename from crates/asap-physical-operators/src/sources/mod.rs rename to crates/executor/src/sources/mod.rs index 9713c2a6f..621fe692b 100644 --- a/crates/asap-physical-operators/src/sources/mod.rs +++ b/crates/executor/src/sources/mod.rs @@ -7,11 +7,9 @@ use crate::{ Error, }; use futures::{stream, StreamExt}; +use planner_types::ir::operator::Source; +use planner_types::ir::schema::{DataType, FieldDataType as SummaryFamilyType}; use planner_types::ir::{NonASAPOp, OperatorNode}; -use planner_types::{ - post_asap::FieldDataType as SummaryFamilyType, - pre_asap::{DataType, Source}, -}; use std::sync::Arc; /// A bound data source. Metadata must be stable for the lifetime of the binding. diff --git a/crates/asap-physical-operators/src/statistic.rs b/crates/executor/src/statistic.rs similarity index 100% rename from crates/asap-physical-operators/src/statistic.rs rename to crates/executor/src/statistic.rs diff --git a/crates/asap-physical-operators/src/summary_kernels/count_min_sketch.rs b/crates/executor/src/summary_kernels/count_min_sketch.rs similarity index 96% rename from crates/asap-physical-operators/src/summary_kernels/count_min_sketch.rs rename to crates/executor/src/summary_kernels/count_min_sketch.rs index af0815c25..20f49f092 100644 --- a/crates/asap-physical-operators/src/summary_kernels/count_min_sketch.rs +++ b/crates/executor/src/summary_kernels/count_min_sketch.rs @@ -1,7 +1,7 @@ //! Count-Min Sketch frequency summary over `asap_sketchlib::CountMinSketch`. use crate::{AggregateCore, KernelError, KeyByLabelValues}; use asap_sketchlib::CountMinSketch; -use planner_types::post_asap::SketchStatistic; +use planner_types::ir::schema::SketchStatistic; #[derive(Debug, Clone)] pub struct CountMinSketchAccumulator { @@ -87,7 +87,7 @@ mod tests { #[test] fn bare_count_reads_total_weight() { let bare_count = SketchStatistic::PointCount { - key: planner_types::pre_asap::ColumnRef::SampleValue, + key: planner_types::ir::scalar::ColumnRef::SampleValue, value: None, }; let mut state = CountMinSketchAccumulator::new(2, 1); diff --git a/crates/asap-physical-operators/src/summary_kernels/datasketches_kll.rs b/crates/executor/src/summary_kernels/datasketches_kll.rs similarity index 98% rename from crates/asap-physical-operators/src/summary_kernels/datasketches_kll.rs rename to crates/executor/src/summary_kernels/datasketches_kll.rs index bc60f8586..24f3bdff6 100644 --- a/crates/asap-physical-operators/src/summary_kernels/datasketches_kll.rs +++ b/crates/executor/src/summary_kernels/datasketches_kll.rs @@ -1,7 +1,7 @@ //! KLL quantile summary over `asap_sketchlib::KllSketch`. use crate::{AggregateCore, KernelError}; use asap_sketchlib::KllSketch; -use planner_types::post_asap::SketchStatistic; +use planner_types::ir::schema::SketchStatistic; #[derive(Clone)] pub struct DatasketchesKLLAccumulator { diff --git a/crates/asap-physical-operators/src/summary_kernels/dd_sketch.rs b/crates/executor/src/summary_kernels/dd_sketch.rs similarity index 96% rename from crates/asap-physical-operators/src/summary_kernels/dd_sketch.rs rename to crates/executor/src/summary_kernels/dd_sketch.rs index d2f21cf29..735effce7 100644 --- a/crates/asap-physical-operators/src/summary_kernels/dd_sketch.rs +++ b/crates/executor/src/summary_kernels/dd_sketch.rs @@ -1,7 +1,7 @@ //! DDSketch quantile summary over `asap_sketchlib::DdSketch`. use crate::{AggregateCore, KernelError}; use asap_sketchlib::DdSketch; -use planner_types::post_asap::SketchStatistic; +use planner_types::ir::schema::SketchStatistic; #[derive(Debug, Clone)] pub struct DDSketchAccumulator { @@ -59,7 +59,7 @@ impl AggregateCore for DDSketchAccumulator { #[cfg(test)] mod tests { use super::*; - use planner_types::pre_asap::ColumnRef; + use planner_types::ir::scalar::ColumnRef; fn bare_count() -> SketchStatistic { SketchStatistic::PointCount { diff --git a/crates/asap-physical-operators/src/summary_kernels/exact.rs b/crates/executor/src/summary_kernels/exact.rs similarity index 99% rename from crates/asap-physical-operators/src/summary_kernels/exact.rs rename to crates/executor/src/summary_kernels/exact.rs index 609338d95..5f64aec53 100644 --- a/crates/asap-physical-operators/src/summary_kernels/exact.rs +++ b/crates/executor/src/summary_kernels/exact.rs @@ -2,7 +2,7 @@ use super::increase::IncreaseAccumulator; use crate::Statistic; use crate::{AggregateCore, KeyByLabelValues, Measurement}; -use planner_types::post_asap::{ExactKind, ExactParams, FieldDataType as SummaryFamilyType}; +use planner_types::ir::schema::{ExactKind, ExactParams, FieldDataType as SummaryFamilyType}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; diff --git a/crates/asap-physical-operators/src/summary_kernels/factory.rs b/crates/executor/src/summary_kernels/factory.rs similarity index 96% rename from crates/asap-physical-operators/src/summary_kernels/factory.rs rename to crates/executor/src/summary_kernels/factory.rs index 4e6c06c1f..be9c21d5c 100644 --- a/crates/asap-physical-operators/src/summary_kernels/factory.rs +++ b/crates/executor/src/summary_kernels/factory.rs @@ -6,7 +6,9 @@ use crate::summary_kernels::{ HydraKllSketchAccumulator, }; use crate::{AggregateCore, KeyByLabelValues}; -use planner_types::post_asap::{FieldDataType as SummaryFamilyType, SketchAlgorithm, SketchParams}; +use planner_types::ir::schema::{ + FieldDataType as SummaryFamilyType, SketchAlgorithm, SketchParams, +}; /// Generate the clone-based `AccumulatorUpdater` methods for updaters whose /// inner `acc` field implements `Clone + AggregateCore`. @@ -381,15 +383,15 @@ fn cms_dims(params: &SketchParams) -> (usize, usize) { /// tags participate in this dispatch and unsupported payloads are errors. pub fn create_planner_accumulator( family: &SummaryFamilyType, - input: &planner_types::post_asap::SummaryUpdate, - grouping: &planner_types::post_asap::GroupingStrategy, + input: &planner_types::ir::schema::SummaryUpdate, + grouping: &planner_types::ir::schema::GroupingStrategy, ) -> Result, String> { if input.item.is_some() && matches!( input.weight_domain, - planner_types::post_asap::WeightDomain::NonNegative { + planner_types::ir::schema::WeightDomain::NonNegative { proof: - planner_types::post_asap::NonNegativeWeightProof::ResetAwareCounterDerivative + planner_types::ir::schema::NonNegativeWeightProof::ResetAwareCounterDerivative } ) { @@ -397,7 +399,7 @@ pub fn create_planner_accumulator( } crate::capability::validate_summary_kernel(family, input, grouping)?; - use planner_types::post_asap::GroupingStrategy; + use planner_types::ir::schema::GroupingStrategy; if grouping != &GroupingStrategy::PerSubpopulationInstance { return Err( "shared grouping has one state for all groups; build it with `create_hydra_cms`".into(), @@ -419,7 +421,7 @@ pub fn create_planner_accumulator( } let updater: Box = match (kind.algorithm(), kind.params()) { (SketchAlgorithm::Kll, SketchParams::Kll { k, .. }) => { - if *k > planner_types::post_asap::sketch::KLL_MAX_K { + if *k > planner_types::ir::schema::sketch::KLL_MAX_K { return Err("KLL k exceeds runtime bound".into()); } Box::new(KllAccumulatorUpdater::new( @@ -583,7 +585,7 @@ impl AccumulatorUpdater for UnivMonUpdater { #[cfg(test)] mod planner_parameter_regression { use super::*; - use planner_types::post_asap::{SketchKind, SummaryInputExpr, SummaryUpdate}; + use planner_types::ir::schema::{SketchKind, SummaryInputExpr, SummaryUpdate}; // Planner width is the bucket count; depth is the independent hash-row count. #[test] @@ -626,7 +628,7 @@ mod planner_parameter_regression { ); let update = SummaryUpdate { item: Some(SummaryInputExpr::Column( - planner_types::pre_asap::ColumnRef::Named("host".into()), + planner_types::ir::scalar::ColumnRef::Named("host".into()), )), weight: SummaryInputExpr::Constant(1.0), weight_domain: Default::default(), @@ -662,7 +664,7 @@ mod planner_parameter_regression { // a per-group updater, and its family must be the matching Count-Min. #[test] fn hydra_cms_builds_one_shared_grid() { - use planner_types::post_asap::{ + use planner_types::ir::schema::{ GroupingStrategy, HydraKind, HydraParams, NonNegativeWeightProof, WeightDomain, }; let grouping = |width| GroupingStrategy::SharedMultiSubpopulation { @@ -693,7 +695,7 @@ mod planner_parameter_regression { assert!(create_hydra_cms(&family(64)).is_err()); let update = SummaryUpdate { item: Some(SummaryInputExpr::Column( - planner_types::pre_asap::ColumnRef::Named("host".into()), + planner_types::ir::scalar::ColumnRef::Named("host".into()), )), weight: SummaryInputExpr::Constant(1.0), weight_domain: WeightDomain::NonNegative { diff --git a/crates/asap-physical-operators/src/summary_kernels/hll_sketch.rs b/crates/executor/src/summary_kernels/hll_sketch.rs similarity index 98% rename from crates/asap-physical-operators/src/summary_kernels/hll_sketch.rs rename to crates/executor/src/summary_kernels/hll_sketch.rs index 993f9d9b6..166d8c2db 100644 --- a/crates/asap-physical-operators/src/summary_kernels/hll_sketch.rs +++ b/crates/executor/src/summary_kernels/hll_sketch.rs @@ -1,7 +1,7 @@ //! HyperLogLog distinct-count summary over `asap_sketchlib::HllSketch`. use crate::{AggregateCore, KernelError}; use asap_sketchlib::{HllSketch, HllVariant}; -use planner_types::post_asap::SketchStatistic; +use planner_types::ir::schema::SketchStatistic; #[derive(Debug, Clone)] pub struct HllSketchAccumulator { diff --git a/crates/asap-physical-operators/src/summary_kernels/hydra_cms.rs b/crates/executor/src/summary_kernels/hydra_cms.rs similarity index 98% rename from crates/asap-physical-operators/src/summary_kernels/hydra_cms.rs rename to crates/executor/src/summary_kernels/hydra_cms.rs index e3393bab9..85cea5394 100644 --- a/crates/asap-physical-operators/src/summary_kernels/hydra_cms.rs +++ b/crates/executor/src/summary_kernels/hydra_cms.rs @@ -10,7 +10,7 @@ //! does not offer, so it is not answered here. use crate::{AggregateCore, KernelError}; use asap_sketchlib::{input::HydraCounter, CountMin, DataInput, FastPath, Hydra, Vector2D}; -use planner_types::post_asap::SketchStatistic; +use planner_types::ir::schema::SketchStatistic; use std::sync::Arc; /// The single Hydra key column: a group's whole label tuple is one @@ -246,7 +246,7 @@ mod tests { group: group.into(), }; let checkout = SketchStatistic::PointCount { - key: planner_types::pre_asap::ColumnRef::Named("item".into()), + key: planner_types::ir::scalar::ColumnRef::Named("item".into()), value: Some("checkout".into()), }; assert_eq!(view("batch").estimate(&checkout).unwrap(), 1.0); @@ -254,7 +254,7 @@ mod tests { assert_eq!(merged.estimate(&checkout).unwrap(), 2.0); assert!(view("api").merge_with(&view("batch")).is_err()); let total = SketchStatistic::PointCount { - key: planner_types::pre_asap::ColumnRef::SampleValue, + key: planner_types::ir::scalar::ColumnRef::SampleValue, value: None, }; assert!(view("api").estimate(&total).is_err()); diff --git a/crates/asap-physical-operators/src/summary_kernels/hydra_kll.rs b/crates/executor/src/summary_kernels/hydra_kll.rs similarity index 100% rename from crates/asap-physical-operators/src/summary_kernels/hydra_kll.rs rename to crates/executor/src/summary_kernels/hydra_kll.rs diff --git a/crates/asap-physical-operators/src/summary_kernels/increase.rs b/crates/executor/src/summary_kernels/increase.rs similarity index 100% rename from crates/asap-physical-operators/src/summary_kernels/increase.rs rename to crates/executor/src/summary_kernels/increase.rs diff --git a/crates/asap-physical-operators/src/summary_kernels/kmv.rs b/crates/executor/src/summary_kernels/kmv.rs similarity index 98% rename from crates/asap-physical-operators/src/summary_kernels/kmv.rs rename to crates/executor/src/summary_kernels/kmv.rs index a1153b9de..8ca1a0891 100644 --- a/crates/asap-physical-operators/src/summary_kernels/kmv.rs +++ b/crates/executor/src/summary_kernels/kmv.rs @@ -6,7 +6,7 @@ use crate::summary_kernels::factory::AccumulatorUpdater; use crate::{AggregateCore, KernelError, KeyByLabelValues}; use asap_sketchlib::{DataInput, KMV}; -use planner_types::post_asap::SketchStatistic; +use planner_types::ir::schema::SketchStatistic; /// The smallest `k` the estimate `(k - 1) / U(k)` is defined for. pub const KMV_MIN_K: u32 = 2; @@ -130,7 +130,7 @@ mod tests { assert_eq!(cardinality(&kmv), 500.0); assert_eq!( kmv.estimate(&SketchStatistic::PointCount { - key: planner_types::pre_asap::ColumnRef::SampleValue, + key: planner_types::ir::scalar::ColumnRef::SampleValue, value: None, }) .unwrap(), diff --git a/crates/asap-physical-operators/src/summary_kernels/mod.rs b/crates/executor/src/summary_kernels/mod.rs similarity index 100% rename from crates/asap-physical-operators/src/summary_kernels/mod.rs rename to crates/executor/src/summary_kernels/mod.rs diff --git a/crates/asap-physical-operators/src/summary_kernels/traits.rs b/crates/executor/src/summary_kernels/traits.rs similarity index 95% rename from crates/asap-physical-operators/src/summary_kernels/traits.rs rename to crates/executor/src/summary_kernels/traits.rs index 3ca12f4b0..2bbf707ea 100644 --- a/crates/asap-physical-operators/src/summary_kernels/traits.rs +++ b/crates/executor/src/summary_kernels/traits.rs @@ -1,4 +1,4 @@ -use planner_types::post_asap::SketchStatistic; +use planner_types::ir::schema::SketchStatistic; pub type KernelError = Box; @@ -69,7 +69,7 @@ mod tests { .unwrap(); dd.inner.update(3.0); let count = SketchStatistic::PointCount { - key: planner_types::pre_asap::ColumnRef::SampleValue, + key: planner_types::ir::scalar::ColumnRef::SampleValue, value: None, }; assert_eq!(state.estimate(&count).unwrap(), 1.0); diff --git a/crates/asap-physical-operators/src/summary_kernels/univmon.rs b/crates/executor/src/summary_kernels/univmon.rs similarity index 99% rename from crates/asap-physical-operators/src/summary_kernels/univmon.rs rename to crates/executor/src/summary_kernels/univmon.rs index c647be0a1..e47e2c846 100644 --- a/crates/asap-physical-operators/src/summary_kernels/univmon.rs +++ b/crates/executor/src/summary_kernels/univmon.rs @@ -2,7 +2,8 @@ use crate::AggregateCore; use asap_sketchlib::{DataInput, UnivMon}; -use planner_types::{post_asap::SketchStatistic, pre_asap::ColumnRef}; +use planner_types::ir::scalar::ColumnRef; +use planner_types::ir::schema::SketchStatistic; type Error = Box; diff --git a/crates/asap-physical-operators/src/summary_kernels/weighted_frequency.rs b/crates/executor/src/summary_kernels/weighted_frequency.rs similarity index 98% rename from crates/asap-physical-operators/src/summary_kernels/weighted_frequency.rs rename to crates/executor/src/summary_kernels/weighted_frequency.rs index b6134db31..b3a5e4b8e 100644 --- a/crates/asap-physical-operators/src/summary_kernels/weighted_frequency.rs +++ b/crates/executor/src/summary_kernels/weighted_frequency.rs @@ -45,9 +45,9 @@ pub struct WeightedFrequency { } impl WeightedFrequency { pub(crate) fn configuration( - kind: &planner_types::post_asap::SketchKind, + kind: &planner_types::ir::schema::SketchKind, ) -> Result<(FrequencyAlgorithm, usize, usize, usize), Error> { - use planner_types::post_asap::{SketchAlgorithm as A, SketchParams as P}; + use planner_types::ir::schema::{SketchAlgorithm as A, SketchParams as P}; let (algorithm, width, depth, capacity) = match (kind.algorithm(), kind.params()) { ( A::CmsWithHeap, diff --git a/crates/asap-physical-operators/src/values.rs b/crates/executor/src/values.rs similarity index 97% rename from crates/asap-physical-operators/src/values.rs rename to crates/executor/src/values.rs index 0c5c25ad9..84a77ccf6 100644 --- a/crates/asap-physical-operators/src/values.rs +++ b/crates/executor/src/values.rs @@ -1,9 +1,8 @@ //! Runtime values preserve Planner schemas; summary states are typed values too. use crate::AggregateCore; use crate::Error; -use planner_types::{ - post_asap::{Field as SummaryField, FieldDataType as SummaryFamilyType, Schema}, - pre_asap::DataType, +use planner_types::ir::schema::{ + DataType, Field as SummaryField, FieldDataType as SummaryFamilyType, Schema, }; use std::{cmp::Ordering, sync::Arc}; /// Shared ownership of schema metadata; the field model is identical at planning @@ -279,7 +278,7 @@ fn validate_state(family: &SummaryFamilyType, state: &dyn AggregateCore) -> Resu dd_sketch::DDSketchAccumulator, exact::ExactAccumulator, hll_sketch::HllSketchAccumulator, kmv::KmvAccumulator, univmon::UnivMonAccumulator, }; - use planner_types::post_asap::SketchParams; + use planner_types::ir::schema::SketchParams; validate_family(family)?; let valid = match family { SummaryFamilyType::Sketch(kind, _) @@ -299,7 +298,7 @@ fn validate_state(family: &SummaryFamilyType, state: &dyn AggregateCore) -> Resu } SummaryFamilyType::Sketch(_, grouping) if grouping - != &planner_types::post_asap::GroupingStrategy::PerSubpopulationInstance => + != &planner_types::ir::schema::GroupingStrategy::PerSubpopulationInstance => { let shape = crate::capability::hydra_cms_shape(family).map_err(Error::Invalid)?; state @@ -407,7 +406,7 @@ pub(crate) fn plain(schema: &SchemaRef, column: usize) -> Result<(&DataType, boo mod weighted_state_tests { use super::*; use crate::summary_kernels::weighted_frequency::{FrequencyAlgorithm, WeightedFrequency}; - use planner_types::post_asap::{SketchAlgorithm, SketchKind, SketchParams}; + use planner_types::ir::schema::{SketchAlgorithm, SketchKind, SketchParams}; // A state cannot acquire a different family or shape merely by relabeling its batch. #[test] diff --git a/crates/asap-physical-operators/tests/blocking_resources.rs b/crates/executor/tests/blocking_resources.rs similarity index 94% rename from crates/asap-physical-operators/tests/blocking_resources.rs rename to crates/executor/tests/blocking_resources.rs index 24bc1244b..cf7309a36 100644 --- a/crates/asap-physical-operators/tests/blocking_resources.rs +++ b/crates/executor/tests/blocking_resources.rs @@ -1,5 +1,5 @@ //! Blocking operators enforce resources before returning their first batch. -use asap_physical_operators::{ +use asap_executor::{ operators::Operator, plan::{PhysicalDAG, PhysicalOperator}, runtime::{Limits, RunContext, Scope}, @@ -7,16 +7,15 @@ use asap_physical_operators::{ Error, }; use futures::{executor::block_on, FutureExt, StreamExt}; +use planner_types::ir::operator::JoinKind; +use planner_types::ir::scalar::ScalarValue; +use planner_types::ir::schema::{DataType, Field, FieldDataType}; use planner_types::ir::Predicate; use planner_types::ir::ScalarExpr; -use planner_types::{ - post_asap::{Field, FieldDataType}, - pre_asap::{DataType, JoinKind, ScalarValue}, -}; use std::sync::Arc; fn schema(width: usize) -> SchemaRef { - Arc::new(planner_types::pre_asap::Schema { + Arc::new(planner_types::ir::schema::Schema { unique_keys: vec![], closed: false, fields: (0..width) @@ -108,7 +107,7 @@ fn join_yields_during_computation_and_observes_cancellation() { // Sorting and grouping yield even for one large batch. #[test] fn blocking_reductions_yield_and_release_memory_on_cancellation() { - use asap_physical_operators::{ + use asap_executor::{ operators::{Reduction, SortKey}, plan::PhysicalOperator, }; @@ -144,7 +143,7 @@ fn blocking_reductions_yield_and_release_memory_on_cancellation() { // Merge-sort rounds preserve input order for tied keys across chunk boundaries. #[test] fn cooperative_sort_preserves_ties_across_chunks() { - use asap_physical_operators::operators::SortKey; + use asap_executor::operators::SortKey; let batch = Batch::try_new( schema(2), (0..1025) @@ -188,9 +187,9 @@ fn cooperative_sort_preserves_ties_across_chunks() { // The integrated weighted-summary path obeys the same cooperative cancellation contract. #[test] fn weighted_summary_build_yields_within_a_batch() { - use planner_types::post_asap::{SketchAlgorithm, SketchKind, SketchParams}; + use planner_types::ir::schema::{SketchAlgorithm, SketchKind, SketchParams}; - let input = Arc::new(planner_types::pre_asap::Schema { + let input = Arc::new(planner_types::ir::schema::Schema { unique_keys: vec![], closed: false, fields: vec![ @@ -253,7 +252,7 @@ fn weighted_summary_build_yields_within_a_batch() { // their reservations on failure. #[test] fn frequency_dictionary_enforces_memory_budget() { - use asap_physical_operators::operators::Reduction; + use asap_executor::operators::Reduction; let input = schema(1); let mut sources = PhysicalDAG::default(); sources diff --git a/crates/asap-physical-operators/tests/common/mod.rs b/crates/executor/tests/common/mod.rs similarity index 87% rename from crates/asap-physical-operators/tests/common/mod.rs rename to crates/executor/tests/common/mod.rs index b1e894942..9bd25ef46 100644 --- a/crates/asap-physical-operators/tests/common/mod.rs +++ b/crates/executor/tests/common/mod.rs @@ -21,7 +21,7 @@ pub fn compile_physical_asap_dag( /// with the built-in models. pub fn selected_dag( root: Rc, - accuracy: planner_types::types::AccuracyTarget, + accuracy: planner_types::ir::properties::AccuracyTarget, ) -> Rc { let run = run_stage1_to_stage3(root, accuracy); let [QueryRoot::Operator(selected)] = run.selected_logical().expect("a plan is selected") @@ -36,7 +36,7 @@ pub fn selected_dag( /// compose panics, so none is silently skipped. pub fn logical_candidates( root: Rc, - accuracy: planner_types::types::AccuracyTarget, + accuracy: planner_types::ir::properties::AccuracyTarget, ) -> Vec> { run_stage1_to_stage3(root, accuracy) .stage1_logical_asap_optimization @@ -56,8 +56,8 @@ pub fn logical_candidates( fn run_stage1_to_stage3( root: Rc, - accuracy: planner_types::types::AccuracyTarget, -) -> asap_aware_mapping::Stage1ToStage3Run { + accuracy: planner_types::ir::properties::AccuracyTarget, +) -> asap_plan_selection::Stage1ToStage3Run { use planner_types::workload::{DataArrival, DataWorkload}; let data = DataWorkload { arrival: DataArrival::Ingesting { @@ -65,7 +65,7 @@ fn run_stage1_to_stage3( }, ..Default::default() }; - asap_aware_mapping::run_stage1_to_stage3( + asap_plan_selection::run_stage1_to_stage3( vec![(0, QueryRoot::Operator(root))], &[planner_types::workload::QueryWorkloadEntry { requirements: planner_types::workload::QueryRequirements { @@ -75,7 +75,7 @@ fn run_stage1_to_stage3( ..Default::default() }], &data, - asap_aware_mapping::PlanningModels::builtin(), + asap_plan_selection::PlanningModels::builtin(), ) .expect("Pass 1 accepts the query") } diff --git a/crates/asap-physical-operators/tests/current_series_heap.rs b/crates/executor/tests/current_series_heap.rs similarity index 93% rename from crates/asap-physical-operators/tests/current_series_heap.rs rename to crates/executor/tests/current_series_heap.rs index 8a4767f89..0d89edd8e 100644 --- a/crates/asap-physical-operators/tests/current_series_heap.rs +++ b/crates/executor/tests/current_series_heap.rs @@ -1,6 +1,6 @@ //! Spatial heap weights come from a fresh instant vector, never sample history. mod common; -use asap_physical_operators::{ +use asap_executor::{ operators::Operator, physical_planner::{ promql_rows::{decode_series_identity, series_row, SERIES_IDENTITY_COLUMN}, @@ -12,12 +12,12 @@ use asap_physical_operators::{ use common::compile_physical_asap_dag; use futures::{executor::block_on, StreamExt}; use planner_types::ir::physical_export::PhysicalASAPOperatorPayload; -use planner_types::types::AccuracyTarget; -use planner_types::{post_asap::*, pre_asap::DataType}; +use planner_types::ir::properties::AccuracyTarget; +use planner_types::ir::schema::{DataType, *}; use std::{collections::BTreeMap, sync::Arc}; fn schema() -> Arc { - Arc::new(planner_types::pre_asap::Schema { + Arc::new(planner_types::ir::schema::Schema { unique_keys: vec![], closed: false, fields: [ @@ -172,7 +172,7 @@ fn spatial_heap_ranks_latest_values_in_independent_runs() { }; let family = FieldDataType::Sketch(SketchKind::new(algorithm, params), Default::default()); let build = Operator::keyed_summary_build(schema(), family, 1, vec![3], vec![2]).unwrap(); - let output = Arc::new(planner_types::pre_asap::Schema { + let output = Arc::new(planner_types::ir::schema::Schema { unique_keys: vec![], closed: false, fields: vec![ @@ -234,7 +234,7 @@ fn spatial_heap_ranks_latest_values_in_independent_runs() { // Blocking membership selection shares the run's cancellation and byte budget. #[test] fn current_series_observes_resource_limits() { - use asap_physical_operators::Error; + use asap_executor::Error; let plan = snapshot_plan(); for cancelled in [false, true] { let data = input(&[("one", 50_000, 1.)]); @@ -273,7 +273,7 @@ fn current_series_observes_resource_limits() { #[test] fn identity_encoding_is_lossless_and_rejects_noncanonical_inputs() { - use asap_physical_operators::physical_planner::promql_rows::encode_series_identity; + use asap_executor::physical_planner::promql_rows::encode_series_identity; let labels = BTreeMap::from([ ("a".into(), "quote\"slash\\".into()), ("other".into(), "".into()), @@ -296,7 +296,7 @@ fn identity_encoding_is_lossless_and_rejects_noncanonical_inputs() { // test does not manually assemble the computation or its dependency edges. #[test] fn planner_current_series_candidate_compiles_with_dynamic_identity() { - use asap_physical_operators::physical_planner::{compile, promql_rows::with_series_identity}; + use asap_executor::physical_planner::{compile, promql_rows::with_series_identity}; use planner_types::workload::*; use std::rc::Rc; let workload = PlanningWorkload { @@ -324,13 +324,13 @@ fn planner_current_series_candidate_compiles_with_dynamic_identity() { .remove(0); let open_root = Rc::new(original.clone()); let open_selected = - asap_aware_mapping::maintained_population::MaintainedPopulationStrategy::new( + asap_logical_asap_optimization::maintained_population::MaintainedPopulationStrategy::new( std::slice::from_ref(&open_root), ) .candidate(&open_root) .unwrap(); let snapshot_program = - asap_physical_operators::physical_planner::promql_rows::compile_current_series_evaluation( + asap_executor::physical_planner::promql_rows::compile_current_series_evaluation( &open_selected, ) .unwrap(); @@ -342,11 +342,12 @@ fn planner_current_series_candidate_compiles_with_dynamic_identity() { assert!(encoded.contains("Sort") && encoded.contains("Limit")); assert_eq!(snapshot_program.input_contracts().count(), 1); let root = Rc::new(with_series_identity(&original).unwrap()); - let selected = asap_aware_mapping::maintained_population::MaintainedPopulationStrategy::new( - std::slice::from_ref(&root), - ) - .candidate(&root) - .unwrap(); + let selected = + asap_logical_asap_optimization::maintained_population::MaintainedPopulationStrategy::new( + std::slice::from_ref(&root), + ) + .candidate(&root) + .unwrap(); let logical = compile_physical_asap_dag(&selected).unwrap(); let raw = logical .nodes diff --git a/crates/asap-physical-operators/tests/deployment.rs b/crates/executor/tests/deployment.rs similarity index 90% rename from crates/asap-physical-operators/tests/deployment.rs rename to crates/executor/tests/deployment.rs index e1779c074..857cb5d58 100644 --- a/crates/asap-physical-operators/tests/deployment.rs +++ b/crates/executor/tests/deployment.rs @@ -1,11 +1,11 @@ //! Exercise the public library without a backend server, store, or scheduler. -use asap_physical_operators::planner::{ - post_asap::{ +use asap_executor::planner::{ + ir::scalar::ColumnRef, + ir::schema::{ FieldDataType, GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, SummaryUpdate, }, - pre_asap::ColumnRef, }; -use asap_physical_operators::{factory::create_planner_accumulator, AggregateCore}; +use asap_executor::{factory::create_planner_accumulator, AggregateCore}; fn family(k: u32) -> FieldDataType { FieldDataType::Sketch( @@ -34,9 +34,7 @@ fn build(values: &[f64]) -> Box { } fn read(state: &dyn AggregateCore) -> f64 { state - .estimate( - &asap_physical_operators::planner::post_asap::SketchStatistic::Quantile { q: 0.5 }, - ) + .estimate(&asap_executor::planner::ir::schema::SketchStatistic::Quantile { q: 0.5 }) .unwrap() } @@ -70,8 +68,8 @@ fn invalid_kll_parameters_are_rejected_at_binding() { // a packed-wire column-bit budget must not be imposed on this constructor. #[test] fn native_count_sketch_dimensions_are_not_packed_wire_dimensions() { - use asap_physical_operators::planner::post_asap::SummaryInputExpr; - use asap_physical_operators::KeyByLabelValues; + use asap_executor::planner::ir::schema::SummaryInputExpr; + use asap_executor::KeyByLabelValues; let family = FieldDataType::Sketch( SketchKind::new( SketchAlgorithm::CountSketchWithHeap, @@ -91,11 +89,11 @@ fn native_count_sketch_dimensions_are_not_packed_wire_dimensions() { let state = operator.into_accumulator(); let state = state .as_any() - .downcast_ref::() + .downcast_ref::() .unwrap(); assert_eq!( state - .point_estimate(&[asap_physical_operators::values::Value::Utf8("a".into())]) + .point_estimate(&[asap_executor::values::Value::Utf8("a".into())]) .unwrap(), 7.0 ); @@ -111,8 +109,8 @@ fn plain_count_sketch(width: u32, depth: u32) -> FieldDataType { ) } -fn host(i: usize) -> asap_physical_operators::KeyByLabelValues { - asap_physical_operators::KeyByLabelValues::new_with_labels(vec![format!("host-{i}")]) +fn host(i: usize) -> asap_executor::KeyByLabelValues { + asap_executor::KeyByLabelValues::new_with_labels(vec![format!("host-{i}")]) } /// Build a plain CountSketch over unit events, one per key index. @@ -128,7 +126,7 @@ fn build_weighted_frequency( family: &FieldDataType, updates: impl IntoIterator, ) -> Box { - use asap_physical_operators::planner::post_asap::SummaryInputExpr; + use asap_executor::planner::ir::schema::SummaryInputExpr; let mut update = SummaryUpdate::column(ColumnRef::SampleValue); update.item = Some(SummaryInputExpr::Column(ColumnRef::Named("host".into()))); let mut operator = create_planner_accumulator(family, &update, &Default::default()).unwrap(); @@ -140,9 +138,7 @@ fn build_weighted_frequency( } fn plain_estimate(state: &dyn AggregateCore, i: usize) -> f64 { - use asap_physical_operators::{ - summary_kernels::weighted_frequency::WeightedFrequency, values::Value, - }; + use asap_executor::{summary_kernels::weighted_frequency::WeightedFrequency, values::Value}; state .as_any() .downcast_ref::() @@ -175,7 +171,7 @@ fn assert_rows_hash_independently(family: &FieldDataType, width: usize, depth: u ); let state = state .as_any() - .downcast_ref::() + .downcast_ref::() .expect("the per-row-seeded weighted frequency kernel"); let cells = serde_json::to_value(state).unwrap()["cells"].clone(); for (index, cell) in cells.as_array().unwrap().iter().enumerate() { @@ -257,8 +253,8 @@ fn deep_heap_sketch_rows_hash_independently() { // summary value, and it keeps fractional weights exactly. #[test] fn heap_sketch_states_load_as_dag_summary_values() { - use asap_physical_operators::{ - planner::post_asap::{Field, Schema}, + use asap_executor::{ + planner::ir::schema::{Field, Schema}, summary_kernels::weighted_frequency::WeightedFrequency, values::{Batch, Value}, }; @@ -299,9 +295,7 @@ fn heap_sketch_states_load_as_dag_summary_values() { // interchangeable: merged either way, they equal one build over both inputs. #[test] fn factory_and_dag_heap_states_are_interchangeable() { - use asap_physical_operators::{ - summary_kernels::weighted_frequency::WeightedFrequency, values::Value, - }; + use asap_executor::{summary_kernels::weighted_frequency::WeightedFrequency, values::Value}; use asap_sketchlib::FrequencyAlgorithm; let estimate = |state: &dyn AggregateCore, i: usize| { state @@ -342,7 +336,7 @@ fn factory_and_dag_heap_states_are_interchangeable() { // validation, never silently dropped. #[test] fn count_min_heap_rejects_negative_inputs() { - use asap_physical_operators::planner::post_asap::SummaryInputExpr; + use asap_executor::planner::ir::schema::SummaryInputExpr; let mut update = SummaryUpdate::column(ColumnRef::SampleValue); update.item = Some(SummaryInputExpr::Column(ColumnRef::Named("host".into()))); let family = heap_sketch(SketchAlgorithm::CmsWithHeap, 64, 3, 4); @@ -386,7 +380,7 @@ fn plain_count_sketch_merge_equals_union_build() { serde_json::to_value( state .as_any() - .downcast_ref::() + .downcast_ref::() .unwrap(), ) .unwrap()["cells"] diff --git a/crates/asap-physical-operators/tests/deployment_computation.rs b/crates/executor/tests/deployment_computation.rs similarity index 96% rename from crates/asap-physical-operators/tests/deployment_computation.rs rename to crates/executor/tests/deployment_computation.rs index d8f0686f3..6fe112fd1 100644 --- a/crates/asap-physical-operators/tests/deployment_computation.rs +++ b/crates/executor/tests/deployment_computation.rs @@ -1,7 +1,7 @@ //! Planner-selected PromQL computation compiles from the timed DAG alone; //! the deployment supplies only raw rows at the ingestion frontier. mod common; -use asap_physical_operators::{ +use asap_executor::{ operators::Operator, physical_planner::{compile, promql_rows, CompiledPhysicalDAG, InputContract, Source}, runtime::{Limits, RunContext, Scope}, @@ -10,8 +10,10 @@ use asap_physical_operators::{ use common::{compile_physical_asap_dag, logical_candidates, selected_dag}; use futures::{executor::block_on, StreamExt}; use planner_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPOperatorPayload}; +use planner_types::ir::properties::AccuracyTarget; +use planner_types::ir::schema::*; use planner_types::ir::ASAPOp; -use planner_types::{post_asap::*, types::AccuracyTarget, workload::*}; +use planner_types::workload::*; use std::{collections::BTreeMap, rc::Rc, sync::Arc}; fn lower(query: &str) -> Rc { @@ -53,11 +55,12 @@ fn exact_dag(query: &str) -> PhysicalASAPDAG { fn population_dag(query: &str) -> PhysicalASAPDAG { let root = promql_rows::with_series_identity(&lower(query)).unwrap(); - let selected = asap_aware_mapping::maintained_population::MaintainedPopulationStrategy::new( - std::slice::from_ref(&root), - ) - .candidate(&root) - .unwrap(); + let selected = + asap_logical_asap_optimization::maintained_population::MaintainedPopulationStrategy::new( + std::slice::from_ref(&root), + ) + .candidate(&root) + .unwrap(); compile_physical_asap_dag(&selected).unwrap() } @@ -74,7 +77,7 @@ fn raw_inputs(dag: &PhysicalASAPDAG) -> Vec<(u64, Arc, String)> { let n = dag.nodes.iter().find(|n| n.id == id)?; if let PhysicalASAPOperatorPayload::NonASAP( planner_types::ir::NonASAPOp::Scan { - source: planner_types::pre_asap::Source::TimeSeries { metric }, + source: planner_types::ir::operator::Source::TimeSeries { metric }, .. }, ) = &n.payload @@ -101,7 +104,7 @@ fn execute( dag: &PhysicalASAPDAG, samples: &[Sample], end: i64, -) -> Result>, String> { +) -> Result>, String> { execute_relabeled(dag, samples, end, &BTreeMap::new()) } @@ -112,7 +115,7 @@ fn execute_relabeled( samples: &[Sample], end: i64, relabel: &BTreeMap<&str, (&str, &str)>, -) -> Result>, String> { +) -> Result>, String> { let inputs = raw_inputs(dag); let program = compile( dag, @@ -321,7 +324,7 @@ fn exact_count_finalizes_to_declared_float_value() { /// `dag` with its Binary operator replaced by `kind`. fn with_kind( mut dag: PhysicalASAPDAG, - kind: planner_types::pre_asap::BinaryOpKind, + kind: planner_types::ir::operator::BinaryOpKind, bool_result: bool, ) -> PhysicalASAPDAG { for node in &mut dag.nodes { @@ -364,7 +367,8 @@ fn grouped_comparisons_filter_or_return_bool() { // drops the metric name; a filter keeps the surviving left value. #[test] fn per_series_comparisons_filter_or_return_bool() { - use planner_types::pre_asap::{BinaryOpKind::*, CompareOpKind::*}; + use planner_types::ir::operator::BinaryOpKind::*; + use planner_types::ir::scalar::CompareOpKind::*; let samples = counter("a", "api", 10., 10.) .chain(counter("a", "db", 10., 10.)) @@ -526,7 +530,7 @@ fn per_series_scalar_arithmetic_rejects_label_sets_equal_without_the_name() { fn with_vector_match( mut dag: PhysicalASAPDAG, - kind: planner_types::pre_asap::VectorMatchKind, + kind: planner_types::ir::operator::VectorMatchKind, labels: &[&str], ) -> PhysicalASAPDAG { for node in &mut dag.nodes { @@ -535,7 +539,7 @@ fn with_vector_match( .. }) = &mut node.payload { - operator.vector_match = Some(planner_types::pre_asap::VectorMatch { + operator.vector_match = Some(planner_types::ir::operator::VectorMatch { kind: kind.clone(), labels: labels.iter().map(|l| l.to_string()).collect(), grouping: None, @@ -549,7 +553,7 @@ fn with_vector_match( // the result's labels; a duplicate match group on either side is an error. #[test] fn per_series_vector_matching_follows_on_and_ignoring() { - use planner_types::pre_asap::VectorMatchKind; + use planner_types::ir::operator::VectorMatchKind; let samples = counter("a", "api", 10., 10.) .chain(counter("b", "api", 5., 5.)) .chain(counter("b", "db", 5., 5.)) @@ -609,7 +613,7 @@ fn population_sums_and_averages_are_compensated() { // returns the sketch's total update weight, including colliding items. #[test] fn stored_count_min_bare_count_compiles_to_a_evaluation() { - use asap_physical_operators::summary_kernels::CountMinSketchAccumulator; + use asap_executor::summary_kernels::CountMinSketchAccumulator; let accuracy = AccuracyTarget::max_error(0.02); let root = lower_with("count(up)", accuracy.clone()); let dag = logical_candidates(root, accuracy) diff --git a/crates/asap-physical-operators/tests/hydra_cms_execution.rs b/crates/executor/tests/hydra_cms_execution.rs similarity index 97% rename from crates/asap-physical-operators/tests/hydra_cms_execution.rs rename to crates/executor/tests/hydra_cms_execution.rs index 4fcb43615..99c0d758e 100644 --- a/crates/asap-physical-operators/tests/hydra_cms_execution.rs +++ b/crates/executor/tests/hydra_cms_execution.rs @@ -1,6 +1,6 @@ //! A Hydra-grouped Count-Min `SummaryAgg` compiles through the physical //! planner and answers each group's item point counts from one shared grid. -use asap_physical_operators::{ +use asap_executor::{ dag::{ operators::Operator, values::{Batch, SchemaRef, Value}, @@ -9,14 +9,16 @@ use asap_physical_operators::{ physical_planner::{compile_node, CompiledPhysicalDAG, InputContract, Source}, }; use futures::{executor::block_on, StreamExt}; +use planner_types::ir::operator::{GroupKeys, Reduction}; use planner_types::ir::physical_export::{ PhysicalASAPDAGNode, PhysicalASAPOperatorPayload as Payload, }; -use planner_types::ir::{ASAPOp, ExprSemantics, Materialization, Predicate, ScalarExpr}; -use planner_types::post_asap::*; -use planner_types::pre_asap::{ - ColumnRef, CompareOpKind, DataType, GroupKeys, Reduction, ScalarValue, -}; +use planner_types::ir::properties::*; +use planner_types::ir::scalar::{ColumnRef, CompareOpKind, ScalarValue}; +use planner_types::ir::schema::DataType; +use planner_types::ir::schema::*; +use planner_types::ir::{ASAPOp, ExprSemantics, Predicate, ScalarExpr}; +use planner_types::physical::Materialization; use std::{collections::BTreeMap, sync::Arc}; const WIDTH: u32 = 64; diff --git a/crates/asap-physical-operators/tests/node_ids.rs b/crates/executor/tests/node_ids.rs similarity index 91% rename from crates/asap-physical-operators/tests/node_ids.rs rename to crates/executor/tests/node_ids.rs index 7fe40f1a0..461498d04 100644 --- a/crates/asap-physical-operators/tests/node_ids.rs +++ b/crates/executor/tests/node_ids.rs @@ -1,9 +1,10 @@ //! Compilation rejects Planner node IDs outside the range its helper IDs avoid. mod common; -use asap_physical_operators::physical_planner::compile; +use asap_executor::physical_planner::compile; use common::compile_physical_asap_dag; +use planner_types::ir::scalar::ScalarValue; +use planner_types::ir::schema::{DataType, Field, Schema}; use planner_types::ir::{NonASAPOp, Operator, OperatorNode, Predicate, ScalarExpr}; -use planner_types::pre_asap::{DataType, Field, ScalarValue, Schema}; use std::collections::BTreeMap; /// A deserialized DAG may carry any ID; one above u32 is an error, not a diff --git a/crates/asap-physical-operators/tests/physical_dag.rs b/crates/executor/tests/physical_dag.rs similarity index 92% rename from crates/asap-physical-operators/tests/physical_dag.rs rename to crates/executor/tests/physical_dag.rs index 9252c4a43..0513fbb18 100644 --- a/crates/asap-physical-operators/tests/physical_dag.rs +++ b/crates/executor/tests/physical_dag.rs @@ -1,5 +1,5 @@ //! Acceptance tests use the library directly, without either backend engine. -use asap_physical_operators::{ +use asap_executor::{ dag::{ operators::{Expression, Operator, Reduction, SortKey}, values::{Batch, SchemaRef, Value}, @@ -12,17 +12,14 @@ use planner_types::ir::physical_export::{ EdgeRole, GroupingEdgeCompatibility, PhysicalASAPDAG, PhysicalASAPDAGEdge, PhysicalASAPDAGNode, PhysicalASAPOperatorPayload, WindowEdgeCompatibility, }; +use planner_types::ir::schema::{DataType, ExactKind, ExactParams, Field, FieldDataType}; use planner_types::ir::ASAPOp; use planner_types::ir::BinaryOperator; use planner_types::ir::NonASAPOp; use planner_types::ir::ScalarExpr; -use planner_types::{ - post_asap::{ExactKind, ExactParams, Field, FieldDataType}, - pre_asap::DataType, -}; use std::sync::Arc; fn schema(fields: &[(&str, DataType, bool)]) -> SchemaRef { - Arc::new(planner_types::pre_asap::Schema { + Arc::new(planner_types::ir::schema::Schema { unique_keys: vec![], closed: false, fields: fields @@ -151,8 +148,8 @@ fn summary_construction_merge_and_evaluation_at_both_phases() { Operator::evaluation( state, 0, - asap_physical_operators::operators::SummaryEvaluation::Exact( - asap_physical_operators::summary_kernels::exact::ExactEvaluation { + asap_executor::operators::SummaryEvaluation::Exact( + asap_executor::summary_kernels::exact::ExactEvaluation { statistic: Statistic::Sum, lookback_ms: None, offset_ms: 0, @@ -307,8 +304,8 @@ fn binding_rejects_unsupported_operations() { assert!(Operator::evaluation( sum.schema(), 0, - asap_physical_operators::operators::SummaryEvaluation::Sketch( - planner_types::post_asap::SketchStatistic::Quantile { q: 0.5 } + asap_executor::operators::SummaryEvaluation::Sketch( + planner_types::ir::schema::SketchStatistic::Quantile { q: 0.5 } ) ) .is_err()); @@ -318,7 +315,7 @@ fn binding_rejects_unsupported_operations() { // KLL is one family example: precomputation changes input sources, not operators. #[test] fn kll_raw_partial_and_precomputed_are_native_dags() { - use planner_types::post_asap::{GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams}; + use planner_types::ir::schema::{GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams}; let input = schema(&[("value", DataType::Float64, false)]); let family = FieldDataType::Sketch( @@ -415,8 +412,8 @@ fn kll_raw_partial_and_precomputed_are_native_dags() { Operator::evaluation( state.clone(), 0, - asap_physical_operators::operators::SummaryEvaluation::Sketch( - planner_types::post_asap::SketchStatistic::Quantile { q: 0.5 }, + asap_executor::operators::SummaryEvaluation::Sketch( + planner_types::ir::schema::SketchStatistic::Quantile { q: 0.5 }, ), ) .unwrap(), @@ -435,11 +432,11 @@ fn kll_raw_partial_and_precomputed_are_native_dags() { // Exact state must match its declared family; a mislabeled state is rejected. #[test] fn exact_state_and_family_validation() { - use asap_physical_operators::summary_kernels::exact::ExactAccumulator; + use asap_executor::summary_kernels::exact::ExactAccumulator; let family = FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum); let mut acc = ExactAccumulator::new(family.clone(), false).unwrap(); acc.update(None, 7., 0); - let schema = Arc::new(planner_types::pre_asap::Schema { + let schema = Arc::new(planner_types::ir::schema::Schema { unique_keys: vec![], closed: false, fields: vec![Field { @@ -471,8 +468,8 @@ fn exact_state_and_family_validation() { Operator::evaluation( schema.clone(), 0, - asap_physical_operators::operators::SummaryEvaluation::Exact( - asap_physical_operators::summary_kernels::exact::ExactEvaluation { + asap_executor::operators::SummaryEvaluation::Exact( + asap_executor::summary_kernels::exact::ExactEvaluation { statistic: Statistic::Sum, lookback_ms: None, offset_ms: 0, @@ -501,11 +498,9 @@ fn exact_state_and_family_validation() { // Planner binding rejects unknown computation instead of accepting a fallback. #[test] fn bind_post_asap_before_execution() { - use asap_physical_operators::dag::planner::bind; - use planner_types::{ - post_asap::ExecutionDataState, - pre_asap::{ArithmeticOpKind, ScalarValue}, - }; + use asap_executor::dag::planner::bind; + use planner_types::ir::properties::ExecutionDataState; + use planner_types::ir::scalar::{ArithmeticOpKind, ScalarValue}; use std::collections::BTreeMap; let schema = schema(&[("value", DataType::Float64, false)]); let node = |id, payload| PhysicalASAPDAGNode { @@ -522,7 +517,7 @@ fn bind_post_asap_before_execution() { 0, PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Values { rows: vec![vec![planner_types::ir::ScalarExpr::Literal( - planner_types::pre_asap::ScalarValue::Float64(1.), + planner_types::ir::scalar::ScalarValue::Float64(1.), )]], schema: (*schema).clone(), }), @@ -558,7 +553,7 @@ fn bind_post_asap_before_execution() { roots: vec![1], scalar_roots: vec![], }; - let sources = || -> BTreeMap> { + let sources = || -> BTreeMap> { BTreeMap::from([( 0, Box::new( @@ -567,7 +562,7 @@ fn bind_post_asap_before_execution() { vec![Batch::try_new(schema.clone(), vec![vec![Value::Float64(1.)]]).unwrap()], ) .unwrap(), - ) as asap_physical_operators::dag::planner::Source<'static>, + ) as asap_executor::dag::planner::Source<'static>, )]) }; let native = bind(&dag, sources(), &[1]).unwrap(); @@ -597,8 +592,8 @@ fn empty_exact_count_is_an_integer_state_evaluation() { let read = Operator::evaluation( build.schema(), 0, - asap_physical_operators::operators::SummaryEvaluation::Exact( - asap_physical_operators::summary_kernels::exact::ExactEvaluation { + asap_executor::operators::SummaryEvaluation::Exact( + asap_executor::summary_kernels::exact::ExactEvaluation { statistic: Statistic::Count, lookback_ms: None, offset_ms: 0, @@ -617,8 +612,8 @@ fn empty_exact_count_is_an_integer_state_evaluation() { // A deployment source cannot pass a different row shape to bound expressions. #[test] fn source_batches_must_match_the_bound_schema() { - use asap_physical_operators::dag::{self, PhysicalOperator}; - use planner_types::post_asap::ExecutionDataState; + use asap_executor::dag::{self, PhysicalOperator}; + use planner_types::ir::properties::ExecutionDataState; use std::{cell::Cell, collections::BTreeMap, rc::Rc}; struct WrongSource { schema: SchemaRef, @@ -656,7 +651,7 @@ fn source_batches_must_match_the_bound_schema() { id: 0, payload: PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Values { rows: vec![vec![planner_types::ir::ScalarExpr::Literal( - planner_types::pre_asap::ScalarValue::Float64(1.), + planner_types::ir::scalar::ScalarValue::Float64(1.), )]], schema: (*expected).clone(), }), @@ -725,11 +720,12 @@ fn extrema_preserve_numeric_values_in_the_presence_of_nan() { // Planner wire nodes, including grouping and edge roles, are executable at either phase. #[test] fn planner_semijoin_sort_limit_contract_at_both_phases() { - use asap_physical_operators::dag::planner::{bind, Source}; - use planner_types::{ - post_asap::*, - pre_asap::{CompareOpKind, GroupKeys, JoinKind}, - }; + use asap_executor::dag::planner::{bind, Source}; + use planner_types::ir::properties::*; + use planner_types::ir::schema::*; + + use planner_types::ir::operator::{GroupKeys, JoinKind}; + use planner_types::ir::scalar::CompareOpKind; use std::collections::BTreeMap; let rows_schema = schema(&[ ("group", DataType::Utf8, false), @@ -737,17 +733,16 @@ fn planner_semijoin_sort_limit_contract_at_both_phases() { ("score", DataType::Float64, false), ]); let keys_schema = schema(&[("key", DataType::Utf8, false)]); - let node = - |id, payload, schema: &asap_physical_operators::values::SchemaRef| PhysicalASAPDAGNode { - id, - payload, - output_schema: (**schema).clone(), - output_state: ExecutionDataState::QUERY_ROWS, - materialization: Default::default(), - guarantee: None, - }; - let edge = |producer, consumer, role, schema: &asap_physical_operators::values::SchemaRef| { - PhysicalASAPDAGEdge { + let node = |id, payload, schema: &asap_executor::values::SchemaRef| PhysicalASAPDAGNode { + id, + payload, + output_schema: (**schema).clone(), + output_state: ExecutionDataState::QUERY_ROWS, + materialization: Default::default(), + guarantee: None, + }; + let edge = + |producer, consumer, role, schema: &asap_executor::values::SchemaRef| PhysicalASAPDAGEdge { producer, consumer, role, @@ -755,8 +750,7 @@ fn planner_semijoin_sort_limit_contract_at_both_phases() { data_state: ExecutionDataState::QUERY_ROWS, grouping: GroupingEdgeCompatibility::NotApplicable, window: WindowEdgeCompatibility::NotApplicable, - } - }; + }; let groups = GroupKeys::by(vec![0]); let dag = PhysicalASAPDAG { nodes: vec![ @@ -764,7 +758,7 @@ fn planner_semijoin_sort_limit_contract_at_both_phases() { 0, PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Values { rows: vec![vec![planner_types::ir::ScalarExpr::Literal( - planner_types::pre_asap::ScalarValue::Float64(0.), + planner_types::ir::scalar::ScalarValue::Float64(0.), )]], schema: (*rows_schema).clone(), }), @@ -774,7 +768,7 @@ fn planner_semijoin_sort_limit_contract_at_both_phases() { 1, PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Values { rows: vec![vec![planner_types::ir::ScalarExpr::Literal( - planner_types::pre_asap::ScalarValue::Float64(0.), + planner_types::ir::scalar::ScalarValue::Float64(0.), )]], schema: (*keys_schema).clone(), }), @@ -894,8 +888,8 @@ fn planner_semijoin_sort_limit_contract_at_both_phases() { // Planner scalar signatures, collection access and null predicates share native execution. #[test] fn planner_expressions_preserve_collection_and_nullable_types() { - use asap_physical_operators::dag::expressions::CompiledExpression; - use planner_types::pre_asap::{CompareOpKind, ScalarValue}; + use asap_executor::dag::expressions::CompiledExpression; + use planner_types::ir::scalar::{CompareOpKind, ScalarValue}; let input_schema = schema(&[( "items", @@ -971,8 +965,9 @@ fn planner_expressions_preserve_collection_and_nullable_types() { // Outer, semi and anti joins share Planner predicates and preserve SQL null behavior. #[test] fn native_relational_join_kinds_preserve_unmatched_rows() { + use planner_types::ir::operator::JoinKind; + use planner_types::ir::scalar::CompareOpKind; use planner_types::ir::Predicate; - use planner_types::pre_asap::{CompareOpKind, JoinKind}; let input = schema(&[("key", DataType::Int64, true)]); let predicate = Predicate(ScalarExpr::Compare { @@ -1053,7 +1048,7 @@ fn weighted_rate_topk_preserves_partitions_fractional_scores_and_evaluation_scop } } fn assert_weighted_rate_topk(count_sketch: bool) { - use planner_types::post_asap::{SketchAlgorithm, SketchKind, SketchParams}; + use planner_types::ir::schema::{SketchAlgorithm, SketchKind, SketchParams}; let raw = schema(&[ ("service", DataType::Utf8, false), @@ -1085,7 +1080,7 @@ fn assert_weighted_rate_topk(count_sketch: bool) { } let rates = Operator::window( raw.clone(), - planner_types::pre_asap::AggIntent::Rate, + planner_types::ir::operator::AggIntent::Rate, 3, 4, vec![0, 1, 2], @@ -1175,19 +1170,18 @@ fn assert_weighted_rate_topk(count_sketch: bool) { // The grouped temporal reducer's sample schema must survive physical Sort/Limit binding. #[test] fn grouped_temporal_schema_compiles_and_executes_topk() { - use asap_physical_operators::physical_planner::{ + use asap_executor::physical_planner::{ compile_node, CompiledPhysicalDAG, InputContract, Source, }; use planner_types::ir::physical_export::{PhysicalASAPDAGNode, PhysicalASAPOperatorPayload}; - use planner_types::post_asap::ExecutionDataState; + use planner_types::ir::properties::ExecutionDataState; - use planner_types::pre_asap::{ - aggregate_output_schema, AggIntent, GroupKeys, Reduction as IrReduction, Schema as IrSchema, - }; + use planner_types::ir::operator::{AggIntent, GroupKeys, Reduction as IrReduction}; + use planner_types::ir::schema::{aggregate_output_schema, Schema as IrSchema}; let grouped = IrSchema::new(vec![ - planner_types::pre_asap::Field::plain("job", DataType::Utf8, false), - planner_types::pre_asap::Field::plain("sum", DataType::Float64, false), + planner_types::ir::schema::Field::plain("job", DataType::Utf8, false), + planner_types::ir::schema::Field::plain("sum", DataType::Float64, false), ]); let output = aggregate_output_schema( &grouped, @@ -1287,13 +1281,14 @@ fn grouped_temporal_schema_compiles_and_executes_topk() { // A certified candidate set must have authoritative values for every key, including after recovery. #[test] fn certified_pruning_rejects_missing_authoritative_values_after_recovery() { - use asap_physical_operators::physical_planner::{ + use asap_executor::physical_planner::{ compile_node, CompiledPhysicalDAG, InputContract, Source, }; - use planner_types::{ - post_asap::*, - pre_asap::{CompareOpKind, JoinKind}, - }; + use planner_types::ir::properties::*; + use planner_types::ir::schema::*; + + use planner_types::ir::operator::JoinKind; + use planner_types::ir::scalar::CompareOpKind; use std::collections::BTreeMap; let schema = schema(&[("key", DataType::Utf8, false)]); for certified in [false, true] { @@ -1381,7 +1376,7 @@ fn certified_pruning_rejects_missing_authoritative_values_after_recovery() { while let Some(batch) = stream.next().await { rows.extend(batch?.rows().iter().cloned()); } - Ok::<_, asap_physical_operators::Error>(rows) + Ok::<_, asap_executor::Error>(rows) }); if certified && !complete { assert!(result @@ -1400,13 +1395,14 @@ fn certified_pruning_rejects_missing_authoritative_values_after_recovery() { // Precompute arithmetic must match population/window identities, never zip arrival order. #[test] fn compiled_ingestion_binary_preserves_alignment_and_rejects_missing_updates() { - use asap_physical_operators::physical_planner::{ + use asap_executor::physical_planner::{ compile_node, CompiledPhysicalDAG, InputContract, Source, }; - use planner_types::{ - post_asap::*, - pre_asap::{ArithmeticOpKind, BinaryOpKind}, - }; + use planner_types::ir::properties::*; + use planner_types::ir::schema::*; + + use planner_types::ir::operator::BinaryOpKind; + use planner_types::ir::scalar::ArithmeticOpKind; use std::collections::BTreeMap; let input = schema(&[ ("population", DataType::Utf8, false), @@ -1492,7 +1488,7 @@ fn compiled_ingestion_binary_preserves_alignment_and_rejects_missing_updates() { while let Some(batch) = stream.next().await { rows.extend(batch?.rows().iter().cloned()); } - Ok::<_, asap_physical_operators::Error>(rows) + Ok::<_, asap_executor::Error>(rows) }); match expected { Some(values) => assert_eq!(floats(&result.unwrap(), 2), values), diff --git a/crates/asap-physical-operators/tests/physical_plan_recovery.rs b/crates/executor/tests/physical_plan_recovery.rs similarity index 93% rename from crates/asap-physical-operators/tests/physical_plan_recovery.rs rename to crates/executor/tests/physical_plan_recovery.rs index e755090e6..68a6e4672 100644 --- a/crates/asap-physical-operators/tests/physical_plan_recovery.rs +++ b/crates/executor/tests/physical_plan_recovery.rs @@ -1,17 +1,14 @@ //! Deserialized physical plans recover selected operators without logical lowering. //! Deployments choose the encoding; JSON is used here only as a test format. -use asap_physical_operators::{ +use asap_executor::{ operators::{Operator, SortKey}, physical_planner::{CompiledPhysicalDAG, InputContract}, }; -use planner_types::{ - post_asap::{Field, FieldDataType}, - pre_asap::DataType, -}; +use planner_types::ir::schema::{DataType, Field, FieldDataType}; use std::{collections::BTreeMap, sync::Arc}; fn sorted() -> CompiledPhysicalDAG { - let schema = Arc::new(planner_types::pre_asap::Schema { + let schema = Arc::new(planner_types::ir::schema::Schema { unique_keys: vec![], closed: false, fields: vec![Field { @@ -74,7 +71,7 @@ fn recovery_retains_selected_operator_and_rejects_invalid_contracts() { #[test] fn candidate_recovery_preserves_materialization_boundary() { - use asap_physical_operators::physical_planner::CompiledPhysicalPlan; + use asap_executor::physical_planner::CompiledPhysicalPlan; let precompute = sorted(); let output = InputContract::bounded(precompute.output_contract(1).unwrap().schema); let query = CompiledPhysicalDAG::from_operators( diff --git a/crates/asap-physical-operators/tests/physical_semantics.rs b/crates/executor/tests/physical_semantics.rs similarity index 93% rename from crates/asap-physical-operators/tests/physical_semantics.rs rename to crates/executor/tests/physical_semantics.rs index 88c6cd2a6..178c7f1a1 100644 --- a/crates/asap-physical-operators/tests/physical_semantics.rs +++ b/crates/executor/tests/physical_semantics.rs @@ -1,7 +1,7 @@ //! Contract tests inspired by DataFusion's limit, sort and join test matrices. //! Expectations follow ASAP's IR (notably row-count and IEEE NaN equality). //! Reference: apache/datafusion e2ca7f3, physical-plan/src/{limit.rs,sorts/sort.rs}. -use asap_physical_operators::{ +use asap_executor::{ expressions::CompiledExpression, operators::{Expression, Operator, Reduction, SortKey}, plan::PhysicalDAG, @@ -9,18 +9,17 @@ use asap_physical_operators::{ values::{Batch, SchemaRef, Value}, }; use futures::{executor::block_on, StreamExt}; +use planner_types::ir::operator::JoinKind; use planner_types::ir::physical_export::{PhysicalASAPDAGNode, PhysicalASAPOperatorPayload}; +use planner_types::ir::scalar::CompareOpKind; +use planner_types::ir::schema::{DataType, Field, FieldDataType}; use planner_types::ir::NonASAPOp; use planner_types::ir::Predicate; use planner_types::ir::ScalarExpr; -use planner_types::{ - post_asap::{Field, FieldDataType}, - pre_asap::{CompareOpKind, DataType, JoinKind}, -}; use std::sync::Arc; fn schema(fields: &[(&str, DataType, bool)]) -> SchemaRef { - Arc::new(planner_types::pre_asap::Schema { + Arc::new(planner_types::ir::schema::Schema { unique_keys: vec![], closed: false, fields: fields @@ -347,23 +346,23 @@ fn projection_rejects_expression_bound_to_another_schema() { // A valid Planner MIN/MAX schema must bind even for a non-null input column. #[test] fn global_extrema_bind_with_planner_derived_schema() { - use asap_physical_operators::physical_planner::compile_node; - use planner_types::{ - post_asap::*, - pre_asap::{AggIntent, GroupKeys, Reduction as PlanReduction}, - }; + use asap_executor::physical_planner::compile_node; + use planner_types::ir::properties::*; + use planner_types::ir::schema::*; + + use planner_types::ir::operator::{AggIntent, GroupKeys, Reduction as PlanReduction}; let input = schema(&[("v", DataType::Int64, false)]); for measure in [ AggIntent::Min { col: Some(0) }, AggIntent::Max { col: Some(0) }, ] { let planner_input = - planner_types::pre_asap::Schema::new(vec![planner_types::pre_asap::Field::plain( + planner_types::ir::schema::Schema::new(vec![planner_types::ir::schema::Field::plain( "v", DataType::Int64, false, )]); - let derived = planner_types::pre_asap::aggregate_output_schema( + let derived = planner_types::ir::schema::aggregate_output_schema( &planner_input, &PlanReduction::Reduce(GroupKeys::by(vec![])), std::slice::from_ref(&measure), @@ -555,7 +554,7 @@ fn boolean_truth_tables_agree_between_expression_paths() { // Partial/final execution must agree with one build for an uncompacted KLL population. #[test] fn kll_partial_merge_and_multiple_evaluations_preserve_population() { - use planner_types::post_asap::{SketchAlgorithm, SketchKind, SketchParams}; + use planner_types::ir::schema::{SketchAlgorithm, SketchKind, SketchParams}; let input = schema(&[("v", DataType::Float64, false)]); let family = FieldDataType::Sketch( @@ -609,8 +608,8 @@ fn kll_partial_merge_and_multiple_evaluations_preserve_population() { Operator::evaluation( state.clone(), 0, - asap_physical_operators::operators::SummaryEvaluation::Sketch( - planner_types::post_asap::SketchStatistic::Quantile { q }, + asap_executor::operators::SummaryEvaluation::Sketch( + planner_types::ir::schema::SketchStatistic::Quantile { q }, ), ) .unwrap(), @@ -629,8 +628,8 @@ fn kll_partial_merge_and_multiple_evaluations_preserve_population() { )); for (pair, expected) in outputs.chunks(2).zip([0., 64., 127.]) { let value = |batches: &[Result< - asap_physical_operators::runtime::SharedValue, - asap_physical_operators::Error, + asap_executor::runtime::SharedValue, + asap_executor::Error, >]| { assert_eq!(batches.len(), 1); match batches[0].as_ref().unwrap().rows()[0][0] { @@ -650,12 +649,12 @@ fn kll_partial_merge_and_multiple_evaluations_preserve_population() { // single-build rank error: 60 panes of 1000 values at k = 200. #[test] fn compacted_kll_panes_merge_within_the_calibrated_rank_error() { - use planner_types::post_asap::{SketchAlgorithm, SketchKind, SketchParams, SketchStatistic}; + use planner_types::ir::schema::{SketchAlgorithm, SketchKind, SketchParams, SketchStatistic}; const PANES: u64 = 60; const PER_PANE: u64 = 1000; const K: u32 = 200; // The planner's empirical 99% KLL rank error, `2.296 / k^0.9723` - // (`asap_aware_mapping`'s `kll_rank_error_99`). + // (`asap_logical_asap_optimization`'s `kll_rank_error_99`). let bound = 2.296 / f64::from(K).powf(0.9723); let n = (PANES * PER_PANE) as f64; let input = schema(&[("v", DataType::Float64, false)]); @@ -714,9 +713,9 @@ fn compacted_kll_panes_merge_within_the_calibrated_rank_error() { Operator::evaluation( state.clone(), 0, - asap_physical_operators::operators::SummaryEvaluation::Sketch( - SketchStatistic::Quantile { q }, - ), + asap_executor::operators::SummaryEvaluation::Sketch(SketchStatistic::Quantile { + q, + }), ) .unwrap(), ) @@ -735,8 +734,8 @@ fn compacted_kll_panes_merge_within_the_calibrated_rank_error() { // and a counter reset inside B. #[test] fn counter_panes_merge_in_any_order() { - use asap_physical_operators::summary_kernels::exact::ExactAccumulator; - use planner_types::post_asap::{ExactKind, ExactParams}; + use asap_executor::summary_kernels::exact::ExactAccumulator; + use planner_types::ir::schema::{ExactKind, ExactParams}; let input = schema(&[ ("v", DataType::Float64, false), ("t", DataType::Timestamp, false), @@ -763,7 +762,7 @@ fn counter_panes_merge_in_any_order() { .as_any() .downcast_ref::() .unwrap() - .evaluation(asap_physical_operators::Statistic::Increase, None, None) + .evaluation(asap_executor::Statistic::Increase, None, None) .unwrap() .unwrap() }; @@ -793,7 +792,7 @@ fn counter_panes_merge_in_any_order() { // Retained zero-column rows still own Vec headers and must consume the output budget. #[test] fn zero_column_output_obeys_memory_limit() { - use asap_physical_operators::Error; + use asap_executor::Error; let input = schema(&[]); let batch = Batch::try_new(input.clone(), vec![vec![]; 200]).unwrap(); let mut dag = PhysicalDAG::default(); @@ -822,8 +821,8 @@ fn zero_column_output_obeys_memory_limit() { // Empty exact-state finalization must preserve ordinary global MIN/MAX null semantics. #[test] fn empty_exact_summary_extrema_agree_with_ordinary_aggregation() { - use asap_physical_operators::Statistic; - use planner_types::post_asap::{ExactKind, ExactParams}; + use asap_executor::Statistic; + use planner_types::ir::schema::{ExactKind, ExactParams}; let input = schema(&[("v", DataType::Float64, false)]); for (kind, params, statistic) in [ @@ -849,8 +848,8 @@ fn empty_exact_summary_extrema_agree_with_ordinary_aggregation() { Operator::evaluation( state, 0, - asap_physical_operators::operators::SummaryEvaluation::Exact( - asap_physical_operators::summary_kernels::exact::ExactEvaluation { + asap_executor::operators::SummaryEvaluation::Exact( + asap_executor::summary_kernels::exact::ExactEvaluation { statistic, lookback_ms: None, offset_ms: 0, @@ -923,9 +922,9 @@ fn planner_node( PhysicalASAPDAGNode { id: 1, payload, - output_state: planner_types::post_asap::ExecutionDataState::QUERY_ROWS, + output_state: planner_types::ir::properties::ExecutionDataState::QUERY_ROWS, materialization: Default::default(), - output_schema: planner_types::pre_asap::Schema::new( + output_schema: planner_types::ir::schema::Schema::new( fields .iter() .map(|(name, dtype, nullable)| Field { @@ -950,9 +949,9 @@ fn round_trip(operator: Operator) -> Operator { /// serialization. #[test] fn measure_filters_keep_groups_without_matching_rows() { - use asap_physical_operators::physical_planner::compile_node; - use planner_types::pre_asap::{AggIntent, GroupKeys, Reduction as Plan}; - use planner_types::types::AccuracyTarget; + use asap_executor::physical_planner::compile_node; + use planner_types::ir::operator::{AggIntent, GroupKeys, Reduction as Plan}; + use planner_types::ir::properties::AccuracyTarget; let (input, rows) = filtered_input(); let plain = |t: DataType| FieldDataType::Plain(t); let node = planner_node( @@ -990,8 +989,8 @@ fn measure_filters_keep_groups_without_matching_rows() { /// declares it non-null does not bind. #[test] fn filtered_sum_declared_non_null_is_rejected() { - use asap_physical_operators::physical_planner::compile_node; - use planner_types::pre_asap::{AggIntent, GroupKeys, Reduction as Plan}; + use asap_executor::physical_planner::compile_node; + use planner_types::ir::operator::{AggIntent, GroupKeys, Reduction as Plan}; let (input, _) = filtered_input(); let node = planner_node( PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Aggregate { @@ -1016,13 +1015,14 @@ fn filtered_sum_declared_non_null_is_rejected() { /// filter survives serialization. #[test] fn filtered_summary_build_keeps_empty_groups() { - use asap_physical_operators::physical_planner::compile_node; - use planner_types::ir::ASAPOp; - use planner_types::post_asap::{ + use asap_executor::physical_planner::compile_node; + use planner_types::ir::operator::{GroupKeys, Reduction as Plan}; + use planner_types::ir::scalar::ColumnRef; + use planner_types::ir::schema::{ ExactKind, ExactParams, SketchAlgorithm, SketchKind, SketchParams, SketchStatistic, SummaryUpdate, }; - use planner_types::pre_asap::{ColumnRef, GroupKeys, Reduction as Plan}; + use planner_types::ir::ASAPOp; let (input, rows) = filtered_input(); let kll = FieldDataType::Sketch( SketchKind::new( @@ -1097,10 +1097,12 @@ fn filtered_summary_build_keeps_empty_groups() { /// query-time build applies its filter. The same build unfiltered binds. #[test] fn filtered_summary_build_is_rejected_at_ingestion_time() { - use asap_physical_operators::physical_planner::compile_node; + use asap_executor::physical_planner::compile_node; + use planner_types::ir::operator::{GroupKeys, Reduction as Plan}; + use planner_types::ir::properties::ExecutionDataState; + use planner_types::ir::scalar::ColumnRef; + use planner_types::ir::schema::{ExactKind, ExactParams, SummaryUpdate}; use planner_types::ir::ASAPOp; - use planner_types::post_asap::{ExactKind, ExactParams, ExecutionDataState, SummaryUpdate}; - use planner_types::pre_asap::{ColumnRef, GroupKeys, Reduction as Plan}; let (input, _) = filtered_input(); let family = FieldDataType::ExactAggregate(ExactKind::Count, ExactParams::Count); let build = |filter| { @@ -1133,7 +1135,7 @@ fn filtered_summary_build_is_rejected_at_ingestion_time() { fn measure_filters_are_rejected_over_promql_series_identity() { let mut input = (*schema(&[ ( - planner_types::pre_asap::schema::PROMQL_SERIES_IDENTITY, + planner_types::ir::schema::PROMQL_SERIES_IDENTITY, DataType::Utf8, false, ), @@ -1160,14 +1162,12 @@ fn measure_filters_are_rejected_over_promql_series_identity() { /// An exact aggregate node over `input` with one `measure` named `name`, /// whose output is one non-null `dtype` column. fn exact_aggregate_node( - measure: planner_types::pre_asap::AggIntent, + measure: planner_types::ir::operator::AggIntent, name: &str, dtype: DataType, ) -> PhysicalASAPDAGNode { - use planner_types::{ - post_asap::ExecutionDataState, - pre_asap::{GroupKeys, Reduction as PlanReduction}, - }; + use planner_types::ir::operator::{GroupKeys, Reduction as PlanReduction}; + use planner_types::ir::properties::ExecutionDataState; PhysicalASAPDAGNode { id: 1, payload: PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Aggregate { @@ -1189,9 +1189,9 @@ fn exact_aggregate_node( // sketch or numeric key conversion; empty and all-NULL input give 0. #[test] fn exact_frequency_intents_execute_typed_keys_and_empty_input() { - use asap_physical_operators::physical_planner::compile_node; - use planner_types::pre_asap::AggIntent; - use planner_types::types::AccuracyTarget; + use asap_executor::physical_planner::compile_node; + use planner_types::ir::operator::AggIntent; + use planner_types::ir::properties::AccuracyTarget; for (dtype, values) in [ ( DataType::Utf8, @@ -1324,9 +1324,9 @@ fn exact_frequency_rejects_non_finite_identities() { // and returns zero on no input. #[test] fn exact_cardinality_binds_and_executes_typed_tuples() { - use asap_physical_operators::physical_planner::compile_node; - use planner_types::pre_asap::AggIntent; - use planner_types::types::AccuracyTarget; + use asap_executor::physical_planner::compile_node; + use planner_types::ir::operator::AggIntent; + use planner_types::ir::properties::AccuracyTarget; let input = schema(&[ ("key", DataType::Int64, true), ("tag", DataType::Utf8, true), diff --git a/crates/asap-physical-operators/tests/plan_properties.rs b/crates/executor/tests/plan_properties.rs similarity index 88% rename from crates/asap-physical-operators/tests/plan_properties.rs rename to crates/executor/tests/plan_properties.rs index b34b9f4b3..030f33164 100644 --- a/crates/asap-physical-operators/tests/plan_properties.rs +++ b/crates/executor/tests/plan_properties.rs @@ -1,5 +1,5 @@ //! Finite-input contracts are validated before source execution. -use asap_physical_operators::{ +use asap_executor::{ operators::{Operator, SortKey}, plan::{Boundedness, Emission, PhysicalDAG}, runtime::{Limits, OutputStream, RunContext, Scope}, @@ -7,10 +7,8 @@ use asap_physical_operators::{ values::{Batch, SchemaRef}, Error, }; -use planner_types::{ - post_asap::{Field, FieldDataType}, - pre_asap::{DataType, Schema, Source}, -}; +use planner_types::ir::operator::Source; +use planner_types::ir::schema::{DataType, Field, FieldDataType, Schema}; use std::sync::{ atomic::{AtomicUsize, Ordering}, Arc, @@ -35,7 +33,7 @@ impl RawSource for DeclaredSource { // A blocking parent must reject unknown and unbounded Scan inputs without opening a reader. #[test] fn blocking_inputs_require_an_explicit_finite_source() { - let schema = Arc::new(planner_types::pre_asap::Schema { + let schema = Arc::new(planner_types::ir::schema::Schema { unique_keys: vec![], closed: false, fields: vec![Field { @@ -71,7 +69,7 @@ fn blocking_inputs_require_an_explicit_finite_source() { &planner_types::ir::OperatorNode::new_shared(planner_types::ir::Operator::NonASAP( planner_types::ir::NonASAPOp::Scan { source: identity, - schema: Schema::new(vec![planner_types::pre_asap::Field::plain( + schema: Schema::new(vec![planner_types::ir::schema::Field::plain( "v", DataType::Int64, false, @@ -124,12 +122,12 @@ fn blocking_inputs_require_an_explicit_finite_source() { /// A KLL kernel accepts `k` only up to the 26 602 `asap_sketchlib` caps it at. #[test] fn kll_kernel_rejects_a_k_above_the_library_cap() { - use asap_physical_operators::capability::validate_summary_kernel; - use planner_types::post_asap::{ + use asap_executor::capability::validate_summary_kernel; + use planner_types::ir::scalar::ColumnRef; + use planner_types::ir::schema::{ GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, SummaryInputExpr, SummaryUpdate, }; - use planner_types::pre_asap::ColumnRef; let grouping = GroupingStrategy::default(); let update = SummaryUpdate { item: None, @@ -156,14 +154,12 @@ fn kll_kernel_rejects_a_k_above_the_library_cap() { /// kernel construction both require a positive odd depth. #[test] fn count_sketch_kernel_requires_an_odd_depth() { - use asap_physical_operators::{ - capability::validate_summary_kernel, factory::create_planner_accumulator, - }; - use planner_types::post_asap::{ + use asap_executor::{capability::validate_summary_kernel, factory::create_planner_accumulator}; + use planner_types::ir::scalar::ColumnRef; + use planner_types::ir::schema::{ GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, SummaryInputExpr, SummaryUpdate, }; - use planner_types::pre_asap::ColumnRef; let grouping = GroupingStrategy::default(); let mut update = SummaryUpdate::column(ColumnRef::SampleValue); update.item = Some(SummaryInputExpr::Column(ColumnRef::Named("host".into()))); @@ -192,13 +188,13 @@ fn count_sketch_kernel_requires_an_odd_depth() { // Kernel support must not be mistaken for executable native state/evaluation support. #[test] fn summary_capability_levels_are_distinct() { - use asap_physical_operators::{ + use asap_executor::{ capability::{validate_native_family, validate_sketch_evaluation, validate_summary_kernel}, - planner::post_asap::SketchStatistic, + planner::ir::schema::SketchStatistic, }; - use planner_types::{ - post_asap::{GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, SummaryUpdate}, - pre_asap::ColumnRef, + use planner_types::ir::scalar::ColumnRef; + use planner_types::ir::schema::{ + GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, SummaryUpdate, }; let grouping = GroupingStrategy::default(); let cms = FieldDataType::Sketch( @@ -212,10 +208,10 @@ fn summary_capability_levels_are_distinct() { grouping.clone(), ); let update = SummaryUpdate { - item: Some(planner_types::post_asap::SummaryInputExpr::Column( + item: Some(planner_types::ir::schema::SummaryInputExpr::Column( ColumnRef::Named("host".into()), )), - weight: planner_types::post_asap::SummaryInputExpr::Constant(1.0), + weight: planner_types::ir::schema::SummaryInputExpr::Constant(1.0), weight_domain: Default::default(), }; assert!(validate_summary_kernel(&cms, &update, &grouping).is_ok()); diff --git a/crates/asap-physical-operators/tests/planspace_series_identity_heap.rs b/crates/executor/tests/planspace_series_identity_heap.rs similarity index 86% rename from crates/asap-physical-operators/tests/planspace_series_identity_heap.rs rename to crates/executor/tests/planspace_series_identity_heap.rs index 15b9dcddd..c3271e5c2 100644 --- a/crates/asap-physical-operators/tests/planspace_series_identity_heap.rs +++ b/crates/executor/tests/planspace_series_identity_heap.rs @@ -5,13 +5,11 @@ mod common; use common::{compile_physical_asap_dag, selected_dag}; use planner_types::ir::OperatorNode; -use asap_physical_operators::physical_planner::promql_rows::SERIES_IDENTITY_COLUMN; -use planner_types::{ - types::AccuracyTarget, - workload::{ - BatchEntry, DataWorkload, DurationMs, PlanningWorkload, Predictability, Query, - QueryLanguage, QueryRequirements, QueryWorkload, - }, +use asap_executor::physical_planner::promql_rows::SERIES_IDENTITY_COLUMN; +use planner_types::ir::properties::AccuracyTarget; +use planner_types::workload::{ + BatchEntry, DataWorkload, DurationMs, PlanningWorkload, Predictability, Query, QueryLanguage, + QueryRequirements, QueryWorkload, }; use std::rc::Rc; diff --git a/crates/asap-physical-operators/tests/precompute.rs b/crates/executor/tests/precompute.rs similarity index 87% rename from crates/asap-physical-operators/tests/precompute.rs rename to crates/executor/tests/precompute.rs index 5435dbb7d..70b0359c3 100644 --- a/crates/asap-physical-operators/tests/precompute.rs +++ b/crates/executor/tests/precompute.rs @@ -1,9 +1,10 @@ //! A precompute boundary reads raw samples only from a time series scan. mod common; -use asap_physical_operators::physical_planner::precompute::{boundary_schema, raw_sample_schema}; +use asap_executor::physical_planner::precompute::{boundary_schema, raw_sample_schema}; use common::compile_physical_asap_dag; +use planner_types::ir::operator::Source; +use planner_types::ir::schema::{DataType, Field, Schema}; use planner_types::ir::{NonASAPOp, Operator, OperatorNode, TimeRangeKind}; -use planner_types::pre_asap::{DataType, Field, Schema, Source}; use std::{collections::BTreeMap, rc::Rc, time::Duration}; /// `TimeRange(1m)` over a scan of `source` with columns `ts, value, key`. @@ -32,7 +33,7 @@ fn time_range_over(source: Source) -> Rc { fn boundary( root: &Rc, -) -> Result { +) -> Result { let dag = compile_physical_asap_dag(root).unwrap(); let nodes = dag .nodes diff --git a/crates/asap-physical-operators/tests/precompute_candidates.rs b/crates/executor/tests/precompute_candidates.rs similarity index 94% rename from crates/asap-physical-operators/tests/precompute_candidates.rs rename to crates/executor/tests/precompute_candidates.rs index 4d1805e2e..d34506b1a 100644 --- a/crates/asap-physical-operators/tests/precompute_candidates.rs +++ b/crates/executor/tests/precompute_candidates.rs @@ -1,6 +1,6 @@ //! Materialized frontiers are compiled by Planner, never rewritten by deployment. mod common; -use asap_physical_operators::{ +use asap_executor::{ factory::create_planner_accumulator, operators::Operator, physical_planner::{ @@ -14,9 +14,10 @@ use asap_physical_operators::{ use common::{compile_physical_asap_dag, logical_candidates, selected_dag}; use futures::{executor::block_on, StreamExt}; use planner_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPOperatorPayload}; +use planner_types::ir::properties::AccuracyTarget; +use planner_types::ir::schema::{DataType, *}; use planner_types::ir::ASAPOp; -use planner_types::types::AccuracyTarget; -use planner_types::{post_asap::*, pre_asap::DataType, workload::*}; +use planner_types::workload::*; use std::{collections::BTreeMap, sync::Arc}; /// `sum by(job)(rate(m[1m]))` with its rows carrying the series identity. @@ -49,7 +50,7 @@ fn exact_promql_root(query: &str) -> std::rc::Rc PhysicalASAPDAG { @@ -129,9 +130,9 @@ fn grouped_rate_can_be_materialized_before_or_after_grouped_sum() { let state = accumulator.into_accumulator(); expected_rate_sum += state .as_any() - .downcast_ref::() + .downcast_ref::() .unwrap() - .evaluation(asap_physical_operators::Statistic::Rate, range_ms, None) + .evaluation(asap_executor::Statistic::Rate, range_ms, None) .unwrap() .unwrap(); let summary = Value::Summary { @@ -164,7 +165,7 @@ fn grouped_rate_can_be_materialized_before_or_after_grouped_sum() { let root = dag.roots[0] as u64; let state_id = state.id as u64; let rate_id = evaluation.id as u64; - let frontiers = asap_physical_operators::physical_planner::enumerate_frontiers( + let frontiers = asap_executor::physical_planner::enumerate_frontiers( &dag, &BTreeMap::from([(state_id, InputContract::bounded(input_schema.clone()))]), &[root], @@ -175,15 +176,13 @@ fn grouped_rate_can_be_materialized_before_or_after_grouped_sum() { assert!(frontiers.contains(&vec![rate_id])); assert!(frontiers.contains(&vec![root])); assert!(!frontiers.contains(&vec![root, rate_id])); - assert!( - asap_physical_operators::physical_planner::enumerate_frontiers( - &dag, - &BTreeMap::from([(state_id, InputContract::bounded(input_schema.clone()))]), - &[root], - 1, - ) - .is_err() - ); + assert!(asap_executor::physical_planner::enumerate_frontiers( + &dag, + &BTreeMap::from([(state_id, InputContract::bounded(input_schema.clone()))]), + &[root], + 1, + ) + .is_err()); let candidates = compile_candidates( &dag, BTreeMap::from([(state_id, InputContract::bounded(input_schema))]), @@ -248,15 +247,13 @@ fn grouped_rate_can_be_materialized_before_or_after_grouped_sum() { InputContract::bounded(Arc::new(state.output_schema.clone())), )]); for frontier in [vec![rate_id, rate_id], vec![root, rate_id], vec![999]] { - assert!( - asap_physical_operators::physical_planner::compile_candidate( - &dag, - contracts.clone(), - &[root], - &frontier - ) - .is_err() - ); + assert!(asap_executor::physical_planner::compile_candidate( + &dag, + contracts.clone(), + &[root], + &frontier + ) + .is_err()); } let inventory = compile_candidates( &dag, @@ -357,9 +354,9 @@ fn grouped_rate_can_be_materialized_before_or_after_grouped_sum() { let rate_of_sum = wrong_order .into_accumulator() .as_any() - .downcast_ref::() + .downcast_ref::() .unwrap() - .evaluation(asap_physical_operators::Statistic::Rate, range_ms, None) + .evaluation(asap_executor::Statistic::Rate, range_ms, None) .unwrap() .unwrap(); assert_ne!( @@ -372,7 +369,7 @@ fn grouped_rate_can_be_materialized_before_or_after_grouped_sum() { /// persistence; an explicit Rate-state input retains its original semantics. #[test] fn bounded_inventory_exposes_grouped_rate_physical_frontiers() { - use asap_physical_operators::physical_planner::enumerate_frontiers; + use asap_executor::physical_planner::enumerate_frontiers; let dag = grouped_rate(); let state = dag .nodes @@ -552,8 +549,8 @@ fn recompiled_candidate( inputs: &BTreeMap, roots: &[u64], frontier: &[u64], -) -> Result { - use asap_physical_operators::plan::Emission; +) -> Result { + use asap_executor::plan::Emission; if frontier.is_empty() { return Ok(CompiledPhysicalPlan { precompute: None, @@ -647,13 +644,13 @@ fn population_topk_cuts_equal_per_frontier_compilation() { .unwrap() .remove(0); let root = - asap_physical_operators::physical_planner::promql_rows::with_series_identity(&original) - .unwrap(); - let selected = asap_aware_mapping::maintained_population::MaintainedPopulationStrategy::new( - std::slice::from_ref(&root), - ) - .candidate(&root) - .unwrap(); + asap_executor::physical_planner::promql_rows::with_series_identity(&original).unwrap(); + let selected = + asap_logical_asap_optimization::maintained_population::MaintainedPopulationStrategy::new( + std::slice::from_ref(&root), + ) + .candidate(&root) + .unwrap(); let dag = compile_physical_asap_dag(&selected).unwrap(); let raw = dag .nodes diff --git a/crates/asap-physical-operators/tests/precompute_population.rs b/crates/executor/tests/precompute_population.rs similarity index 90% rename from crates/asap-physical-operators/tests/precompute_population.rs rename to crates/executor/tests/precompute_population.rs index 154ffe5f6..a1751c46d 100644 --- a/crates/asap-physical-operators/tests/precompute_population.rs +++ b/crates/executor/tests/precompute_population.rs @@ -1,5 +1,5 @@ //! Persisted precompute DAGs preserve group/window identity and execute state-to-state computation. -use asap_physical_operators::{ +use asap_executor::{ factory::create_planner_accumulator, operators::Operator, physical_planner::{precompute, CompiledPhysicalDAG, Source}, @@ -8,23 +8,23 @@ use asap_physical_operators::{ Statistic, }; use futures::{executor::block_on, StreamExt}; +use planner_types::ir::operator::{BinaryOpKind, GroupKeys, Reduction}; use planner_types::ir::physical_export::{ EdgeRole, GroupingEdgeCompatibility, PhysicalASAPDAG, PhysicalASAPDAGEdge, PhysicalASAPDAGNode, PhysicalASAPOperatorPayload, WindowEdgeCompatibility, }; +use planner_types::ir::properties::*; +use planner_types::ir::scalar::{ArithmeticOpKind, ColumnRef}; +use planner_types::ir::schema::{DataType, *}; use planner_types::ir::ASAPOp; use planner_types::ir::BinaryOperator; -use planner_types::{ - post_asap::*, - pre_asap::{ArithmeticOpKind, BinaryOpKind, ColumnRef, DataType, GroupKeys, Reduction}, -}; use std::{collections::BTreeMap, sync::Arc}; // Typed series identity survives finalization and derived precompute through population metadata. #[test] fn finalized_shared_panes_rebuild_one_global_summary_after_recovery() { let family = FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum); - let schema = |dtype| planner_types::pre_asap::Schema { + let schema = |dtype| planner_types::ir::schema::Schema { unique_keys: vec![], closed: false, fields: vec![Field { @@ -46,7 +46,7 @@ fn finalized_shared_panes_rebuild_one_global_summary_after_recovery() { value_schema.time_index = Some(1); value_schema.fields.push(Field { table: None, - name: planner_types::pre_asap::schema::PROMQL_SERIES_IDENTITY.into(), + name: planner_types::ir::schema::PROMQL_SERIES_IDENTITY.into(), dtype: FieldDataType::Plain(DataType::Utf8), nullable: false, }); @@ -61,8 +61,8 @@ fn finalized_shared_panes_rebuild_one_global_summary_after_recovery() { children: vec![], }), output_state: ExecutionDataState::INGESTION_SUMMARY, - materialization: planner_types::post_asap::Materialization::MatAtIngestion( - planner_types::post_asap::Medium::Memory, + materialization: planner_types::physical::Materialization::MatAtIngestion( + planner_types::physical::Medium::Memory, ), output_schema: state_schema.clone(), guarantee: None, @@ -111,8 +111,8 @@ fn finalized_shared_panes_rebuild_one_global_summary_after_recovery() { child: 2, }), output_state: ExecutionDataState::INGESTION_SUMMARY, - materialization: planner_types::post_asap::Materialization::MatAtIngestion( - planner_types::post_asap::Medium::Memory, + materialization: planner_types::physical::Materialization::MatAtIngestion( + planner_types::physical::Medium::Memory, ), output_schema: state_schema.clone(), guarantee: None, @@ -171,9 +171,9 @@ fn finalized_shared_panes_rebuild_one_global_summary_after_recovery() { unreachable!() }; *filter = Some(planner_types::ir::Predicate( - planner_types::ir::ScalarExpr::Literal(planner_types::pre_asap::ScalarValue::Boolean( - true, - )), + planner_types::ir::ScalarExpr::Literal( + planner_types::ir::scalar::ScalarValue::Boolean(true), + ), )); assert!(precompute::compile(&filtered, &[0], &[3]).is_err()); let program = precompute::compile(&dag, &[0], &[3]).unwrap(); @@ -244,7 +244,7 @@ fn finalized_shared_panes_rebuild_one_global_summary_after_recovery() { assert_eq!( state .as_any() - .downcast_ref::() + .downcast_ref::() .unwrap() .evaluation(Statistic::Sum, None, None) .unwrap() @@ -256,7 +256,7 @@ fn finalized_shared_panes_rebuild_one_global_summary_after_recovery() { } fn logical_schema(family: FieldDataType) -> Schema { - planner_types::pre_asap::Schema { + planner_types::ir::schema::Schema { unique_keys: vec![], closed: false, fields: vec![Field { @@ -279,13 +279,13 @@ fn try_state_dag( family: FieldDataType, target: Option, merge: bool, -) -> Result { +) -> Result { let mut nodes = vec![PhysicalASAPDAGNode { id: 0, payload: PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryMerge { children: vec![] }), output_state: ExecutionDataState::INGESTION_SUMMARY, - materialization: planner_types::post_asap::Materialization::MatAtIngestion( - planner_types::post_asap::Medium::Memory, + materialization: planner_types::physical::Materialization::MatAtIngestion( + planner_types::physical::Medium::Memory, ), output_schema: logical_schema(family.clone()), guarantee: None, @@ -320,8 +320,8 @@ fn try_state_dag( child: read_id, }), output_state: ExecutionDataState::INGESTION_SUMMARY, - materialization: planner_types::post_asap::Materialization::MatAtIngestion( - planner_types::post_asap::Medium::Memory, + materialization: planner_types::physical::Materialization::MatAtIngestion( + planner_types::physical::Medium::Memory, ), output_schema: logical_schema(target), guarantee: None, @@ -353,9 +353,9 @@ fn try_state_dag( fn native_run( program: &CompiledPhysicalDAG, family: FieldDataType, - states: Vec>, + states: Vec>, context: RunContext, -) -> Result>, asap_physical_operators::Error> { +) -> Result>, asap_executor::Error> { let program = serde_json::from_slice::(&serde_json::to_vec(&program).unwrap()) .unwrap(); @@ -398,11 +398,11 @@ fn ingestion_context(limits: Limits) -> RunContext { ) .unwrap() } -fn sum_state(value: f64) -> Arc { - let mut state = asap_physical_operators::summary_kernels::exact::ExactAccumulator::new( - planner_types::post_asap::FieldDataType::ExactAggregate( - planner_types::post_asap::ExactKind::Sum, - planner_types::post_asap::ExactParams::Sum, +fn sum_state(value: f64) -> Arc { + let mut state = asap_executor::summary_kernels::exact::ExactAccumulator::new( + planner_types::ir::schema::FieldDataType::ExactAggregate( + planner_types::ir::schema::ExactKind::Sum, + planner_types::ir::schema::ExactParams::Sum, ), false, ) @@ -480,7 +480,7 @@ fn precompute_rejects_nonfinite_and_nonpositive_dds_updates() { // An exact count must not silently lose units when exposed through Float64 rows. #[test] fn precompute_count_conversion_checks_precision() { - use asap_physical_operators::summary_kernels::exact::ExactAccumulator; + use asap_executor::summary_kernels::exact::ExactAccumulator; let family = FieldDataType::ExactAggregate(ExactKind::Count, ExactParams::Count); let program = state_dag(family.clone(), None, false); for (count, valid) in [(3u64, true), ((1u64 << 53) + 1, false)] { diff --git a/crates/asap-physical-operators/tests/promql_binary.rs b/crates/executor/tests/promql_binary.rs similarity index 93% rename from crates/asap-physical-operators/tests/promql_binary.rs rename to crates/executor/tests/promql_binary.rs index 9d83df7e7..adcfe901b 100644 --- a/crates/asap-physical-operators/tests/promql_binary.rs +++ b/crates/executor/tests/promql_binary.rs @@ -1,25 +1,25 @@ //! Binary computation must be fully compiled before deployment binds values. -use asap_physical_operators::{ +use asap_executor::{ operators::Operator, physical_planner::{compile_node, CompiledPhysicalDAG, InputContract, Source}, runtime::{Limits, RunContext, Scope}, values::{Batch, SchemaRef, Value}, }; use futures::{executor::block_on, StreamExt}; +use planner_types::ir::operator::BinaryOpKind; use planner_types::ir::physical_export::{ EdgeRole, GroupingEdgeCompatibility, PhysicalASAPDAG, PhysicalASAPDAGEdge, PhysicalASAPDAGNode, PhysicalASAPOperatorPayload, WindowEdgeCompatibility, }; +use planner_types::ir::properties::ExecutionDataState; +use planner_types::ir::scalar::ArithmeticOpKind; +use planner_types::ir::schema::{DataType, Field, FieldDataType}; use planner_types::ir::ASAPOp; use planner_types::ir::BinaryOperator; -use planner_types::{ - post_asap::{ExecutionDataState, Field, FieldDataType}, - pre_asap::{ArithmeticOpKind, BinaryOpKind, DataType}, -}; use std::{collections::BTreeMap, sync::Arc}; fn schema() -> SchemaRef { - Arc::new(planner_types::pre_asap::Schema { + Arc::new(planner_types::ir::schema::Schema { unique_keys: vec![], closed: false, fields: vec![ @@ -97,14 +97,14 @@ fn program_for_bool(operator: BinaryOperator, return_bool: bool) -> CompiledPhys fn evaluate( left: Vec>, right: Vec>, -) -> Result>, asap_physical_operators::Error> { +) -> Result>, asap_executor::Error> { evaluate_with(program(), left, right) } fn evaluate_with( physical_dag: CompiledPhysicalDAG, left: Vec>, right: Vec>, -) -> Result>, asap_physical_operators::Error> { +) -> Result>, asap_executor::Error> { let sources = [left, right] .into_iter() .enumerate() @@ -164,11 +164,11 @@ fn duplicate_matching_identity_is_rejected() { // Scalar broadcasting and comparison filtering keep the vector operand's value. #[test] fn scalar_broadcast_and_bool_comparison_are_distinct() { - use asap_physical_operators::physical_planner::promql_values; - use planner_types::pre_asap::CompareOpKind; + use asap_executor::physical_planner::promql_values; + use planner_types::ir::scalar::CompareOpKind; for return_bool in [false, true] { let physical_dag = promql_values::compile_binary( - &asap_physical_operators::expressions::binary::BinaryOperator::from_logical( + &asap_executor::expressions::binary::BinaryOperator::from_logical( &BinaryOperator { kind: BinaryOpKind::Compare(CompareOpKind::Lt), vector_match: None, @@ -291,8 +291,8 @@ fn binary_obeys_memory_and_cancellation() { }); assert!(matches!( (cancel, result), - (true, Err(asap_physical_operators::Error::Cancelled)) - | (false, Err(asap_physical_operators::Error::MemoryLimit)) + (true, Err(asap_executor::Error::Cancelled)) + | (false, Err(asap_executor::Error::MemoryLimit)) )); } } @@ -302,7 +302,7 @@ fn binary_obeys_memory_and_cancellation() { fn label_map_bool_comparison_drops_the_name() { let program = program_for_bool( BinaryOperator { - kind: BinaryOpKind::Compare(planner_types::pre_asap::CompareOpKind::Gt), + kind: BinaryOpKind::Compare(planner_types::ir::scalar::CompareOpKind::Gt), vector_match: None, checked_relative_division: false, checked_finite_division: false, @@ -330,20 +330,20 @@ fn label_map_bool_comparison_drops_the_name() { // Stored temporal evaluations drop metric names before filter comparisons and set matching. #[test] fn stored_series_evaluations_support_filters_and_sets() { - use asap_physical_operators::{ - physical_planner::compile, summary_kernels::exact::ExactAccumulator, - }; - use planner_types::post_asap::*; - use planner_types::pre_asap::{ - schema::PROMQL_SERIES_IDENTITY, CompareOpKind, PromQLVectorSetOpKind, - }; + use asap_executor::{physical_planner::compile, summary_kernels::exact::ExactAccumulator}; + use planner_types::ir::properties::*; + use planner_types::ir::schema::*; + + use planner_types::ir::operator::PromQLVectorSetOpKind; + use planner_types::ir::scalar::CompareOpKind; + use planner_types::ir::schema::PROMQL_SERIES_IDENTITY; for (exact_kind, params) in [ (ExactKind::Sum, ExactParams::Sum), (ExactKind::Count, ExactParams::Count), ] { let family = FieldDataType::ExactAggregate(exact_kind.clone(), params); - let state_schema = Arc::new(planner_types::pre_asap::Schema { + let state_schema = Arc::new(planner_types::ir::schema::Schema { unique_keys: vec![], closed: false, fields: vec![ diff --git a/crates/asap-physical-operators/tests/promql_values.rs b/crates/executor/tests/promql_values.rs similarity index 95% rename from crates/asap-physical-operators/tests/promql_values.rs rename to crates/executor/tests/promql_values.rs index 7339bec76..554bddb6e 100644 --- a/crates/asap-physical-operators/tests/promql_values.rs +++ b/crates/executor/tests/promql_values.rs @@ -1,14 +1,15 @@ //! Compile, persist and rebind dynamic-label computation without deployment lowering. -use asap_physical_operators::expressions::binary::{BinaryOpKind, BinaryOperator}; +use asap_executor::expressions::binary::{BinaryOpKind, BinaryOperator}; -use asap_physical_operators::{ +use asap_executor::{ operators::Operator, physical_planner::{promql_values::*, CompiledPhysicalDAG, Source}, runtime::{Limits, RunContext, Scope}, values::{Batch, Value}, }; use futures::{executor::block_on, StreamExt}; -use planner_types::pre_asap::{AggIntent, ColumnRef, GroupKeys}; +use planner_types::ir::operator::{AggIntent, GroupKeys}; +use planner_types::ir::scalar::ColumnRef; use std::collections::BTreeMap; @@ -30,7 +31,7 @@ fn run(dag: CompiledPhysicalDAG, rows: Vec>) -> Vec> { fn run_inputs( dag: CompiledPhysicalDAG, batches: Vec, -) -> Result>, asap_physical_operators::Error> { +) -> Result>, asap_executor::Error> { let dag = serde_json::from_slice::(&serde_json::to_vec(&dag).unwrap()).unwrap(); let sources = batches @@ -99,7 +100,7 @@ fn grouping_preserves_unenumerated_labels_and_empty_label_semantics() { run( compile_aggregate( &AggIntent::Count { - accuracy: planner_types::types::AccuracyTarget::exact(), + accuracy: planner_types::ir::properties::AccuracyTarget::exact(), }, &GroupKeys::by(vec![ColumnRef::Named("job".into())]), ) @@ -210,13 +211,13 @@ fn histogram_quantile_keeps_each_label_group() { // Linking an ensemble preserves its shared producer and every selected operator. #[test] fn composed_ensemble_shares_a_producer_across_roots() { - use asap_physical_operators::{ + use asap_executor::{ physical_planner::InputContract, plan::{PhysicalOperator, PlanProperties}, runtime::{Input, OutputStream}, values::SchemaRef, }; - use planner_types::pre_asap::ArithmeticOpKind; + use planner_types::ir::scalar::ArithmeticOpKind; struct Counted { source: Operator, starts: std::rc::Rc>, @@ -241,7 +242,7 @@ fn composed_ensemble_shares_a_producer_across_roots() { &'a self, inputs: Vec>, context: RunContext, - ) -> Result, asap_physical_operators::Error> { + ) -> Result, asap_executor::Error> { self.starts.set(self.starts.get() + 1); self.source.start(inputs, context) } @@ -325,7 +326,7 @@ fn compiled_constant_needs_no_deployment_source() { // arithmetic or bool comparisons remove the metric name. #[test] fn scalar_broadcast_rejects_colliding_result_labels_after_recovery() { - use planner_types::pre_asap::{ArithmeticOpKind, CompareOpKind}; + use planner_types::ir::scalar::{ArithmeticOpKind, CompareOpKind}; for left_scalar in [false, true] { for names in [["a", "a"], ["a", "b"]] { for (kind, return_bool) in [ @@ -399,8 +400,9 @@ fn scalar_broadcast_rejects_colliding_result_labels_after_recovery() { // finalize each population, and preserve the requested metric-name semantics. #[test] fn exact_state_evaluations_recover_and_finalize_panes() { - use asap_physical_operators::factory::create_planner_accumulator; - use planner_types::post_asap::*; + use asap_executor::factory::create_planner_accumulator; + use planner_types::ir::schema::*; + use std::sync::Arc; for (kind, params, expected) in [ (ExactKind::Sum, ExactParams::Sum, 12.), @@ -449,8 +451,9 @@ fn exact_state_evaluations_recover_and_finalize_panes() { #[test] fn recovered_exact_counter_uses_window_and_omits_insufficient_samples() { - use asap_physical_operators::factory::create_planner_accumulator; - use planner_types::post_asap::*; + use asap_executor::factory::create_planner_accumulator; + use planner_types::ir::schema::*; + use std::sync::Arc; for (kind, params, expected) in [ (ExactKind::Rate, ExactParams::Rate, 1.), diff --git a/crates/asap-physical-operators/tests/raw_scan.rs b/crates/executor/tests/raw_scan.rs similarity index 95% rename from crates/asap-physical-operators/tests/raw_scan.rs rename to crates/executor/tests/raw_scan.rs index e8b77f5c1..996d83a3b 100644 --- a/crates/asap-physical-operators/tests/raw_scan.rs +++ b/crates/executor/tests/raw_scan.rs @@ -1,21 +1,20 @@ //! Scan acceptance uses the public connector contract and Planner physical DAGs. -use asap_physical_operators::dag::{ +use asap_executor::dag::{ planner::bind_with_data_sources, scan::{DataSources, MemorySource, RawSource}, values::{Batch, SchemaRef, Value}, Error, Limits, OutputStream, RunContext, Scope, }; use futures::{executor::block_on, stream, StreamExt}; +use planner_types::ir::operator::{GroupKeys, Source}; use planner_types::ir::physical_export::{ EdgeRole, GroupingEdgeCompatibility, PhysicalASAPDAG, PhysicalASAPDAGEdge, PhysicalASAPDAGNode, PhysicalASAPOperatorPayload, WindowEdgeCompatibility, }; +use planner_types::ir::properties::*; +use planner_types::ir::schema::{DataType, Field, *}; use planner_types::ir::NonASAPOp; use planner_types::ir::Predicate; -use planner_types::{ - post_asap::*, - pre_asap::{DataType, Field, GroupKeys, Source}, -}; use std::{ collections::BTreeMap, sync::{ @@ -25,12 +24,13 @@ use std::{ }; fn fixture() -> (planner_types::ir::NonASAPOp, SchemaRef, Vec) { - let schema = planner_types::pre_asap::Schema::new(vec![planner_types::pre_asap::Field::plain( - "value", - DataType::Int64, - true, - )]); - let output = Arc::new(planner_types::pre_asap::Schema { + let schema = + planner_types::ir::schema::Schema::new(vec![planner_types::ir::schema::Field::plain( + "value", + DataType::Int64, + true, + )]); + let output = Arc::new(planner_types::ir::schema::Schema { unique_keys: vec![], closed: false, fields: vec![Field { @@ -189,8 +189,8 @@ struct CountingSource { fail: bool, } impl RawSource for CountingSource { - fn boundedness(&self) -> asap_physical_operators::plan::Boundedness { - asap_physical_operators::plan::Boundedness::Bounded + fn boundedness(&self) -> asap_executor::plan::Boundedness { + asap_executor::plan::Boundedness::Bounded } fn schema(&self) -> SchemaRef { self.schema.clone() @@ -322,7 +322,7 @@ fn schema_drift_and_memory_limits_fail_the_scan() { // An empty table is a valid empty scan; nullable comparisons retain only TRUE. #[test] fn empty_sources_and_three_valued_predicates() { - use planner_types::pre_asap::{CompareOpKind, ScalarValue}; + use planner_types::ir::scalar::{CompareOpKind, ScalarValue}; let (mut scan, schema, batches) = fixture(); if let planner_types::ir::NonASAPOp::Scan { @@ -367,7 +367,7 @@ fn empty_sources_and_three_valued_predicates() { // A physical candidate can be compiled once without readers and rebound per run. #[test] fn compile_without_readers_and_rebind_inputs() { - use asap_physical_operators::{ + use asap_executor::{ operators::Operator, physical_planner::{compile, InputContract, Source}, }; @@ -403,7 +403,7 @@ fn compile_without_readers_and_rebind_inputs() { // Input boundedness must be proved during compilation, before readers exist. #[test] fn compilation_rejects_unknown_boundedness_for_sort() { - use asap_physical_operators::{ + use asap_executor::{ physical_planner::{compile, InputContract}, plan::{Boundedness, Emission, PlanProperties}, }; diff --git a/crates/asap-physical-operators/tests/summary_projection.rs b/crates/executor/tests/summary_projection.rs similarity index 91% rename from crates/asap-physical-operators/tests/summary_projection.rs rename to crates/executor/tests/summary_projection.rs index bdd2d92d7..c88d4b520 100644 --- a/crates/asap-physical-operators/tests/summary_projection.rs +++ b/crates/executor/tests/summary_projection.rs @@ -1,5 +1,5 @@ //! Opaque state travels through a retained physical projection without scalar decoding. -use asap_physical_operators::{ +use asap_executor::{ expressions::Expression, factory::create_planner_accumulator, operators::Operator, @@ -12,12 +12,11 @@ use planner_types::ir::physical_export::{ EdgeRole, GroupingEdgeCompatibility, PhysicalASAPDAG, PhysicalASAPDAGEdge, PhysicalASAPDAGNode, PhysicalASAPOperatorPayload, WindowEdgeCompatibility, }; +use planner_types::ir::properties::*; +use planner_types::ir::scalar::ColumnRef; +use planner_types::ir::schema::{DataType, *}; use planner_types::ir::ASAPOp; use planner_types::ir::NonASAPOp; -use planner_types::{ - post_asap::*, - pre_asap::{ColumnRef, DataType}, -}; use std::{collections::BTreeMap, sync::Arc}; // A Post-ASAP projection may reorder/rename summary columns; recovery must retain @@ -25,7 +24,7 @@ use std::{collections::BTreeMap, sync::Arc}; #[test] fn post_asap_summary_projection_survives_recovery() { let family = FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum); - let schema = Arc::new(planner_types::pre_asap::Schema { + let schema = Arc::new(planner_types::ir::schema::Schema { unique_keys: vec![], closed: false, fields: vec![ @@ -44,7 +43,7 @@ fn post_asap_summary_projection_survives_recovery() { ], time_index: None, }); - let output = planner_types::pre_asap::Schema { + let output = planner_types::ir::schema::Schema { unique_keys: vec![], closed: false, fields: vec![ @@ -65,8 +64,8 @@ fn post_asap_summary_projection_survives_recovery() { }), output_schema: (*schema).clone(), output_state: ExecutionDataState::INGESTION_SUMMARY, - materialization: planner_types::post_asap::Materialization::MatAtIngestion( - planner_types::post_asap::Medium::Memory, + materialization: planner_types::physical::Materialization::MatAtIngestion( + planner_types::physical::Medium::Memory, ), guarantee: None, }, @@ -85,8 +84,8 @@ fn post_asap_summary_projection_survives_recovery() { }), output_schema: output.clone(), output_state: ExecutionDataState::INGESTION_SUMMARY, - materialization: planner_types::post_asap::Materialization::MatAtIngestion( - planner_types::post_asap::Medium::Memory, + materialization: planner_types::physical::Materialization::MatAtIngestion( + planner_types::physical::Medium::Memory, ), guarantee: None, }, @@ -184,7 +183,7 @@ fn post_asap_summary_projection_survives_recovery() { #[test] fn summary_merge_without_inputs_is_rejected() { let family = FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum); - let schema = planner_types::pre_asap::Schema { + let schema = planner_types::ir::schema::Schema { unique_keys: vec![], closed: false, fields: vec![Field { diff --git a/crates/asap-physical-operators/tests/univmon_execution.rs b/crates/executor/tests/univmon_execution.rs similarity index 93% rename from crates/asap-physical-operators/tests/univmon_execution.rs rename to crates/executor/tests/univmon_execution.rs index fc8725117..3eed50efb 100644 --- a/crates/asap-physical-operators/tests/univmon_execution.rs +++ b/crates/executor/tests/univmon_execution.rs @@ -1,6 +1,6 @@ //! One native UnivMon build supplies the three statistics of #509 Example 2: //! a distinct count, an entropy and an L2 norm of one identity column. -use asap_physical_operators::{ +use asap_executor::{ operators::{Operator, SummaryEvaluation}, physical_planner::compile_node, plan::{PhysicalDAG, PhysicalOperator}, @@ -9,14 +9,16 @@ use asap_physical_operators::{ Error, }; use futures::{executor::block_on, StreamExt}; +use planner_types::ir::operator::{GroupKeys, Reduction}; use planner_types::ir::physical_export::{PhysicalASAPDAGNode, PhysicalASAPOperatorPayload}; -use planner_types::ir::ASAPOp; -use planner_types::post_asap::{ - ExecutionDataState, Field, FieldDataType, GroupingStrategy, NonNegativeWeightProof, - SketchAlgorithm, SketchKind, SketchParams, SketchStatistic, SummaryInputExpr, SummaryUpdate, - WeightDomain, +use planner_types::ir::properties::ExecutionDataState; +use planner_types::ir::scalar::ColumnRef; +use planner_types::ir::schema::{DataType, Schema}; +use planner_types::ir::schema::{ + Field, FieldDataType, GroupingStrategy, NonNegativeWeightProof, SketchAlgorithm, SketchKind, + SketchParams, SketchStatistic, SummaryInputExpr, SummaryUpdate, WeightDomain, }; -use planner_types::pre_asap::{ColumnRef, DataType, GroupKeys, Reduction, Schema}; +use planner_types::ir::ASAPOp; use std::sync::Arc; fn family() -> FieldDataType { diff --git a/crates/asap-physical-operators/tests/weighted_topk_binding.rs b/crates/executor/tests/weighted_topk_binding.rs similarity index 95% rename from crates/asap-physical-operators/tests/weighted_topk_binding.rs rename to crates/executor/tests/weighted_topk_binding.rs index 2cd1973e1..183781df7 100644 --- a/crates/asap-physical-operators/tests/weighted_topk_binding.rs +++ b/crates/executor/tests/weighted_topk_binding.rs @@ -1,6 +1,6 @@ //! Planner output binds directly to the shared runtime at a declared rate-value frontier. mod common; -use asap_physical_operators::dag::{ +use asap_executor::dag::{ operators::Operator, planner::{compile, InputContract, Source}, values::{Batch, Value}, @@ -9,8 +9,9 @@ use asap_physical_operators::dag::{ use common::{compile_physical_asap_dag, logical_candidates}; use futures::{executor::block_on, StreamExt}; use planner_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPOperatorPayload}; +use planner_types::ir::properties::{AccuracyTarget, *}; +use planner_types::ir::schema::{DataType, *}; use planner_types::ir::ASAPOp; -use planner_types::{post_asap::*, pre_asap::DataType, types::AccuracyTarget}; use std::{collections::BTreeMap, rc::Rc, sync::Arc}; #[test] #[ignore = "the whole-expression heap's SummaryEstimate{TopK} output is not the keyed-evaluation \ @@ -223,21 +224,19 @@ fn rate_updates_cannot_enter_integer_heap_factory() { ); let input = SummaryUpdate { item: Some(SummaryInputExpr::Column( - planner_types::pre_asap::ColumnRef::Named("service".into()), + planner_types::ir::scalar::ColumnRef::Named("service".into()), )), - weight: SummaryInputExpr::Column(planner_types::pre_asap::ColumnRef::SampleValue), + weight: SummaryInputExpr::Column(planner_types::ir::scalar::ColumnRef::SampleValue), weight_domain: WeightDomain::NonNegative { proof: NonNegativeWeightProof::ResetAwareCounterDerivative, }, }; - assert!( - asap_physical_operators::factory::create_planner_accumulator( - &family, - &input, - &Default::default() - ) - .is_err() - ); + assert!(asap_executor::factory::create_planner_accumulator( + &family, + &input, + &Default::default() + ) + .is_err()); } // A per-series rate feeds a heap sketch directly, without a grouped Sum @@ -249,7 +248,7 @@ fn rate_updates_cannot_enter_integer_heap_factory() { #[ignore = "the runtime's keyed top-k evaluation returns the selected rows, but the planner's \ top-k estimate declares [keys.., topk_k]; the canonical top-k schema is #672"] fn direct_rate_topk_preserves_dynamic_unreferenced_labels() { - use asap_physical_operators::physical_planner::promql_rows::{ + use asap_executor::physical_planner::promql_rows::{ decode_series_identity, series_row, with_series_identity, SERIES_IDENTITY_COLUMN, }; let logical = lower_promql( @@ -261,8 +260,7 @@ fn direct_rate_topk_preserves_dynamic_unreferenced_labels() { for algorithm in [SketchAlgorithm::CountSketchWithHeap] { let candidate = &heap_candidate(Rc::clone(&root), &algorithm); let (source, ranked) = - asap_physical_operators::physical_planner::promql_rows::compile_rate_ranking(candidate) - .unwrap(); + asap_executor::physical_planner::promql_rows::compile_rate_ranking(candidate).unwrap(); assert!(matches!( source.operator, planner_types::ir::Operator::ASAP( @@ -316,10 +314,9 @@ fn direct_rate_topk_preserves_dynamic_unreferenced_labels() { ) .unwrap(); let bytes = serde_json::to_vec(&raw_compiled).unwrap(); - let raw_compiled = serde_json::from_slice::< - asap_physical_operators::physical_planner::CompiledPhysicalDAG, - >(&bytes) - .unwrap(); + let raw_compiled = + serde_json::from_slice::(&bytes) + .unwrap(); // Each evaluation receives a complete raw window. A reset, a stopped // series and an expired leader must not retain last run's heap weights. for (end, series, expected) in [ @@ -520,9 +517,10 @@ fn direct_rate_topk_preserves_dynamic_unreferenced_labels() { /// it upstream of a query-time node, so the pin is lifted first. fn continuously_maintained_dag(candidate: &Rc) -> PhysicalASAPDAG { use planner_types::ir::{ - apply_materialization_timings, Materialization, MaterializationAssignment, Medium, - Operator as IrOperator, OperatorNode, TimingMemo, + apply_materialization_timings, MaterializationAssignment, Operator as IrOperator, + OperatorNode, TimingMemo, }; + use planner_types::physical::{Materialization, Medium}; fn unpinned(node: &Rc) -> Rc { let mut copy = (**node).clone(); copy.operator = node.operator.map_children(unpinned); @@ -552,9 +550,7 @@ fn continuously_maintained_dag(candidate: &Rc) #[ignore = "the runtime's keyed top-k evaluation returns the selected rows, but the planner's \ top-k estimate declares [keys.., topk_k]; the canonical top-k schema is #672"] fn maintained_rate_heap_compiles_fixed_window_precompute() { - use asap_physical_operators::physical_planner::{ - compile_candidate, promql_rows::with_series_identity, - }; + use asap_executor::physical_planner::{compile_candidate, promql_rows::with_series_identity}; let root = Rc::new( with_series_identity( &lower_promql( @@ -610,14 +606,18 @@ fn maintained_rate_heap_compiles_fixed_window_precompute() { &[heap.id as u64], ) .unwrap(); - let exported = asap_physical_operators::physical_planner::promql_rows::compile_fixed_window_rate_aggregation(&dag).unwrap(); + let exported = + asap_executor::physical_planner::promql_rows::compile_fixed_window_rate_aggregation( + &dag, + ) + .unwrap(); assert_eq!( serde_json::to_vec(&exported).unwrap(), serde_json::to_vec(&physical).unwrap() ); // Execute the selected split across a state serialization boundary. // Each run builds fresh weights from that window's counters. - let execute = |plan: &asap_physical_operators::physical_planner::CompiledPhysicalDAG, + let execute = |plan: &asap_executor::physical_planner::CompiledPhysicalDAG, input: Batch, scope: Scope| { let id = plan.input_contracts().next().unwrap().0; @@ -667,10 +667,8 @@ fn maintained_rate_heap_compiles_fixed_window_precompute() { .zip(["a", "b", "c"]) .map(|(samples, label)| { let mut accumulator = - asap_physical_operators::factory::create_planner_accumulator( - family, input, grouping, - ) - .unwrap(); + asap_executor::factory::create_planner_accumulator(family, input, grouping) + .unwrap(); for (offset, value) in [10_000, 30_000, 50_000].into_iter().zip(samples) { accumulator.update_single(value, end - 60_000 + offset); } diff --git a/crates/frontend-common/src/lib.rs b/crates/frontend-common/src/lib.rs index f756bc46a..47684f668 100644 --- a/crates/frontend-common/src/lib.rs +++ b/crates/frontend-common/src/lib.rs @@ -2,7 +2,7 @@ //! and its resolver into the unified IR. //! //! A front end builds an [`UnresolvedOp`] tree (column references are -//! name-based [`ColumnRef`](asap_types::pre_asap::ColumnRef)s) during its +//! name-based [`ColumnRef`](asap_types::ir::scalar::ColumnRef)s) during its //! own `interpret` step and calls [`resolve_root`], which binds every //! reference to a positional `ColumnId` and returns the //! [`OperatorNode`](asap_types::ir::OperatorNode) DAG. diff --git a/crates/frontend-common/src/resolve.rs b/crates/frontend-common/src/resolve.rs index 24b354cf1..5fb09d9c5 100644 --- a/crates/frontend-common/src/resolve.rs +++ b/crates/frontend-common/src/resolve.rs @@ -15,18 +15,18 @@ //! label names its enclosing scope references (issue #52): the `job` in //! `sum by (job)(a or b)` appears in neither side's own matchers. -use asap_types::ir::aggregate_schema::aggregate_output_schema; +use asap_types::ir::schema::aggregate_schema::aggregate_output_schema; use std::rc::Rc; use thiserror::Error; -use asap_types::ir::operator_properties::ConcatDiscriminatorKey; +use asap_types::ir::operator::operator_properties::ConcatDiscriminatorKey; +use asap_types::ir::operator::{AggIntent, GroupKeys, Reduction}; +use asap_types::ir::scalar::column_resolution::resolve_group_keys_promql; +use asap_types::ir::scalar::{resolve_column_ref, resolve_column_refs, ColumnRef, ResolveError}; +use asap_types::ir::schema::{ColumnId, Schema}; +use asap_types::ir::SchemaDerivationError; use asap_types::ir::{NonASAPOp, OperatorNode, Predicate, ProjectItem, ScalarExpr, SortKey}; -use asap_types::pre_asap::column_resolution::resolve_group_keys_promql; -use asap_types::pre_asap::{ - resolve_column_ref, resolve_column_refs, AggIntent, ColumnId, ColumnRef, GroupKeys, Reduction, - ResolveError, Schema, SchemaDerivationError, -}; use crate::schema_resolver::{collect_referenced_columns, SchemaResolver}; use crate::unresolved::{UnresolvedOp, UnresolvedScalar, UnresolvedSortKey}; @@ -157,7 +157,7 @@ fn resolve( let child = if child.schema.closed { child } else { - asap_types::ir::schema_support::with_promql_series_identity(&child) + asap_types::ir::schema::schema_support::with_promql_series_identity(&child) .map_err(SchemaDerivationError::InvalidScalarSignature)? }; let scalar = resolve_expr(scalar, &Schema::default())?; @@ -172,7 +172,7 @@ fn resolve( let child = if child.schema.closed { child } else { - asap_types::ir::schema_support::with_promql_series_identity(&child) + asap_types::ir::schema::schema_support::with_promql_series_identity(&child) .map_err(SchemaDerivationError::InvalidScalarSignature)? }; let sample = resolve_expr(sample, &child.schema)?; @@ -733,12 +733,14 @@ pub fn resolve_scalar_root(tree: &UnresolvedScalar) -> Result, scalar: ScalarExpr, - op: &asap_types::pre_asap::BinaryOpKind, + op: &asap_types::ir::operator::BinaryOpKind, scalar_left: bool, return_bool: bool, ) -> Result, ResolveDAGError> { + use asap_types::ir::operator::BinaryOpKind; + use asap_types::ir::scalar::ScalarValue; + use asap_types::ir::schema::DataType; use asap_types::ir::ExprSemantics; - use asap_types::pre_asap::{BinaryOpKind, DataType, ScalarValue}; let value = child .schema .column_id("value") @@ -807,7 +809,7 @@ fn project_sample( computed: ScalarExpr, drop_metric_name: bool, ) -> Result, ResolveDAGError> { - let value = asap_types::pre_asap::column_resolution::resolve_column_ref( + let value = asap_types::ir::scalar::column_resolution::resolve_column_ref( &ColumnRef::SampleValue, &child.schema, )?; @@ -820,9 +822,7 @@ fn project_sample( .map(|(i, f)| { let expr = if i == value { computed.clone() - } else if drop_metric_name - && f.name == asap_types::pre_asap::schema::PROMQL_SERIES_IDENTITY - { + } else if drop_metric_name && f.name == asap_types::ir::schema::PROMQL_SERIES_IDENTITY { ScalarExpr::FunctionCall { name: "promql_drop_metric_name".into(), args: vec![ScalarExpr::Column(i)], @@ -847,13 +847,14 @@ fn project_sample( mod tests { use super::*; use crate::unresolved::UnresolvedPredicate; + use asap_types::ir::operator::{ + BinaryOpKind, JoinKind, PromQLVectorSetOpKind, Source, VectorMatch, + }; + use asap_types::ir::properties::AccuracyTarget; + use asap_types::ir::scalar::{CompareOpKind, ScalarValue}; + use asap_types::ir::schema::{DataType, Field}; use asap_types::ir::BinaryOperator; use asap_types::ir::ExprSemantics; - use asap_types::pre_asap::{ - BinaryOpKind, CompareOpKind, DataType, Field, JoinKind, PromQLVectorSetOpKind, ScalarValue, - Source, VectorMatch, - }; - use asap_types::types::AccuracyTarget; fn scan(metric: &str) -> UnresolvedOp { UnresolvedOp::Scan { @@ -1134,7 +1135,7 @@ mod tests { let unresolved = UnresolvedOp::PromqlScalarOp { child: Rc::new(scan("m")), scalar: UnresolvedScalar::PromqlScalarFromVector(Rc::new(x)), - op: BinaryOpKind::Arithmetic(asap_types::pre_asap::ArithmeticOpKind::Mul), + op: BinaryOpKind::Arithmetic(asap_types::ir::scalar::ArithmeticOpKind::Mul), scalar_left: false, return_bool: false, }; diff --git a/crates/frontend-common/src/schema_resolver.rs b/crates/frontend-common/src/schema_resolver.rs index 07037766f..dd2862458 100644 --- a/crates/frontend-common/src/schema_resolver.rs +++ b/crates/frontend-common/src/schema_resolver.rs @@ -9,7 +9,9 @@ //! observability domain (metric label sets are open-ended). A registry-backed //! `SchemaCatalog` is future work; only the catalog impl swaps when it lands. -use asap_types::pre_asap::{AggIntent, ColumnRef, DataType, Field, GroupKeys, Reduction, Schema}; +use asap_types::ir::operator::{AggIntent, GroupKeys, Reduction}; +use asap_types::ir::scalar::ColumnRef; +use asap_types::ir::schema::{DataType, Field, Schema}; use crate::unresolved::{UnresolvedOp, UnresolvedScalar}; @@ -128,7 +130,7 @@ fn push_ref_name(c: &ColumnRef, out: &mut Vec) { /// (never the operators referenced from scalar positions: those are bound in /// their own scope). fn leftmost_scan_name(tree: &UnresolvedOp) -> Option<&str> { - use asap_types::pre_asap::Source; + use asap_types::ir::operator::Source; use UnresolvedOp as U; match tree { U::Scan { source, .. } => Some(match source { @@ -319,7 +321,7 @@ pub fn collect_referenced_columns(tree: &UnresolvedOp) -> Vec { mod tests { use std::rc::Rc; - use asap_types::pre_asap::{AggIntent, Reduction, Source}; + use asap_types::ir::operator::{AggIntent, Reduction, Source}; use super::*; use crate::unresolved::UnresolvedSortKey; diff --git a/crates/frontend-common/src/unresolved.rs b/crates/frontend-common/src/unresolved.rs index 0bca427cc..eb6ab3587 100644 --- a/crates/frontend-common/src/unresolved.rs +++ b/crates/frontend-common/src/unresolved.rs @@ -15,14 +15,15 @@ use std::time::Duration; use serde::{Deserialize, Serialize}; -use asap_types::ir::operator_properties::ConcatDiscriminatorKey; +use asap_types::ir::operator::operator_properties::ConcatDiscriminatorKey; +use asap_types::ir::operator::{ + AggIntent, GroupKeys, InfoMatcher, JoinKind, Reduction, RelationalSetOpKind, SampleKind, + Source, TimeShift, WindowFrame, WindowFuncKind, +}; +use asap_types::ir::scalar::{ArithmeticOpKind, ColumnRef, CompareOpKind, ScalarValue}; +use asap_types::ir::schema::{DataType, Schema}; use asap_types::ir::BinaryOperator; use asap_types::ir::{ExprSemantics, TimeRangeKind}; -use asap_types::pre_asap::{ - AggIntent, ArithmeticOpKind, ColumnRef, CompareOpKind, DataType, GroupKeys, InfoMatcher, - JoinKind, Reduction, RelationalSetOpKind, SampleKind, ScalarValue, Schema, Source, TimeShift, - WindowFrame, WindowFuncKind, -}; /// A row-level filter predicate (WHERE clause / PromQL label matcher). #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -221,7 +222,7 @@ pub enum UnresolvedOp { PromqlScalarOp { child: Rc, scalar: UnresolvedScalar, - op: asap_types::pre_asap::BinaryOpKind, + op: asap_types::ir::operator::BinaryOpKind, scalar_left: bool, return_bool: bool, }, diff --git a/crates/frontend-metricsql/src/lib.rs b/crates/frontend-metricsql/src/lib.rs index e983f135d..125409cd9 100644 --- a/crates/frontend-metricsql/src/lib.rs +++ b/crates/frontend-metricsql/src/lib.rs @@ -5,12 +5,12 @@ use std::{rc::Rc, time::Duration}; use asap_frontend_common::{ resolve_root, UnresolvedOp as U, UnresolvedPredicate, UnresolvedScalar, }; -use asap_types::ir::{BinaryOperator, ExprSemantics, OperatorNode, TimeRangeKind}; -use asap_types::pre_asap::{ - AggIntent, ArithmeticOpKind, BinaryOpKind, ColumnRef, CompareOpKind, GroupKeys, - PromQLVectorSetOpKind, Reduction, ScalarValue, Source, +use asap_types::ir::operator::{ + AggIntent, BinaryOpKind, GroupKeys, PromQLVectorSetOpKind, Reduction, Source, }; -use asap_types::types::AccuracyTarget; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::scalar::{ArithmeticOpKind, ColumnRef, CompareOpKind, ScalarValue}; +use asap_types::ir::{BinaryOperator, ExprSemantics, OperatorNode, TimeRangeKind}; use asap_types::workload::{DurationMs, PlanningWorkload, QueryLanguage, WorkloadError}; use metricsql_parser::ast::{AggregateModifier, DurationExpr, Expr, MetricExpr, RollupExpr}; use metricsql_parser::functions::{AggregateFunction, BuiltinFunction, RollupFunction}; diff --git a/crates/frontend-metricsql/tests/lowering.rs b/crates/frontend-metricsql/tests/lowering.rs index a4c8dd689..04e15268a 100644 --- a/crates/frontend-metricsql/tests/lowering.rs +++ b/crates/frontend-metricsql/tests/lowering.rs @@ -4,9 +4,9 @@ use std::time::Duration; use asap_frontend_metricsql::{ canonical_metricsql, lower_metricsql, parse_metricsql, MetricsqlError, }; +use asap_types::ir::operator::{AggIntent, Reduction, Source}; +use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::{NonASAPOp, OperatorNode, TimeRangeKind}; -use asap_types::pre_asap::{AggIntent, Reduction, Source}; -use asap_types::types::AccuracyTarget; /// Every metric's sample interval, the lookback of a bare selector. const INTERVAL: Duration = Duration::from_secs(15); diff --git a/crates/frontend-metricsql/tests/lowering_semantics.rs b/crates/frontend-metricsql/tests/lowering_semantics.rs index 1ae78e857..3a2e28a5d 100644 --- a/crates/frontend-metricsql/tests/lowering_semantics.rs +++ b/crates/frontend-metricsql/tests/lowering_semantics.rs @@ -2,9 +2,9 @@ use std::time::Duration; use asap_frontend_metricsql::{lower_metricsql_query_workload, MetricsqlError}; +use asap_types::ir::operator::AggIntent; +use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::{NonASAPOp, OperatorNode, QueryRoot, TimeRangeKind}; -use asap_types::pre_asap::AggIntent; -use asap_types::types::AccuracyTarget; use asap_types::workload::{ BatchEntry, DataArrival, DataWorkload, DurationMs, PlanningWorkload, Query, QueryLanguage, QueryRequirements, QueryWorkload, SourceFacts, WorkloadError, diff --git a/crates/frontend-promql/Cargo.toml b/crates/frontend-promql/Cargo.toml index 569c87c9a..40eff896d 100644 --- a/crates/frontend-promql/Cargo.toml +++ b/crates/frontend-promql/Cargo.toml @@ -16,7 +16,8 @@ asap-frontend-common = { path = "../frontend-common" } promql-parser = { git = "https://github.com/ProjectASAP/promql-parser", rev = "9fede7eecca923c9882fe256484d00d37f8706cb" } [dev-dependencies] -asap-aware-mapping = { path = "../asap-aware-mapping" } +asap-logical-asap-optimization = { path = "../logical-asap-optimization" } +asap-plan-selection = { path = "../plan-selection" } # Corpus tests live in domain folders (observability) but must still be declared # as test targets — Cargo only auto-discovers `.rs` files directly in `tests/`. diff --git a/crates/frontend-promql/src/lib.rs b/crates/frontend-promql/src/lib.rs index c5254f104..c4ec52e84 100644 --- a/crates/frontend-promql/src/lib.rs +++ b/crates/frontend-promql/src/lib.rs @@ -114,8 +114,8 @@ fn lower_promql_query_workload_inner( mod tests { use std::time::Duration; + use asap_types::ir::operator::Source; use asap_types::ir::{NonASAPOp, OperatorNode, TimeRangeKind}; - use asap_types::pre_asap::Source; use asap_types::workload::{ BatchEntry, DataArrival, DataWorkload, PlanningWorkload, Query, QueryRequirements, QueryWorkload, SourceFacts, WorkloadError, diff --git a/crates/frontend-promql/src/promql.rs b/crates/frontend-promql/src/promql.rs index e510a2e08..f05dc4474 100644 --- a/crates/frontend-promql/src/promql.rs +++ b/crates/frontend-promql/src/promql.rs @@ -70,17 +70,16 @@ use promql_parser::parser::{ use asap_frontend_common::{ UnresolvedOp as Unresolved, UnresolvedPredicate, UnresolvedScalar as Scalar, UnresolvedSortKey, }; -use asap_types::ir::operator_properties::{ +use asap_types::ir::operator::agg_intent::{topk, AggIntent, TimeFunc}; +use asap_types::ir::operator::operator_properties::{ AtModifier, BinaryOpKind, GroupKeys, GroupSide, PromQLVectorSetOpKind, Reduction, Source, TimeShift, VectorGrouping, VectorMatch, VectorMatchKind, }; use asap_types::ir::{BinaryOperator, ExprSemantics, TimeRangeKind}; -use asap_types::pre_asap::agg_intent::{topk, AggIntent, TimeFunc}; -use asap_types::pre_asap::{ - ArithmeticOpKind, ColumnRef, CompareOpKind, InfoMatcher, SampleKind, ScalarValue, -}; -use asap_types::types::AccuracyTarget; +use asap_types::ir::operator::{InfoMatcher, SampleKind}; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::scalar::{ArithmeticOpKind, ColumnRef, CompareOpKind, ScalarValue}; use asap_types::workload::WorkloadError; /// Every scalar expression this front end builds follows PromQL's numeric rules. diff --git a/crates/frontend-promql/tests/count_planning.rs b/crates/frontend-promql/tests/count_planning.rs index 35046932d..fafadc70d 100644 --- a/crates/frontend-promql/tests/count_planning.rs +++ b/crates/frontend-promql/tests/count_planning.rs @@ -1,12 +1,12 @@ //! Query text through summary selection: counts use observations, never value weights. mod support; use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; -use asap_types::ir::{ASAPOp, Operator, OperatorNode}; -use asap_types::post_asap::{ +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::schema::{ ExactKind, FieldDataType, GroupingStrategy, NonNegativeWeightProof, SketchAlgorithm, SummaryInputExpr, WeightDomain, }; -use asap_types::types::AccuracyTarget; +use asap_types::ir::{ASAPOp, Operator, OperatorNode}; use std::rc::Rc; use support::{logical_candidates, lower_promql, post_asap_dag, selected_dag}; @@ -29,7 +29,7 @@ fn grouped_count_offers_hydra_but_selection_keeps_per_group_state() { let target = AccuracyTarget::max_error(0.01).failure_probability(0.01); // Hydra hashes a non-null item per row: the series identity (PromQL // labels are nullable). - let root = asap_types::ir::schema_support::with_promql_series_identity( + let root = asap_types::ir::schema::schema_support::with_promql_series_identity( &lower_promql("count by(job)(up)", target.clone()).unwrap(), ) .unwrap(); @@ -136,8 +136,8 @@ fn frequency_count_candidates_use_unit_weights() { // This narrow test oracle interprets the emitted aggregate, not Prometheus ingestion, // staleness, or scrape scheduling. Unsupported plan shapes fail explicitly. fn aggregate_fixture(query: &str, series: &[Vec]) -> Vec { + use asap_types::ir::operator::{AggIntent, Reduction}; use asap_types::ir::NonASAPOp; - use asap_types::pre_asap::{AggIntent, Reduction}; let root = lower_promql(query, AccuracyTarget::exact()).unwrap(); let NonASAPOp::Aggregate { reduction, @@ -226,7 +226,7 @@ fn count_over_time_counts_scrapes_not_sample_values() { // This checks the planner's numerical update contract, not a sketch-library runtime. #[test] fn cms_count_updates_total_ten_for_zero_positive_and_negative_samples() { - use asap_types::pre_asap::ColumnRef; + use asap_types::ir::scalar::ColumnRef; let accuracy = AccuracyTarget::max_error(0.02); let root = lower_promql("count_over_time(up[5m])", accuracy.clone()).unwrap(); let candidates = logical_candidates(root, accuracy); diff --git a/crates/frontend-promql/tests/histogram_metadata.rs b/crates/frontend-promql/tests/histogram_metadata.rs index e5f585f69..35ca65ca2 100644 --- a/crates/frontend-promql/tests/histogram_metadata.rs +++ b/crates/frontend-promql/tests/histogram_metadata.rs @@ -7,9 +7,9 @@ use asap_frontend_promql::{HistogramCatalog, HistogramKind}; mod support; +use asap_types::ir::operator::AggIntent; +use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::{NonASAPOp, OperatorNode}; -use asap_types::pre_asap::AggIntent; -use asap_types::types::AccuracyTarget; use support::{lower_promql, lower_promql_with_histograms}; /// The histogram/quantile intent kind in the lowered DAG: `"HQ"` for the diff --git a/crates/frontend-promql/tests/maintained_population_horizon.rs b/crates/frontend-promql/tests/maintained_population_horizon.rs index 9d4e6a837..baff0db16 100644 --- a/crates/frontend-promql/tests/maintained_population_horizon.rs +++ b/crates/frontend-promql/tests/maintained_population_horizon.rs @@ -1,8 +1,8 @@ mod support; -use asap_aware_mapping::maintained_population::MaintainedPopulationStrategy; +use asap_logical_asap_optimization::maintained_population::MaintainedPopulationStrategy; +use asap_types::ir::operator::maintained_population::PopulationInput; +use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::{ASAPOp, NonASAPOp, Operator}; -use asap_types::post_asap::maintained_population::PopulationInput; -use asap_types::types::AccuracyTarget; // A population for a one-second selector must expire members after one second. #[test] diff --git a/crates/frontend-promql/tests/observability/awesome_prometheus_alerts.rs b/crates/frontend-promql/tests/observability/awesome_prometheus_alerts.rs index 85cca5f52..eb1795f20 100644 --- a/crates/frontend-promql/tests/observability/awesome_prometheus_alerts.rs +++ b/crates/frontend-promql/tests/observability/awesome_prometheus_alerts.rs @@ -31,9 +31,10 @@ use asap_frontend_promql::PromqlError as LoweringError; mod support; use std::rc::Rc; +use asap_types::ir::operator::{AggIntent, BinaryOpKind, Reduction}; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::scalar::{CompareOpKind, ScalarValue}; use asap_types::ir::{BinaryOperator, NonASAPOp, OperatorNode, ScalarExpr}; -use asap_types::pre_asap::{AggIntent, BinaryOpKind, CompareOpKind, Reduction, ScalarValue}; -use asap_types::types::AccuracyTarget; use support::lower_promql; const CORPUS: &str = include_str!("data/awesome_prometheus_alerts.txt"); diff --git a/crates/frontend-promql/tests/observability/metrics_observability.rs b/crates/frontend-promql/tests/observability/metrics_observability.rs index 8c640b374..e080dfe38 100644 --- a/crates/frontend-promql/tests/observability/metrics_observability.rs +++ b/crates/frontend-promql/tests/observability/metrics_observability.rs @@ -7,7 +7,7 @@ use asap_frontend_promql::PromqlError; #[path = "../support.rs"] mod support; -use asap_types::types::AccuracyTarget; +use asap_types::ir::properties::AccuracyTarget; use support::{logical_plan, lower_promql}; const CORPORA: &[(&str, &str)] = &[ diff --git a/crates/frontend-promql/tests/observability/o11y_bench_promql.rs b/crates/frontend-promql/tests/observability/o11y_bench_promql.rs index 5fbbcfce0..53d125bc1 100644 --- a/crates/frontend-promql/tests/observability/o11y_bench_promql.rs +++ b/crates/frontend-promql/tests/observability/o11y_bench_promql.rs @@ -21,7 +21,7 @@ use asap_frontend_promql::PromqlError as LoweringError; #[path = "../support.rs"] mod support; -use asap_types::types::AccuracyTarget; +use asap_types::ir::properties::AccuracyTarget; use support::lower_promql; const CORPUS: &str = include_str!("data/o11y_bench_promql.txt"); diff --git a/crates/frontend-promql/tests/observability/promql_corpus.rs b/crates/frontend-promql/tests/observability/promql_corpus.rs index 6f96c901e..954af5966 100644 --- a/crates/frontend-promql/tests/observability/promql_corpus.rs +++ b/crates/frontend-promql/tests/observability/promql_corpus.rs @@ -16,7 +16,7 @@ use asap_frontend_promql::PromqlError as LoweringError; #[path = "../support.rs"] mod support; -use asap_types::types::AccuracyTarget; +use asap_types::ir::properties::AccuracyTarget; use support::{logical_plan, lower_promql}; const DOCS: &str = include_str!("data/promql_corpus_docs.txt"); diff --git a/crates/frontend-promql/tests/promql_binding_regressions.rs b/crates/frontend-promql/tests/promql_binding_regressions.rs index 6e3c8b444..7cb58a766 100644 --- a/crates/frontend-promql/tests/promql_binding_regressions.rs +++ b/crates/frontend-promql/tests/promql_binding_regressions.rs @@ -1,5 +1,5 @@ mod support; -use asap_types::types::AccuracyTarget; +use asap_types::ir::properties::AccuracyTarget; use support::lower_promql; /// Prometheus treats these quantile parameters as valid queries returning special values. @@ -32,8 +32,8 @@ fn irate_and_rate_have_distinct_canonical_intents() { /// PromQL count counts series even when two sample values are equal. #[test] fn count_is_row_count_not_distinct_sample_value_count() { + use asap_types::ir::operator::AggIntent; use asap_types::ir::NonASAPOp; - use asap_types::pre_asap::AggIntent; let tree = lower_promql("count(smoke_gauge)", AccuracyTarget::exact()).unwrap(); let NonASAPOp::Aggregate { measures, .. } = tree.expect_non_asap() else { panic!("expected aggregate") diff --git a/crates/frontend-promql/tests/promql_conformance.rs b/crates/frontend-promql/tests/promql_conformance.rs index 0a06627e9..a259a2931 100644 --- a/crates/frontend-promql/tests/promql_conformance.rs +++ b/crates/frontend-promql/tests/promql_conformance.rs @@ -36,15 +36,16 @@ use std::time::Duration; use asap_frontend_promql::PromqlError as LoweringError; mod support; +use asap_types::ir::operator::{ + AggIntent, AtModifier, BinaryOpKind, PromQLVectorSetOpKind, Reduction, SampleKind, Source, + TimeFunc, +}; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::scalar::{ArithmeticOpKind, CompareOpKind, ScalarValue}; +use asap_types::ir::schema::DataType; use asap_types::ir::{ BinaryOperator, ExprSemantics, NonASAPOp, OperatorNode, ScalarExpr, TimeRangeKind, }; -use asap_types::pre_asap::schema::DataType; -use asap_types::pre_asap::{ - AggIntent, ArithmeticOpKind, AtModifier, BinaryOpKind, CompareOpKind, PromQLVectorSetOpKind, - Reduction, SampleKind, ScalarValue, Source, TimeFunc, -}; -use asap_types::types::AccuracyTarget; use support::{lower_promql, promql_scalar}; // ── harness helpers ───────────────────────────────────────────────────────────── diff --git a/crates/frontend-promql/tests/promql_equivalence.rs b/crates/frontend-promql/tests/promql_equivalence.rs index 9f7f3d395..79de910c5 100644 --- a/crates/frontend-promql/tests/promql_equivalence.rs +++ b/crates/frontend-promql/tests/promql_equivalence.rs @@ -20,8 +20,8 @@ use std::rc::Rc; mod support; +use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::OperatorNode; -use asap_types::types::AccuracyTarget; use support::lower_promql; fn lo(q: &str) -> Rc { diff --git a/crates/frontend-promql/tests/promql_lowering.rs b/crates/frontend-promql/tests/promql_lowering.rs index ddd5eddd1..1569cbddc 100644 --- a/crates/frontend-promql/tests/promql_lowering.rs +++ b/crates/frontend-promql/tests/promql_lowering.rs @@ -3,13 +3,12 @@ use std::rc::Rc; use std::time::Duration; +use asap_types::ir::operator::{AggIntent, BinaryOpKind, Reduction, Source}; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::scalar::{ArithmeticOpKind, CompareOpKind, ScalarValue}; use asap_types::ir::{ BinaryOperator, ExprSemantics, NonASAPOp, OperatorNode, ScalarExpr, TimeRangeKind, }; -use asap_types::pre_asap::{ - AggIntent, ArithmeticOpKind, BinaryOpKind, CompareOpKind, Reduction, ScalarValue, Source, -}; -use asap_types::types::AccuracyTarget; use asap_types::workload::{ BatchEntry, PlanningWorkload, Predictability, Query, QueryLanguage, QueryRequirements, QueryWorkload, @@ -836,7 +835,7 @@ fn binary_op_with_on_grouping() { panic!("expected BinaryOp, got {qe:?}"); }; let vm = vector_match.as_ref().expect("vector_match present"); - use asap_types::pre_asap::VectorMatchKind; + use asap_types::ir::operator::VectorMatchKind; assert_eq!(vm.kind, VectorMatchKind::On); assert_eq!(vm.labels, vec!["host".to_string()]); } diff --git a/crates/frontend-promql/tests/scalar_design.rs b/crates/frontend-promql/tests/scalar_design.rs index 888d7a086..ab8a49cac 100644 --- a/crates/frontend-promql/tests/scalar_design.rs +++ b/crates/frontend-promql/tests/scalar_design.rs @@ -1,8 +1,8 @@ //! Scalar expressions never become constant-wrapper operators. mod support; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::scalar::{ArithmeticOpKind, CompareOpKind, ScalarValue}; use asap_types::ir::{NonASAPOp, QueryRoot, ScalarExpr}; -use asap_types::pre_asap::{ArithmeticOpKind, CompareOpKind, ScalarValue}; -use asap_types::types::AccuracyTarget; fn root(query: &str) -> QueryRoot { asap_frontend_promql::lower_promql_query_workload(&support::workload( diff --git a/crates/frontend-promql/tests/support.rs b/crates/frontend-promql/tests/support.rs index 1c730b5ea..445f33875 100644 --- a/crates/frontend-promql/tests/support.rs +++ b/crates/frontend-promql/tests/support.rs @@ -3,9 +3,9 @@ use std::rc::Rc; use asap_frontend_promql::{ lower_promql_workload, lower_promql_workload_with_histograms, HistogramCatalog, PromqlError, }; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::scalar::ScalarValue; use asap_types::ir::{NonASAPOp, OperatorNode, ScalarExpr}; -use asap_types::pre_asap::ScalarValue; -use asap_types::types::AccuracyTarget; use asap_types::workload::{ BatchEntry, DataWorkload, DurationMs, PlanningWorkload, Predictability, Query, QueryLanguage, QueryRequirements, QueryWorkload, @@ -112,8 +112,8 @@ pub fn sample_expression(node: &OperatorNode) -> &ScalarExpr { pub fn run_stage1_to_stage3( root: Rc, accuracy: AccuracyTarget, -) -> asap_aware_mapping::Stage1ToStage3Run { - asap_aware_mapping::run_stage1_to_stage3( +) -> asap_plan_selection::Stage1ToStage3Run { + asap_plan_selection::run_stage1_to_stage3( vec![(0, asap_types::ir::QueryRoot::Operator(root))], &[asap_types::workload::QueryWorkloadEntry { requirements: asap_types::workload::QueryRequirements { @@ -123,7 +123,7 @@ pub fn run_stage1_to_stage3( ..Default::default() }], &DataWorkload::default(), - asap_aware_mapping::PlanningModels::builtin(), + asap_plan_selection::PlanningModels::builtin(), ) .expect("Pass 1 accepts the query") } @@ -170,17 +170,17 @@ pub fn logical_candidates( /// compose (a candidate its input cannot feed), and selection skips it. #[allow(dead_code)] pub fn logical_plan(root: &Rc) -> Result, String> { - let run = asap_aware_mapping::run_stage1_to_stage3( + let run = asap_plan_selection::run_stage1_to_stage3( vec![(0, asap_types::ir::QueryRoot::Operator(Rc::clone(root)))], &[asap_types::workload::QueryWorkloadEntry { requirements: asap_types::workload::QueryRequirements { - accuracy: asap_types::types::AccuracyTarget::exact(), + accuracy: asap_types::ir::properties::AccuracyTarget::exact(), max_latency_ms: None, }, ..Default::default() }], &DataWorkload::default(), - asap_aware_mapping::PlanningModels::builtin(), + asap_plan_selection::PlanningModels::builtin(), ) .map_err(|e| e.to_string())?; let operator = |roots: &[asap_types::ir::QueryRoot]| match &roots[0] { diff --git a/crates/frontend-promql/tests/univmon_candidates.rs b/crates/frontend-promql/tests/univmon_candidates.rs index 6a6076e98..a84cc3adc 100644 --- a/crates/frontend-promql/tests/univmon_candidates.rs +++ b/crates/frontend-promql/tests/univmon_candidates.rs @@ -1,13 +1,12 @@ use std::rc::Rc; -use asap_aware_mapping::accuracy::{AccuracyModel, DefaultAccuracyModel}; +use asap_logical_asap_optimization::accuracy::{AccuracyModel, DefaultAccuracyModel}; mod support; use asap_types::ir::cse::share_common_sub_dags; +use asap_types::ir::properties::AccuracyMetric; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::schema::{FieldDataType, SketchAlgorithm, SketchStatistic, SummaryInputExpr}; use asap_types::ir::{ASAPOp, Operator, OperatorNode}; -use asap_types::post_asap::{ - AccuracyMetric, FieldDataType, SketchAlgorithm, SketchStatistic, SummaryInputExpr, -}; -use asap_types::types::AccuracyTarget; use support::{logical_candidates, lower_promql, post_asap_dag, run_stage1_to_stage3}; /// The summary evaluation over a UnivMon state that `root` reads, if any. diff --git a/crates/frontend-sql/Cargo.toml b/crates/frontend-sql/Cargo.toml index 3f81f1104..ed92af0e0 100644 --- a/crates/frontend-sql/Cargo.toml +++ b/crates/frontend-sql/Cargo.toml @@ -17,7 +17,8 @@ datafusion = "54" serde_json = "1" [dev-dependencies] -asap-aware-mapping = { path = "../asap-aware-mapping" } +asap-logical-asap-optimization = { path = "../logical-asap-optimization" } +asap-plan-selection = { path = "../plan-selection" } tokio = { version = "1", features = ["rt", "macros", "rt-multi-thread"] } # bgp_jan2024_workload corpus is sourced verbatim as YAML (ASAPQuery PR #561) # rather than transcribed into the flat .sql shape the other corpora use. diff --git a/crates/frontend-sql/src/frequency.rs b/crates/frontend-sql/src/frequency.rs index d923a6b18..a2108faff 100644 --- a/crates/frontend-sql/src/frequency.rs +++ b/crates/frontend-sql/src/frequency.rs @@ -10,15 +10,16 @@ //! bits, the SQL idiom's `LN` gives nats. Integer products and nullable //! keys are refused, because SQL integer overflow and NULL groups are //! observable; such queries stay relational. -use asap_types::ir::non_asap::any_measure_filtered; +use asap_types::ir::operator::non_asap::any_measure_filtered; +use asap_types::ir::operator::{ + AggIntent, GroupKeys, JoinKind, Reduction, WindowFrameBound, WindowFrameOffset, WindowFuncKind, +}; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::scalar::{ArithmeticOpKind, CompareOpKind, ScalarValue}; +use asap_types::ir::schema::DataType; use asap_types::ir::{ ExprSemantics, NonASAPOp, Operator, OperatorNode, Predicate, ProjectItem, ScalarExpr, }; -use asap_types::pre_asap::{ - AggIntent, ArithmeticOpKind, CompareOpKind, DataType, GroupKeys, JoinKind, Reduction, - ScalarValue, WindowFrameBound, WindowFrameOffset, WindowFuncKind, -}; -use asap_types::types::AccuracyTarget; use std::rc::Rc; /// The recognized form of a query whose root is the L2 or natural-log entropy diff --git a/crates/frontend-sql/src/lib.rs b/crates/frontend-sql/src/lib.rs index 97eac2be8..a09f80547 100644 --- a/crates/frontend-sql/src/lib.rs +++ b/crates/frontend-sql/src/lib.rs @@ -3,7 +3,7 @@ //! (issue #179) → [`resolve_root`]. //! //! Emits the shared front-end tree (`UnresolvedOp` / `UnresolvedScalar`, -//! name-based [`ColumnRef`](asap_types::pre_asap::ColumnRef)s) directly, rather +//! name-based [`ColumnRef`](asap_types::ir::scalar::ColumnRef)s) directly, rather //! than a separate per-language relational tree; `resolve_root` binds it into //! the unified [`OperatorNode`] IR, deriving every schema on the way. //! Depends on DataFusion only — never on the PromQL parser. @@ -15,8 +15,8 @@ pub mod sql; use std::rc::Rc; use asap_frontend_common::resolve_root; +use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::OperatorNode; -use asap_types::types::AccuracyTarget; use asap_types::workload::{QueryLanguage, QueryWorkload, SqlDialect}; pub use error::SqlError; diff --git a/crates/frontend-sql/src/sql/collection_udfs.rs b/crates/frontend-sql/src/sql/collection_udfs.rs index 63c862b2b..17711eb83 100644 --- a/crates/frontend-sql/src/sql/collection_udfs.rs +++ b/crates/frontend-sql/src/sql/collection_udfs.rs @@ -5,10 +5,10 @@ //! into IR scalar functions, and DataFusion never executes them (evaluation //! is the query engine's responsibility). use super::types::{arrow_to_dtype, dtype_to_arrow, scalar_value_to_asap}; +use asap_types::ir::scalar::scalar_type_rules::MapScalarFunction; use asap_types::ir::scalar::{element_access_type, struct_field_type}; +use asap_types::ir::schema::{Field, Schema}; use asap_types::ir::ScalarExpr; -use asap_types::pre_asap::scalar_type_rules::MapScalarFunction; -use asap_types::pre_asap::{Field, Schema}; use datafusion::arrow::datatypes::{DataType, Field as ArrowField, FieldRef}; use datafusion::common::{DataFusionError, Result, ScalarValue as DfScalarValue}; use datafusion::logical_expr::{ diff --git a/crates/frontend-sql/src/sql/expr.rs b/crates/frontend-sql/src/sql/expr.rs index 92360bcec..a95e7ba27 100644 --- a/crates/frontend-sql/src/sql/expr.rs +++ b/crates/frontend-sql/src/sql/expr.rs @@ -3,8 +3,8 @@ use std::rc::Rc; use datafusion::logical_expr::{BinaryExpr, Expr, Operator}; use asap_frontend_common::UnresolvedScalar as Unresolved; +use asap_types::ir::scalar::{ArithmeticOpKind, ColumnRef, CompareOpKind, ScalarValue}; use asap_types::ir::ExprSemantics; -use asap_types::pre_asap::{ArithmeticOpKind, ColumnRef, CompareOpKind, ScalarValue}; use crate::error::SqlError as LoweringError; @@ -63,7 +63,7 @@ impl SqlLowerer<'_> { }; Ok(Unresolved::Cast { expr: Box::new(Unresolved::Literal(text)), - to: asap_types::pre_asap::schema::DataType::Date, + to: asap_types::ir::schema::DataType::Date, try_cast: false, }) } @@ -330,7 +330,7 @@ pub(super) fn split_disjuncts(expr: &Expr) -> Vec<&Expr> { mod tests { use super::*; use crate::sql::SqlCatalog; - use asap_types::pre_asap::schema::DataType; + use asap_types::ir::schema::DataType; use datafusion::common::ScalarValue as DfScalarValue; // Typed Arrow dates normalize to the same typed form as SQL date casts. diff --git a/crates/frontend-sql/src/sql/mod.rs b/crates/frontend-sql/src/sql/mod.rs index c19190ca2..208b45ab3 100644 --- a/crates/frontend-sql/src/sql/mod.rs +++ b/crates/frontend-sql/src/sql/mod.rs @@ -57,19 +57,17 @@ use asap_frontend_common::{ UnresolvedProjectItem as ProjectItem, UnresolvedScalar as Scalar, UnresolvedSortKey as SortKey, }; use asap_sql_function_catalog::{AggSemantic, Arity, RewriteKind}; -use asap_types::ir::operator_properties::{ +use asap_types::ir::operator::agg_intent::AggIntent; +use asap_types::ir::operator::operator_properties::{ GroupKeys, Reduction, Source, WindowFrame, WindowFrameBound, WindowFrameOffset, WindowFrameUnits, }; +use asap_types::ir::schema::{DataType, FieldDataType, Schema}; use asap_types::ir::TimeRangeKind; -use asap_types::pre_asap::agg_intent::AggIntent; -use asap_types::pre_asap::schema::{DataType, FieldDataType, Schema}; -use asap_types::pre_asap::{ - resolve_column_ref, ColumnRef, CompareOpKind, JoinKind, RelationalSetOpKind, ScalarValue, - WindowFuncKind, -}; -use asap_types::types::AccuracyTarget; +use asap_types::ir::operator::{JoinKind, RelationalSetOpKind, WindowFuncKind}; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::scalar::{resolve_column_ref, ColumnRef, CompareOpKind, ScalarValue}; use asap_types::workload::SqlDialect; use crate::error::SqlError as LoweringError; @@ -512,7 +510,7 @@ impl<'a> SqlLowerer<'a> { .fields() .iter() .map(|f| { - Ok(asap_types::pre_asap::Field::plain( + Ok(asap_types::ir::schema::Field::plain( f.name().clone(), arrow_to_dtype(f.data_type())?, f.is_nullable(), diff --git a/crates/frontend-sql/src/sql/types.rs b/crates/frontend-sql/src/sql/types.rs index eeb56ff1f..90e980bba 100644 --- a/crates/frontend-sql/src/sql/types.rs +++ b/crates/frontend-sql/src/sql/types.rs @@ -9,8 +9,8 @@ use datafusion::arrow::datatypes::{ }; use datafusion::common::ScalarValue as DfScalarValue; -use asap_types::pre_asap::schema::{DataType, Field, Schema}; -use asap_types::pre_asap::ScalarValue; +use asap_types::ir::scalar::ScalarValue; +use asap_types::ir::schema::{DataType, Field, Schema}; use crate::error::SqlError as LoweringError; @@ -386,9 +386,10 @@ mod bottom_map_tests { use super::*; #[test] fn empty_map_bottom_types_roundtrip_without_string_defaults() { - let (map, nullable) = asap_types::pre_asap::scalar_type_rules::MapScalarFunction::Construct - .output_type(&[]) - .unwrap(); + let (map, nullable) = + asap_types::ir::scalar::scalar_type_rules::MapScalarFunction::Construct + .output_type(&[]) + .unwrap(); assert!(!nullable); let arrow = dtype_to_arrow(&map); assert_eq!(arrow_to_dtype(&arrow).unwrap(), map); diff --git a/crates/frontend-sql/tests/bgp_analytics/bgp_analytics.rs b/crates/frontend-sql/tests/bgp_analytics/bgp_analytics.rs index 31894624f..7185e8f8a 100644 --- a/crates/frontend-sql/tests/bgp_analytics/bgp_analytics.rs +++ b/crates/frontend-sql/tests/bgp_analytics/bgp_analytics.rs @@ -38,10 +38,10 @@ use std::rc::Rc; use asap_frontend_sql::{lower_sql_dialect, SqlCatalog, SqlError as LoweringError}; +use asap_types::ir::operator::{AggIntent, GroupKeys}; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::{NonASAPOp, OperatorNode}; -use asap_types::pre_asap::schema::{DataType, Field, Schema}; -use asap_types::pre_asap::{AggIntent, GroupKeys}; -use asap_types::types::AccuracyTarget; use asap_types::workload::SqlDialect; use datafusion::error::DataFusionError; diff --git a/crates/frontend-sql/tests/bgp_jan2024_workload/bgp_jan2024_workload.rs b/crates/frontend-sql/tests/bgp_jan2024_workload/bgp_jan2024_workload.rs index 8238150f5..bc7353f9e 100644 --- a/crates/frontend-sql/tests/bgp_jan2024_workload/bgp_jan2024_workload.rs +++ b/crates/frontend-sql/tests/bgp_jan2024_workload/bgp_jan2024_workload.rs @@ -14,8 +14,8 @@ //! tally** by outcome category -- see the module doc on [`Category`] for why. use asap_frontend_sql::{lower_sql_dialect, SqlCatalog, SqlError}; -use asap_types::pre_asap::schema::{DataType, Field, Schema}; -use asap_types::types::AccuracyTarget; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::workload::SqlDialect; use datafusion::error::DataFusionError; use serde::Deserialize; diff --git a/crates/frontend-sql/tests/data_quality_check/synthetic_packet_trace.rs b/crates/frontend-sql/tests/data_quality_check/synthetic_packet_trace.rs index 5833e236f..53ffbbfed 100644 --- a/crates/frontend-sql/tests/data_quality_check/synthetic_packet_trace.rs +++ b/crates/frontend-sql/tests/data_quality_check/synthetic_packet_trace.rs @@ -22,10 +22,10 @@ use std::rc::Rc; use asap_frontend_sql::{lower_sql, SqlCatalog, SqlError as LoweringError}; +use asap_types::ir::operator::{AggIntent, GroupKeys}; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::{NonASAPOp, OperatorNode}; -use asap_types::pre_asap::schema::{DataType, Field, Schema}; -use asap_types::pre_asap::{AggIntent, GroupKeys}; -use asap_types::types::AccuracyTarget; const CORPUS: &str = include_str!("data/synthetic_packet_trace_queries.sql"); diff --git a/crates/frontend-sql/tests/data_quality_check/tpch_deequ.rs b/crates/frontend-sql/tests/data_quality_check/tpch_deequ.rs index 340594521..a7ad616f1 100644 --- a/crates/frontend-sql/tests/data_quality_check/tpch_deequ.rs +++ b/crates/frontend-sql/tests/data_quality_check/tpch_deequ.rs @@ -23,8 +23,8 @@ //! which is the same narrowing sidra's own catalog file makes. use asap_frontend_sql::{lower_sql, SqlCatalog, SqlError as LoweringError}; -use asap_types::pre_asap::schema::{DataType, Field, Schema}; -use asap_types::types::AccuracyTarget; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::schema::{DataType, Field, Schema}; const CORPUS: &str = include_str!("data/tpch_deequ_queries.sql"); diff --git a/crates/frontend-sql/tests/frequency_entropy.rs b/crates/frontend-sql/tests/frequency_entropy.rs index 334958660..c17083edc 100644 --- a/crates/frontend-sql/tests/frequency_entropy.rs +++ b/crates/frontend-sql/tests/frequency_entropy.rs @@ -2,11 +2,10 @@ //! `FrequencyEntropy` intent (#509 Example 2's Q2). use asap_frontend_common::resolve_root; use asap_frontend_sql::{lower_sql, SqlCatalog, SqlLowerer}; -use asap_types::{ - ir::{NonASAPOp, OperatorNode}, - pre_asap::{AggIntent, DataType, Field, Schema}, - types::AccuracyTarget, -}; +use asap_types::ir::operator::AggIntent; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::schema::{DataType, Field, Schema}; +use asap_types::ir::{NonASAPOp, OperatorNode}; fn catalog(nullable: bool) -> SqlCatalog { SqlCatalog::new().with_table( diff --git a/crates/frontend-sql/tests/frequency_l2.rs b/crates/frontend-sql/tests/frequency_l2.rs index e8304bb59..604af0a3d 100644 --- a/crates/frontend-sql/tests/frequency_l2.rs +++ b/crates/frontend-sql/tests/frequency_l2.rs @@ -2,11 +2,10 @@ //! (#509 Example 2's Q3, in its Float64 form). use asap_frontend_common::resolve_root; use asap_frontend_sql::{lower_sql, lower_sql_batch, SqlCatalog, SqlLowerer}; -use asap_types::{ - ir::{NonASAPOp, OperatorNode}, - pre_asap::{AggIntent, DataType, Field, Schema}, - types::AccuracyTarget, -}; +use asap_types::ir::operator::AggIntent; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::schema::{DataType, Field, Schema}; +use asap_types::ir::{NonASAPOp, OperatorNode}; use std::rc::Rc; fn catalog(nullable: bool) -> SqlCatalog { @@ -149,7 +148,7 @@ async fn batch_lowering_recognizes_frequency_l2() { /// recognized L2 intent. #[tokio::test] async fn stage1_offers_exact_and_summary_l2_candidates() { - use asap_aware_mapping::per_computation_candidates::{ + use asap_logical_asap_optimization::per_computation_candidates::{ generate_per_computation_candidates, ComputationCandidate, }; use asap_types::ir::QueryRoot; diff --git a/crates/frontend-sql/tests/maintained_population.rs b/crates/frontend-sql/tests/maintained_population.rs index d3a4b7493..35a2534dc 100644 --- a/crates/frontend-sql/tests/maintained_population.rs +++ b/crates/frontend-sql/tests/maintained_population.rs @@ -1,15 +1,14 @@ //! SQL and PromQL use the same shared-state rule without sharing membership semantics. -use asap_aware_mapping::maintained_population::MaintainedPopulationStrategy; use asap_frontend_sql::{lower_sql, SqlCatalog}; -use asap_types::types::AccuracyTarget; -use asap_types::{ - ir::{ - apply_materialization_timings, cse::share_common_sub_dags, - physical_export::compile_physical_asap_dag, ASAPOp, MaterializationAssignment, NonASAPOp, - Operator, OperatorNode, TimingMemo, - }, - post_asap::maintained_population::{MaintainedPopulation, PopulationInput}, - pre_asap::{DataType, Field, Schema}, +use asap_logical_asap_optimization::maintained_population::MaintainedPopulationStrategy; +use asap_types::ir::cse::share_common_sub_dags; +use asap_types::ir::operator::maintained_population::{MaintainedPopulation, PopulationInput}; +use asap_types::ir::physical_export::compile_physical_asap_dag; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::schema::{DataType, Field, Schema}; +use asap_types::ir::{ + apply_materialization_timings, ASAPOp, MaterializationAssignment, NonASAPOp, Operator, + OperatorNode, TimingMemo, }; use std::rc::Rc; diff --git a/crates/frontend-sql/tests/netflow/netflow.rs b/crates/frontend-sql/tests/netflow/netflow.rs index fd57143b4..596f911dd 100644 --- a/crates/frontend-sql/tests/netflow/netflow.rs +++ b/crates/frontend-sql/tests/netflow/netflow.rs @@ -7,10 +7,10 @@ use std::rc::Rc; use asap_frontend_sql::{lower_sql, SqlCatalog}; +use asap_types::ir::operator::{AggIntent, GroupKeys}; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::{NonASAPOp, OperatorNode}; -use asap_types::pre_asap::schema::{DataType, Field, Schema}; -use asap_types::pre_asap::{AggIntent, GroupKeys}; -use asap_types::types::AccuracyTarget; const CORPUS: &str = include_str!("data/netflow.sql"); diff --git a/crates/frontend-sql/tests/pearson_corr.rs b/crates/frontend-sql/tests/pearson_corr.rs index 717096abd..df8f81fac 100644 --- a/crates/frontend-sql/tests/pearson_corr.rs +++ b/crates/frontend-sql/tests/pearson_corr.rs @@ -2,12 +2,13 @@ use std::rc::Rc; use asap_frontend_sql::{lower_sql, SqlCatalog}; +use asap_types::ir::operator::AggIntent; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::{ apply_materialization_timings, physical_export::compile_physical_asap_dag, MaterializationAssignment, NonASAPOp, OperatorNode, ScalarExpr, TimingMemo, }; -use asap_types::pre_asap::{AggIntent, DataType, Field, Schema}; -use asap_types::types::AccuracyTarget; fn catalog() -> SqlCatalog { let schema = Schema::new(vec![ @@ -148,17 +149,17 @@ async fn corr_filter_is_a_measure_filter() { #[tokio::test] async fn corr_survives_exact_plan_compilation() { let query = lower("SELECT corr(x, y) AS r FROM a").await; - let run = asap_aware_mapping::run_stage1_to_stage3( + let run = asap_plan_selection::run_stage1_to_stage3( vec![(0, asap_types::ir::QueryRoot::Operator(query.clone()))], &[asap_types::workload::QueryWorkloadEntry { requirements: asap_types::workload::QueryRequirements { - accuracy: asap_types::types::AccuracyTarget::exact(), + accuracy: asap_types::ir::properties::AccuracyTarget::exact(), max_latency_ms: None, }, ..Default::default() }], &Default::default(), - asap_aware_mapping::PlanningModels::builtin(), + asap_plan_selection::PlanningModels::builtin(), ) .unwrap(); let [asap_types::ir::QueryRoot::Operator(plan)] = run.selected_logical().unwrap() else { diff --git a/crates/frontend-sql/tests/sql_lowering.rs b/crates/frontend-sql/tests/sql_lowering.rs index 4193d2ee5..ae867667c 100644 --- a/crates/frontend-sql/tests/sql_lowering.rs +++ b/crates/frontend-sql/tests/sql_lowering.rs @@ -14,13 +14,14 @@ use asap_frontend_common::{UnresolvedOp, UnresolvedScalar}; use asap_frontend_sql::{ lower_sql, lower_sql_dialect, SqlCatalog, SqlError as LoweringError, SqlLowerer, }; -use asap_types::ir::{ExprSemantics, NonASAPOp, OperatorNode, ScalarExpr}; -use asap_types::pre_asap::schema::{DataType, Field, FieldDataType, Schema}; -use asap_types::pre_asap::{ - AggIntent, ColumnRef, CompareOpKind, GroupKeys, JoinKind, Reduction, ScalarValue, Source, - WindowFrameBound, WindowFrameOffset, WindowFrameUnits, WindowFuncKind, +use asap_types::ir::operator::{ + AggIntent, GroupKeys, JoinKind, Reduction, Source, WindowFrameBound, WindowFrameOffset, + WindowFrameUnits, WindowFuncKind, }; -use asap_types::types::AccuracyTarget; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::scalar::{ColumnRef, CompareOpKind, ScalarValue}; +use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; +use asap_types::ir::{ExprSemantics, NonASAPOp, OperatorNode, ScalarExpr}; use asap_types::workload::SqlDialect; fn col(name: &str, dtype: DataType) -> Field { diff --git a/crates/frontend-sql/tests/temporal_types.rs b/crates/frontend-sql/tests/temporal_types.rs index 22a94d26f..e0380c31b 100644 --- a/crates/frontend-sql/tests/temporal_types.rs +++ b/crates/frontend-sql/tests/temporal_types.rs @@ -1,6 +1,6 @@ use asap_frontend_sql::{lower_sql, SqlCatalog}; -use asap_types::pre_asap::schema::{DataType, Field, Schema}; -use asap_types::types::AccuracyTarget; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::schema::{DataType, Field, Schema}; fn catalog() -> SqlCatalog { SqlCatalog::new().with_table( "t", diff --git a/crates/integration-tests/Cargo.toml b/crates/integration-tests/Cargo.toml index dcd2a1e05..5787b8aec 100644 --- a/crates/integration-tests/Cargo.toml +++ b/crates/integration-tests/Cargo.toml @@ -7,7 +7,9 @@ edition = "2021" asap-types = { path = "../types" } asap-frontend-promql = { path = "../frontend-promql" } asap-frontend-sql = { path = "../frontend-sql" } -asap-aware-mapping = { path = "../asap-aware-mapping" } +asap-logical-asap-optimization = { path = "../logical-asap-optimization" } +asap-physical-asap-optimization = { path = "../physical-asap-optimization" } +asap-plan-selection = { path = "../plan-selection" } [dev-dependencies] asap-devtools = { path = "../devtools" } @@ -16,5 +18,5 @@ asap_sketchlib = { workspace = true } serde_json = "1" tokio = { version = "1", features = ["rt", "macros", "rt-multi-thread"] } -asap-physical-operators = { path = "../asap-physical-operators" } +asap-executor = { path = "../executor" } futures = "0.3" diff --git a/crates/integration-tests/src/lib.rs b/crates/integration-tests/src/lib.rs index 0618b7441..a37ce7cc5 100644 --- a/crates/integration-tests/src/lib.rs +++ b/crates/integration-tests/src/lib.rs @@ -13,9 +13,9 @@ pub mod fixtures { + use asap_types::ir::properties::AccuracyTarget; + use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::OperatorNode; - use asap_types::pre_asap::schema::{DataType, Field, Schema}; - use asap_types::types::AccuracyTarget; use asap_types::workload::{ BatchEntry, DataWorkload, DurationMs, PlanningWorkload, Predictability, Query, QueryLanguage, QueryRequirements, QueryWorkload, @@ -97,9 +97,9 @@ pub mod fixtures { pub mod post_asap { use asap_types::ir::physical_export::{compile_physical_asap_dag, PhysicalASAPDAG}; use asap_types::ir::{ - apply_materialization_timings, Materialization, MaterializationAssignment, Medium, - OperatorNode, TimingMemo, + apply_materialization_timings, MaterializationAssignment, OperatorNode, TimingMemo, }; + use asap_types::physical::{Materialization, Medium}; use std::rc::Rc; /// Time `root` under the default assignment (every sub-DAG @@ -141,8 +141,8 @@ pub mod post_asap { /// Plans and candidates from the planner pipeline (`run_stage1_to_stage3`) for one /// query, with the built-in models. pub mod planning { + use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::{OperatorNode, QueryRoot}; - use asap_types::types::AccuracyTarget; use asap_types::workload::{DataWorkload, QueryRequirements, QueryWorkloadEntry}; use std::rc::Rc; @@ -150,8 +150,8 @@ pub mod planning { pub fn run_stage1_to_stage3( root: Rc, accuracy: AccuracyTarget, - ) -> asap_aware_mapping::Stage1ToStage3Run { - asap_aware_mapping::run_stage1_to_stage3( + ) -> asap_plan_selection::Stage1ToStage3Run { + asap_plan_selection::run_stage1_to_stage3( vec![(0, QueryRoot::Operator(root))], &[QueryWorkloadEntry { requirements: QueryRequirements { @@ -161,7 +161,7 @@ pub mod planning { ..Default::default() }], &DataWorkload::default(), - asap_aware_mapping::PlanningModels::builtin(), + asap_plan_selection::PlanningModels::builtin(), ) .expect("Pass 1 accepts the query") } diff --git a/crates/integration-tests/tests/aggregate.rs b/crates/integration-tests/tests/aggregate.rs index a85c5b025..8aa849b8c 100644 --- a/crates/integration-tests/tests/aggregate.rs +++ b/crates/integration-tests/tests/aggregate.rs @@ -13,9 +13,9 @@ use std::time::Duration; use asap_integration_tests::fixtures::lower_promql; use asap_integration_tests::fixtures::metric_schema; +use asap_types::ir::operator::{AggIntent, Reduction, Source}; +use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::{NonASAPOp, OperatorNode, TimeRangeKind}; -use asap_types::pre_asap::{AggIntent, Reduction, Source}; -use asap_types::types::AccuracyTarget; fn lower(q: &str) -> Rc { lower_promql(q, AccuracyTarget::exact()) diff --git a/crates/integration-tests/tests/binary_op.rs b/crates/integration-tests/tests/binary_op.rs index a8d32f407..a98f187bb 100644 --- a/crates/integration-tests/tests/binary_op.rs +++ b/crates/integration-tests/tests/binary_op.rs @@ -11,12 +11,13 @@ use std::time::Duration; use asap_integration_tests::fixtures::lower_promql; use asap_integration_tests::fixtures::metric_schema; -use asap_types::ir::{BinaryOperator, NonASAPOp, OperatorNode, ScalarExpr, TimeRangeKind}; -use asap_types::pre_asap::{ - AggIntent, ArithmeticOpKind, BinaryOpKind, CompareOpKind, GroupSide, Reduction, Source, - VectorGrouping, VectorMatch, VectorMatchKind, +use asap_types::ir::operator::{ + AggIntent, BinaryOpKind, GroupSide, Reduction, Source, VectorGrouping, VectorMatch, + VectorMatchKind, }; -use asap_types::types::AccuracyTarget; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::scalar::{ArithmeticOpKind, CompareOpKind}; +use asap_types::ir::{BinaryOperator, NonASAPOp, OperatorNode, ScalarExpr, TimeRangeKind}; fn lower(q: &str) -> Rc { lower_promql(q, AccuracyTarget::exact()) diff --git a/crates/integration-tests/tests/deployment_capabilities.rs b/crates/integration-tests/tests/deployment_capabilities.rs index c83e7427b..4c8abc6e3 100644 --- a/crates/integration-tests/tests/deployment_capabilities.rs +++ b/crates/integration-tests/tests/deployment_capabilities.rs @@ -4,12 +4,13 @@ mod planner_layering_common; -use asap_aware_mapping::stage3_plan_selection::PlanSelectionSettings; -use asap_aware_mapping::{DeploymentCapabilities, SupportedSummary}; use asap_devtools::planning_stages::lower_sql_roots; +use asap_logical_asap_optimization::{DeploymentCapabilities, SupportedSummary}; +use asap_plan_selection::PlanSelectionSettings; use asap_types::ir::flat::FlatDag; -use asap_types::ir::{ASAPOp, Materialization, Operator}; -use asap_types::post_asap::{SketchAlgorithm, StatisticKind}; +use asap_types::ir::schema::{SketchAlgorithm, StatisticKind}; +use asap_types::ir::{ASAPOp, Operator}; +use asap_types::physical::Materialization; use planner_layering_common::*; /// A deployment whose executor supports only `supported`: each family with diff --git a/crates/integration-tests/tests/exact_composition.rs b/crates/integration-tests/tests/exact_composition.rs index dffe440b8..b8ecdefed 100644 --- a/crates/integration-tests/tests/exact_composition.rs +++ b/crates/integration-tests/tests/exact_composition.rs @@ -11,14 +11,15 @@ use std::rc::Rc; use asap_integration_tests::planning::logical_candidates; use asap_integration_tests::post_asap::{at_ingestion, post_asap_dag, timed}; -use asap_types::ir::operator_properties::{Reduction, Source}; +use asap_types::ir::operator::agg_intent::{default_quantile, AggIntent}; +use asap_types::ir::operator::operator_properties::{Reduction, Source}; use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; +use asap_types::ir::properties::ExecutionTiming; +use asap_types::ir::schema::{DataType, Field, Schema}; +use asap_types::ir::schema::{ExactKind, FieldDataType, SummaryUpdate}; use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, TimeRangeKind}; -use asap_types::post_asap::{ExactKind, ExecutionTiming, FieldDataType, SummaryUpdate}; -use asap_types::pre_asap::agg_intent::{default_quantile, AggIntent}; -use asap_types::pre_asap::schema::{DataType, Field, Schema}; -use asap_types::types::AccuracyTarget; +use asap_types::ir::properties::AccuracyTarget; // ── fixtures ──────────────────────────────────────────────────────────── @@ -295,9 +296,9 @@ fn summary_construction_follows_its_value_input_phase() { child: post, family: FieldDataType::ExactAggregate( ExactKind::Max, - asap_types::post_asap::ExactParams::Max, + asap_types::ir::schema::ExactParams::Max, ), - input: SummaryUpdate::column(asap_types::pre_asap::ColumnRef::SampleValue), + input: SummaryUpdate::column(asap_types::ir::scalar::ColumnRef::SampleValue), reduction: Reduction::by(vec![]), grouping: Default::default(), filter: None, diff --git a/crates/integration-tests/tests/filtered_aggregates.rs b/crates/integration-tests/tests/filtered_aggregates.rs index a348a2896..b44d49e10 100644 --- a/crates/integration-tests/tests/filtered_aggregates.rs +++ b/crates/integration-tests/tests/filtered_aggregates.rs @@ -7,19 +7,19 @@ use std::collections::BTreeMap; use std::rc::Rc; use std::sync::Arc; -use asap_frontend_sql::{lower_sql, SqlCatalog}; -use asap_integration_tests::planning::{logical_candidates, selected_dag}; -use asap_physical_operators::{ +use asap_executor::{ physical_planner::{compile, InputContract, Source}, runtime::{Limits, RunContext, Scope}, sources::{DataSources, MemorySource}, values::{Batch, Value}, }; +use asap_frontend_sql::{lower_sql, SqlCatalog}; +use asap_integration_tests::planning::{logical_candidates, selected_dag}; use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::schema::{DataType, Field, Schema}; +use asap_types::ir::schema::{ExactKind, FieldDataType, SketchAlgorithm}; use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, Predicate}; -use asap_types::post_asap::{ExactKind, FieldDataType, SketchAlgorithm}; -use asap_types::pre_asap::schema::{DataType, Field, Schema}; -use asap_types::types::AccuracyTarget; use futures::StreamExt; fn catalog() -> SqlCatalog { diff --git a/crates/integration-tests/tests/frequency_execution.rs b/crates/integration-tests/tests/frequency_execution.rs index 74e4879fa..94439da74 100644 --- a/crates/integration-tests/tests/frequency_execution.rs +++ b/crates/integration-tests/tests/frequency_execution.rs @@ -6,20 +6,20 @@ use std::collections::BTreeMap; use std::rc::Rc; use std::sync::Arc; -use asap_integration_tests::fixtures::lower_promql; -use asap_integration_tests::planning::logical_candidates; -use asap_physical_operators::physical_planner::{ +use asap_executor::physical_planner::{ compile_candidate, frontier_from_timing, promql_rows, InputContract, }; -use asap_physical_operators::runtime::Scope; -use asap_physical_operators::values::{Batch, Value}; +use asap_executor::runtime::Scope; +use asap_executor::values::{Batch, Value}; +use asap_integration_tests::fixtures::lower_promql; +use asap_integration_tests::planning::logical_candidates; use asap_types::ir::physical_export::{compile_physical_asap_dag, PhysicalASAPOperatorPayload}; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::schema::{FieldDataType, SketchAlgorithm}; use asap_types::ir::{ apply_materialization_timings, ASAPOp, MaterializationAssignment, NonASAPOp, OperatorNode, TimingMemo, }; -use asap_types::post_asap::{FieldDataType, SketchAlgorithm}; -use asap_types::types::AccuracyTarget; mod physical_common; diff --git a/crates/integration-tests/tests/frontend_timestamps.rs b/crates/integration-tests/tests/frontend_timestamps.rs index 91c0bde8d..5a1fd282b 100644 --- a/crates/integration-tests/tests/frontend_timestamps.rs +++ b/crates/integration-tests/tests/frontend_timestamps.rs @@ -2,9 +2,9 @@ use asap_frontend_sql::{lower_sql, SqlCatalog}; use asap_integration_tests::fixtures::lower_promql_root; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::{NonASAPOp, ScalarExpr}; -use asap_types::pre_asap::schema::{DataType, Field, Schema}; -use asap_types::types::AccuracyTarget; /// PromQL exposes its evaluation time as Unix seconds, whereas SQL exposes /// `CURRENT_TIMESTAMP` as a timestamp. They represent related clock concepts diff --git a/crates/integration-tests/tests/kll_pane_execution.rs b/crates/integration-tests/tests/kll_pane_execution.rs index c7aa5194f..e4b5c819a 100644 --- a/crates/integration-tests/tests/kll_pane_execution.rs +++ b/crates/integration-tests/tests/kll_pane_execution.rs @@ -1,6 +1,6 @@ //! Maintenance -> stored pane state -> independently bound query execution. mod physical_common; -use asap_physical_operators::{ +use asap_executor::{ operators::{Operator, SummaryEvaluation}, physical_planner::{CompiledPhysicalDAG, InputContract, Source}, plan::{PhysicalDAG, PhysicalOperator, PlanProperties}, @@ -9,11 +9,9 @@ use asap_physical_operators::{ values::{Batch, SchemaRef, Value}, AggregateCore, Error, }; -use asap_types::{ - post_asap::{ - Field, FieldDataType, Schema, SketchAlgorithm, SketchKind, SketchParams, SketchStatistic, - }, - pre_asap::DataType, +use asap_types::ir::schema::{ + DataType, Field, FieldDataType, Schema, SketchAlgorithm, SketchKind, SketchParams, + SketchStatistic, }; use futures::{executor::block_on, StreamExt}; use std::{ @@ -171,15 +169,15 @@ fn five_panes_roundtrip_and_shared_merge_runs_once() { vec![6, 7, 8], ) .unwrap(); - let layout = asap_types::post_asap::PaneLayout { + let layout = asap_types::physical::PaneLayout { pane_width_ms: 60_000, pane_origin_ms: Some(0), }; assert!( - asap_types::post_asap::validate_pane_coverage( + asap_types::physical::validate_pane_coverage( &layout, Some(330_000), - &asap_types::post_asap::WindowEdgeCoverage::PaneAligned + &asap_types::physical::WindowEdgeCoverage::PaneAligned ) .is_err(), "moving window edges require residual computation" @@ -187,10 +185,10 @@ fn five_panes_roundtrip_and_shared_merge_runs_once() { for offset in [0, 1] { let restored = restore(schema.clone(), &panes[offset..offset + 5]); let evaluation_time_ms = (5 + offset as i64) * 60_000; - asap_types::post_asap::validate_pane_coverage( + asap_types::physical::validate_pane_coverage( &layout, Some(evaluation_time_ms), - &asap_types::post_asap::WindowEdgeCoverage::PaneAligned, + &asap_types::physical::WindowEdgeCoverage::PaneAligned, ) .unwrap(); let inputs: BTreeMap<_, _> = (0..5) diff --git a/crates/integration-tests/tests/materialization_e2e.rs b/crates/integration-tests/tests/materialization_e2e.rs index d687640dd..6eaae094e 100644 --- a/crates/integration-tests/tests/materialization_e2e.rs +++ b/crates/integration-tests/tests/materialization_e2e.rs @@ -7,25 +7,26 @@ use std::collections::BTreeMap; use std::rc::Rc; use std::sync::Arc; -use asap_aware_mapping::maintained_population::MaintainedPopulationStrategy; -use asap_integration_tests::fixtures::lower_promql; -use asap_integration_tests::planning::{logical_candidates, selected_dag}; -use asap_physical_operators::physical_planner::{ +use asap_executor::physical_planner::{ compile_candidate, frontier_from_timing, promql_rows, CompiledPhysicalDAG, InputContract, }; -use asap_physical_operators::runtime::Scope; -use asap_physical_operators::values::{Batch, SchemaRef, Value}; +use asap_executor::runtime::Scope; +use asap_executor::values::{Batch, SchemaRef, Value}; +use asap_integration_tests::fixtures::lower_promql; +use asap_integration_tests::planning::{logical_candidates, selected_dag}; +use asap_logical_asap_optimization::maintained_population::MaintainedPopulationStrategy; +use asap_types::ir::operator::maintained_population::PopulationInput; +use asap_types::ir::operator::Source; use asap_types::ir::physical_export::{ compile_physical_asap_dag, PhysicalASAPDAG, PhysicalASAPOperatorPayload, }; +use asap_types::ir::properties::ExecutionTiming; +use asap_types::ir::properties::{AccuracyMetric, AccuracyTarget}; use asap_types::ir::{ - apply_materialization_timings, ASAPOp, Materialization, MaterializationAssignment, Medium, - NonASAPOp, OperatorNode, TimingMemo, + apply_materialization_timings, ASAPOp, MaterializationAssignment, NonASAPOp, OperatorNode, + TimingMemo, }; -use asap_types::post_asap::maintained_population::PopulationInput; -use asap_types::post_asap::ExecutionTiming; -use asap_types::pre_asap::Source; -use asap_types::types::{AccuracyMetric, AccuracyTarget}; +use asap_types::physical::{Materialization, Medium}; mod physical_common; @@ -67,7 +68,7 @@ fn sketch_candidate(query: &str) -> Rc { matches!( node.asap(), Some(ASAPOp::SummaryAgg { - family: asap_types::post_asap::FieldDataType::Sketch(..), + family: asap_types::ir::schema::FieldDataType::Sketch(..), .. }) ) @@ -217,11 +218,11 @@ fn values_1_to_100(schema: &SchemaRef, _metric: &str) -> Batch { .fields .iter() .map(|field| match field.dtype { - asap_types::post_asap::FieldDataType::Plain( - asap_types::pre_asap::DataType::Float64, + asap_types::ir::schema::FieldDataType::Plain( + asap_types::ir::schema::DataType::Float64, ) => Value::Float64(f64::from(value)), - asap_types::post_asap::FieldDataType::Plain( - asap_types::pre_asap::DataType::Timestamp, + asap_types::ir::schema::FieldDataType::Plain( + asap_types::ir::schema::DataType::Timestamp, ) => Value::Timestamp(300_000), _ => panic!("unexpected field {field:?}"), }) @@ -272,7 +273,7 @@ fn mixed_placement_answers_like_query_time() { matches!( node.asap(), Some(ASAPOp::SummaryAgg { - family: asap_types::post_asap::FieldDataType::Sketch(..), + family: asap_types::ir::schema::FieldDataType::Sketch(..), .. }) ) diff --git a/crates/integration-tests/tests/nested.rs b/crates/integration-tests/tests/nested.rs index 3e385824b..bcaaaf9ed 100644 --- a/crates/integration-tests/tests/nested.rs +++ b/crates/integration-tests/tests/nested.rs @@ -13,14 +13,15 @@ use std::time::Duration; use asap_integration_tests::fixtures::lower_promql; use asap_integration_tests::fixtures::metric_schema; +use asap_types::ir::operator::{ + AggIntent, AtModifier, BinaryOpKind, GroupKeys, PromQLVectorSetOpKind, Reduction, Source, + TimeShift, VectorMatch, VectorMatchKind, +}; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::scalar::{ArithmeticOpKind, CompareOpKind, ScalarValue}; use asap_types::ir::{ BinaryOperator, ExprSemantics, NonASAPOp, OperatorNode, Predicate, ScalarExpr, TimeRangeKind, }; -use asap_types::pre_asap::{ - AggIntent, ArithmeticOpKind, AtModifier, BinaryOpKind, CompareOpKind, GroupKeys, - PromQLVectorSetOpKind, Reduction, ScalarValue, Source, TimeShift, VectorMatch, VectorMatchKind, -}; -use asap_types::types::AccuracyTarget; fn lower(q: &str) -> Rc { lower_promql(q, AccuracyTarget::exact()) diff --git a/crates/integration-tests/tests/operator_design_examples.rs b/crates/integration-tests/tests/operator_design_examples.rs index c59d57de9..ac07a7864 100644 --- a/crates/integration-tests/tests/operator_design_examples.rs +++ b/crates/integration-tests/tests/operator_design_examples.rs @@ -1,11 +1,13 @@ //! #511 examples: source text → unified dag → summary rewrite → flat export. use asap_frontend_sql::{lower_sql, SqlCatalog}; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, ScalarExpr}; -use asap_types::post_asap::{ +use asap_types::ir::operator::AggIntent; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::scalar::ColumnRef; +use asap_types::ir::schema::{DataType, Field, Schema}; +use asap_types::ir::schema::{ ExactKind, ExactParams, FieldDataType, GroupingStrategy, SummaryUpdate, }; -use asap_types::pre_asap::{AggIntent, ColumnRef, DataType, Field, Schema}; -use asap_types::types::AccuracyTarget; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, ScalarExpr}; use std::rc::Rc; mod physical_common; @@ -139,7 +141,7 @@ async fn sql_scalar_subquery_retains_its_cardinality_contract() { /// Execute the SQL SUM example for nonempty, empty and all-NULL populations. #[tokio::test] async fn sql_sum_example_executes_with_sql_null_semantics() { - use asap_physical_operators::{ + use asap_executor::{ physical_planner::{compile, InputContract, Source}, runtime::{Limits, RunContext, Scope}, sources::{DataSources, MemorySource}, @@ -261,12 +263,12 @@ async fn sql_window_and_filtered_aggregate_types() { /// state. No replacement dag is constructed by the test. #[tokio::test] async fn batch_planning_replaces_summary_operators() { - use asap_aware_mapping::pass::PlanningModels; - use asap_physical_operators::{ + use asap_executor::{ physical_planner::{compile, InputContract}, runtime::Scope, values::{Batch, Value}, }; + use asap_plan_selection::pass::PlanningModels; use asap_planner::{e2e_plan, FrontendInput, UserInput}; use asap_types::workload::*; use std::{collections::BTreeMap, sync::Arc}; @@ -322,7 +324,7 @@ async fn batch_planning_replaces_summary_operators() { }; assert!(matches!( state.materialization, - asap_types::ir::Materialization::MatAtQuery(_) + asap_types::physical::Materialization::MatAtQuery(_) )); for (plan, expected) in output.plans.iter().zip([31.0, 60.0]) { let root = &plan.root; @@ -395,6 +397,6 @@ async fn batch_planning_replaces_summary_operators() { }; assert!(matches!( state.materialization, - asap_types::ir::Materialization::MatAtQuery(_) + asap_types::physical::Materialization::MatAtQuery(_) )); } diff --git a/crates/integration-tests/tests/operator_sharing.rs b/crates/integration-tests/tests/operator_sharing.rs index be6a1fe23..5aa27f9e7 100644 --- a/crates/integration-tests/tests/operator_sharing.rs +++ b/crates/integration-tests/tests/operator_sharing.rs @@ -11,9 +11,9 @@ use std::rc::Rc; use asap_frontend_sql::{lower_sql, SqlCatalog}; use asap_integration_tests::planning::selected_dag; use asap_integration_tests::post_asap::post_asap_dag; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode}; -use asap_types::pre_asap::schema::{DataType, Field, Schema}; -use asap_types::types::AccuracyTarget; fn col(name: &str, dtype: DataType) -> Field { Field::plain(name, dtype, false) diff --git a/crates/integration-tests/tests/per_computation_candidates_sql.rs b/crates/integration-tests/tests/per_computation_candidates_sql.rs index 5c1e0823e..9db7233f6 100644 --- a/crates/integration-tests/tests/per_computation_candidates_sql.rs +++ b/crates/integration-tests/tests/per_computation_candidates_sql.rs @@ -2,16 +2,16 @@ //! the guarantees composed over estimates, and what a Top-K heap sketch //! ranks. -use asap_aware_mapping::per_computation_candidates::{ +use asap_frontend_sql::{lower_sql, SqlCatalog}; +use asap_logical_asap_optimization::per_computation_candidates::{ compose_logical_candidate, generate_per_computation_candidates, ComputationCandidate, ComputationTarget, HavingPlacement, }; -use asap_frontend_sql::{lower_sql, SqlCatalog}; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::scalar::ColumnRef; +use asap_types::ir::schema::{DataType, Field, Schema}; +use asap_types::ir::schema::{FieldDataType, SketchAlgorithm, SummaryInputExpr}; use asap_types::ir::{ASAPOp, OperatorNode, QueryRoot}; -use asap_types::post_asap::{FieldDataType, SketchAlgorithm, SummaryInputExpr}; -use asap_types::pre_asap::schema::{DataType, Field, Schema}; -use asap_types::pre_asap::ColumnRef; -use asap_types::types::AccuracyTarget; async fn targets(sql: &str) -> Vec { let catalog = SqlCatalog::new().with_table( @@ -66,14 +66,14 @@ async fn mixed_key_and_value_having_stays_above() { /// Whether `node` reads a quantile sketch (KLL or DDSketch) anywhere below it. fn reads_quantile_sketch(node: &std::rc::Rc) -> bool { - use asap_types::post_asap::SketchAlgorithm; + use asap_types::ir::schema::SketchAlgorithm; asap_types::ir::OperatorNode::reachable(node) .iter() .any(|node| { matches!( node.asap(), Some(asap_types::ir::ASAPOp::SummaryAgg { - family: asap_types::post_asap::FieldDataType::Sketch(kind, _), + family: asap_types::ir::schema::FieldDataType::Sketch(kind, _), .. }) if matches!(kind.algorithm(), SketchAlgorithm::Kll | SketchAlgorithm::DDSketch) ) @@ -136,14 +136,14 @@ async fn a_filter_or_group_key_over_an_estimate_has_no_guarantee() { /// Whether `node` reads a DDSketch anywhere below it. fn reads_ddsketch(node: &std::rc::Rc) -> bool { - use asap_types::post_asap::SketchAlgorithm; + use asap_types::ir::schema::SketchAlgorithm; asap_types::ir::OperatorNode::reachable(node) .iter() .any(|node| { matches!( node.asap(), Some(asap_types::ir::ASAPOp::SummaryAgg { - family: asap_types::post_asap::FieldDataType::Sketch(kind, _), + family: asap_types::ir::schema::FieldDataType::Sketch(kind, _), .. }) if matches!(kind.algorithm(), SketchAlgorithm::DDSketch) ) diff --git a/crates/integration-tests/tests/physical_common/mod.rs b/crates/integration-tests/tests/physical_common/mod.rs index 37eb6edfe..49b514e87 100644 --- a/crates/integration-tests/tests/physical_common/mod.rs +++ b/crates/integration-tests/tests/physical_common/mod.rs @@ -1,4 +1,4 @@ -use asap_physical_operators::{ +use asap_executor::{ operators::Operator, physical_planner::{CompiledPhysicalDAG, Source}, runtime::{Limits, RunContext, Scope}, @@ -58,7 +58,9 @@ pub fn compile_at_ingestion_physical_asap_dag( compile_with( root, &asap_types::ir::MaterializationAssignment::with_default( - asap_types::ir::Materialization::MatAtIngestion(asap_types::ir::Medium::Memory), + asap_types::physical::Materialization::MatAtIngestion( + asap_types::physical::Medium::Memory, + ), ), ) } @@ -80,9 +82,9 @@ fn compile_with( #[allow(dead_code)] pub fn execute_raw_rows( root: &std::rc::Rc, - rows: Vec>, -) -> Vec> { - use asap_physical_operators::dag::{ + rows: Vec>, +) -> Vec> { + use asap_executor::dag::{ planner::bind_with_data_sources, scan::{DataSources, MemorySource}, }; diff --git a/crates/integration-tests/tests/planner_layering_common/mod.rs b/crates/integration-tests/tests/planner_layering_common/mod.rs index dafee9bb9..3eff77cc4 100644 --- a/crates/integration-tests/tests/planner_layering_common/mod.rs +++ b/crates/integration-tests/tests/planner_layering_common/mod.rs @@ -15,17 +15,16 @@ use std::collections::{BTreeMap, BTreeSet, HashSet}; -use asap_aware_mapping::stage2_physical_asap_optimization::PhysicalCandidate; -use asap_aware_mapping::stage3_plan_selection::PlanSelectionSettings; -use asap_aware_mapping::stage3_plan_selection::Selection; -use asap_aware_mapping::DeploymentCapabilities; use asap_devtools::planning_stages::{lower_promql_roots, run_planning_stages_over}; +use asap_logical_asap_optimization::DeploymentCapabilities; +use asap_physical_asap_optimization::PhysicalCandidate; +use asap_plan_selection::PlanSelectionSettings; +use asap_plan_selection::Selection; use asap_types::ir::flat::{flatten, FlatDag, NodeId}; use asap_types::ir::physical_export::PhysicalASAPDAG; +use asap_types::ir::schema::{FieldDataType, SketchAlgorithm, SketchParams, SketchStatistic}; use asap_types::ir::{ASAPOp, NonASAPOp, Operator, QueryRoot}; -use asap_types::post_asap::{ - FieldDataType, Materialization, SketchAlgorithm, SketchParams, SketchStatistic, -}; +use asap_types::physical::Materialization; use asap_types::workload::{PlanningWorkload, QueryRecurrence, RepeatedDemand}; pub use asap_devtools::planner_layering_examples::*; @@ -643,7 +642,7 @@ pub fn materialization(p: &Physical, node: NodeId) -> MaterializationOption { pub fn runs_at_ingestion(p: &Physical, node: NodeId) -> bool { let n = p.dag().nodes.iter().find(|n| n.id == node).expect("node"); - n.output_state.timing == asap_types::post_asap::ExecutionTiming::IngestionTime + n.output_state.timing == asap_types::ir::properties::ExecutionTiming::IngestionTime } /// Every node upstream of an ingestion-time node also runs at ingestion time. diff --git a/crates/integration-tests/tests/planner_layering_example1.rs b/crates/integration-tests/tests/planner_layering_example1.rs index 74e2b1e27..8a58806d0 100644 --- a/crates/integration-tests/tests/planner_layering_example1.rs +++ b/crates/integration-tests/tests/planner_layering_example1.rs @@ -20,12 +20,12 @@ mod planner_layering_common; use std::collections::{BTreeMap, BTreeSet, HashSet}; -use asap_aware_mapping::stage3_plan_selection::select_plan; -use asap_aware_mapping::PlanningModels; +use asap_plan_selection::select_plan; +use asap_plan_selection::PlanningModels; use asap_types::ir::flat::NodeId; +use asap_types::ir::schema::{FieldDataType, GroupingStrategy, HydraKind, SketchAlgorithm}; use asap_types::ir::{ASAPOp, NonASAPOp, Operator, QueryRoot, ScalarExpr, SortKey}; -use asap_types::post_asap::{FieldDataType, GroupingStrategy, HydraKind, SketchAlgorithm}; -use asap_types::post_asap::{Materialization, Medium}; +use asap_types::physical::{Materialization, Medium}; use planner_layering_common::*; // ── Example 1 workload ─────────────────────────────────────────────────── @@ -502,8 +502,8 @@ fn stage2_physical_asap_optimization_materializes_only_panes_at_ingestion() { if matches!( family, FieldDataType::ExactAggregate( - asap_types::post_asap::ExactKind::Rate - | asap_types::post_asap::ExactKind::Increase, + asap_types::ir::schema::ExactKind::Rate + | asap_types::ir::schema::ExactKind::Increase, _ ) ) { @@ -535,7 +535,7 @@ fn stage2_physical_asap_optimization_materializes_only_panes_at_ingestion() { /// Compile `p` in the physical planner (the runtime capability check), with /// the samples each time range reads as its input. fn compile_in_runtime(p: &Physical) -> Result<(), String> { - use asap_physical_operators::physical_planner::{compile, InputContract}; + use asap_executor::physical_planner::{compile, InputContract}; use std::sync::Arc; let dag = p.dag(); let mut inputs = BTreeMap::new(); diff --git a/crates/integration-tests/tests/planner_layering_example2.rs b/crates/integration-tests/tests/planner_layering_example2.rs index 7077122be..0a6be821d 100644 --- a/crates/integration-tests/tests/planner_layering_example2.rs +++ b/crates/integration-tests/tests/planner_layering_example2.rs @@ -19,12 +19,11 @@ mod planner_layering_common; use std::collections::{BTreeMap, BTreeSet}; use asap_frontend_sql::{lower_sql_dialect, SqlCatalog}; +use asap_types::ir::properties::ProbabilityExpr; +use asap_types::ir::properties::{AccuracyScope, AccuracyTarget}; +use asap_types::ir::schema::DataType; +use asap_types::ir::schema::{Field, Schema, SketchAlgorithm, SketchParams, SketchStatistic}; use asap_types::ir::QueryRoot; -use asap_types::post_asap::{ - Field, ProbabilityExpr, Schema, SketchAlgorithm, SketchParams, SketchStatistic, -}; -use asap_types::pre_asap::DataType; -use asap_types::types::{AccuracyScope, AccuracyTarget}; use asap_types::workload::{ DataArrival, DataWorkload, DurationMs, PlanningWorkload, Predictability, Query, QueryLanguage, QueryRequirements, QueryWorkload, RepeatedDemand, RepeatingEntry, RepetitionInterval, @@ -929,11 +928,11 @@ fn stage3_plan_selection_selected_plan_has_at_most_one_univmon() { /// binding"). #[test] fn stage3_plan_selection_selected_kmv_plan_executes_end_to_end() { - use asap_aware_mapping::stage3_plan_selection::select_plan; - use asap_aware_mapping::PlanningModels; - use asap_physical_operators::physical_planner::{compile, promql_rows, InputContract}; - use asap_physical_operators::runtime::Scope; - use asap_physical_operators::values::{Batch, Value}; + use asap_executor::physical_planner::{compile, promql_rows, InputContract}; + use asap_executor::runtime::Scope; + use asap_executor::values::{Batch, Value}; + use asap_plan_selection::select_plan; + use asap_plan_selection::PlanningModels; use std::sync::Arc; let workload = standin_workload(); diff --git a/crates/integration-tests/tests/planner_layering_example3.rs b/crates/integration-tests/tests/planner_layering_example3.rs index 40a097acb..32fc43311 100644 --- a/crates/integration-tests/tests/planner_layering_example3.rs +++ b/crates/integration-tests/tests/planner_layering_example3.rs @@ -13,7 +13,7 @@ mod planner_layering_common; use std::collections::BTreeSet; use asap_types::ir::flat::NodeId; -use asap_types::post_asap::{SketchAlgorithm, SketchParams, SketchStatistic}; +use asap_types::ir::schema::{SketchAlgorithm, SketchParams, SketchStatistic}; use asap_types::workload::DataArrival; use planner_layering_common::*; diff --git a/crates/integration-tests/tests/planner_layering_example4.rs b/crates/integration-tests/tests/planner_layering_example4.rs index bec54824c..136618a57 100644 --- a/crates/integration-tests/tests/planner_layering_example4.rs +++ b/crates/integration-tests/tests/planner_layering_example4.rs @@ -15,13 +15,13 @@ mod planner_layering_common; use std::collections::{BTreeMap, BTreeSet}; -use asap_aware_mapping::stage3_plan_selection::{ +use asap_logical_asap_optimization::DeploymentCapabilities; +use asap_plan_selection::{ ClusterLimits, ClusterObjective, CostModel, PlanSelectionSettings, RejectionCause, ResourceUnitCost, }; -use asap_aware_mapping::DeploymentCapabilities; use asap_types::ir::flat::NodeId; -use asap_types::post_asap::SketchAlgorithm; +use asap_types::ir::schema::SketchAlgorithm; use asap_types::workload::{DataArrival, PlanningWorkload}; use planner_layering_common::*; diff --git a/crates/integration-tests/tests/precompute_raw_samples.rs b/crates/integration-tests/tests/precompute_raw_samples.rs index ed9331ce5..f14bc886a 100644 --- a/crates/integration-tests/tests/precompute_raw_samples.rs +++ b/crates/integration-tests/tests/precompute_raw_samples.rs @@ -7,9 +7,7 @@ use asap_types::ir::OperatorNode; use physical_common::compile_at_ingestion_physical_asap_dag; use std::{collections::BTreeMap, collections::BTreeSet, rc::Rc, sync::Arc}; -use asap_integration_tests::fixtures::lower_promql; -use asap_integration_tests::planning::logical_candidates; -use asap_physical_operators::{ +use asap_executor::{ factory::create_planner_accumulator, operators::Operator, physical_planner::{precompute, Source}, @@ -18,12 +16,15 @@ use asap_physical_operators::{ values::{Batch, Value}, AggregateCore, KeyByLabelValues, Statistic, }; -use asap_types::post_asap::{ +use asap_integration_tests::fixtures::lower_promql; +use asap_integration_tests::planning::logical_candidates; +use asap_types::ir::operator::Reduction; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::scalar::expr_ir::ColumnRef; +use asap_types::ir::schema::{ EntityIdentity, ExactKind, FieldDataType, SketchAlgorithm, SketchStatistic, SummaryInputExpr, SummaryUpdate, }; -use asap_types::pre_asap::{expr_ir::ColumnRef, Reduction}; -use asap_types::types::AccuracyTarget; use futures::{executor::block_on, StreamExt}; type Series = BTreeMap; @@ -54,7 +55,7 @@ fn canonical(labels: &Series) -> Series { /// requirement that its weights are counter samples, which the executor /// needs before it builds a Count-Min heap. fn candidates(query: &str, accuracy: AccuracyTarget) -> Vec> { - use asap_aware_mapping::stage3_plan_selection::with_confirmed_weights; + use asap_plan_selection::with_confirmed_weights; use asap_types::ir::QueryRoot; use asap_types::workload::{DataWorkload, MetricType, SourceFacts}; let root = lower_promql(query, accuracy.clone()).expect("lowering failed"); @@ -139,9 +140,9 @@ fn execute( .map(|n| (&n.output_schema, &n.payload)) ); }); - let program = serde_json::from_slice::< - asap_physical_operators::physical_planner::CompiledPhysicalDAG, - >(&serde_json::to_vec(&program).unwrap()) + let program = serde_json::from_slice::( + &serde_json::to_vec(&program).unwrap(), + ) .unwrap(); let schema = precompute::raw_sample_schema(); let batch = Batch::try_new( @@ -324,8 +325,8 @@ fn check( Reduction::PerEntity => vec![], }; let stored_only = matches!(family, FieldDataType::Sketch(kind, _) - if kind.algorithm() == &asap_types::post_asap::SketchAlgorithm::Cms); - if stored_only || asap_physical_operators::capability::validate_native_family(family).is_err() { + if kind.algorithm() == &asap_types::ir::schema::SketchAlgorithm::Cms); + if stored_only || asap_executor::capability::validate_native_family(family).is_err() { // Families without a native state, or with native stored state only (plain CMS), are outside precompute execution; // their compile must fail. assert!(precompute::compile(dag, &[source], &[root]).is_err()); @@ -376,7 +377,7 @@ fn check( } } let mut expected = - BTreeMap::>::new(); + BTreeMap::>::new(); for (labels, time, value) in rows { let updater = expected .entry(population(reduction, &keys, labels)) @@ -520,10 +521,8 @@ fn grouped_raw_summary(family: FieldDataType, input: SummaryUpdate) -> (Physical // estimate each item's exact total; invalid weight contracts do not compile. #[test] fn raw_sample_heaps_resolve_items_from_labels() { - use asap_types::post_asap::{ - GroupingStrategy::PerSubpopulationInstance, NonNegativeWeightProof, SketchKind, - SketchParams, WeightDomain, - }; + use asap_types::ir::schema::GroupingStrategy::PerSubpopulationInstance; + use asap_types::ir::schema::{NonNegativeWeightProof, SketchKind, SketchParams, WeightDomain}; let heap = |algorithm, params| { FieldDataType::Sketch(SketchKind::new(algorithm, params), PerSubpopulationInstance) }; @@ -604,9 +603,9 @@ fn raw_sample_heaps_resolve_items_from_labels() { // `without` grouping over raw samples drops the listed labels and `__name__`. #[test] fn raw_sample_without_grouping_drops_labels_and_name() { - use asap_types::pre_asap::GroupKeys; + use asap_types::ir::operator::GroupKeys; let family = - FieldDataType::ExactAggregate(ExactKind::Sum, asap_types::post_asap::ExactParams::Sum); + FieldDataType::ExactAggregate(ExactKind::Sum, asap_types::ir::schema::ExactParams::Sum); let (mut dag, source, root) = grouped_raw_summary(family, SummaryUpdate::column(ColumnRef::SampleValue)); let service = dag diff --git a/crates/integration-tests/tests/promql_numeric_regressions.rs b/crates/integration-tests/tests/promql_numeric_regressions.rs index 5924ffa1f..6d93b1a8a 100644 --- a/crates/integration-tests/tests/promql_numeric_regressions.rs +++ b/crates/integration-tests/tests/promql_numeric_regressions.rs @@ -3,10 +3,11 @@ use asap_integration_tests::fixtures::lower_promql; use asap_integration_tests::planning::logical_candidates; use asap_integration_tests::post_asap::post_asap_dag; +use asap_types::ir::operator::Reduction; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::scalar::ColumnRef; +use asap_types::ir::schema::{ExactKind, FieldDataType, SummaryInputExpr, SummaryUpdate}; use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode}; -use asap_types::post_asap::{ExactKind, FieldDataType, SummaryInputExpr, SummaryUpdate}; -use asap_types::pre_asap::{ColumnRef, Reduction}; -use asap_types::types::AccuracyTarget; use std::rc::Rc; /// The first logical candidate (in enumeration order) with an ASAP operator, @@ -173,7 +174,7 @@ fn quantile_over_temporal_average_keeps_a_legal_candidate() { #[test] fn sketch_counts_use_unit_weights_and_signed_sums_keep_value_weights() { - use asap_types::post_asap::{NonNegativeWeightProof, SketchAlgorithm, WeightDomain}; + use asap_types::ir::schema::{NonNegativeWeightProof, SketchAlgorithm, WeightDomain}; for is_count in [true, false] { let query = if is_count { "topk(1, count_over_time(up[5m]))" diff --git a/crates/integration-tests/tests/promql_to_post_asap.rs b/crates/integration-tests/tests/promql_to_post_asap.rs index ea12f8fe2..1aa326df6 100644 --- a/crates/integration-tests/tests/promql_to_post_asap.rs +++ b/crates/integration-tests/tests/promql_to_post_asap.rs @@ -8,24 +8,24 @@ use std::rc::Rc; -use asap_aware_mapping::accuracy::{AccuracyEvidenceProvider, OperandDomain}; -use asap_aware_mapping::stage2_physical_asap_optimization::PhysicalCandidate; -use asap_aware_mapping::stage3_plan_selection::Selection; -use asap_aware_mapping::PlanningModels; use asap_integration_tests::fixtures::lower_promql; use asap_integration_tests::planning::logical_candidates; use asap_integration_tests::post_asap::{at_ingestion, at_ingestion_post_asap_dag, post_asap_dag}; -use asap_types::ir::operator_properties::Reduction; +use asap_logical_asap_optimization::accuracy::{AccuracyEvidenceProvider, OperandDomain}; +use asap_physical_asap_optimization::PhysicalCandidate; +use asap_plan_selection::PlanningModels; +use asap_plan_selection::Selection; +use asap_types::ir::operator::operator_properties::Reduction; use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, QueryRoot, ScalarExpr}; -use asap_types::post_asap::{ +use asap_types::ir::properties::AccuracyMetric; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::scalar::expr_ir::ColumnRef; +use asap_types::ir::schema::DataType; +use asap_types::ir::schema::{ EntityIdentity, ExactKind, ExactParams, FieldDataType, GroupingStrategy, Schema, SketchAlgorithm, SketchKind, SketchParams, SketchStatistic, SummaryInputExpr, SummaryUpdate, }; -use asap_types::pre_asap::expr_ir::ColumnRef; -use asap_types::pre_asap::schema::DataType; -use asap_types::types::AccuracyMetric; -use asap_types::types::AccuracyTarget; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, QueryRoot, ScalarExpr}; use asap_types::workload::{DataWorkload, QueryRequirements, QueryWorkloadEntry}; /// The first logical candidate (Stage 1, in enumeration order) of `root` as @@ -334,7 +334,7 @@ fn execute_topk_reference(plan: &OperatorNode) -> Vec<(String, f64)> { panic!("expected temporal input") }; let Some(NonASAPOp::Scan { - source: asap_types::pre_asap::Source::TimeSeries { metric }, + source: asap_types::ir::operator::Source::TimeSeries { metric }, predicates, .. }) = child.non_asap() @@ -626,7 +626,7 @@ fn nested_summary_explicitly_finalizes_exact_child_at_ingestion_time() { }; assert_eq!( child.timing, - Some(asap_types::post_asap::ExecutionTiming::IngestionTime) + Some(asap_types::ir::properties::ExecutionTiming::IngestionTime) ); assert!(matches!( source.asap(), @@ -651,7 +651,7 @@ fn nested_summary_explicitly_finalizes_exact_child_at_ingestion_time() { #[test] fn physical_node_owns_phase_independently_of_binary_payload() { - use asap_types::post_asap::ExecutionTiming; + use asap_types::ir::properties::ExecutionTiming; // The one-selector case (both operands over the same series, so the // BinaryOp may run at ingestion) is `arithmetic_over_one_selector_executes` // in materialization_e2e.rs, ignored until #580. @@ -705,7 +705,7 @@ fn ddsketch_ratio_selection( evidence: &dyn AccuracyEvidenceProvider, ) -> (Vec, Selection) { let root = lower_promql(query, planned).unwrap(); - let run = asap_aware_mapping::run_stage1_to_stage3( + let run = asap_plan_selection::run_stage1_to_stage3( vec![(0, QueryRoot::Operator(root))], &[QueryWorkloadEntry { requirements: QueryRequirements { @@ -751,7 +751,7 @@ fn ddsketch_ratio_rejects_one_invalid_domain_when_the_other_is_missing() { }; matches!( measures.as_slice(), - [asap_types::pre_asap::agg_intent::AggIntent::Quantile { q, .. }] if *q == 0.9 + [asap_types::ir::operator::agg_intent::AggIntent::Quantile { q, .. }] if *q == 0.9 ) .then(|| OperandDomain { lower: -1.0, diff --git a/crates/integration-tests/tests/reuse_rules.rs b/crates/integration-tests/tests/reuse_rules.rs index b88a86f80..cd5892063 100644 --- a/crates/integration-tests/tests/reuse_rules.rs +++ b/crates/integration-tests/tests/reuse_rules.rs @@ -8,24 +8,25 @@ use std::collections::BTreeMap; use std::rc::Rc; use std::sync::Arc; -use asap_aware_mapping::pass::Pass2Candidate; -use asap_aware_mapping::stage3_plan_selection::select_plan; -use asap_aware_mapping::{run_stage1_to_stage3, PlanningModels, Stage1ToStage3Run}; use asap_devtools::candidate_labels::pass2_label; use asap_devtools::planner_layering_examples::{shared_data_workload, SHARED_ROWS_PER_SECOND}; +use asap_executor::physical_planner::{compile, promql_rows, InputContract}; +use asap_executor::runtime::Scope; +use asap_executor::values::{Batch, Value}; use asap_integration_tests::fixtures::lower_promql; -use asap_physical_operators::physical_planner::{compile, promql_rows, InputContract}; -use asap_physical_operators::runtime::Scope; -use asap_physical_operators::values::{Batch, Value}; +use asap_logical_asap_optimization::workload_candidates::Pass2Candidate; +use asap_plan_selection::select_plan; +use asap_plan_selection::{run_stage1_to_stage3, PlanningModels, Stage1ToStage3Run}; use asap_types::ir::physical_export::{ compile_physical_asap_dag_with_node_ids, PhysicalASAPOperatorPayload, }; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::schema::Schema; use asap_types::ir::{ - apply_materialization_timings, ASAPOp, Materialization, MaterializationAssignment, Medium, - NonASAPOp, Operator, OperatorNode, QueryRoot, TimingMemo, + apply_materialization_timings, ASAPOp, MaterializationAssignment, NonASAPOp, Operator, + OperatorNode, QueryRoot, TimingMemo, }; -use asap_types::post_asap::Schema; -use asap_types::types::AccuracyTarget; +use asap_types::physical::{Materialization, Medium}; use asap_types::workload::{ DataArrival, DataWorkload, QueryRecurrence, QueryWorkloadEntry, RepeatedDemand, RepetitionInterval, diff --git a/crates/integration-tests/tests/scan.rs b/crates/integration-tests/tests/scan.rs index e380713e7..1828779aa 100644 --- a/crates/integration-tests/tests/scan.rs +++ b/crates/integration-tests/tests/scan.rs @@ -11,11 +11,12 @@ use std::time::Duration; use asap_integration_tests::fixtures::lower_promql; use asap_integration_tests::fixtures::metric_schema; +use asap_types::ir::operator::Source; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::scalar::{CompareOpKind, ScalarValue}; use asap_types::ir::{ ExprSemantics, NonASAPOp, OperatorNode, Predicate, ScalarExpr, TimeRangeKind, }; -use asap_types::pre_asap::{CompareOpKind, ScalarValue, Source}; -use asap_types::types::AccuracyTarget; fn lower(q: &str) -> Rc { lower_promql(q, AccuracyTarget::exact()) diff --git a/crates/integration-tests/tests/schema.rs b/crates/integration-tests/tests/schema.rs index 6028139b9..b496782ea 100644 --- a/crates/integration-tests/tests/schema.rs +++ b/crates/integration-tests/tests/schema.rs @@ -10,7 +10,7 @@ //! are label-preserving and keep the schema open. use asap_integration_tests::fixtures::lower_promql; -use asap_types::types::AccuracyTarget; +use asap_types::ir::properties::AccuracyTarget; fn lower(q: &str) -> std::rc::Rc { lower_promql(q, AccuracyTarget::exact()) diff --git a/crates/integration-tests/tests/sql_cardinality.rs b/crates/integration-tests/tests/sql_cardinality.rs index 4b8dcf023..c138a873a 100644 --- a/crates/integration-tests/tests/sql_cardinality.rs +++ b/crates/integration-tests/tests/sql_cardinality.rs @@ -1,11 +1,9 @@ //! The design's SQL distinct query executes natively and exactly. mod physical_common; +use asap_executor::values::Value; use asap_frontend_sql::{lower_sql, SqlCatalog}; -use asap_physical_operators::values::Value; -use asap_types::{ - pre_asap::{DataType, Field, Schema}, - types::AccuracyTarget, -}; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::schema::{DataType, Field, Schema}; // COUNT(DISTINCT src_ip) skips NULL and keeps the WHERE, Utf8 identities and // zero on no input. diff --git a/crates/integration-tests/tests/sql_frequency.rs b/crates/integration-tests/tests/sql_frequency.rs index 61224230d..39bea1015 100644 --- a/crates/integration-tests/tests/sql_frequency.rs +++ b/crates/integration-tests/tests/sql_frequency.rs @@ -2,12 +2,10 @@ //! natively and exactly: entropy in nats (SQL's `LN`), L2 of the //! per-source counts, and NULL on empty input, decided by an exact count. mod physical_common; +use asap_executor::values::Value; use asap_frontend_sql::{lower_sql, SqlCatalog}; -use asap_physical_operators::values::Value; -use asap_types::{ - pre_asap::{DataType, Field, Schema}, - types::AccuracyTarget, -}; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::schema::{DataType, Field, Schema}; const ENTROPY: &str = "SELECT -SUM(p * LN(p)) FROM (\ SELECT COUNT(*) * 1.0 / SUM(COUNT(*)) OVER () AS p FROM flows GROUP BY src_ip)"; diff --git a/crates/integration-tests/tests/sql_to_physical.rs b/crates/integration-tests/tests/sql_to_physical.rs index e122c3d12..f5c229a46 100644 --- a/crates/integration-tests/tests/sql_to_physical.rs +++ b/crates/integration-tests/tests/sql_to_physical.rs @@ -1,20 +1,17 @@ //! SQL frontend, the planner pipeline's selection, physical compilation and //! fresh-run execution. mod physical_common; -use asap_frontend_sql::{lower_sql, SqlCatalog}; -use asap_integration_tests::planning::selected_dag; -use asap_physical_operators::{ +use asap_executor::{ physical_planner::{compile, InputContract, Source}, runtime::{Limits, RunContext, Scope}, sources::{DataSources, MemorySource}, values::{Batch, Value}, }; +use asap_frontend_sql::{lower_sql, SqlCatalog}; +use asap_integration_tests::planning::selected_dag; use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; -use asap_types::types::AccuracyTarget; -use asap_types::{ - post_asap::FieldDataType, - pre_asap::{DataType, Field, Schema}, -}; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; use futures::StreamExt; use physical_common::compile_physical_asap_dag; use std::{collections::BTreeMap, sync::Arc}; diff --git a/crates/integration-tests/tests/sql_to_post_asap.rs b/crates/integration-tests/tests/sql_to_post_asap.rs index e55e9def4..c663b70d5 100644 --- a/crates/integration-tests/tests/sql_to_post_asap.rs +++ b/crates/integration-tests/tests/sql_to_post_asap.rs @@ -22,16 +22,16 @@ use std::rc::Rc; use asap_frontend_sql::{lower_sql, lower_sql_dialect, SqlCatalog}; use asap_integration_tests::planning::logical_candidates; use asap_integration_tests::post_asap::post_asap_dag; -use asap_types::ir::operator_properties::Reduction; +use asap_types::ir::operator::operator_properties::Reduction; use asap_types::ir::physical_export::{EdgeRole, PhysicalASAPNodeId, PhysicalASAPOperatorPayload}; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, Predicate, ScalarExpr}; -use asap_types::post_asap::{ +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::scalar::expr_ir::ColumnRef; +use asap_types::ir::schema::{DataType, Field, Schema}; +use asap_types::ir::schema::{ ExactKind, ExactParams, FieldDataType, GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, SketchStatistic, SummaryUpdate, }; -use asap_types::pre_asap::expr_ir::ColumnRef; -use asap_types::pre_asap::schema::{DataType, Field, Schema}; -use asap_types::types::AccuracyTarget; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, Predicate, ScalarExpr}; use asap_types::workload::SqlDialect; /// The first logical candidate (Stage 1, in enumeration order) of `root` as @@ -309,7 +309,7 @@ async fn sql_join_recursively_binds_both_temporal_aggregate_children() { assert!(matches!( &cols[1].expr, ScalarExpr::Arithmetic { - op: asap_types::pre_asap::ArithmeticOpKind::Div, + op: asap_types::ir::scalar::ArithmeticOpKind::Div, .. } )); @@ -325,12 +325,12 @@ async fn sql_join_recursively_binds_both_temporal_aggregate_children() { join.operator ); }; - assert_eq!(kind, &asap_types::pre_asap::JoinKind::Inner); + assert_eq!(kind, &asap_types::ir::operator::JoinKind::Inner); assert!(matches!( &pred.0, ScalarExpr::Compare { left, - op: asap_types::pre_asap::CompareOpKind::Eq, + op: asap_types::ir::scalar::CompareOpKind::Eq, right, .. } if matches!( diff --git a/crates/integration-tests/tests/stage1_logical_asap_optimization.rs b/crates/integration-tests/tests/stage1_logical_asap_optimization.rs index 171236bed..485e90dc8 100644 --- a/crates/integration-tests/tests/stage1_logical_asap_optimization.rs +++ b/crates/integration-tests/tests/stage1_logical_asap_optimization.rs @@ -6,24 +6,24 @@ use std::collections::BTreeMap; use std::rc::Rc; use std::sync::Arc; -use asap_aware_mapping::per_computation_candidates::{ - compose_logical_candidate, generate_per_computation_candidates, ComputationCandidate, -}; -use asap_frontend_sql::{lower_sql, SqlCatalog}; -use asap_physical_operators::{ +use asap_executor::{ physical_planner::{compile, InputContract, Source}, runtime::{Limits, RunContext, Scope}, sources::{DataSources, MemorySource}, values::{Batch, Value}, }; +use asap_frontend_sql::{lower_sql, SqlCatalog}; +use asap_logical_asap_optimization::per_computation_candidates::{ + compose_logical_candidate, generate_per_computation_candidates, ComputationCandidate, +}; use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; -use asap_types::ir::{ASAPOp, NonASAPOp, OperatorNode, QueryRoot, ScalarExpr}; -use asap_types::post_asap::{ +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::scalar::{ColumnRef, CompareOpKind, ScalarValue}; +use asap_types::ir::schema::{DataType, Field, Schema}; +use asap_types::ir::schema::{ ExactKind, FieldDataType, SketchAlgorithm, SummaryInputExpr, SummaryUpdate, }; -use asap_types::pre_asap::schema::{DataType, Field, Schema}; -use asap_types::pre_asap::{ColumnRef, CompareOpKind, ScalarValue}; -use asap_types::types::AccuracyTarget; +use asap_types::ir::{ASAPOp, NonASAPOp, OperatorNode, QueryRoot, ScalarExpr}; use asap_types::workload::{ BatchEntry, DataWorkload, DurationMs, PlanningWorkload, Query, QueryLanguage, QueryRequirements, QueryWorkload, @@ -571,7 +571,7 @@ async fn every_candidate( ComputationCandidate, Result< Vec<(i32, QueryRoot)>, - asap_aware_mapping::per_computation_candidates::PerComputationCandidateError, + asap_logical_asap_optimization::per_computation_candidates::PerComputationCandidateError, >, )> { let root = lower_sql(query, catalog, accuracy()).await.unwrap(); @@ -603,7 +603,7 @@ fn count_updates(root: &QueryRoot) -> Vec { Some(ASAPOp::SummaryAgg { family, input, .. }) if !matches!( family, - FieldDataType::ExactAggregate(asap_types::post_asap::ExactKind::Sum, _) + FieldDataType::ExactAggregate(asap_types::ir::schema::ExactKind::Sum, _) ) => { Some(input.clone()) @@ -734,7 +734,7 @@ async fn sql_count_star_without_a_non_null_column_is_unsupported() { every_candidate(&catalog, "SELECT k, count(*) FROM t GROUP BY k").await { match composed { - Err(asap_aware_mapping::per_computation_candidates::PerComputationCandidateError::Unsupported(_)) => {} + Err(asap_logical_asap_optimization::per_computation_candidates::PerComputationCandidateError::Unsupported(_)) => {} other => panic!("{columns:?} {candidate:?}: {other:?}"), } } diff --git a/crates/integration-tests/tests/stage2_physical_asap_optimization_sql.rs b/crates/integration-tests/tests/stage2_physical_asap_optimization_sql.rs index bc07e3c75..0f3476c82 100644 --- a/crates/integration-tests/tests/stage2_physical_asap_optimization_sql.rs +++ b/crates/integration-tests/tests/stage2_physical_asap_optimization_sql.rs @@ -3,15 +3,15 @@ //! measure, descending, whatever the group keys are called or typed, and no //! valid candidate is pruned. -use asap_aware_mapping::per_computation_candidates::{ +use asap_frontend_sql::{lower_sql, SqlCatalog}; +use asap_logical_asap_optimization::per_computation_candidates::{ compose_logical_candidate, generate_per_computation_candidates, ComputationCandidate, }; -use asap_aware_mapping::stage2_physical_asap_optimization::optimize_physical_asap; -use asap_frontend_sql::{lower_sql, SqlCatalog}; +use asap_physical_asap_optimization::optimize_physical_asap; +use asap_types::ir::operator::AggIntent; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::{NonASAPOp, OperatorNode, QueryRoot, ScalarExpr}; -use asap_types::pre_asap::schema::{DataType, Field, Schema}; -use asap_types::pre_asap::AggIntent; -use asap_types::types::AccuracyTarget; fn approximate() -> AccuracyTarget { AccuracyTarget::max_error(0.01).failure_probability(0.001) @@ -117,7 +117,7 @@ async fn check_every_choice( &roots, &[], &asap_types::workload::DataWorkload::default(), - &asap_aware_mapping::DeploymentCapabilities::default(), + &asap_logical_asap_optimization::DeploymentCapabilities::default(), ) .unwrap_or_else(|e| panic!("{sql}, choice {choice:?} pruned: {e}")) .remove(0); diff --git a/crates/integration-tests/tests/summary_coverage_e2e.rs b/crates/integration-tests/tests/summary_coverage_e2e.rs index 4614e0d10..937447ad7 100644 --- a/crates/integration-tests/tests/summary_coverage_e2e.rs +++ b/crates/integration-tests/tests/summary_coverage_e2e.rs @@ -12,12 +12,12 @@ use std::rc::Rc; use asap_frontend_sql::{lower_sql, SqlCatalog}; use asap_integration_tests::fixtures::lower_promql; use asap_integration_tests::planning::logical_candidates; -use asap_types::ir::summary_coverage::{ColumnIdentity, Constraint, SelectionBox}; +use asap_types::ir::properties::summary_coverage::{ColumnIdentity, Constraint, SelectionBox}; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::scalar::expr_ir::ScalarValue; +use asap_types::ir::schema::{DataType, Field, Schema}; +use asap_types::ir::schema::{FieldDataType, SketchAlgorithm}; use asap_types::ir::{ASAPOp, NonASAPOp, OperatorNode, Predicate, ScalarExpr}; -use asap_types::post_asap::{FieldDataType, SketchAlgorithm}; -use asap_types::pre_asap::expr_ir::ScalarValue; -use asap_types::pre_asap::schema::{DataType, Field, Schema}; -use asap_types::types::AccuracyTarget; fn accuracy() -> AccuracyTarget { AccuracyTarget::max_error(0.01).failure_probability(0.01) diff --git a/crates/integration-tests/tests/time_range.rs b/crates/integration-tests/tests/time_range.rs index 9d826e141..6d7f8b8dc 100644 --- a/crates/integration-tests/tests/time_range.rs +++ b/crates/integration-tests/tests/time_range.rs @@ -12,9 +12,9 @@ use std::time::Duration; use asap_integration_tests::fixtures::lower_promql; use asap_integration_tests::fixtures::metric_schema; +use asap_types::ir::operator::{AggIntent, Reduction, Source}; +use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::{NonASAPOp, OperatorNode, TimeRangeKind}; -use asap_types::pre_asap::{AggIntent, Reduction, Source}; -use asap_types::types::AccuracyTarget; fn lower(q: &str) -> Rc { lower_promql(q, AccuracyTarget::exact()) diff --git a/crates/integration-tests/tests/tumbling_pane_merge.rs b/crates/integration-tests/tests/tumbling_pane_merge.rs index 633915a83..b2b2f48fb 100644 --- a/crates/integration-tests/tests/tumbling_pane_merge.rs +++ b/crates/integration-tests/tests/tumbling_pane_merge.rs @@ -8,23 +8,24 @@ use std::collections::BTreeMap; use std::rc::Rc; use std::sync::Arc; -use asap_aware_mapping::asap_aware_cse::{pane_candidates, window_composition_pass2_candidates}; +use asap_executor::physical_planner::{compile, promql_rows, CompiledPhysicalDAG, InputContract}; +use asap_executor::runtime::Scope; +use asap_executor::values::{Batch, Value}; use asap_integration_tests::fixtures::lower_promql; use asap_integration_tests::planning::selected_dag; -use asap_physical_operators::physical_planner::{ - compile, promql_rows, CompiledPhysicalDAG, InputContract, +use asap_logical_asap_optimization::asap_aware_cse::{ + pane_candidates, window_composition_pass2_candidates, }; -use asap_physical_operators::runtime::Scope; -use asap_physical_operators::values::{Batch, Value}; -use asap_types::ir::operator_properties::TimeShift; -use asap_types::ir::pane_chain::expand_pane_chains; +use asap_types::ir::operator::operator_properties::TimeShift; +use asap_types::ir::operator::pane_chain::expand_pane_chains; use asap_types::ir::physical_export::{compile_physical_asap_dag_with_node_ids, PhysicalASAPDAG}; +use asap_types::ir::properties::{AccuracyMetric, AccuracyTarget}; +use asap_types::ir::schema::Schema; use asap_types::ir::{ - apply_materialization_timings, ASAPOp, Materialization, MaterializationAssignment, Medium, - NonASAPOp, Operator, OperatorNode, QueryRoot, TimingMemo, + apply_materialization_timings, ASAPOp, MaterializationAssignment, NonASAPOp, Operator, + OperatorNode, QueryRoot, TimingMemo, }; -use asap_types::post_asap::Schema; -use asap_types::types::{AccuracyMetric, AccuracyTarget}; +use asap_types::physical::{Materialization, Medium}; const EVALUATION_MS: i64 = 300_000; const PANE_MS: i64 = 60_000; diff --git a/crates/logical-asap-optimization/Cargo.toml b/crates/logical-asap-optimization/Cargo.toml new file mode 100644 index 000000000..c7708aed1 --- /dev/null +++ b/crates/logical-asap-optimization/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "asap-logical-asap-optimization" +version = "0.1.0" +edition = "2021" + +# #509 Stage 1, logical ASAP-aware optimization: Pass 1 per-computation +# candidates and Pass 2 ASAP-aware CSE over asap-types' operator DAG, with +# the analytical accuracy model. Depends on no front end and on no later +# stage. +[dependencies] +asap_sketchlib = { workspace = true } +asap-types = { path = "../types" } +thiserror = "2" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +# Only for `test-support`: lowering PromQL test queries. +asap-frontend-promql = { path = "../frontend-promql", optional = true } + +[features] +# The test helpers (`test_support`) the later stage crates' tests share. +test-support = ["dep:asap-frontend-promql"] + +[dev-dependencies] +asap-frontend-promql = { path = "../frontend-promql" } diff --git a/crates/asap-aware-mapping/src/accuracy/budget.rs b/crates/logical-asap-optimization/src/accuracy/budget.rs similarity index 100% rename from crates/asap-aware-mapping/src/accuracy/budget.rs rename to crates/logical-asap-optimization/src/accuracy/budget.rs diff --git a/crates/asap-aware-mapping/src/accuracy/composition.rs b/crates/logical-asap-optimization/src/accuracy/composition.rs similarity index 98% rename from crates/asap-aware-mapping/src/accuracy/composition.rs rename to crates/logical-asap-optimization/src/accuracy/composition.rs index d91ba5764..dcac1f393 100644 --- a/crates/asap-aware-mapping/src/accuracy/composition.rs +++ b/crates/logical-asap-optimization/src/accuracy/composition.rs @@ -6,7 +6,7 @@ use super::*; /// Rows an exact aggregation folds (per group): bounded by the series /// count of the sources the query reads. -pub(crate) const INPUT_ROW_COUNT: &str = "input_row_count"; +pub const INPUT_ROW_COUNT: &str = "input_row_count"; /// Rows one group of a SQL fold over an approximate relation folds: not /// bounded by the series count of the sources it reads, so plan selection /// leaves it unbound. @@ -15,16 +15,16 @@ pub(crate) const GROUP_ROW_COUNT: &str = "group_row_count"; pub(crate) const VALUES_NON_NEGATIVE: &str = "values_non_negative"; /// The relative bound of a quantile ratio, given enforced operand domains /// (`OperandDomain`). -pub(crate) const DIVISION_OPERAND_DOMAINS: &str = "division_operand_domains"; +pub const DIVISION_OPERAND_DOMAINS: &str = "division_operand_domains"; /// The absolute bound of a product of two estimates, /// `|x|·e_y + |y|·e_x + e_x·e_y`, given enforced operand domains. -pub(crate) const PRODUCT_OPERAND_DOMAINS: &str = "product_operand_domains"; +pub const PRODUCT_OPERAND_DOMAINS: &str = "product_operand_domains"; /// The absolute bound of a quotient with an approximate denominator, /// `(|x|·e_y + |y|·e_x) / (|y|·(|y| − e_y))`, given enforced operand domains /// with the denominator's bounded away from zero. pub(crate) const QUOTIENT_OPERAND_DOMAINS: &str = "quotient_operand_domains"; /// `‖f‖₁` of the stream a Count-Min bound is normalized by. -pub(crate) const STREAM_L1_NORM: &str = "stream_l1_norm"; +pub const STREAM_L1_NORM: &str = "stream_l1_norm"; /// `‖f‖₂` of the stream a Count Sketch bound is normalized by. pub(crate) const STREAM_L2_NORM: &str = "stream_l2_norm"; /// Rows of one source's stream a summary folds: with a constant update @@ -53,12 +53,12 @@ pub(crate) fn source_norm_statistic(norm: &str, source: &str) -> String { /// `values_non_negative[m]`: an indicator, 1 when proven. Plan selection /// binds it to 1 when the data workload declares `metric` a counter, and /// leaves it unbound otherwise. -pub(crate) fn samples_sign_statistic(metric: &str) -> String { +pub fn samples_sign_statistic(metric: &str) -> String { format!("{VALUES_NON_NEGATIVE}[{metric}]") } /// The metric a [`samples_sign_statistic`] names. -pub(crate) fn parse_samples_sign_statistic(statistic: &str) -> Option<&str> { +pub fn parse_samples_sign_statistic(statistic: &str) -> Option<&str> { statistic .strip_prefix(VALUES_NON_NEGATIVE)? .strip_prefix('[')? @@ -66,7 +66,7 @@ pub(crate) fn parse_samples_sign_statistic(statistic: &str) -> Option<&str> { } /// The norm and source a [`source_norm_statistic`] names. -pub(crate) fn parse_source_norm_statistic(statistic: &str) -> Option<(&'static str, &str)> { +pub fn parse_source_norm_statistic(statistic: &str) -> Option<(&'static str, &str)> { [STREAM_L1_NORM, STREAM_L2_NORM] .into_iter() .find_map(|norm| { @@ -547,7 +547,7 @@ impl DefaultAccuracyModel { /// Whether `guarantee` is a sketch's distinct-count estimate. fn estimates_distinct_count(guarantee: &AccuracyGuarantee) -> bool { - let cardinality = format!("{:?}", asap_types::post_asap::SketchStatistic::Cardinality); + let cardinality = format!("{:?}", asap_types::ir::schema::SketchStatistic::Cardinality); guarantee.provenance.iter().any(|source| { matches!(source, GuaranteeSource::SketchEvaluation { query, .. } if *query == cardinality) }) diff --git a/crates/asap-aware-mapping/src/accuracy/estimators/cardinality.rs b/crates/logical-asap-optimization/src/accuracy/estimators/cardinality.rs similarity index 99% rename from crates/asap-aware-mapping/src/accuracy/estimators/cardinality.rs rename to crates/logical-asap-optimization/src/accuracy/estimators/cardinality.rs index 5e69aa1c5..e9cd9848f 100644 --- a/crates/asap-aware-mapping/src/accuracy/estimators/cardinality.rs +++ b/crates/logical-asap-optimization/src/accuracy/estimators/cardinality.rs @@ -128,7 +128,7 @@ pub(crate) fn theta_k_99(eps: f64) -> u32 { #[cfg(test)] mod tests { use super::*; - use asap_types::pre_asap::agg_intent::default_cardinality; + use asap_types::ir::operator::agg_intent::default_cardinality; fn error_and_delta(params: &SketchParams) -> (f64, f64) { let g = guarantee(&SketchAlgorithm::Kmv, params, &SketchStatistic::Cardinality).unwrap(); diff --git a/crates/asap-aware-mapping/src/accuracy/estimators/cms.rs b/crates/logical-asap-optimization/src/accuracy/estimators/cms.rs similarity index 93% rename from crates/asap-aware-mapping/src/accuracy/estimators/cms.rs rename to crates/logical-asap-optimization/src/accuracy/estimators/cms.rs index 41bec6a78..cbf1b5a6a 100644 --- a/crates/asap-aware-mapping/src/accuracy/estimators/cms.rs +++ b/crates/logical-asap-optimization/src/accuracy/estimators/cms.rs @@ -42,8 +42,8 @@ mod tests { #[test] fn local_guarantee_inverts_frequency_sizing() { use crate::realization::default_size_params; - use asap_types::post_asap::{GroupingStrategy, SketchKind}; - let c = asap_types::pre_asap::agg_intent::default_cardinality(); + use asap_types::ir::schema::{GroupingStrategy, SketchKind}; + let c = asap_types::ir::operator::agg_intent::default_cardinality(); let params = default_size_params(SketchAlgorithm::Cms, &c, 0.01, 0.001); let g = DefaultAccuracyModel .local_guarantee( @@ -60,13 +60,13 @@ mod tests { &AccuracyTarget::max_error(0.01) .failure_probability(0.001) .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1) - .accuracy_scope(asap_types::types::AccuracyScope::PerValue) + .accuracy_scope(asap_types::ir::properties::AccuracyScope::PerValue) )); } #[test] fn heap_evaluation_retains_frequency_metric() { - use asap_types::post_asap::{GroupingStrategy, SketchKind}; + use asap_types::ir::schema::{GroupingStrategy, SketchKind}; let cms_heap = SketchParams::CmsWithHeap { width: 272, depth: 5, diff --git a/crates/asap-aware-mapping/src/accuracy/estimators/count_sketch.rs b/crates/logical-asap-optimization/src/accuracy/estimators/count_sketch.rs similarity index 94% rename from crates/asap-aware-mapping/src/accuracy/estimators/count_sketch.rs rename to crates/logical-asap-optimization/src/accuracy/estimators/count_sketch.rs index 6a6d60137..e5bf5c447 100644 --- a/crates/asap-aware-mapping/src/accuracy/estimators/count_sketch.rs +++ b/crates/logical-asap-optimization/src/accuracy/estimators/count_sketch.rs @@ -71,8 +71,8 @@ mod tests { #[test] fn count_sketch_uses_an_l2_guarantee() { use crate::realization::default_size_params; - use asap_types::post_asap::{GroupingStrategy, SketchKind}; - use asap_types::pre_asap::agg_intent::default_cardinality; + use asap_types::ir::operator::agg_intent::default_cardinality; + use asap_types::ir::schema::{GroupingStrategy, SketchKind}; let intent = default_cardinality(); let count_sketch = default_size_params(SketchAlgorithm::CountSketch, &intent, 0.01, 0.01); let guarantee = DefaultAccuracyModel @@ -82,7 +82,7 @@ mod tests { GroupingStrategy::default(), ), &SketchStatistic::PointCount { - key: asap_types::pre_asap::expr_ir::ColumnRef::SampleValue, + key: asap_types::ir::scalar::expr_ir::ColumnRef::SampleValue, value: None, }, ) @@ -96,7 +96,7 @@ mod tests { &AccuracyTarget::max_error(0.01) .failure_probability(0.01) .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L2) - .accuracy_scope(asap_types::types::AccuracyScope::PerValue) + .accuracy_scope(asap_types::ir::properties::AccuracyScope::PerValue) )); } diff --git a/crates/asap-aware-mapping/src/accuracy/estimators/ddsketch.rs b/crates/logical-asap-optimization/src/accuracy/estimators/ddsketch.rs similarity index 100% rename from crates/asap-aware-mapping/src/accuracy/estimators/ddsketch.rs rename to crates/logical-asap-optimization/src/accuracy/estimators/ddsketch.rs diff --git a/crates/asap-aware-mapping/src/accuracy/estimators/hll.rs b/crates/logical-asap-optimization/src/accuracy/estimators/hll.rs similarity index 97% rename from crates/asap-aware-mapping/src/accuracy/estimators/hll.rs rename to crates/logical-asap-optimization/src/accuracy/estimators/hll.rs index fc49b3ae4..76b59d832 100644 --- a/crates/asap-aware-mapping/src/accuracy/estimators/hll.rs +++ b/crates/logical-asap-optimization/src/accuracy/estimators/hll.rs @@ -39,7 +39,7 @@ const MAX_DISTINCT: &str = "max_distinct"; /// The relative error a classic HLL's guarantee states: the one its /// precision was sized for, else its relative standard error. -pub(crate) fn classic_relative_error(precision: u8, relative_error: Option) -> f64 { +pub fn classic_relative_error(precision: u8, relative_error: Option) -> f64 { relative_error.unwrap_or_else(|| relative_standard_error(precision)) } @@ -47,11 +47,7 @@ pub(crate) fn classic_relative_error(precision: u8, relative_error: Option) /// `relative_error` over `source` is left as ([`classic_guarantee`]). /// `relative_error` is written in Rust's shortest round-trip form, so it /// parses back to the same value. -pub(crate) fn classic_failure_statistic( - precision: u8, - relative_error: f64, - source: &str, -) -> String { +pub fn classic_failure_statistic(precision: u8, relative_error: f64, source: &str) -> String { format!( "{CLASSIC_FAILURE_PROBABILITY}[p={precision},eps={relative_error}]({MAX_DISTINCT}[{source}])" ) @@ -59,7 +55,7 @@ pub(crate) fn classic_failure_statistic( /// The precision, relative error and source a [`classic_failure_statistic`] /// names. -pub(crate) fn parse_classic_failure_statistic(statistic: &str) -> Option<(u8, f64, &str)> { +pub fn parse_classic_failure_statistic(statistic: &str) -> Option<(u8, f64, &str)> { let rest = statistic .strip_prefix(CLASSIC_FAILURE_PROBABILITY)? .strip_prefix("[p=")?; @@ -119,11 +115,7 @@ pub(crate) fn classic_guarantee( /// The failure probability a [`classic_failure_statistic`] names, given the /// source's distinct bound: `1` (no certificate) when the bound is outside /// the contract's range for this precision and relative error. -pub(crate) fn classic_failure_probability( - precision: u8, - relative_error: f64, - max_distinct: u64, -) -> f64 { +pub fn classic_failure_probability(precision: u8, relative_error: f64, max_distinct: u64) -> f64 { u32::try_from(max_distinct) .ok() .and_then(|n| ClassicHllConfidence::new(n, relative_error)) @@ -232,7 +224,7 @@ impl ClassicHllConfidence { /// distinct items is at most `failure_probability`; `at_least` when none up /// to 18 is. Plan selection's enlargement (#669b) applies this rule to the /// source's bound, Pass 1's sizing ([`hll_precision`]) to a bound of `⌈1/ε⌉`. -pub(crate) fn min_precision( +pub fn min_precision( relative_error: f64, failure_probability: f64, max_distinct: u64, @@ -462,7 +454,7 @@ mod tests { #[test] fn precision_is_sized_from_epsilon_and_delta() { use crate::realization::default_size_params; - use asap_types::pre_asap::agg_intent::default_cardinality; + use asap_types::ir::operator::agg_intent::default_cardinality; for (eps, delta, precision) in [(0.05, 0.01, 15), (0.02, 0.01, 17)] { let params = default_size_params(SketchAlgorithm::Hll, &default_cardinality(), eps, delta); @@ -480,8 +472,8 @@ mod tests { #[test] fn generic_rse_sizing_does_not_certify_confidence() { use crate::realization::default_size_params; - use asap_types::post_asap::{GroupingStrategy, SketchKind}; - use asap_types::pre_asap::agg_intent::default_cardinality; + use asap_types::ir::operator::agg_intent::default_cardinality; + use asap_types::ir::schema::{GroupingStrategy, SketchKind}; let c = default_cardinality(); let params = default_size_params(SketchAlgorithm::Hll, &c, 0.01, 0.01); let g = DefaultAccuracyModel diff --git a/crates/asap-aware-mapping/src/accuracy/estimators/kll.rs b/crates/logical-asap-optimization/src/accuracy/estimators/kll.rs similarity index 96% rename from crates/asap-aware-mapping/src/accuracy/estimators/kll.rs rename to crates/logical-asap-optimization/src/accuracy/estimators/kll.rs index 95150dda7..b9b19e2b0 100644 --- a/crates/asap-aware-mapping/src/accuracy/estimators/kll.rs +++ b/crates/logical-asap-optimization/src/accuracy/estimators/kll.rs @@ -2,7 +2,7 @@ //! and, below a 0.01 failure probability, that calibration scaled by the //! Gaussian quantile ratio `z(1 − δ/2) / z(1 − 0.01/2)`. use super::*; -use asap_types::post_asap::sketch::KLL_MAX_K; +use asap_types::ir::schema::sketch::KLL_MAX_K; pub(super) fn guarantee( algorithm: &SketchAlgorithm, @@ -187,8 +187,8 @@ mod tests { #[test] fn local_guarantee_inverts_rank_sizing() { use crate::realization::default_size_params; - use asap_types::post_asap::{GroupingStrategy, SketchKind}; - use asap_types::pre_asap::agg_intent::default_quantile; + use asap_types::ir::operator::agg_intent::default_quantile; + use asap_types::ir::schema::{GroupingStrategy, SketchKind}; let q = default_quantile(0.99); let params = default_size_params(SketchAlgorithm::Kll, &q, 0.01, 0.01); let g = DefaultAccuracyModel @@ -211,7 +211,7 @@ mod tests { &AccuracyTarget::max_error(0.01) .failure_probability(0.01) .accuracy_metric(AccuracyMetric::Rank) - .accuracy_scope(asap_types::types::AccuracyScope::PerValue) + .accuracy_scope(asap_types::ir::properties::AccuracyScope::PerValue) )); assert_eq!(g.approximate_layer_count(), 1); assert!(g.provenance.iter().any(|source| matches!( @@ -226,7 +226,7 @@ mod tests { #[test] fn sizing_meets_a_failure_probability_below_one_percent() { use crate::realization::default_size_params; - use asap_types::pre_asap::agg_intent::default_quantile; + use asap_types::ir::operator::agg_intent::default_quantile; let q = default_quantile(0.99); let k = |delta| match default_size_params(SketchAlgorithm::Kll, &q, 0.01, delta) { SketchParams::Kll { k, .. } => k, @@ -245,7 +245,7 @@ mod tests { &AccuracyTarget::max_error(0.01) .failure_probability(0.001) .accuracy_metric(AccuracyMetric::Rank) - .accuracy_scope(asap_types::types::AccuracyScope::PerValue) + .accuracy_scope(asap_types::ir::properties::AccuracyScope::PerValue) )); assert_eq!(g.spec.failure_probability.evaluate(), Some(0.001)); } @@ -353,7 +353,7 @@ mod tests { #[test] fn accuracy_target_drives_kll_k() { use crate::realization::default_size_params; - use asap_types::pre_asap::agg_intent::default_quantile; + use asap_types::ir::operator::agg_intent::default_quantile; let q = default_quantile(0.99); assert_eq!( default_size_params(SketchAlgorithm::Kll, &q, 0.01, 0.01), @@ -375,7 +375,7 @@ mod tests { #[test] fn degenerate_epsilon_saturates_to_tightest_kll_params() { use crate::realization::default_size_params; - use asap_types::pre_asap::agg_intent::default_quantile; + use asap_types::ir::operator::agg_intent::default_quantile; assert_eq!( default_size_params(SketchAlgorithm::Kll, &default_quantile(0.99), 0.0, 0.01), SketchParams::Kll { diff --git a/crates/asap-aware-mapping/src/accuracy/estimators/mod.rs b/crates/logical-asap-optimization/src/accuracy/estimators/mod.rs similarity index 98% rename from crates/asap-aware-mapping/src/accuracy/estimators/mod.rs rename to crates/logical-asap-optimization/src/accuracy/estimators/mod.rs index 9ba54fd03..052180d11 100644 --- a/crates/asap-aware-mapping/src/accuracy/estimators/mod.rs +++ b/crates/logical-asap-optimization/src/accuracy/estimators/mod.rs @@ -1,6 +1,6 @@ //! Dispatch committed estimator parameters to their accuracy models. use super::*; -use asap_types::pre_asap::AggIntent; +use asap_types::ir::operator::AggIntent; pub mod cardinality; pub mod cms; @@ -81,7 +81,7 @@ pub(super) fn local_guarantee( /// ([`hll::min_precision`]); Theta is calibrated at a fixed 99%; a KMV not /// sized for a relative error and an L2 or entropy UnivMon are not /// enlarged. -pub(crate) fn with_failure_probability(params: &SketchParams, delta: f64) -> SketchParams { +pub fn with_failure_probability(params: &SketchParams, delta: f64) -> SketchParams { if !(delta > 0.0 && delta < 1.0) { return params.clone(); } @@ -491,7 +491,7 @@ mod shared_size_tests { #[cfg(test)] mod tests { use super::*; - use asap_types::pre_asap::agg_intent::{default_cardinality, default_quantile}; + use asap_types::ir::operator::agg_intent::{default_cardinality, default_quantile}; fn delta_and_error(params: &SketchParams, query: &SketchStatistic) -> (f64, f64) { let algorithm = match params { @@ -515,7 +515,7 @@ mod tests { #[test] fn enlarging_meets_the_failure_probability_at_no_larger_error() { let point = SketchStatistic::PointCount { - key: asap_types::pre_asap::expr_ir::ColumnRef::SampleValue, + key: asap_types::ir::scalar::expr_ir::ColumnRef::SampleValue, value: None, }; let quantile = SketchStatistic::Quantile { q: 0.99 }; diff --git a/crates/asap-aware-mapping/src/accuracy/estimators/univmon.rs b/crates/logical-asap-optimization/src/accuracy/estimators/univmon.rs similarity index 98% rename from crates/asap-aware-mapping/src/accuracy/estimators/univmon.rs rename to crates/logical-asap-optimization/src/accuracy/estimators/univmon.rs index 13c5dcb1e..82592d91f 100644 --- a/crates/asap-aware-mapping/src/accuracy/estimators/univmon.rs +++ b/crates/logical-asap-optimization/src/accuracy/estimators/univmon.rs @@ -16,7 +16,7 @@ //! (OctoSketch aggregators fed promoted deltas) are not covered: the //! executor rejects them. use super::*; -use asap_types::pre_asap::AggIntent; +use asap_types::ir::operator::AggIntent; /// Layers `asap_sketchlib` builds: layer `i` hashes at seed index `i`, and /// seed 19 picks the layers a key reaches. @@ -191,7 +191,7 @@ pub(crate) fn incomplete_top_layer_statistic(heap_size: u32, layers: u8, source: } /// The heap, layers and source an [`incomplete_top_layer_statistic`] names. -pub(crate) fn parse_incomplete_top_layer_statistic(statistic: &str) -> Option<(u32, u8, &str)> { +pub fn parse_incomplete_top_layer_statistic(statistic: &str) -> Option<(u32, u8, &str)> { let rest = statistic .strip_prefix(INCOMPLETE_TOP_LAYER)? .strip_prefix("[k=")?; @@ -213,11 +213,7 @@ pub(crate) fn parse_incomplete_top_layer_statistic(statistic: &str) -> Option<(u /// `F0_max ≤ k · 2^(layers−2)`, so at most `k/2` keys reach the top layer /// in expectation; /// - 1 (no certificate) otherwise: the pyramid is too shallow. -pub(crate) fn incomplete_top_layer_probability( - heap_size: u32, - layers: u8, - max_distinct: u64, -) -> f64 { +pub fn incomplete_top_layer_probability(heap_size: u32, layers: u8, max_distinct: u64) -> f64 { let (k, max_distinct) = (u128::from(heap_size), u128::from(max_distinct)); if max_distinct <= k { 0.0 @@ -348,7 +344,7 @@ pub(super) fn size_params(intent: &AggIntent, eps: f64, delta: f64) -> SketchPar #[cfg(test)] mod tests { use super::*; - use asap_types::pre_asap::agg_intent::default_cardinality; + use asap_types::ir::operator::agg_intent::default_cardinality; fn l2_intent() -> AggIntent { AggIntent::FrequencyL2 { @@ -607,7 +603,7 @@ mod tests { &SketchAlgorithm::UnivMon, ¶ms, &SketchStatistic::PointCount { - key: asap_types::pre_asap::expr_ir::ColumnRef::SampleValue, + key: asap_types::ir::scalar::expr_ir::ColumnRef::SampleValue, value: None, }, ) diff --git a/crates/asap-aware-mapping/src/accuracy/evidence.rs b/crates/logical-asap-optimization/src/accuracy/evidence.rs similarity index 100% rename from crates/asap-aware-mapping/src/accuracy/evidence.rs rename to crates/logical-asap-optimization/src/accuracy/evidence.rs diff --git a/crates/asap-aware-mapping/src/accuracy/mod.rs b/crates/logical-asap-optimization/src/accuracy/mod.rs similarity index 97% rename from crates/asap-aware-mapping/src/accuracy/mod.rs rename to crates/logical-asap-optimization/src/accuracy/mod.rs index fda3b0b5f..3866a4c09 100644 --- a/crates/asap-aware-mapping/src/accuracy/mod.rs +++ b/crates/logical-asap-optimization/src/accuracy/mod.rs @@ -14,19 +14,19 @@ pub mod evidence; pub use budget::per_input_target; pub use evidence::{AccuracyEvidenceProvider, NoAccuracyEvidence, OperandDomain, PropagationStats}; -use asap_types::ir::OperatorNode; -use asap_types::post_asap::{ +use asap_types::ir::properties::{ AccuracyError, AccuracyGuarantee, AccuracyMetric, BoundExpr, CompositionOperator, - FieldDataType, GuaranteeSource, ProbabilityExpr, SketchAlgorithm, SketchParams, - SketchStatistic, + GuaranteeSource, ProbabilityExpr, }; -use asap_types::types::{AccuracyScope, AccuracyTarget}; +use asap_types::ir::properties::{AccuracyScope, AccuracyTarget}; +use asap_types::ir::schema::{FieldDataType, SketchAlgorithm, SketchParams, SketchStatistic}; +use asap_types::ir::OperatorNode; -/// The deployment-extensible accuracy algebra. `asap-aware-mapping` ships +/// The deployment-extensible accuracy algebra. This crate ships /// [`DefaultAccuracyModel`]; a deployment with a proof for a composition the /// default rejects (a registered cross-metric conversion, say) implements /// this trait and passes it to plan selection in -/// [`crate::PlanningModels::accuracy`]. +/// `asap_plan_selection::PlanningModels::accuracy`. pub trait AccuracyModel { /// The guarantee of reading `query` out of a summary of family `family` /// built over an **exact** input — derived from the family's committed diff --git a/crates/asap-aware-mapping/src/asap_aware_cse.rs b/crates/logical-asap-optimization/src/asap_aware_cse.rs similarity index 98% rename from crates/asap-aware-mapping/src/asap_aware_cse.rs rename to crates/logical-asap-optimization/src/asap_aware_cse.rs index 11f2305fd..c6cb094f9 100644 --- a/crates/asap-aware-mapping/src/asap_aware_cse.rs +++ b/crates/logical-asap-optimization/src/asap_aware_cse.rs @@ -101,17 +101,17 @@ use std::collections::HashSet; use std::rc::Rc; use asap_types::ir::cse::share_common_sub_dags; -use asap_types::ir::operator_properties::{Reduction, Source}; +use asap_types::ir::operator::operator_properties::{Reduction, Source}; +use asap_types::ir::operator::AggIntent; +use asap_types::ir::schema::{ + ExactKind, FamilyCapability, FieldDataType, GroupingStrategy, MergeInputRule, SketchAlgorithm, + SketchKind, +}; use asap_types::ir::SortKey; use asap_types::ir::{ ASAPOp, NonASAPOp, Operator, OperatorNode, OperatorResultKind, QueryRoot, TimeRangeKind, }; use asap_types::ir::{OperatorResultKind as ResultKind, ScalarExpr}; -use asap_types::post_asap::{ - ExactKind, FamilyCapability, FieldDataType, GroupingStrategy, MergeInputRule, SketchAlgorithm, - SketchKind, -}; -use asap_types::pre_asap::AggIntent; use crate::accuracy::estimators::size_for_all_readers; use crate::per_computation_candidates::{recompose_guarantees, PerComputationCandidateError}; @@ -1090,7 +1090,7 @@ fn finer_keys(coarse: &OperatorNode, fine: &OperatorNode) -> Option { type ColumnName = (Option, String); -fn column_name(field: &asap_types::pre_asap::Field) -> ColumnName { +fn column_name(field: &asap_types::ir::schema::Field) -> ColumnName { (field.table.clone(), field.name.clone()) } @@ -1170,7 +1170,7 @@ fn rolled_up( // not summed again. if finer.schema.fields.get(value).map(|f| &f.dtype) != Some(&FieldDataType::Plain( - asap_types::pre_asap::DataType::Float64, + asap_types::ir::schema::DataType::Float64, )) { return None; @@ -1429,7 +1429,7 @@ fn exact_topk(node: &OperatorNode) -> Option<(&Rc, &Reduction, usi let exact = child .guarantee .as_ref() - .is_some_and(asap_types::post_asap::AccuracyGuarantee::is_exact); + .is_some_and(asap_types::ir::properties::AccuracyGuarantee::is_exact); (exact && !by.is_without() && filters.iter().all(Option::is_none)) .then_some((child, reduction, *k)) } @@ -1531,8 +1531,8 @@ fn common_nodes(a: &QueryRoot, b: &QueryRoot) -> HashSet<*const OperatorNode> { mod tests { use super::*; use crate::test_support::lower_promql; - use asap_types::ir::schema_support::with_promql_series_identity; - use asap_types::types::AccuracyTarget; + use asap_types::ir::properties::AccuracyTarget; + use asap_types::ir::schema::schema_support::with_promql_series_identity; fn roots(queries: &[&str]) -> Vec { queries @@ -1692,8 +1692,8 @@ mod tests { #[test] fn twenty_nested_sharing_candidates_enumerate_quickly() { use crate::test_support::{filter, metric_scan_with_keys}; + use asap_types::ir::scalar::expr_ir::CompareOpKind; use asap_types::ir::ScalarExpr; - use asap_types::pre_asap::expr_ir::CompareOpKind; let chain = || { let mut node = metric_scan_with_keys(&["job"], vec![vec![0, 2]]); for i in 0..19 { @@ -1732,7 +1732,7 @@ mod tests { }; let root = lower_promql( "count_over_time(m[5m])", - asap_types::types::AccuracyTarget::max_error(0.01).failure_probability(0.001), + asap_types::ir::properties::AccuracyTarget::max_error(0.01).failure_probability(0.001), ); let inventory = generate_per_computation_candidates(vec![( 0, @@ -1793,9 +1793,9 @@ mod tests { .map(|(i, (q, epsilon))| { let root = lower_promql( q, - asap_types::types::AccuracyTarget::max_error(*epsilon) + asap_types::ir::properties::AccuracyTarget::max_error(*epsilon) .failure_probability(0.01) - .accuracy_metric(asap_types::types::AccuracyMetric::Rank), + .accuracy_metric(asap_types::ir::properties::AccuracyMetric::Rank), ); ( i, @@ -1835,7 +1835,7 @@ mod tests { .. }) = node.asap() { - let asap_types::post_asap::SketchParams::Kll { k, .. } = kind.params() else { + let asap_types::ir::schema::SketchParams::Kll { k, .. } = kind.params() else { panic!("{kind:?}") }; if !out.iter().any(|(p, _)| *p == Rc::as_ptr(&node)) { @@ -2223,7 +2223,7 @@ mod tests { )); assert_eq!( *family, - FieldDataType::ExactAggregate(ExactKind::Sum, asap_types::post_asap::ExactParams::Sum) + FieldDataType::ExactAggregate(ExactKind::Sum, asap_types::ir::schema::ExactParams::Sum) ); for (before, after) in roots.iter().zip(rolled) { let (before, after) = (operator(before), operator(after)); @@ -2232,7 +2232,7 @@ mod tests { assert!(after .guarantee .as_ref() - .is_some_and(asap_types::post_asap::AccuracyGuarantee::is_exact)); + .is_some_and(asap_types::ir::properties::AccuracyGuarantee::is_exact)); } } @@ -2309,7 +2309,7 @@ mod tests { /// guarantee, and no sketch's states can be regrouped in the IR. #[test] fn rollup_is_refused_for_families_without_an_exact_merge_across_groups() { - use asap_types::post_asap::{ExactParams as P, SketchParams}; + use asap_types::ir::schema::{ExactParams as P, SketchParams}; let exact = |kind, params| FieldDataType::ExactAggregate(kind, params); assert_eq!( rollup_kind(&exact(ExactKind::Sum, P::Sum)), @@ -2396,7 +2396,7 @@ mod tests { assert!(after .guarantee .as_ref() - .is_some_and(asap_types::post_asap::AccuracyGuarantee::is_exact)); + .is_some_and(asap_types::ir::properties::AccuracyGuarantee::is_exact)); } } @@ -2435,7 +2435,8 @@ mod tests { .map(|(i, q)| { let root = lower_promql( q, - asap_types::types::AccuracyTarget::max_error(0.01).failure_probability(0.01), + asap_types::ir::properties::AccuracyTarget::max_error(0.01) + .failure_probability(0.01), ); (i, QueryRoot::Operator(root)) }) diff --git a/crates/asap-aware-mapping/src/deployment_capabilities.rs b/crates/logical-asap-optimization/src/deployment_capabilities.rs similarity index 99% rename from crates/asap-aware-mapping/src/deployment_capabilities.rs rename to crates/logical-asap-optimization/src/deployment_capabilities.rs index 838951f13..adce4a2ed 100644 --- a/crates/asap-aware-mapping/src/deployment_capabilities.rs +++ b/crates/logical-asap-optimization/src/deployment_capabilities.rs @@ -31,7 +31,7 @@ //! and keeps no query-time result across evaluations, which is the //! planner's behavior without deployment inputs. -use asap_types::post_asap::{SketchAlgorithm, StatisticKind}; +use asap_types::ir::schema::{SketchAlgorithm, StatisticKind}; use asap_types::workload::{DataArrival, DataWorkload}; use serde::{Deserialize, Serialize}; use thiserror::Error; diff --git a/crates/asap-aware-mapping/src/function_rules.rs b/crates/logical-asap-optimization/src/function_rules.rs similarity index 94% rename from crates/asap-aware-mapping/src/function_rules.rs rename to crates/logical-asap-optimization/src/function_rules.rs index 8c8d7e7eb..fd5eb81d4 100644 --- a/crates/asap-aware-mapping/src/function_rules.rs +++ b/crates/logical-asap-optimization/src/function_rules.rs @@ -1,8 +1,8 @@ //! The accuracy composition rule of each exact function applied to an //! approximate input, shared by Pass 1's guarantee composition and the //! accuracy algebra. -use asap_types::post_asap::CompositionOperator; -use asap_types::pre_asap::AggIntent; +use asap_types::ir::operator::AggIntent; +use asap_types::ir::properties::CompositionOperator; /// The composition operator that propagates an approximate input's guarantee /// through `intent`; `None` for a function with no registered rule over diff --git a/crates/logical-asap-optimization/src/lib.rs b/crates/logical-asap-optimization/src/lib.rs new file mode 100644 index 000000000..863171f5d --- /dev/null +++ b/crates/logical-asap-optimization/src/lib.rs @@ -0,0 +1,72 @@ +//! `asap-logical-asap-optimization` — #509 Stage 1, logical ASAP-aware +//! optimization: from the query-language frontends' logical DAGs to the +//! CandidateLogicalASAPDAGs. +//! +//! Identical-sub-DAG sharing over the logical DAG (`asap_types::ir::cse`) +//! runs before the stages; Stage 1 adds the ASAP-aware sharing that is +//! invisible at the logical level ([`asap_aware_cse`], Pass 2), e.g. +//! `Quantile(x, 0.99)` and `Quantile(x, 0.95)` reading one built sketch. +//! +//! It depends only on `asap-types`, never on a front end or a later stage: +//! Stage 2 (`asap-physical-asap-optimization`) and Stage 3 +//! (`asap-plan-selection`) depend on it, and Cargo enforces that one-way +//! flow. +//! +//! Post-lowering **canonicalization** is *not* here: it lives in +//! `asap_types::ir::canonicalize`, run inside `asap_frontend_common`'s shared +//! `resolve_root` so every front end normalizes before the IR leaves resolution +//! (issue #34, closed). +//! +//! ## Components +//! +//! - [`per_computation_candidates`] — Pass 1: per-computation candidate +//! generation and composition. +//! - [`asap_aware_cse`] — Pass 2: ASAP-aware common-subexpression elimination. +//! - [`workload_candidates`] — the whole-workload Stage 1 candidates: each +//! composed Pass 1 candidate followed by its Pass 2 candidates. +//! - [`realization`] — the summary families that can serve an aggregate +//! intent, the intent's accuracy target and `(ε, δ)` budget, and default +//! sketch sizing, shared by Pass 1 and Pass 2. +//! - [`accuracy`] — the analytical accuracy model: per-family estimators, +//! composition and budget allocation, and the +//! [`AccuracyModel`](accuracy::AccuracyModel) extension point (issue #172) +//! that derives a machine-readable +//! [`AccuracyGuarantee`](asap_types::ir::properties::AccuracyGuarantee) +//! for every finalized value and propagates it through +//! approximate-over-approximate compositions under conservative rules +//! (no independence assumptions, unknown statistics stay unknown). Plan +//! selection checks each candidate's guarantee against its query's +//! `AccuracyTarget` before pricing it. +//! - [`deployment_capabilities`] — the deployment's supported summaries. +//! - [`maintained_population`] — builds a maintained-population sub-DAG for +//! an instant top-k/quantile/sum query; Pass 1 does not offer it yet (#681). + +pub mod accuracy; +/// The deployment's capabilities: supported summaries and what it retains. +pub mod deployment_capabilities; +mod function_rules; +pub mod realization; +/// Test helpers shared by the stage crates' tests (feature `test-support`). +#[cfg(any(test, feature = "test-support"))] +pub mod test_support; + +pub use accuracy::{ + AccuracyEvidenceProvider, AccuracyModel, DefaultAccuracyModel, NoAccuracyEvidence, + PropagationStats, +}; +pub use deployment_capabilities::{ + DeploymentCapabilities, DeploymentCapabilitiesError, SupportedSummary, +}; +pub use realization::summary_candidates; + +pub mod maintained_population; + +/// Stage 1, Pass 1: per-computation candidate generation. No execution timing is +/// assigned: that is a Stage 2 materialization decision. +pub mod per_computation_candidates; + +/// Stage 1, Pass 2: ASAP-aware common-subexpression elimination (the +/// identical-expression rule). +pub mod asap_aware_cse; + +pub mod workload_candidates; diff --git a/crates/asap-aware-mapping/src/maintained_population.rs b/crates/logical-asap-optimization/src/maintained_population.rs similarity index 96% rename from crates/asap-aware-mapping/src/maintained_population.rs rename to crates/logical-asap-optimization/src/maintained_population.rs index 3c2f69a81..96e806a1e 100644 --- a/crates/asap-aware-mapping/src/maintained_population.rs +++ b/crates/logical-asap-optimization/src/maintained_population.rs @@ -1,8 +1,11 @@ //! Shared maintained-population candidates over canonical relational IR. -use asap_types::ir::non_asap::any_measure_filtered; +use asap_types::ir::operator::maintained_population::*; +use asap_types::ir::operator::non_asap::any_measure_filtered; +use asap_types::ir::operator::{AggIntent, Reduction, Source}; +use asap_types::ir::properties::AccuracyGuarantee; +use asap_types::ir::scalar::{CompareOpKind, ScalarValue}; +use asap_types::ir::schema::DataType; use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, ScalarExpr}; -use asap_types::post_asap::{maintained_population::*, AccuracyGuarantee}; -use asap_types::pre_asap::{AggIntent, CompareOpKind, DataType, Reduction, ScalarValue, Source}; use std::rc::Rc; fn strip_projection(mut root: &OperatorNode) -> &OperatorNode { @@ -314,7 +317,7 @@ mod tests { use crate::test_support::lower_promql; use asap_types::ir::cse::share_common_sub_dags; use asap_types::ir::physical_export::compile_physical_asap_dag as export_timed; - use asap_types::ir::timing::{ + use asap_types::ir::properties::timing::{ apply_materialization_timings, MaterializationAssignment, TimingMemo, }; @@ -333,7 +336,7 @@ mod tests { } fn lower(q: &str) -> Rc { - lower_promql(q, asap_types::types::AccuracyTarget::exact()) + lower_promql(q, asap_types::ir::properties::AccuracyTarget::exact()) } // Instant scalar aggregations share the same retractable series population. @@ -442,7 +445,7 @@ mod tests { /// `root`, a `Limit`, with its partition replaced by `partition_by`. fn with_limit_partition( root: &Rc, - partition_by: asap_types::pre_asap::GroupKeys, + partition_by: asap_types::ir::operator::GroupKeys, ) -> Rc { let Some(NonASAPOp::Limit { n, offset, child, .. @@ -478,7 +481,7 @@ mod tests { }; assert!(!by_job.is_empty()); let global_limit_over_per_job_sort = - with_limit_partition(&per_job, asap_types::pre_asap::GroupKeys::none()); + with_limit_partition(&per_job, asap_types::ir::operator::GroupKeys::none()); let per_job_limit_over_global_sort = with_limit_partition(&global, by_job.clone()); assert!(recognize(&global_limit_over_per_job_sort).is_none()); assert!(recognize(&per_job_limit_over_global_sort).is_none()); @@ -489,8 +492,8 @@ mod tests { // always runs at query time. #[test] fn population_timing_follows_its_materialization() { - use asap_types::ir::{Materialization, Medium}; - use asap_types::post_asap::ExecutionTiming::{IngestionTime, QueryTime}; + use asap_types::ir::properties::ExecutionTiming::{IngestionTime, QueryTime}; + use asap_types::physical::{Materialization, Medium}; let root = lower("topk(5,a)"); let candidate = MaintainedPopulationStrategy::new(std::slice::from_ref(&root)) .candidate(&root) diff --git a/crates/asap-aware-mapping/src/per_computation_candidates.rs b/crates/logical-asap-optimization/src/per_computation_candidates.rs similarity index 98% rename from crates/asap-aware-mapping/src/per_computation_candidates.rs rename to crates/logical-asap-optimization/src/per_computation_candidates.rs index 37f42e628..257d94640 100644 --- a/crates/asap-aware-mapping/src/per_computation_candidates.rs +++ b/crates/logical-asap-optimization/src/per_computation_candidates.rs @@ -90,26 +90,27 @@ use std::collections::{HashMap, HashSet}; use std::rc::Rc; -use asap_types::ir::operator_properties::{BinaryOpKind, JoinKind, Reduction, Source}; -use asap_types::ir::pane_chain::{merged_chain, whole_window_build}; -use asap_types::ir::{ - ASAPOp, BinaryOperator, ExprSemantics, NonASAPOp, Operator, OperatorNode, OperatorResultKind, - Predicate, ProjectItem, QueryRoot, ScalarExpr, SchemaDerivationError, -}; -use asap_types::post_asap::guarantee::{ +use asap_types::ir::operator::operator_properties::{BinaryOpKind, JoinKind, Reduction, Source}; +use asap_types::ir::operator::pane_chain::{merged_chain, whole_window_build}; +use asap_types::ir::operator::AggIntent; +use asap_types::ir::properties::guarantee::{ AccuracyGuarantee, AccuracyMetric, BoundExpr, CompositionOperator, GuaranteeSource, ProbabilityExpr, }; -use asap_types::post_asap::{ +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; +use asap_types::ir::scalar::{resolve_column_ref, ColumnRef, CompareOpKind, ScalarValue}; +use asap_types::ir::schema::{DataType, Schema}; +use asap_types::ir::schema::{ EntityIdentity, ExactKind, ExactParams, FamilyCapability, FieldDataType, GroupingStrategy, - Materialization, MergedError, NonNegativeWeightProof, SketchAlgorithm, SketchKind, - SketchParams, SketchStatistic, SummaryInputExpr, SummaryUpdate, WeightDomain, + MergedError, NonNegativeWeightProof, SketchAlgorithm, SketchKind, SketchParams, + SketchStatistic, SummaryInputExpr, SummaryUpdate, WeightDomain, }; -use asap_types::pre_asap::expr_ir::ArithmeticOpKind; -use asap_types::pre_asap::{ - resolve_column_ref, AggIntent, ColumnRef, CompareOpKind, DataType, ScalarValue, Schema, +use asap_types::ir::{ + ASAPOp, BinaryOperator, ExprSemantics, NonASAPOp, Operator, OperatorNode, OperatorResultKind, + Predicate, ProjectItem, QueryRoot, ScalarExpr, SchemaDerivationError, }; -use asap_types::types::AccuracyTarget; +use asap_types::physical::Materialization; use thiserror::Error; use crate::accuracy::composition::samples_sign_statistic; @@ -705,7 +706,7 @@ fn add_nested_sum_targets( /// directly beneath it, neither filtered nor with `HAVING`, and not grouped /// by the metric name a per-series inner sum drops. fn nested_sum_input(target: &OperatorNode) -> Option<(Rc, AggIntent, Reduction)> { - use asap_types::ir::non_asap::any_measure_filtered; + use asap_types::ir::operator::non_asap::any_measure_filtered; let Some(NonASAPOp::Aggregate { child: inner, reduction: reduction @ Reduction::Reduce(by), @@ -852,7 +853,7 @@ fn whole_expression_input( let [intent @ AggIntent::TopK { .. }] = measures.as_slice() else { return None; }; - if asap_types::ir::non_asap::any_measure_filtered(filters) { + if asap_types::ir::operator::non_asap::any_measure_filtered(filters) { return None; } // The rule reads only the family's algorithm, and rejects Count-Min over @@ -862,7 +863,7 @@ fn whole_expression_input( let family = FieldDataType::Sketch( SketchKind::new( SketchAlgorithm::CountSketchWithHeap, - asap_types::post_asap::SketchParams::CountSketchWithHeap { + asap_types::ir::schema::SketchParams::CountSketchWithHeap { width: 1, depth: 1, heap_size: 1, @@ -881,7 +882,7 @@ fn whole_expression_input( ); if per_series && realized.child.schema.has_promql_series_identity() { input.item = Some(SummaryInputExpr::Column(ColumnRef::Named( - asap_types::pre_asap::schema::PROMQL_SERIES_IDENTITY.into(), + asap_types::ir::schema::PROMQL_SERIES_IDENTITY.into(), ))); } else if realized.child.schema.closed { // An encoded label set needs an open PromQL schema. @@ -1070,7 +1071,7 @@ fn budget_shares( /// guarantee that is not exact ([`root_guarantee`]). Empty when it carries /// none (no rule composes the inputs) or an exact one. Plan selection sizes /// each for its share of the query's failure probability too. -pub(crate) fn approximate_inputs(root: &QueryRoot) -> Vec> { +pub fn approximate_inputs(root: &QueryRoot) -> Vec> { if root_guarantee(root).is_none_or(|g| g.is_exact()) { return Vec::new(); } @@ -1103,7 +1104,7 @@ pub(crate) fn approximate_inputs(root: &QueryRoot) -> Vec> { /// The theoretical guarantee of a query's output, as composed: the root /// node's own, or a scalar root's composed over the operators it reads /// ([`scalar_guarantee`]). -pub(crate) fn root_guarantee(root: &QueryRoot) -> Option { +pub fn root_guarantee(root: &QueryRoot) -> Option { match root { QueryRoot::Operator(node) => node.guarantee.clone(), QueryRoot::Scalar(expr) => scalar_guarantee(expr, &|_| None), @@ -1116,7 +1117,7 @@ pub(crate) fn root_guarantee(root: &QueryRoot) -> Option { /// ([`composed_guarantee`]; a node that had none keeps none). Timing, /// materialization and sharing are kept. Plan selection enlarges summaries /// for a failure probability this way. -pub(crate) fn with_resized_summaries( +pub fn with_resized_summaries( roots: &[QueryRoot], resize: &dyn Fn(&OperatorNode, &SketchParams) -> Option, ) -> Result, SchemaDerivationError> { @@ -1198,7 +1199,7 @@ pub(crate) fn with_resized_summaries( /// rows of, as [`source_distinct_guarantee`] reads it: the source whose /// distinct bound gives its failure probability. `None` for a state over /// computed rows or several sources. -pub(crate) fn distinct_bound_source(state: &OperatorNode) -> Option { +pub fn distinct_bound_source(state: &OperatorNode) -> Option { let Some(ASAPOp::SummaryAgg { child, .. } | ASAPOp::PaneChain { child, .. }) = state.asap() else { return None; @@ -1365,7 +1366,7 @@ fn with_guarantees( /// failure probability per point ([`AccuracyScope::PerValue`]). Plan /// selection states it over a whole result of `G` points as `G·δ`. /// -/// [`AccuracyScope::PerValue`]: asap_types::types::AccuracyScope::PerValue +/// [`AccuracyScope::PerValue`]: asap_types::ir::properties::AccuracyScope::PerValue fn composed_guarantee(node: &OperatorNode) -> Option { if let Some(guarantee) = merged_panes_guarantee(node) { return guarantee; @@ -1956,7 +1957,7 @@ fn case_guarantee( /// value has the child's as an operand ([`operand_guarantee`]) and other /// columns (labels, timestamps) are exact. `None` when the child has no /// sample value or the column's guarantee is unknown. -pub(crate) fn project_column_guarantee( +pub fn project_column_guarantee( child: &Rc, column: usize, ) -> Option { @@ -2042,7 +2043,7 @@ pub(crate) fn scalar_guarantee( /// The guarantee of a scalar operand as an input to arithmetic or a /// function: a planned operator's as [`operand_guarantee`], anything else /// as [`scalar_guarantee`]. -pub(crate) fn operand_of( +pub fn operand_of( expr: &ScalarExpr, column: &dyn Fn(usize) -> Option, ) -> Option { @@ -2058,7 +2059,7 @@ pub(crate) fn operand_of( /// Count-Min or Count Sketch bound restated as an absolute error over the /// stream norm its summary's update weight allows ([`stream_norm`], /// [`additive_as_absolute`]). -pub(crate) fn operand_guarantee(node: &Rc) -> Option { +pub fn operand_guarantee(node: &Rc) -> Option { let guarantee = node.guarantee.clone()?; Some(additive_as_absolute(guarantee, stream_norm(node).as_ref())) } @@ -2283,7 +2284,7 @@ pub(crate) fn arithmetic_guarantee( /// any other operation, or when an operand's error has no value form (a /// rank error). Plan selection (Stage 3) finds the operations to bind with /// this same rule. -pub(crate) fn operand_domain_statistic( +pub fn operand_domain_statistic( op: &ArithmeticOpKind, l: &AccuracyGuarantee, r: &AccuracyGuarantee, @@ -2486,10 +2487,10 @@ fn estimate_bound_over_samples( return None; } let mut local = DefaultAccuracyModel.local_guarantee(family, query)?; - local.spec.error = asap_types::post_asap::BoundExpr::Product { + local.spec.error = asap_types::ir::properties::BoundExpr::Product { factors: vec![ local.spec.error, - asap_types::post_asap::BoundExpr::Unknown { + asap_types::ir::properties::BoundExpr::Unknown { statistic: samples_sign_statistic(metric), }, ], @@ -3170,7 +3171,7 @@ fn summary_update( // column, the item form the runtime builds keyed summaries // from. SummaryInputExpr::Column(ColumnRef::Named( - asap_types::pre_asap::schema::PROMQL_SERIES_IDENTITY.into(), + asap_types::ir::schema::PROMQL_SERIES_IDENTITY.into(), )) } else if child.closed { // A closed child (an aggregate's output, say) lists every @@ -3256,8 +3257,8 @@ fn statistic(intent: &AggIntent) -> Result AccuracyGuarantee { AccuracyGuarantee::new( @@ -3918,8 +3919,8 @@ mod tests { ); // Per-series state ranks each series' identity, which the rows carry // as a column; a query over aggregated rows is planned over labels. - let root = - asap_types::ir::schema_support::with_promql_series_identity(&root).unwrap_or(root); + let root = asap_types::ir::schema::schema_support::with_promql_series_identity(&root) + .unwrap_or(root); generate_per_computation_candidates(vec![(0, QueryRoot::Operator(root))]).unwrap() } @@ -4457,7 +4458,7 @@ mod tests { let family = FieldDataType::Sketch( SketchKind::new( SketchAlgorithm::Cms, - asap_types::post_asap::SketchParams::Cms { width: 8, depth: 2 }, + asap_types::ir::schema::SketchParams::Cms { width: 8, depth: 2 }, ), GroupingStrategy::default(), ); @@ -4497,7 +4498,7 @@ mod tests { let family = FieldDataType::Sketch( SketchKind::new( SketchAlgorithm::Cms, - asap_types::post_asap::SketchParams::Cms { + asap_types::ir::schema::SketchParams::Cms { width: 272, depth: 5, }, @@ -4539,7 +4540,7 @@ mod tests { /// every group, whose distinct items the source's bound does not bound. #[test] fn classic_hll_confidence_needs_one_instance_over_a_sources_own_rows() { - use asap_types::post_asap::{HydraKind, HydraParams}; + use asap_types::ir::schema::{HydraKind, HydraParams}; let hll = |grouping| { FieldDataType::Sketch( SketchKind::new( @@ -4719,7 +4720,7 @@ mod tests { /// Mergeable intents give exactly their accumulator (`avg`: sum, count); other exact-only intents give `Exact`. #[test] fn intent_candidate_matrix() { - use asap_types::pre_asap::agg_intent::{MathFunc, TimeFunc}; + use asap_types::ir::operator::agg_intent::{MathFunc, TimeFunc}; use AggIntent as A; use ExactKind as E; let accumulator = |kinds: Vec| ComputationCandidate::ExactAccumulator(kinds); @@ -4986,7 +4987,7 @@ mod tests { ( with_identity, SummaryInputExpr::Column(ColumnRef::Named( - asap_types::pre_asap::schema::PROMQL_SERIES_IDENTITY.into(), + asap_types::ir::schema::PROMQL_SERIES_IDENTITY.into(), )), ), ( diff --git a/crates/asap-aware-mapping/src/realization.rs b/crates/logical-asap-optimization/src/realization.rs similarity index 96% rename from crates/asap-aware-mapping/src/realization.rs rename to crates/logical-asap-optimization/src/realization.rs index a3060af1f..9eec31df5 100644 --- a/crates/asap-aware-mapping/src/realization.rs +++ b/crates/logical-asap-optimization/src/realization.rs @@ -4,17 +4,17 @@ //! the column/tuple a summary consumes, the column a top-k ranks, and the //! keyed-additive rule for a heap sketch that reads a larger sub-DAG. -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, ScalarExpr}; -use asap_types::post_asap::{ +use asap_types::ir::operator::agg_intent::AggIntent; +use asap_types::ir::operator::Reduction; +use asap_types::ir::properties::{AccuracyMetric, AccuracyTarget}; +use asap_types::ir::scalar::column_resolution::{resolve_column_ref, ResolveError}; +use asap_types::ir::scalar::expr_ir::ColumnRef; +use asap_types::ir::schema::ColumnId; +use asap_types::ir::schema::{ EntityIdentity, Field, FieldDataType, NonNegativeWeightProof, Schema, SketchAlgorithm, SketchParams, SummaryInputExpr, SummaryUpdate, WeightDomain, }; -use asap_types::pre_asap::agg_intent::AggIntent; -use asap_types::pre_asap::column_resolution::{resolve_column_ref, ResolveError}; -use asap_types::pre_asap::expr_ir::ColumnRef; -use asap_types::pre_asap::schema::ColumnId; -use asap_types::pre_asap::Reduction; -use asap_types::types::{AccuracyMetric, AccuracyTarget}; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, ScalarExpr}; use std::rc::Rc; /// Confidence δ assumed when the target carries only an ε @@ -150,7 +150,7 @@ pub(crate) fn summarised_column(intent: &AggIntent, child_schema: &Schema) -> Co /// `FinalizeExactAccumulator` or `SummaryEstimate`, under the `Project` /// restoring its names (a renaming) and a value `HAVING`'s `Filter`. Over /// per-series rows (PromQL `topk by (..) (k, rows)`) it is the sample value. -pub(crate) fn ranked_column(child: &OperatorNode) -> Result { +pub fn ranked_column(child: &OperatorNode) -> Result { let mut node = child; let reduction = loop { match &node.operator { @@ -282,7 +282,7 @@ pub(crate) fn realize_keyed_additive_summary_input( }; // The heap sketch consumes every raw row; a filtered measure counts only // some of them (#466). - if asap_types::ir::non_asap::any_measure_filtered(filters) { + if asap_types::ir::operator::non_asap::any_measure_filtered(filters) { return None; } let counter_input = matches!(measures.as_slice(), [AggIntent::Sum { .. }]) @@ -388,9 +388,9 @@ pub(crate) fn schema_column_ref(child: &OperatorNode, index: usize) -> Option Rc { +pub fn lower_promql(query: &str, accuracy: AccuracyTarget) -> Rc { let workload = PlanningWorkload { query_workload: QueryWorkload { language: QueryLanguage::PromQL, @@ -46,26 +47,24 @@ pub(crate) fn lower_promql(query: &str, accuracy: AccuracyTarget) -> Rc Rc { +pub fn metric_scan(labels: &[&str]) -> Rc { metric_scan_with_keys(labels, vec![]) } /// [`metric_scan`] with explicit `unique_keys` (a `[[0]]` key makes CSE /// willing to hoist the scan). -pub(crate) fn metric_scan_with_keys( - labels: &[&str], - unique_keys: Vec>, -) -> Rc { +pub fn metric_scan_with_keys(labels: &[&str], unique_keys: Vec>) -> Rc { let mut columns = vec![ Field::plain("ts", DataType::Timestamp, false), Field::plain("value", DataType::Float64, false), @@ -79,7 +78,7 @@ pub(crate) fn metric_scan_with_keys( } /// A predicate-free `TimeSeries(metric)` scan with the given schema. -pub(crate) fn scan(metric: &str, schema: Schema) -> Rc { +pub fn scan(metric: &str, schema: Schema) -> Rc { scan_from( Source::TimeSeries { metric: metric.into(), @@ -88,7 +87,7 @@ pub(crate) fn scan(metric: &str, schema: Schema) -> Rc { ) } -pub(crate) fn scan_from(source: Source, schema: Schema) -> Rc { +pub fn scan_from(source: Source, schema: Schema) -> Rc { OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Scan { source, predicates: vec![], @@ -98,7 +97,7 @@ pub(crate) fn scan_from(source: Source, schema: Schema) -> Rc { } /// A general aggregate node. -pub(crate) fn aggregate( +pub fn aggregate( reduction: Reduction, measures: Vec, output_names: Vec, @@ -117,16 +116,12 @@ pub(crate) fn aggregate( } /// `intent by (by)` — a single-measure, `HAVING`-free grouped aggregate. -pub(crate) fn agg( - by: Vec, - intent: AggIntent, - child: Rc, -) -> Rc { +pub fn agg(by: Vec, intent: AggIntent, child: Rc) -> Rc { aggregate(Reduction::by(by), vec![intent], vec![], None, child) } /// `intent without (excluded)`. -pub(crate) fn without_agg( +pub fn without_agg( excluded: Vec, intent: AggIntent, child: Rc, @@ -140,7 +135,7 @@ pub(crate) fn without_agg( ) } -pub(crate) fn filter(pred: ScalarExpr, child: Rc) -> Rc { +pub fn filter(pred: ScalarExpr, child: Rc) -> Rc { OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Filter { pred: Predicate(pred), child, @@ -148,7 +143,7 @@ pub(crate) fn filter(pred: ScalarExpr, child: Rc) -> Rc, child: Rc) -> Rc { +pub fn dedup(cols: Vec, child: Rc) -> Rc { OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Dedup { cols, child, @@ -157,7 +152,7 @@ pub(crate) fn dedup(cols: Vec, child: Rc) -> Rc) -> Rc { +pub fn time_range(range: Duration, child: Rc) -> Rc { OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::TimeRange { range, kind: TimeRangeKind::Range, @@ -168,7 +163,7 @@ pub(crate) fn time_range(range: Duration, child: Rc) -> Rc) -> Rc { +pub fn timed(root: &Rc) -> Rc { apply_materialization_timings( root, &MaterializationAssignment::default(), @@ -178,7 +173,7 @@ pub(crate) fn timed(root: &Rc) -> Rc { } /// `root` timed with every sub-DAG `MatAtIngestion` where the DAG allows it. -pub(crate) fn at_ingestion(root: &Rc) -> Rc { +pub fn at_ingestion(root: &Rc) -> Rc { apply_materialization_timings( root, &MaterializationAssignment::with_default(Materialization::MatAtIngestion(Medium::Memory)), @@ -188,45 +183,46 @@ pub(crate) fn at_ingestion(root: &Rc) -> Rc { } /// The Count-Min summary of a Stage 1 candidate of `query`. -pub(crate) fn cms_summary(query: &str) -> OperatorNode { +pub fn cms_summary(query: &str) -> OperatorNode { + use crate::deployment_capabilities::DeploymentCapabilities; + use crate::per_computation_candidates::generate_per_computation_candidates; + use crate::workload_candidates::stage1_logical_asap_optimization; + use asap_types::ir::schema::{FieldDataType, SketchAlgorithm}; use asap_types::ir::{ASAPOp, QueryRoot}; - use asap_types::post_asap::{FieldDataType, SketchAlgorithm}; let target = AccuracyTarget::max_error(0.01).failure_probability(0.01); - let root = lower_promql(query, target.clone()); - let run = crate::pass::run_stage1_to_stage3( - vec![(0, QueryRoot::Operator(root))], - &[entry_with( - target, - asap_types::workload::QueryRecurrence::Unknown, - )], - &DataWorkload::default(), - crate::PlanningModels::builtin(), + let root = lower_promql(query, target); + // The Stage 1 candidates the planner pipeline enumerates for `query` + // with the built-in deployment and an unknown recurrence. + let per_computation = generate_per_computation_candidates(vec![(0, QueryRoot::Operator(root))]) + .unwrap() + .without_unsupported_summaries(&DeploymentCapabilities::default()); + let summary = stage1_logical_asap_optimization( + &per_computation, + &[asap_types::workload::QueryRecurrence::Unknown], ) - .unwrap(); - run.stage1_logical_asap_optimization - .into_iter() - .filter_map(|candidate| candidate.roots.ok()) - .flat_map(|roots| match &roots[0] { - QueryRoot::Operator(node) => OperatorNode::reachable(node), - QueryRoot::Scalar(_) => vec![], - }) - .find(|node| { - matches!( - node.asap(), - Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) if *kind.algorithm() == SketchAlgorithm::Cms - ) - }) - .map(|node| (*node).clone()) - .unwrap_or_else(|| panic!("{query}: a Count-Min summary")) + .filter_map(|candidate| candidate.roots.ok()) + .flat_map(|roots| match &roots[0] { + QueryRoot::Operator(node) => OperatorNode::reachable(node), + QueryRoot::Scalar(_) => vec![], + }) + .find(|node| { + matches!( + node.asap(), + Some(ASAPOp::SummaryAgg { + family: FieldDataType::Sketch(kind, _), + .. + }) if *kind.algorithm() == SketchAlgorithm::Cms + ) + }) + .map(|node| (*node).clone()) + .unwrap_or_else(|| panic!("{query}: a Count-Min summary")); + summary } /// `summary` with its update weight replaced by `weight`. -pub(crate) fn weighted( +pub fn weighted( mut summary: OperatorNode, - weight: asap_types::post_asap::SummaryInputExpr, + weight: asap_types::ir::schema::SummaryInputExpr, ) -> Rc { use asap_types::ir::{ASAPOp, Operator}; if let Operator::ASAP(ASAPOp::SummaryAgg { input, .. }) = &mut summary.operator { @@ -236,13 +232,13 @@ pub(crate) fn weighted( } /// A point-count estimate over `summary` carrying `guarantee`. -pub(crate) fn estimate_over( +pub fn estimate_over( summary: Rc, - guarantee: asap_types::post_asap::AccuracyGuarantee, + guarantee: asap_types::ir::properties::AccuracyGuarantee, ) -> Rc { + use asap_types::ir::scalar::ColumnRef; + use asap_types::ir::schema::SketchStatistic; use asap_types::ir::{ASAPOp, Operator}; - use asap_types::post_asap::SketchStatistic; - use asap_types::pre_asap::ColumnRef; Rc::new( OperatorNode::new(Operator::ASAP(ASAPOp::SummaryEstimate { summary_input: summary, @@ -258,7 +254,7 @@ pub(crate) fn estimate_over( /// The workload entry of a query evaluated as `recurrence` says, exact and /// with no latency bound. -pub(crate) fn entry( +pub fn entry( recurrence: asap_types::workload::QueryRecurrence, ) -> asap_types::workload::QueryWorkloadEntry { entry_with(AccuracyTarget::exact(), recurrence) @@ -266,7 +262,7 @@ pub(crate) fn entry( /// The workload entry of a query with accuracy target `accuracy`, evaluated /// as `recurrence` says, with no latency bound. -pub(crate) fn entry_with( +pub fn entry_with( accuracy: AccuracyTarget, recurrence: asap_types::workload::QueryRecurrence, ) -> asap_types::workload::QueryWorkloadEntry { diff --git a/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs b/crates/logical-asap-optimization/src/workload_candidates.rs similarity index 52% rename from crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs rename to crates/logical-asap-optimization/src/workload_candidates.rs index 20971d318..e84518899 100644 --- a/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs +++ b/crates/logical-asap-optimization/src/workload_candidates.rs @@ -1,53 +1,15 @@ -//! Stage 1 (logical ASAP-aware optimization) → Stage 2 (physical ASAP-aware -//! optimization) → Stage 3 (plan selection), starting from the query-language -//! frontends' roots; Stage 0 runs before it in the facade. -//! -//! This is the planner pipeline (#509 MVP). [`Stage1ToStage3Pass`] is the -//! [`OptimizationPass`](super::OptimizationPass) that runs it. -//! -//! [`run_stage1_to_stage3`] runs, for one workload: -//! 1. Stage 1, logical ASAP-aware optimization: Pass 1 per-computation candidate -//! generation, then composition of one whole-workload candidate per choice -//! of a per-computation candidate for every target (the Cartesian product, -//! in mixed-radix order, the last target varying fastest), each followed -//! by its Pass 2 candidates: those of the identical-expression rule -//! ([`identical_expression_pass2_candidates`]: one per subset of its -//! sharing candidates), then those of the summary-capability rule -//! ([`summary_capability_pass2_candidates`]: one per choice of shared -//! summary builds), each followed by its own identical-expression -//! candidates, and, for the composed candidate and each -//! summary-capability candidate, those of the window-composition rule -//! ([`window_composition_pass2_candidates`]: one per non-empty subset of -//! the builds a repeating query could build as tumbling panes), each -//! followed by its own identical-expression candidates, and, for the -//! composed candidate, those of the reuse rules (exact top-k reuse, then -//! each rollup Pass 2 candidate followed by its own exact top-k reuse -//! Pass 2 candidates: [`topk_reuse_pass2_candidates`], -//! [`rollup_pass2_candidates`]), each followed by its own -//! identical-expression candidates; every distinct candidate once; the -//! independent candidate is kept; -//! 2. Stage 2, physical ASAP-aware optimization of every Stage 1 candidate -//! ([`optimize_physical_asap`]): one physical candidate per -//! materialization assignment of its summary builds; -//! 3. Stage 3, plan selection over the physical candidates ([`select_plan`]). -//! -//! Stage 0, the query-language frontends, runs before it (the `asap-planner` -//! facade lowers the workload). The window-composition rule offers -//! per-query tumbling panes only; sliding windows, Exponential Histograms -//! and a pane width shared across queries are not planned yet. -//! -//! Every combination is enumerated and plan selection prices every -//! candidate; nothing is capped. A search that avoids building every -//! candidate is future work (issue #671 surveys the options). +//! Stage 1 (logical ASAP-aware optimization) over the Pass 1 inventory: +//! the whole-workload CandidateLogicalASAPDAGs, one at a time, each +//! composed Pass 1 candidate followed by its Pass 2 candidates +//! ([`stage1_logical_asap_optimization`]). Stage 2 and Stage 3 run over +//! them in `asap-plan-selection`'s Stage 1 to Stage 3 driver. use std::rc::Rc; use asap_types::ir::flat::flatten; -use asap_types::ir::schema_support::with_promql_series_identity; use asap_types::ir::{OperatorNode, QueryRoot}; -use asap_types::workload::{DataWorkload, QueryLanguage, QueryRecurrence, QueryWorkloadEntry}; +use asap_types::workload::{QueryRecurrence, RepeatedDemand}; -use super::{OptimizationInput, OptimizationPass, OptimizeError, PlanOutput, QueryPlan}; use crate::asap_aware_cse::{ capability_sharing_candidates, identical_expression_pass2_candidates, pane_candidates, rollup_candidates, rollup_pass2_candidates, sharing_candidates, @@ -56,28 +18,8 @@ use crate::asap_aware_cse::{ TopKReuseCandidate, MAX_REUSE_CANDIDATES, }; use crate::per_computation_candidates::{ - compose_logical_candidate, generate_per_computation_candidates, PerComputationCandidateError, - PerComputationCandidates, + compose_logical_candidate, PerComputationCandidateError, PerComputationCandidates, }; -use crate::stage2_physical_asap_optimization::{ - fixed_interval_ms, optimize_physical_asap, PhysicalCandidate, -}; -use crate::stage3_plan_selection::{select_plan, Selection, SelectionError}; -use crate::PlanningModels; - -/// One whole-workload CandidateLogicalASAPDAG: `choice[t]` picks target -/// `t`'s candidate, or its fused computation's -/// (`targets[t].computation_for(choice[t])`), of the Pass 1 inventory. `roots` are the -/// composed query roots in input order, or why the choice did not compose. -#[derive(Debug)] -pub struct LogicalCandidate { - /// `L`, numbered in enumeration order. - pub id: String, - pub choice: Vec, - pub roots: Result, PerComputationCandidateError>, - /// For a Pass 2 candidate: what it changed of its Pass 1 candidate. - pub pass2: Option, -} /// What a Pass 2 candidate changed of the Pass 1 candidate it derives /// from: each set of summary builds in `shared_builds` made one build sized @@ -107,82 +49,6 @@ pub struct Pass2Candidate { pub sharing_candidates: Vec, } -/// A composed logical candidate that physical ASAP-aware optimization could -/// not implement, with the reason. -#[derive(Debug, Clone, PartialEq)] -pub struct PrunedCandidate { - pub from_logical: String, - pub reason: String, -} - -/// The output of Stage 1, Stage 2 and Stage 3 for one workload -/// ([`run_stage1_to_stage3`]). -#[derive(Debug)] -pub struct Stage1ToStage3Run { - /// Stage 1, Pass 1: the per-computation candidates, with the roots they - /// read. - pub stage1_pass1_per_computation: PerComputationCandidates, - /// Stage 1, Pass 1: the number of composed whole-workload candidates, - /// the size of the Cartesian product of the targets' candidates, - /// saturating at `usize::MAX`. Pass 2 adds to them. - pub combinations: usize, - /// Stage 1: every whole-workload logical candidate, in enumeration - /// order: each composed Pass 1 candidate, followed by its Pass 2 - /// candidates (those with [`LogicalCandidate::pass2`]). - pub stage1_logical_asap_optimization: Vec, - /// Stage 2: the physical candidates of each Stage 1 candidate that - /// composed and was implemented, one per materialization assignment: - /// `P` (nothing materialized), then `P-m`, from `L`. - pub stage2_physical_asap_optimization: Vec, - /// Stage 2: the Stage 1 candidates Stage 2 could not implement, with the - /// reasons. - pub stage2_pruned: Vec, - /// Stage 3: plan selection over the Stage 2 candidates. - pub stage3_plan_selection: Result, - /// Stage 3: the selected candidate's logical roots, with the weight - /// requirements it confirmed made proofs ([`Self::selected_logical`]). - pub stage3_selected_logical: Option>, -} - -impl Stage1ToStage3Run { - /// The selected physical candidate, if selection succeeded. - pub fn selected(&self) -> Option<&PhysicalCandidate> { - let selected = &self.stage3_plan_selection.as_ref().ok()?.selected; - self.stage2_physical_asap_optimization - .iter() - .find(|p| &p.id == selected) - } - - /// The composed roots of the logical candidate the selected physical - /// candidate implements: the plan before physical operator - /// implementation and materialization, with the weight requirements - /// Stage 3 confirmed made proofs, as in [`Selection::plan`]. - pub fn selected_logical(&self) -> Option<&[QueryRoot]> { - self.stage3_selected_logical.as_deref() - } -} - -/// Run Stage 1 → Stage 2 → Stage 3 over the frontends' `roots`, enumerating every -/// whole-workload candidate. `entries[i]` is the workload entry of -/// `roots[i]`: its accuracy target, latency bound, recurrence and -/// predictability. Only a Pass 1 failure is -/// an error; a candidate that does not compose or cannot be implemented is -/// listed with its reason, and a failed selection is reported in the run. -pub fn run_stage1_to_stage3( - roots: Vec<(Id, QueryRoot)>, - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - models: PlanningModels<'_>, -) -> Result, PerComputationCandidateError> { - let per_computation = generate_per_computation_candidates(roots)?; - Ok(run_stage1_to_stage3_from_pass1( - per_computation, - entries, - data, - models, - )) -} - /// One Stage 1 candidate in enumeration order, before Stage 2 numbers it. #[derive(Debug)] pub struct Stage1Candidate { @@ -475,112 +341,6 @@ impl Pass2List { } } -/// [`run_stage1_to_stage3`] after Pass 1: enumerates every Stage 1 -/// candidate of `per_computation` (the Pass 1 inventory of the frontends' -/// roots, [`stage1_logical_asap_optimization`]) without the summaries the -/// deployment does not support -/// ([`PerComputationCandidates::without_unsupported_summaries`]), so a -/// caller can read the inventory, e.g. its -/// [`combinations`](PerComputationCandidates::combinations), before -/// enumerating. -pub fn run_stage1_to_stage3_from_pass1( - per_computation: PerComputationCandidates, - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - models: PlanningModels<'_>, -) -> Stage1ToStage3Run { - let per_computation = - per_computation.without_unsupported_summaries(models.deployment_capabilities); - let recurrence: Vec<_> = entries.iter().map(|e| e.recurrence.clone()).collect(); - let stage1 = stage1_logical_asap_optimization(&per_computation, &recurrence).collect(); - run_stage2_and_stage3(per_computation, stage1, entries, data, models) -} - -/// Stage 2 and Stage 3 over `stage1`, the Stage 1 candidates of -/// `per_computation` in enumeration order (all of them, or the ones a caller -/// enumerated): candidate `n` (from 1) is `L`, implemented as `P` -/// with nothing materialized and as `P-m` (`k` from 1) for each other -/// materialization assignment. -/// -/// It does not prune: `per_computation` must already be pruned for -/// `models.deployment_capabilities` -/// ([`PerComputationCandidates::without_unsupported_summaries`]), with -/// `stage1` enumerated from that pruned inventory, as -/// [`run_stage1_to_stage3_from_pass1`] does. Pruning here would come too -/// late: each of `stage1`'s choices indexes the inventory it was enumerated -/// from, so a candidate with an unsupported summary would already be in -/// `stage1`. -pub fn run_stage2_and_stage3( - per_computation: PerComputationCandidates, - stage1: Vec, - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - models: PlanningModels<'_>, -) -> Stage1ToStage3Run { - let combinations = per_computation.combinations(); - let mut logical: Vec = Vec::new(); - let mut physical = Vec::new(); - let mut pruned = Vec::new(); - for (index, candidate) in stage1.into_iter().enumerate() { - let number = index + 1; - let id = format!("L{number}"); - if let Ok(roots) = &candidate.roots { - match optimize_physical_asap(&id, roots, entries, data, models.deployment_capabilities) - { - Ok(implemented) => { - for (k, mut p) in implemented.into_iter().enumerate() { - p.id = match k { - 0 => format!("P{number}"), - k => format!("P{number}-m{k}"), - }; - physical.push(p); - } - } - Err(error) => pruned.push(PrunedCandidate { - from_logical: id.clone(), - reason: error.to_string(), - }), - } - } - logical.push(LogicalCandidate { - id, - choice: candidate.choice, - roots: candidate.roots, - pass2: candidate.pass2, - }); - } - let selection = select_plan(&physical, entries, data, models); - let targets = &crate::stage3_plan_selection::accuracy_targets(entries); - let selected_logical = selection.as_ref().ok().and_then(|selection| { - let from = &physical - .iter() - .find(|p| p.id == selection.selected)? - .from_logical; - let roots = logical - .iter() - .find(|l| &l.id == from)? - .roots - .as_ref() - .ok()?; - // As Stage 3 sized it for its queries' failure probabilities. - let sized = - crate::stage3_plan_selection::resize_for_failure_probability(roots, targets, data) - .ok()?; - Some(crate::stage3_plan_selection::with_confirmed_weights( - &sized, data, - )) - }); - Stage1ToStage3Run { - stage3_selected_logical: selected_logical, - stage1_pass1_per_computation: per_computation, - combinations, - stage1_logical_asap_optimization: logical, - stage2_physical_asap_optimization: physical, - stage2_pruned: pruned, - stage3_plan_selection: selection, - } -} - /// Mixed-radix increment of `choice`, each digit over its target's /// candidates and then its fused computation's: the last target varies fastest. /// `false` when a carry out of the first target ends the enumeration. @@ -595,82 +355,23 @@ fn next_choice(inventory: &PerComputationCandidates, choice: &mut [usize false } -/// The planner pipeline behind the optimization-pass interface: the -/// workload's roots in entry order, PromQL roots with each series' full -/// identity as a column (the row representation per-series state needs at -/// runtime; a root with no such representation is planned over its labels -/// alone), through [`run_stage1_to_stage3`] with every entry's accuracy target. The -/// output is the selected physical candidate's roots, with the open weight -/// requirements plan selection confirmed made proofs. -#[derive(Debug, Default, Clone, Copy)] -pub struct Stage1ToStage3Pass; - -impl OptimizationPass for Stage1ToStage3Pass { - fn name(&self) -> &'static str { - "stage1-to-stage3" - } - - fn optimize(&self, input: OptimizationInput<'_>) -> Result { - let workload = input.workload; - let promql = workload.query_workload().language == QueryLanguage::PromQL; - let mut roots: Vec<(usize, QueryRoot)> = workload - .operator_indices() - .iter() - .zip(workload.exprs()) - .map(|(&index, node)| { - let node = match promql { - true => with_promql_series_identity(node).unwrap_or_else(|_| Rc::clone(node)), - false => Rc::clone(node), - }; - (index, QueryRoot::Operator(node)) - }) - .chain( - workload - .scalar_roots() - .iter() - .map(|(index, expr)| (*index, QueryRoot::Scalar(expr.clone()))), - ) - .collect(); - roots.sort_by_key(|(index, _)| *index); - let entries: Vec<_> = workload.query_workload().entries().collect(); - let data = workload.data_workload().cloned().unwrap_or_default(); - let run = run_stage1_to_stage3(roots, &entries, &data, input.models)?; - let selection = run.stage3_plan_selection?; - let roots = selection.plan.clone(); - let mut output = PlanOutput::new(Vec::new()); - for (entry_index, root) in roots.iter().enumerate() { - match root { - QueryRoot::Operator(node) => output.plans.push(QueryPlan { - entry_index, - root: Rc::clone(node), - }), - QueryRoot::Scalar(expr) => { - // One plan per distinct operator the scalar reads. - let mut refs: Vec<&Rc> = Vec::new(); - for node in expr.operator_refs() { - if !refs.iter().any(|r| Rc::ptr_eq(r, node)) { - refs.push(node); - } - } - output - .scalar_plans - .extend(refs.into_iter().map(|node| QueryPlan { - entry_index, - root: Rc::clone(node), - })); - output.scalar_roots.push((entry_index, expr.clone())); - } - } - } - output.selection = Some(selection); - Ok(output) +/// The fixed interval at which a query repeats, in milliseconds. +pub fn fixed_interval_ms(recurrence: &QueryRecurrence) -> Option { + match recurrence { + QueryRecurrence::Repeated( + RepeatedDemand::FixedInterval(interval) + | RepeatedDemand::FixedIntervalAt { interval, .. }, + ) => Some(u64::from(interval.0)).filter(|&ms| ms > 0), + _ => None, } } #[cfg(test)] mod tests { use super::*; + use crate::per_computation_candidates::generate_per_computation_candidates; use crate::test_support::lower_promql; + use asap_types::ir::schema::schema_support::with_promql_series_identity; /// The composed candidate of `queries`, every target at its first /// (exact) candidate. @@ -679,7 +380,7 @@ mod tests { .iter() .enumerate() .map(|(i, q)| { - let root = lower_promql(q, asap_types::types::AccuracyTarget::exact()); + let root = lower_promql(q, asap_types::ir::properties::AccuracyTarget::exact()); ( i, QueryRoot::Operator(with_promql_series_identity(&root).unwrap()), diff --git a/crates/asap-aware-mapping/tests/per_computation_candidates.rs b/crates/logical-asap-optimization/tests/per_computation_candidates.rs similarity index 96% rename from crates/asap-aware-mapping/tests/per_computation_candidates.rs rename to crates/logical-asap-optimization/tests/per_computation_candidates.rs index a84107f75..e22039151 100644 --- a/crates/asap-aware-mapping/tests/per_computation_candidates.rs +++ b/crates/logical-asap-optimization/tests/per_computation_candidates.rs @@ -1,20 +1,18 @@ //! Frontend-to-Pass-1 acceptance: candidate discovery precedes empirical selection. -use asap_aware_mapping::per_computation_candidates::{ +use asap_logical_asap_optimization::per_computation_candidates::{ computation_candidates_for_intent, generate_per_computation_candidates, ComputationCandidate, HavingEntry, HavingPlacement, PerComputationCandidateError, PruneReason, }; -use asap_types::{ - ir::operator_properties::{Reduction, Source}, - ir::{ - ASAPOp, ExprSemantics, NonASAPOp, Operator, OperatorNode, Predicate, QueryRoot, ScalarExpr, - }, - post_asap::{ - ExactKind, GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, SummaryUpdate, - }, - pre_asap::{ - AggIntent, ColumnRef, CompareOpKind, DataType, Field, FieldDataType, ScalarValue, Schema, - }, - types::AccuracyTarget, +use asap_types::ir::operator::operator_properties::{Reduction, Source}; +use asap_types::ir::operator::AggIntent; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::scalar::{ColumnRef, CompareOpKind, ScalarValue}; +use asap_types::ir::schema::{ + DataType, ExactKind, Field, FieldDataType, GroupingStrategy, Schema, SketchAlgorithm, + SketchKind, SketchParams, SummaryUpdate, +}; +use asap_types::ir::{ + ASAPOp, ExprSemantics, NonASAPOp, Operator, OperatorNode, Predicate, QueryRoot, ScalarExpr, }; use std::rc::Rc; @@ -241,7 +239,7 @@ fn assigned_physical_annotation_asap_input_and_invalid_accuracy_are_rejected() { accuracy: approximate(), })) .clone(); - producer.timing = Some(asap_types::post_asap::ExecutionTiming::QueryTime); + producer.timing = Some(asap_types::ir::properties::ExecutionTiming::QueryTime); assert!(matches!( generate_per_computation_candidates(vec![( 0, @@ -254,7 +252,7 @@ fn assigned_physical_annotation_asap_input_and_invalid_accuracy_are_rejected() { })) .clone(); stored.materialization = - asap_types::post_asap::Materialization::MatAtQuery(asap_types::post_asap::Medium::Memory); + asap_types::physical::Materialization::MatAtQuery(asap_types::physical::Medium::Memory); assert!(matches!( generate_per_computation_candidates(vec![( 0, @@ -748,7 +746,7 @@ fn example_1_exact_q2_prunes_topk_heaps() { /// The `HAVING` entries of the target whose measure satisfies `is`. fn having_of( - targets: &[asap_aware_mapping::per_computation_candidates::ComputationTarget], + targets: &[asap_logical_asap_optimization::per_computation_candidates::ComputationTarget], is: impl Fn(&AggIntent) -> bool, ) -> Vec { let target = targets @@ -862,9 +860,9 @@ fn having_shared_with_a_project_consumer_stays_above() { #[test] fn without_and_per_entity_having_stays_above() { for reduction in [ - Reduction::Reduce(asap_types::ir::operator_properties::GroupKeys::without( - vec![1], - )), + Reduction::Reduce( + asap_types::ir::operator::operator_properties::GroupKeys::without(vec![1]), + ), Reduction::PerEntity, ] { let node = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Aggregate { @@ -901,7 +899,7 @@ fn without_and_per_entity_having_stays_above() { /// with an exact guarantee. #[test] fn composed_candidate_replaces_chosen_target_with_summary_evaluation() { - use asap_aware_mapping::per_computation_candidates::compose_logical_candidate; + use asap_logical_asap_optimization::per_computation_candidates::compose_logical_candidate; let producer = aggregate(AggIntent::Cardinality { cols: vec![0], accuracy: approximate(), @@ -940,7 +938,7 @@ fn composed_candidate_replaces_chosen_target_with_summary_evaluation() { /// The one root of the workload candidate composed from `roots` and `choice`. fn composed(roots: Vec>, choice: &[usize]) -> Vec> { - use asap_aware_mapping::per_computation_candidates::compose_logical_candidate; + use asap_logical_asap_optimization::per_computation_candidates::compose_logical_candidate; let inventory = generate_per_computation_candidates( roots .into_iter() @@ -982,7 +980,7 @@ fn summary_agg(node: &OperatorNode) -> &ASAPOp { /// measure's accuracy requirement. #[test] fn composed_summary_is_sized_for_its_measure() { - use asap_aware_mapping::realization::default_size_params; + use asap_logical_asap_optimization::realization::default_size_params; let intent = AggIntent::Count { accuracy: approximate(), }; @@ -1157,7 +1155,7 @@ fn composed_guarantees() { /// a subquery in it reads the subquery's substitution. #[test] fn own_having_subquery_is_substituted() { - use asap_aware_mapping::per_computation_candidates::compose_logical_candidate; + use asap_logical_asap_optimization::per_computation_candidates::compose_logical_candidate; let inner = aggregate(AggIntent::Count { accuracy: approximate(), }); @@ -1190,7 +1188,7 @@ fn own_having_subquery_is_substituted() { /// column. #[test] fn per_series_avg_with_series_identity_composes_from_sum_and_count() { - use asap_types::ir::schema_support::with_promql_series_identity; + use asap_types::ir::schema::schema_support::with_promql_series_identity; let [QueryRoot::Operator(root)] = &exact_promql("avg_over_time(m[5m])")[..] else { panic!("one operator root"); }; @@ -1202,7 +1200,7 @@ fn per_series_avg_with_series_identity_composes_from_sum_and_count() { /// Exact per-series `rate` keeps every label and retypes the value column to the Rate state. #[test] fn per_series_rate_keeps_labels_and_retypes_its_state() { - use asap_types::post_asap::ExactParams; + use asap_types::ir::schema::ExactParams; let [QueryRoot::Operator(root)] = &exact_promql("rate(m{job=\"api\"}[5m])")[..] else { panic!("one operator root"); }; diff --git a/crates/physical-asap-optimization/Cargo.toml b/crates/physical-asap-optimization/Cargo.toml new file mode 100644 index 000000000..b06e4bfa2 --- /dev/null +++ b/crates/physical-asap-optimization/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "asap-physical-asap-optimization" +version = "0.1.0" +edition = "2021" + +# #509 Stage 2, physical ASAP-aware optimization: materialization and +# physical operator implementation of each Stage 1 candidate. Depends on +# Stage 1, never on plan selection or the executor. +[dependencies] +asap-types = { path = "../types" } +asap-logical-asap-optimization = { path = "../logical-asap-optimization" } +thiserror = "2" + +[dev-dependencies] +asap-logical-asap-optimization = { path = "../logical-asap-optimization", features = ["test-support"] } diff --git a/crates/asap-aware-mapping/src/stage2_physical_asap_optimization.rs b/crates/physical-asap-optimization/src/lib.rs similarity index 95% rename from crates/asap-aware-mapping/src/stage2_physical_asap_optimization.rs rename to crates/physical-asap-optimization/src/lib.rs index f76dffa6f..26ff57bdd 100644 --- a/crates/asap-aware-mapping/src/stage2_physical_asap_optimization.rs +++ b/crates/physical-asap-optimization/src/lib.rs @@ -71,27 +71,29 @@ use std::collections::{BTreeMap, HashMap}; use std::rc::Rc; -use asap_types::ir::operator_properties::Source; +use asap_types::ir::operator::operator_properties::Source; +use asap_types::ir::operator::{AggIntent, Reduction}; use asap_types::ir::physical_export::{ compile_physical_asap_workload_with_node_ids, PhysicalASAPDAG, PhysicalASAPNodeId, }; +use asap_types::ir::properties::{ExecutionDataStateError, ExecutionTiming}; +use asap_types::ir::scalar::column_resolution::resolve_column_ref; +use asap_types::ir::scalar::expr_ir::ColumnRef; +use asap_types::ir::schema::{ExactKind, FieldDataType}; use asap_types::ir::{ apply_materialization_timings, is_filtered_build, split_shared_by_timing, ASAPOp, - MaterializationAssignment, Medium, NonASAPOp, Operator, OperatorNode, ProjectItem, QueryRoot, + MaterializationAssignment, NonASAPOp, Operator, OperatorNode, ProjectItem, QueryRoot, ScalarExpr, SchemaDerivationError, SortKey, TimingMemo, }; -use asap_types::post_asap::{ - ExactKind, ExecutionDataStateError, ExecutionTiming, FieldDataType, Materialization, -}; -use asap_types::pre_asap::column_resolution::resolve_column_ref; -use asap_types::pre_asap::expr_ir::ColumnRef; -use asap_types::pre_asap::{AggIntent, Reduction}; +use asap_types::physical::Materialization; +use asap_types::physical::Medium; use asap_types::workload::{ DataArrival, DataWorkload, Predictability, QueryRecurrence, QueryWorkloadEntry, RepeatedDemand, }; use thiserror::Error; -use crate::deployment_capabilities::DeploymentCapabilities; +use asap_logical_asap_optimization::deployment_capabilities::DeploymentCapabilities; +use asap_logical_asap_optimization::workload_candidates::fixed_interval_ms; /// One CandidatePhysicalASAPDAG (Stage 2 output), derived from exactly one /// CandidateLogicalASAPDAG (Stage 1 output). It holds planning data only; @@ -218,17 +220,6 @@ impl MaterializationCandidate { } } -/// The fixed interval at which a query repeats, in milliseconds. -pub(crate) fn fixed_interval_ms(recurrence: &QueryRecurrence) -> Option { - match recurrence { - QueryRecurrence::Repeated( - RepeatedDemand::FixedInterval(interval) - | RepeatedDemand::FixedIntervalAt { interval, .. }, - ) => Some(u64::from(interval.0)).filter(|&ms| ms > 0), - _ => None, - } -} - /// Whether a tumbling-pane chain of `pane_width_ms`-wide panes read by /// queries repeating as `readers` builds exactly one new pane per /// evaluation: there is a reader, every reader repeats at a fixed interval @@ -239,7 +230,7 @@ pub(crate) fn fixed_interval_ms(recurrence: &QueryRecurrence) -> Option { /// /// Co-scheduling assumption: readers with the same interval and no known /// phase run at the same instants as each other and as the readers with a -/// known phase, as [`crate::stage3_plan_selection`] assumes when it counts +/// known phase, as plan selection (`asap-plan-selection`) assumes when it counts /// one evaluation per interval. Readers at different known phases would /// each need their own newest pane, so they do not share one kept chain. pub(crate) fn builds_one_pane_per_evaluation<'a>( @@ -659,7 +650,7 @@ fn implement_topk( return Err(unsupported("HAVING")); } let child = memo[&Rc::as_ptr(child)].clone(); - let value = crate::realization::ranked_column(&child) + let value = asap_logical_asap_optimization::realization::ranked_column(&child) .map_err(|e| unsupported(&format!("no ranked column: {e}")))?; let sorted = Rc::new( OperatorNode::new(Operator::NonASAP(NonASAPOp::Sort { @@ -735,12 +726,12 @@ fn rebuilt( #[cfg(test)] mod tests { use super::*; - use crate::test_support::lower_promql; + use asap_logical_asap_optimization::test_support::lower_promql; use asap_types::ir::physical_export::PhysicalASAPNodeId; use asap_types::ir::physical_export::PhysicalASAPOperatorPayload as Payload; + use asap_types::ir::properties::AccuracyTarget; + use asap_types::ir::properties::ExecutionTiming; use asap_types::ir::ASAPOp; - use asap_types::post_asap::ExecutionTiming; - use asap_types::types::AccuracyTarget; /// The all-`NotMaterialized` candidate, with no workload declared. fn first_candidate(roots: &[QueryRoot]) -> PhysicalCandidate { @@ -863,9 +854,9 @@ mod tests { /// measure, `HAVING`) fails, so its candidate is pruned with the reason. #[test] fn topk_with_extra_measure_or_having_fails() { - use crate::test_support::{aggregate, metric_scan}; + use asap_logical_asap_optimization::test_support::{aggregate, metric_scan}; + use asap_types::ir::scalar::expr_ir::ScalarValue; use asap_types::ir::Predicate; - use asap_types::pre_asap::expr_ir::ScalarValue; let topk = AggIntent::TopK { k: 3, accuracy: AccuracyTarget::exact(), @@ -985,7 +976,7 @@ mod tests { "topk by (job) (10, sum_over_time(m[1m]))", AccuracyTarget::max_error(0.01).failure_probability(0.001), ); - let inventory = crate::per_computation_candidates::generate_per_computation_candidates( + let inventory = asap_logical_asap_optimization::per_computation_candidates::generate_per_computation_candidates( vec![(0, QueryRoot::Operator(root))], ) .unwrap(); @@ -996,11 +987,13 @@ mod tests { .map(|t| t.choice_count() - 1) .collect(); let roots: Vec<_> = - crate::per_computation_candidates::compose_logical_candidate(&inventory, &choice) - .unwrap() - .into_iter() - .map(|(_, root)| root) - .collect(); + asap_logical_asap_optimization::per_computation_candidates::compose_logical_candidate( + &inventory, &choice, + ) + .unwrap() + .into_iter() + .map(|(_, root)| root) + .collect(); let candidate = first_candidate(&roots); assert!(candidate .dag @@ -1027,9 +1020,9 @@ mod tests { query, AccuracyTarget::max_error(0.01) .failure_probability(0.001) - .accuracy_metric(asap_types::types::AccuracyMetric::Rank), + .accuracy_metric(asap_types::ir::properties::AccuracyMetric::Rank), ); - let inventory = crate::per_computation_candidates::generate_per_computation_candidates( + let inventory = asap_logical_asap_optimization::per_computation_candidates::generate_per_computation_candidates( vec![(0, QueryRoot::Operator(root))], ) .unwrap(); @@ -1038,20 +1031,28 @@ mod tests { .iter() .map(|t| t.choice_count() - 1) .collect(); - crate::per_computation_candidates::compose_logical_candidate(&inventory, &choice) - .unwrap() - .remove(0) - .1 + asap_logical_asap_optimization::per_computation_candidates::compose_logical_candidate( + &inventory, &choice, + ) + .unwrap() + .remove(0) + .1 } /// `root` with its summary build in 1-min tumbling panes. fn in_panes(root: QueryRoot) -> QueryRoot { let roots = [root]; - let rule_candidates = crate::asap_aware_cse::pane_candidates(&roots, &[Some(MINUTE)]); + let rule_candidates = asap_logical_asap_optimization::asap_aware_cse::pane_candidates( + &roots, + &[Some(MINUTE)], + ); assert_eq!(rule_candidates.len(), 1, "{rule_candidates:?}"); let mut pass2_candidates = - crate::asap_aware_cse::window_composition_pass2_candidates(&roots, &rule_candidates) - .unwrap(); + asap_logical_asap_optimization::asap_aware_cse::window_composition_pass2_candidates( + &roots, + &rule_candidates, + ) + .unwrap(); pass2_candidates.pop().unwrap().roots.unwrap().remove(0) } @@ -1071,9 +1072,9 @@ mod tests { fn every_minute() -> QueryWorkloadEntry { QueryWorkloadEntry { predictability: Predictability::Predictable { known_at: None }, - ..crate::test_support::entry(QueryRecurrence::Repeated(RepeatedDemand::FixedInterval( - RepetitionInterval(MINUTE as u32), - ))) + ..asap_logical_asap_optimization::test_support::entry(QueryRecurrence::Repeated( + RepeatedDemand::FixedInterval(RepetitionInterval(MINUTE as u32)), + )) } } @@ -1344,7 +1345,7 @@ mod tests { reduction, grouping, filter: Some(asap_types::ir::Predicate(ScalarExpr::Literal( - asap_types::pre_asap::expr_ir::ScalarValue::Boolean(true), + asap_types::ir::scalar::expr_ir::ScalarValue::Boolean(true), ))), }), build.schema.clone(), @@ -1371,7 +1372,7 @@ mod tests { let p50 = Rc::new( OperatorNode::new(Operator::ASAP(ASAPOp::SummaryEstimate { summary_input: Rc::clone(&build), - query: asap_types::post_asap::SketchStatistic::Quantile { q: 0.5 }, + query: asap_types::ir::schema::SketchStatistic::Quantile { q: 0.5 }, })) .unwrap(), ); @@ -1417,7 +1418,7 @@ mod tests { child: Rc::clone(&scan), })) .unwrap(); - let exact = crate::test_support::aggregate( + let exact = asap_logical_asap_optimization::test_support::aggregate( Reduction::by(vec![]), vec![AggIntent::Max { col: None }], vec![], diff --git a/crates/plan-selection/Cargo.toml b/crates/plan-selection/Cargo.toml new file mode 100644 index 000000000..abb350b64 --- /dev/null +++ b/crates/plan-selection/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "asap-plan-selection" +version = "0.1.0" +edition = "2021" + +# #509 Stage 3, plan selection: the only stage that prices plans, with the +# planning models (cost and accuracy) and the Stage 1 to Stage 3 driver. +# Depends on Stages 1 and 2, never on the executor. +[dependencies] +asap_sketchlib = { workspace = true } +asap-types = { path = "../types" } +asap-logical-asap-optimization = { path = "../logical-asap-optimization" } +asap-physical-asap-optimization = { path = "../physical-asap-optimization" } +thiserror = "2" +serde = { version = "1", features = ["derive"] } +serde_json = "1" + +[dev-dependencies] +asap-logical-asap-optimization = { path = "../logical-asap-optimization", features = ["test-support"] } +asap-frontend-promql = { path = "../frontend-promql" } diff --git a/crates/asap-aware-mapping/src/analytical_cost.rs b/crates/plan-selection/src/cost/analytical_cost.rs similarity index 98% rename from crates/asap-aware-mapping/src/analytical_cost.rs rename to crates/plan-selection/src/cost/analytical_cost.rs index fd4499666..0cfa20d9f 100644 --- a/crates/asap-aware-mapping/src/analytical_cost.rs +++ b/crates/plan-selection/src/cost/analytical_cost.rs @@ -2,16 +2,18 @@ //! //! This module deliberately keeps CPU work, retained state, and scan I/O //! separate. Plan selection (Stage 3) turns them into one objective with -//! [`crate::stage3_plan_selection::ResourceUnitCost`]. +//! [`crate::ResourceUnitCost`]. use std::collections::{HashMap, HashSet}; -use asap_types::post_asap::{SketchAlgorithm, SketchParams}; -pub use asap_types::resources::{CacheCapacityEvidence, CacheEvidence, CacheProfile, ModeledCpu}; +use asap_types::ir::schema::{SketchAlgorithm, SketchParams}; +pub use asap_types::workload::resources::{ + CacheCapacityEvidence, CacheEvidence, CacheProfile, ModeledCpu, +}; use asap_types::workload::DataArrival; use serde::{Deserialize, Serialize}; -use crate::physical_operator_statistics::{ +use crate::cost::physical_operator_statistics::{ validate_comparison_scopes, ComparisonScope, EdgeStatistics, OperatorStatistics, OperatorStatisticsProvider, PromqlEdgeStatistics, PromqlValueKind, ScanSelection, }; @@ -152,7 +154,7 @@ pub fn cache_hit_ratios( #[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] #[serde(from = "ResourceEstimateWire", into = "ResourceEstimateWire")] pub struct ResourceEstimate { - resources: asap_types::resources::PhysicalResources, + resources: asap_types::workload::resources::PhysicalResources, } // Keep the established three-field JSON format. Required fields make a missing @@ -183,7 +185,7 @@ impl From for ResourceEstimateWire { impl ResourceEstimate { pub const fn new(cpu_ops: f64, peak_memory_bytes: u64, scan_bytes: u64) -> Self { Self { - resources: asap_types::resources::PhysicalResources { + resources: asap_types::workload::resources::PhysicalResources { cpu: ModeledCpu { cpu_ops }, peak_memory_bytes: Some(peak_memory_bytes), retained_memory_bytes: None, @@ -210,7 +212,9 @@ impl ResourceEstimate { .expect("analytical scan bytes are required by construction") } - pub fn resources(&self) -> &asap_types::resources::PhysicalResources { + pub fn resources( + &self, + ) -> &asap_types::workload::resources::PhysicalResources { &self.resources } } @@ -942,7 +946,7 @@ fn checked_cpu_product(rows: u64, operations_per_row: u64) -> Result Result { @@ -963,7 +967,7 @@ fn partitioned_order_estimate( fn validate_partitioning( input: EdgeStatistics, - partitioning: &crate::physical_operator_statistics::PartitionStatistics, + partitioning: &crate::cost::physical_operator_statistics::PartitionStatistics, partitioned: bool, ) -> Result<(), AnalyticalCostError> { let inconsistent = |reason| Err(AnalyticalCostError::InconsistentOperatorStatistics(reason)); @@ -1864,8 +1868,9 @@ pub(crate) fn validate_operator_semantics( } fn require_promql_unary( - edges: &crate::physical_operator_statistics::UnaryEdgeStatistics, -) -> Result { + edges: &crate::cost::physical_operator_statistics::UnaryEdgeStatistics, +) -> Result +{ let promql = edges.promql.ok_or(AnalyticalCostError::MissingOrStale( "promql_edge_statistics", ))?; @@ -1875,8 +1880,11 @@ fn require_promql_unary( } fn require_promql_binary( - edges: &crate::physical_operator_statistics::BinaryEdgeStatistics, -) -> Result { + edges: &crate::cost::physical_operator_statistics::BinaryEdgeStatistics, +) -> Result< + crate::cost::physical_operator_statistics::PromqlBinaryEdgeStatistics, + AnalyticalCostError, +> { let promql = edges.promql.ok_or(AnalyticalCostError::MissingOrStale( "promql_edge_statistics", ))?; @@ -1887,7 +1895,7 @@ fn require_promql_binary( } fn validate_promql_cardinality_preserving_shape( - promql: crate::physical_operator_statistics::PromqlUnaryEdgeStatistics, + promql: crate::cost::physical_operator_statistics::PromqlUnaryEdgeStatistics, ) -> Result<(), AnalyticalCostError> { validate_promql_edge(promql.input)?; validate_promql_edge(promql.output)?; @@ -1901,7 +1909,7 @@ fn validate_promql_cardinality_preserving_shape( } fn validate_promql_filter_shape( - promql: crate::physical_operator_statistics::PromqlUnaryEdgeStatistics, + promql: crate::cost::physical_operator_statistics::PromqlUnaryEdgeStatistics, ) -> Result<(), AnalyticalCostError> { validate_promql_edge(promql.input)?; validate_promql_edge(promql.output)?; @@ -1967,7 +1975,7 @@ fn validate_promql_edge_shape( fn validate_promql_bridge( operator: PhysicalOperator, - edges: &crate::physical_operator_statistics::UnaryEdgeStatistics, + edges: &crate::cost::physical_operator_statistics::UnaryEdgeStatistics, ) -> Result<(), AnalyticalCostError> { let promql = require_promql_unary(edges)?; let valid = match operator { @@ -2001,7 +2009,7 @@ fn validate_promql_binary( cardinality: PromqlVectorCardinality, build_side: Option, matching_key_bytes: u64, - edges: &crate::physical_operator_statistics::BinaryEdgeStatistics, + edges: &crate::cost::physical_operator_statistics::BinaryEdgeStatistics, ) -> Result<(), AnalyticalCostError> { let promql = require_promql_binary(edges)?; validate_instant_vector_rows(edges.output, promql.output)?; @@ -2070,7 +2078,7 @@ fn validate_promql_series_sample( grouping_key_count: u64, group_count: u64, key_bytes: u64, - edges: &crate::physical_operator_statistics::UnaryEdgeStatistics, + edges: &crate::cost::physical_operator_statistics::UnaryEdgeStatistics, ) -> Result<(), AnalyticalCostError> { let promql = require_promql_unary(edges)?; validate_instant_vector_rows(edges.output, promql.output)?; @@ -2170,7 +2178,7 @@ pub enum AnalyticalCostError { #[error("measured_operation_costs has two entries for {operation:?} on {sketch:?}")] DuplicateMeasuredOperationCost { sketch: SketchParams, - operation: crate::stage3_plan_selection::SummaryOperation, + operation: crate::SummaryOperation, }, /// Bad input: a non-empty measured table with `ms_per_cpu_op` not /// positive, so a measured time cannot be counted in CPU operations. @@ -2190,7 +2198,7 @@ fn checked_bytes(parts: &[u64]) -> Result { #[cfg(test)] mod tests { use super::*; - use crate::physical_operator_statistics::{ + use crate::cost::physical_operator_statistics::{ validate_comparison_scopes, BinaryEdgeStatistics, ComparisonScope, EdgeStatistics, OperatorStatistics, PartitionStatistics, PromqlBinaryEdgeStatistics, PromqlEdgeStatistics, PromqlUnaryEdgeStatistics, PromqlValueKind, ScanSelection, UnaryEdgeStatistics, @@ -2204,7 +2212,7 @@ mod tests { assert_eq!(estimate.cpu_ops(), 12.5); assert_eq!(estimate.peak_memory_bytes(), u64::MAX); assert_eq!(estimate.scan_bytes(), 0); - let shared: &asap_types::resources::PhysicalResources = + let shared: &asap_types::workload::resources::PhysicalResources = estimate.resources(); assert_eq!(shared.cpu.cpu_ops, 12.5); assert_eq!(shared.peak_memory_bytes, Some(u64::MAX)); @@ -2972,7 +2980,7 @@ mod tests { } fn comparison_scope() -> ComparisonScope { - use asap_types::ir::operator_properties::Source; + use asap_types::ir::operator::operator_properties::Source; use asap_types::workload::{ DurationMs, QueryRecurrence, RepeatedDemand, RepetitionInterval, TimestampMs, }; @@ -3235,7 +3243,7 @@ mod tests { operator: PhysicalOperator::Scan, children: vec![], scan_selection: Some(ScanSelection { - source: asap_types::ir::operator_properties::Source::Table { + source: asap_types::ir::operator::operator_properties::Source::Table { table_ref: "other_metrics".into(), }, source_snapshot_id: "catalog-version-42".into(), @@ -3300,7 +3308,7 @@ mod tests { let mut scope = comparison_scope(); let coverage = scope.sources[0].clone(); scope.sources.push(ScanSelection { - source: asap_types::ir::operator_properties::Source::Table { + source: asap_types::ir::operator::operator_properties::Source::Table { table_ref: "auxiliary".into(), }, source_snapshot_id: "catalog-version-42".into(), @@ -3666,10 +3674,10 @@ mod tests { // Legacy mapping imports are aliases of the shared schema, not a second type. #[test] fn shared_cache_types_work_through_legacy_mapping_imports() { - let central: asap_types::resources::CacheProfile = cache_profile(100, 50); + let central: asap_types::workload::resources::CacheProfile = cache_profile(100, 50); let legacy: CacheProfile = serde_json::from_value(serde_json::to_value(¢ral).unwrap()).unwrap(); - fn accepts_shared(_: &asap_types::resources::CacheProfile) {} + fn accepts_shared(_: &asap_types::workload::resources::CacheProfile) {} accepts_shared(&legacy); assert_eq!(central, legacy); assert_eq!( diff --git a/crates/plan-selection/src/cost/mod.rs b/crates/plan-selection/src/cost/mod.rs new file mode 100644 index 000000000..96877f873 --- /dev/null +++ b/crates/plan-selection/src/cost/mod.rs @@ -0,0 +1,13 @@ +//! The analytical resource and cost model plan selection prices with. +//! +//! - [`analytical_cost`] and [`physical_operator_statistics`] — the +//! analytical per-operator resource model. +//! - [`physical_handoff_cost`] — transfer and materialization handoffs. +//! - [`query_physical_lowering`] — the physical operators a query lowers +//! to, as the cost model prices them. +//! - [`storage_io`] — storage request and byte costs. +pub mod analytical_cost; +pub mod physical_handoff_cost; +pub mod physical_operator_statistics; +pub mod query_physical_lowering; +pub mod storage_io; diff --git a/crates/asap-aware-mapping/src/physical_handoff_cost.rs b/crates/plan-selection/src/cost/physical_handoff_cost.rs similarity index 97% rename from crates/asap-aware-mapping/src/physical_handoff_cost.rs rename to crates/plan-selection/src/cost/physical_handoff_cost.rs index 207d812f1..a5a442aa5 100644 --- a/crates/asap-aware-mapping/src/physical_handoff_cost.rs +++ b/crates/plan-selection/src/cost/physical_handoff_cost.rs @@ -1,11 +1,11 @@ //! Byte estimates at deployment-declared physical handoffs. -use crate::analytical_cost::{ +use crate::cost::analytical_cost::{ estimate_physical_dag, AnalyticalCostError, EvidenceBackedPhysicalDAG, ExecutionMultiplicity, PhysicalDAGNode, }; -use crate::physical_operator_statistics::{ComparisonScope, OperatorStatistics}; -pub use asap_types::resources::{PhysicalHandoffBytes, PhysicalHandoffKind}; +use crate::cost::physical_operator_statistics::{ComparisonScope, OperatorStatistics}; +pub use asap_types::workload::resources::{PhysicalHandoffBytes, PhysicalHandoffKind}; use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; diff --git a/crates/asap-aware-mapping/src/physical_operator_statistics.rs b/crates/plan-selection/src/cost/physical_operator_statistics.rs similarity index 98% rename from crates/asap-aware-mapping/src/physical_operator_statistics.rs rename to crates/plan-selection/src/cost/physical_operator_statistics.rs index 5a0706e32..19c1b8427 100644 --- a/crates/asap-aware-mapping/src/physical_operator_statistics.rs +++ b/crates/plan-selection/src/cost/physical_operator_statistics.rs @@ -6,7 +6,7 @@ use std::collections::HashMap; -use asap_types::ir::operator_properties::{InfoMatcher, Source}; +use asap_types::ir::operator::operator_properties::{InfoMatcher, Source}; use asap_types::ir::Predicate; use asap_types::workload::{ @@ -15,7 +15,7 @@ use asap_types::workload::{ }; use serde::{Deserialize, Serialize}; -use crate::analytical_cost::AnalyticalCostError; +use crate::cost::analytical_cost::AnalyticalCostError; /// The semantic and workload boundary within which two resource estimates /// may be compared. Canonical workload and query-IR types remain authoritative; @@ -280,7 +280,7 @@ pub struct PartitionStatistics { } /// Workload-dependent evidence for one operator in an already-lowered -/// physical DAG. [`PhysicalOperator`](crate::analytical_cost::PhysicalOperator) +/// physical DAG. [`PhysicalOperator`](crate::cost::analytical_cost::PhysicalOperator) /// is the authoritative operator vocabulary: every one of its variants has a /// matching statistics variant here. /// diff --git a/crates/asap-aware-mapping/src/query_physical_lowering.rs b/crates/plan-selection/src/cost/query_physical_lowering.rs similarity index 95% rename from crates/asap-aware-mapping/src/query_physical_lowering.rs rename to crates/plan-selection/src/cost/query_physical_lowering.rs index 69a67fe1d..6e3a26513 100644 --- a/crates/asap-aware-mapping/src/query_physical_lowering.rs +++ b/crates/plan-selection/src/cost/query_physical_lowering.rs @@ -4,13 +4,13 @@ use std::rc::Rc; use asap_types::ir::{NonASAPOp, OperatorNode, ScalarExpr}; -use crate::analytical_cost::{ +use crate::cost::analytical_cost::{ validate_operator_semantics, AnalyticalCostError, EvidenceBackedPhysicalDAG, ExecutionMultiplicity, HashJoinBuildSide, PhysicalDAGNode, PhysicalNodeEvidence, PhysicalOperator, PromqlBinaryOperandMode, PromqlBinaryOperation, PromqlPresenceKind, PromqlSeriesSampleKind, PromqlVectorCardinality, }; -use crate::physical_operator_statistics::{ +use crate::cost::physical_operator_statistics::{ ComparisonScope, EdgeStatistics, OperatorStatistics, ScanSelection, }; @@ -54,7 +54,7 @@ pub fn lower_query_physical_dag( ) -> Result { use std::collections::HashMap; - use asap_types::pre_asap::{GroupKeys, RelationalSetOpKind}; + use asap_types::ir::operator::{GroupKeys, RelationalSetOpKind}; scope.validate()?; @@ -240,7 +240,7 @@ pub fn lower_query_physical_dag( expr: &ScalarExpr, ) -> Result { match expr { - ScalarExpr::Literal(asap_types::pre_asap::ScalarValue::Float64(_)) + ScalarExpr::Literal(asap_types::ir::scalar::ScalarValue::Float64(_)) | ScalarExpr::EvalTimestamp => self.lower_promql_scalar_leaf(query, occurrence), ScalarExpr::PromqlScalarFromVector(vector) => self.lower_promql_unary( query, @@ -374,12 +374,12 @@ pub fn lower_query_physical_dag( .. } => { if having.is_some() - || asap_types::ir::non_asap::any_measure_filtered(filters) + || asap_types::ir::operator::non_asap::any_measure_filtered(filters) || measures.is_empty() { return Err(AnalyticalCostError::UnsupportedQueryOperator); } - if matches!(reduction, asap_types::pre_asap::Reduction::PerEntity) { + if matches!(reduction, asap_types::ir::operator::Reduction::PerEntity) { if measures.len() != 1 { return Err(AnalyticalCostError::UnsupportedQueryOperator); } @@ -405,7 +405,7 @@ pub fn lower_query_physical_dag( }; return self.lower_promql_unary(query, occurrence, operator, child); } - let asap_types::pre_asap::Reduction::Reduce(grouping) = reduction else { + let asap_types::ir::operator::Reduction::Reduce(grouping) = reduction else { unreachable!("per-entity reduction returned above") }; if grouping.is_without() || !supports_hash_aggregate(reduction, measures) { @@ -546,9 +546,9 @@ pub fn lower_query_physical_dag( if order_by.is_empty() || !matches!( func, - asap_types::pre_asap::WindowFuncKind::RowNumber - | asap_types::pre_asap::WindowFuncKind::Rank - | asap_types::pre_asap::WindowFuncKind::DenseRank + asap_types::ir::operator::WindowFuncKind::RowNumber + | asap_types::ir::operator::WindowFuncKind::Rank + | asap_types::ir::operator::WindowFuncKind::DenseRank ) { return Err(AnalyticalCostError::UnsupportedQueryOperator); @@ -622,21 +622,21 @@ pub fn lower_query_physical_dag( return Err(AnalyticalCostError::UnsupportedQueryOperator); } let kind = match kind { - asap_types::pre_asap::SampleKind::LimitK(k) => { + asap_types::ir::operator::SampleKind::LimitK(k) => { let k = u64::try_from(*k).map_err(|_| AnalyticalCostError::Overflow)?; if k == 0 { return Err(AnalyticalCostError::MissingOrZero("limitk")); } PromqlSeriesSampleKind::LimitK { k } } - asap_types::pre_asap::SampleKind::LimitRatio(ratio) + asap_types::ir::operator::SampleKind::LimitRatio(ratio) if ratio.is_finite() && (-1.0..=1.0).contains(ratio) => { PromqlSeriesSampleKind::LimitRatio { ratio_bits: ratio.to_bits(), } } - asap_types::pre_asap::SampleKind::LimitRatio(_) => { + asap_types::ir::operator::SampleKind::LimitRatio(_) => { return Err(AnalyticalCostError::UnsupportedQueryOperator) } }; @@ -784,7 +784,7 @@ pub fn lower_query_physical_dag( right, } => { let equality_key_count = - if matches!(kind, asap_types::pre_asap::JoinKind::Cross) { + if matches!(kind, asap_types::ir::operator::JoinKind::Cross) { None } else { hash_join_key_count(&pred.0, left, right) @@ -1019,7 +1019,7 @@ fn require_operator_statistics( fn bind_scan_coverage( node_id: &str, - source: &asap_types::pre_asap::Source, + source: &asap_types::ir::operator::Source, predicates: &[asap_types::ir::Predicate], scope: &ComparisonScope, ) -> Result { @@ -1042,10 +1042,11 @@ fn bind_scan_coverage( fn bind_info_coverage( node_id: &str, - selector: &[asap_types::pre_asap::InfoMatcher], + selector: &[asap_types::ir::operator::InfoMatcher], scope: &ComparisonScope, ) -> Result { - use asap_types::pre_asap::{CompareOpKind, Source}; + use asap_types::ir::operator::Source; + use asap_types::ir::scalar::CompareOpKind; let mut metric: Option<&str> = None; for matcher in selector @@ -1089,9 +1090,9 @@ fn duration_millis( } fn promql_binary_operation( - operation: &asap_types::pre_asap::BinaryOpKind, + operation: &asap_types::ir::operator::BinaryOpKind, ) -> PromqlBinaryOperation { - use asap_types::pre_asap::{BinaryOpKind, PromQLVectorSetOpKind}; + use asap_types::ir::operator::{BinaryOpKind, PromQLVectorSetOpKind}; match operation { BinaryOpKind::Set(PromQLVectorSetOpKind::And) => PromqlBinaryOperation::And, BinaryOpKind::Set(PromQLVectorSetOpKind::Or) => PromqlBinaryOperation::Or, @@ -1103,9 +1104,9 @@ fn promql_binary_operation( } fn promql_vector_cardinality( - vector_match: Option<&asap_types::pre_asap::VectorMatch>, + vector_match: Option<&asap_types::ir::operator::VectorMatch>, ) -> PromqlVectorCardinality { - use asap_types::pre_asap::GroupSide; + use asap_types::ir::operator::GroupSide; match vector_match.and_then(|matching| matching.grouping.as_ref()) { Some(grouping) if grouping.side == GroupSide::Left => PromqlVectorCardinality::ManyToOne, Some(_) => PromqlVectorCardinality::OneToMany, @@ -1118,7 +1119,7 @@ fn hash_join_key_count( left: &OperatorNode, right: &OperatorNode, ) -> Option { - use asap_types::pre_asap::CompareOpKind; + use asap_types::ir::scalar::CompareOpKind; let left_width = left.schema.fields.len(); let total_width = left_width.saturating_add(right.schema.fields.len()); @@ -1232,10 +1233,10 @@ fn scalar_operation_count(expr: &ScalarExpr) -> Result } fn supports_hash_aggregate( - reduction: &asap_types::pre_asap::Reduction, - measures: &[asap_types::pre_asap::AggIntent], + reduction: &asap_types::ir::operator::Reduction, + measures: &[asap_types::ir::operator::AggIntent], ) -> bool { - use asap_types::pre_asap::{AggIntent, Reduction}; + use asap_types::ir::operator::{AggIntent, Reduction}; matches!(reduction, Reduction::Reduce(_)) && !measures.is_empty() @@ -1259,19 +1260,20 @@ fn supports_hash_aggregate( }) } -fn presence_intent(intent: &asap_types::pre_asap::AggIntent) -> bool { +fn presence_intent(intent: &asap_types::ir::operator::AggIntent) -> bool { matches!( intent, - asap_types::pre_asap::AggIntent::Absent | asap_types::pre_asap::AggIntent::AbsentOverTime + asap_types::ir::operator::AggIntent::Absent + | asap_types::ir::operator::AggIntent::AbsentOverTime ) } -fn present_over_time_intent(intent: &asap_types::pre_asap::AggIntent) -> bool { - matches!(intent, asap_types::pre_asap::AggIntent::PresentOverTime) +fn present_over_time_intent(intent: &asap_types::ir::operator::AggIntent) -> bool { + matches!(intent, asap_types::ir::operator::AggIntent::PresentOverTime) } -fn fixed_state_per_series_intent(intent: &asap_types::pre_asap::AggIntent) -> bool { - use asap_types::pre_asap::AggIntent; +fn fixed_state_per_series_intent(intent: &asap_types::ir::operator::AggIntent) -> bool { + use asap_types::ir::operator::AggIntent; matches!( intent, AggIntent::Rate @@ -1310,15 +1312,15 @@ fn fixed_state_per_series_intent(intent: &asap_types::pre_asap::AggIntent) -> bo #[cfg(test)] mod tests { use super::*; - use crate::analytical_cost::{ + use crate::cost::analytical_cost::{ estimate_physical_dag, estimate_physical_dag_comparison, PhysicalDAGEstimateRequest, }; - use crate::physical_operator_statistics::{ + use crate::cost::physical_operator_statistics::{ validate_comparison_scopes, BinaryEdgeStatistics, PartitionStatistics, PromqlEdgeStatistics, PromqlUnaryEdgeStatistics, PromqlValueKind, UnaryEdgeStatistics, }; + use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::{BinaryOperator, ExprSemantics, Predicate, SortKey, TimeRangeKind}; - use asap_types::types::AccuracyTarget; use asap_types::workload::{DataArrival, DurationMs, QueryRecurrence, TimestampMs}; use std::collections::HashMap; @@ -1453,7 +1455,10 @@ mod tests { } } - fn coverage(source: asap_types::pre_asap::Source, predicates: Vec) -> ScanSelection { + fn coverage( + source: asap_types::ir::operator::Source, + predicates: Vec, + ) -> ScanSelection { ScanSelection { source, source_snapshot_id: "snapshot-1".into(), @@ -1464,7 +1469,8 @@ mod tests { #[test] fn info_scan_selection_includes_symbolic_selector_matchers() { - use asap_types::pre_asap::{CompareOpKind, InfoMatcher, Source}; + use asap_types::ir::operator::{InfoMatcher, Source}; + use asap_types::ir::scalar::CompareOpKind; let selector = vec![InfoMatcher { label: "cluster".into(), @@ -1500,7 +1506,8 @@ mod tests { // Correlation can be costed as an exact hash aggregate using provider-supplied state size. #[test] fn correlation_lowers_to_physical_hash_aggregate() { - use asap_types::pre_asap::{AggIntent, DataType, Field, Reduction, Schema, Source}; + use asap_types::ir::operator::{AggIntent, Reduction, Source}; + use asap_types::ir::schema::{DataType, Field, Schema}; let source = Source::Table { table_ref: "pairs".into(), }; @@ -1555,8 +1562,9 @@ mod tests { // executor runs them as native reducers over one identity column. #[test] fn exact_frequency_intents_lower_to_physical_hash_aggregates() { - use asap_types::pre_asap::{AggIntent, DataType, Field, Reduction, Schema, Source}; - use asap_types::types::AccuracyTarget; + use asap_types::ir::operator::{AggIntent, Reduction, Source}; + use asap_types::ir::properties::AccuracyTarget; + use asap_types::ir::schema::{DataType, Field, Schema}; let source = Source::Table { table_ref: "flows".into(), }; @@ -1625,8 +1633,8 @@ mod tests { #[test] fn query_lowering_recurses_and_fuses_global_sort_limit() { - use asap_types::pre_asap::{AggIntent, GroupKeys, Reduction, Source}; - use asap_types::pre_asap::{DataType, Field, Schema}; + use asap_types::ir::operator::{AggIntent, GroupKeys, Reduction, Source}; + use asap_types::ir::schema::{DataType, Field, Schema}; use std::rc::Rc; let scan = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Scan { @@ -1634,7 +1642,7 @@ mod tests { table_ref: "events".into(), }, predicates: vec![Predicate(ScalarExpr::Literal( - asap_types::pre_asap::ScalarValue::Boolean(true), + asap_types::ir::scalar::ScalarValue::Boolean(true), ))], schema: Schema::new(vec![ Field::plain("service", DataType::Utf8, false), @@ -1675,7 +1683,7 @@ mod tests { table_ref: "events".into(), }, vec![Predicate(ScalarExpr::Literal( - asap_types::pre_asap::ScalarValue::Boolean(true), + asap_types::ir::scalar::ScalarValue::Boolean(true), ))], ); let scope = scope(vec![scan_coverage]); @@ -1767,8 +1775,9 @@ mod tests { #[test] fn query_lowering_shares_only_provider_identified_physical_nodes() { - use asap_types::pre_asap::{CompareOpKind, DataType, Field, Schema}; - use asap_types::pre_asap::{JoinKind, Source}; + use asap_types::ir::operator::{JoinKind, Source}; + use asap_types::ir::scalar::CompareOpKind; + use asap_types::ir::schema::{DataType, Field, Schema}; use std::rc::Rc; let shared = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Scan { @@ -1834,7 +1843,7 @@ mod tests { }) }; let shared_scope = scope(vec![scan_selection]); - let no_cache = crate::analytical_cost::CacheProfile::no_cache(); + let no_cache = crate::cost::analytical_cost::CacheProfile::no_cache(); let shared_dag = lower_query_physical_dag(&root, &shared_scope, &shared_provider).unwrap(); assert_eq!(shared_dag.nodes.len(), 2); assert_eq!( @@ -1935,8 +1944,9 @@ mod tests { #[test] fn query_lowering_covers_relational_unary_operators() { - use asap_types::pre_asap::{DataType, Field, ScalarValue, Schema}; - use asap_types::pre_asap::{GroupKeys, Source, TimeShift, WindowFuncKind}; + use asap_types::ir::operator::{GroupKeys, Source, TimeShift, WindowFuncKind}; + use asap_types::ir::scalar::ScalarValue; + use asap_types::ir::schema::{DataType, Field, Schema}; let scan = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Scan { source: Source::Table { @@ -2120,8 +2130,8 @@ mod tests { #[test] fn query_lowering_maps_concat_and_union_all_but_rejects_distinct_set_ops() { - use asap_types::pre_asap::{DataType, Field, Schema}; - use asap_types::pre_asap::{RelationalSetOpKind, Source}; + use asap_types::ir::operator::{RelationalSetOpKind, Source}; + use asap_types::ir::schema::{DataType, Field, Schema}; let scan = |name: &str| { OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Scan { @@ -2208,8 +2218,8 @@ mod tests { #[test] fn query_lowering_fails_closed_for_missing_or_inconsistent_statistics() { - use asap_types::pre_asap::Source; - use asap_types::pre_asap::{DataType, Field, Schema}; + use asap_types::ir::operator::Source; + use asap_types::ir::schema::{DataType, Field, Schema}; let scan = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Scan { source: Source::Table { @@ -2326,8 +2336,9 @@ mod tests { #[test] fn query_lowering_accepts_a_consistently_empty_edge() { - use asap_types::pre_asap::{DataType, Field, ScalarValue, Schema}; - use asap_types::pre_asap::{GroupKeys, Source}; + use asap_types::ir::operator::{GroupKeys, Source}; + use asap_types::ir::scalar::ScalarValue; + use asap_types::ir::schema::{DataType, Field, Schema}; let scan = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Scan { source: Source::Table { @@ -2390,9 +2401,9 @@ mod tests { #[test] fn query_lowering_rejects_aggregates_without_a_hash_implementation() { - use asap_types::pre_asap::{AggIntent, GroupKeys, Reduction, Source, WindowFuncKind}; - use asap_types::pre_asap::{DataType, Field, Schema}; - use asap_types::types::AccuracyTarget; + use asap_types::ir::operator::{AggIntent, GroupKeys, Reduction, Source, WindowFuncKind}; + use asap_types::ir::properties::AccuracyTarget; + use asap_types::ir::schema::{DataType, Field, Schema}; let scan = || { OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Scan { @@ -2447,7 +2458,7 @@ mod tests { .unwrap(); let shifted = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::TimeShift { - shift: asap_types::pre_asap::TimeShift { + shift: asap_types::ir::operator::TimeShift { offset_ms: 60_000, at: None, }, @@ -2476,7 +2487,7 @@ mod tests { #[test] fn scalar_work_counts_every_local_predicate_operation() { - use asap_types::pre_asap::{CompareOpKind, ScalarValue}; + use asap_types::ir::scalar::{CompareOpKind, ScalarValue}; let comparison = || ScalarExpr::Compare { left: Box::new(ScalarExpr::Column(0)), @@ -2491,7 +2502,8 @@ mod tests { #[test] fn promql_presence_is_lowered_with_a_per_step_output_bound() { - use asap_types::pre_asap::{AggIntent, DataType, Field, Reduction, Schema, Source}; + use asap_types::ir::operator::{AggIntent, Reduction, Source}; + use asap_types::ir::schema::{DataType, Field, Schema}; let source = Source::TimeSeries { metric: "missing".into(), @@ -2560,7 +2572,8 @@ mod tests { #[test] fn promql_range_and_subquery_preserve_internal_steps() { - use asap_types::pre_asap::{DataType, Field, Schema, Source}; + use asap_types::ir::operator::Source; + use asap_types::ir::schema::{DataType, Field, Schema}; use std::time::Duration; let source = Source::TimeSeries { metric: "m".into() }; @@ -2641,10 +2654,11 @@ mod tests { #[test] fn promql_binary_lowering_keeps_operation_and_matching_cardinality() { - use asap_types::pre_asap::{ - ArithmeticOpKind, BinaryOpKind, DataType, Field, GroupSide, Schema, Source, - VectorGrouping, VectorMatch, VectorMatchKind, + use asap_types::ir::operator::{ + BinaryOpKind, GroupSide, Source, VectorGrouping, VectorMatch, VectorMatchKind, }; + use asap_types::ir::scalar::ArithmeticOpKind; + use asap_types::ir::schema::{DataType, Field, Schema}; let left_source = Source::TimeSeries { metric: "a".into() }; let right_source = Source::TimeSeries { metric: "b".into() }; @@ -2704,7 +2718,7 @@ mod tests { inputs: [left_edge, right_edge], output: output_edge, promql: Some( - crate::physical_operator_statistics::PromqlBinaryEdgeStatistics { + crate::cost::physical_operator_statistics::PromqlBinaryEdgeStatistics { inputs: [left_promql, right_promql], output: output_promql, }, @@ -2732,10 +2746,9 @@ mod tests { #[test] fn promql_relabel_sample_and_per_series_lower_as_a_complete_chain() { - use asap_types::pre_asap::{ - AggIntent, DataType, Field, GroupKeys, Reduction, SampleKind, ScalarValue, Schema, - Source, - }; + use asap_types::ir::operator::{AggIntent, GroupKeys, Reduction, SampleKind, Source}; + use asap_types::ir::scalar::ScalarValue; + use asap_types::ir::schema::{DataType, Field, Schema}; let source = Source::TimeSeries { metric: "requests".into(), @@ -2865,7 +2878,7 @@ mod tests { same_reference: bool, scalar_inputs: usize, ) { - use crate::physical_operator_statistics::ScalarInputStatistics; + use crate::cost::physical_operator_statistics::ScalarInputStatistics; use asap_types::workload::{ BatchEntry, DataWorkload, PlanningWorkload, Predictability, Query, QueryLanguage, QueryRequirements, QueryWorkload, diff --git a/crates/asap-aware-mapping/src/storage_io.rs b/crates/plan-selection/src/cost/storage_io.rs similarity index 98% rename from crates/asap-aware-mapping/src/storage_io.rs rename to crates/plan-selection/src/cost/storage_io.rs index 125ec915b..ba2363ffe 100644 --- a/crates/asap-aware-mapping/src/storage_io.rs +++ b/crates/plan-selection/src/cost/storage_io.rs @@ -5,13 +5,13 @@ use std::collections::{HashMap, HashSet}; use serde::{Deserialize, Serialize}; // Keep the original public import path while sharing the sole type definition. -pub use asap_types::resources::StorageResources; +pub use asap_types::workload::resources::StorageResources; -use crate::analytical_cost::{ +use crate::cost::analytical_cost::{ estimate_physical_dag, AnalyticalCostError, EvidenceBackedPhysicalDAG, ExecutionMultiplicity, PhysicalDAGNode, PhysicalOperator, }; -use crate::physical_operator_statistics::{ComparisonScope, OperatorStatistics}; +use crate::cost::physical_operator_statistics::{ComparisonScope, OperatorStatistics}; pub const STORAGE_IO_MODEL_VERSION: &str = "storage-requests-v1"; diff --git a/crates/asap-aware-mapping/src/stage3_plan_selection.rs b/crates/plan-selection/src/lib.rs similarity index 96% rename from crates/asap-aware-mapping/src/stage3_plan_selection.rs rename to crates/plan-selection/src/lib.rs index c39995931..f81c871c1 100644 --- a/crates/asap-aware-mapping/src/stage3_plan_selection.rs +++ b/crates/plan-selection/src/lib.rs @@ -62,7 +62,7 @@ //! longest query reading it, once per evaluation, which is close to nothing. //! A `MatAtQueryKeptAcrossEvaluations` tumbling-pane chain (#509 Example //! 4's B3, offered by Stage 2 when the deployment keeps query-time results, -//! [`crate::DeploymentCapabilities::query_time_results_retained`]) is kept across +//! [`asap_logical_asap_optimization::DeploymentCapabilities::query_time_results_retained`]) is kept across //! evaluations: each evaluation scans one pane of raw data and builds the //! newest pane, and the `N − 1` older panes, which complete the readers' //! look-back, are kept every second (in memory, all `N` at the peak). When @@ -73,7 +73,7 @@ //! [`ResourceUsage::peak_memory_bytes`], which cost models 2 and 3 read. A //! computation is charged for its CPU operations and scanned bytes only. //! -//! Prices come from [`crate::analytical_cost::estimate_operator`] over edge +//! Prices come from [`crate::cost::analytical_cost::estimate_operator`] over edge //! statistics derived from the [`DataWorkload`] and a fixed default group //! count; summary build and estimation are priced as rows × sketch depth and //! groups × k. A query-time scan reads the longest window above it (a time @@ -135,7 +135,7 @@ //! Disk holds state materialized on `Medium::Disk` (Stage 2 places none //! there yet) and the raw data query-time scans read: per source, the //! longest window a scan reads. A deployment that does not retain raw data -//! ([`crate::DeploymentCapabilities::raw_data_retained`]) keeps no raw samples +//! ([`asap_logical_asap_optimization::DeploymentCapabilities::raw_data_retained`]) keeps no raw samples //! after they arrive, so a candidate with a query-time scan is invalid, //! with the window it would read as the reason. //! @@ -159,50 +159,64 @@ use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::rc::Rc; -use asap_types::ir::operator_properties::{ +use asap_types::ir::operator::operator_properties::{ BinaryOpKind, PromQLVectorSetOpKind, RelationalSetOpKind, }; +use asap_types::ir::operator::{AggIntent, Reduction, Source}; use asap_types::ir::physical_export::compile_physical_asap_workload_with_node_ids; use asap_types::ir::physical_export::{ PhysicalASAPDAG, PhysicalASAPNodeId, PhysicalASAPOperatorPayload as Payload, }; +use asap_types::ir::properties::{ + AccuracyGuarantee, AccuracyMetric, CompositionOperator, ExecutionTiming, +}; +use asap_types::ir::properties::{AccuracyScope, AccuracyTarget}; +use asap_types::ir::scalar::ScalarValue; +use asap_types::ir::schema::sketch::KLL_MAX_K; +use asap_types::ir::schema::{DataType, Schema}; +use asap_types::ir::schema::{ + FieldDataType, GroupingStrategy, SketchAlgorithm, SketchParams, SketchStatistic, WeightDomain, +}; use asap_types::ir::{ ASAPOp, BinaryOperator, NonASAPOp, Operator, OperatorNode, OperatorResultKind, QueryRoot, ScalarExpr, SchemaDerivationError, TimeRangeKind, }; -use asap_types::post_asap::sketch::KLL_MAX_K; -use asap_types::post_asap::{ - AccuracyGuarantee, AccuracyMetric, CompositionOperator, ExecutionTiming, FieldDataType, - GroupingStrategy, Materialization, Medium, SketchAlgorithm, SketchParams, SketchStatistic, - WeightDomain, -}; -use asap_types::pre_asap::{AggIntent, DataType, Reduction, ScalarValue, Schema, Source}; -use asap_types::types::{AccuracyScope, AccuracyTarget}; +use asap_types::physical::{Materialization, Medium}; use asap_types::workload::{ DataWorkload, MetricType, QueryRecurrence, QueryWorkloadEntry, RepeatedDemand, }; use thiserror::Error; -use crate::accuracy::composition::{ +use crate::cost::analytical_cost::{ + estimate_operator, AnalyticalCostError, PhysicalOperator, ResourceEstimate, +}; +use crate::cost::physical_operator_statistics::{ + EdgeStatistics, OperatorStatistics, PartitionStatistics, UnaryEdgeStatistics, +}; +use asap_logical_asap_optimization::accuracy::composition::{ parse_samples_sign_statistic, parse_source_norm_statistic, DIVISION_OPERAND_DOMAINS, INPUT_ROW_COUNT, PRODUCT_OPERAND_DOMAINS, STREAM_L1_NORM, }; -use crate::accuracy::estimators::with_failure_probability; -use crate::accuracy::estimators::{hll, univmon}; -use crate::accuracy::{OperandDomain, PropagationStats}; -use crate::analytical_cost::{ - estimate_operator, AnalyticalCostError, PhysicalOperator, ResourceEstimate, -}; -use crate::deployment_capabilities::DeploymentCapabilitiesError; -use crate::per_computation_candidates::{ +use asap_logical_asap_optimization::accuracy::estimators::with_failure_probability; +use asap_logical_asap_optimization::accuracy::estimators::{hll, univmon}; +use asap_logical_asap_optimization::accuracy::{OperandDomain, PropagationStats}; +use asap_logical_asap_optimization::deployment_capabilities::DeploymentCapabilitiesError; +use asap_logical_asap_optimization::per_computation_candidates::{ approximate_inputs, distinct_bound_source, operand_domain_statistic, operand_guarantee, operand_of, project_column_guarantee, root_guarantee, with_resized_summaries, }; -use crate::physical_operator_statistics::{ - EdgeStatistics, OperatorStatistics, PartitionStatistics, UnaryEdgeStatistics, +use asap_physical_asap_optimization::PhysicalCandidate; + +pub mod cost; +/// Costed reuse of compatible physical pane producers. +pub mod pane_sharing; +/// The Stage 1 to Stage 3 driver and the optimization-pass interface. +pub mod pass; +pub use pass::{ + optimize, run_stage1_to_stage3, OptimizationInput, OptimizationInputError, OptimizationPass, + OptimizeError, PassNameConflict, PassRegistry, PlanOutput, PlanningModels, QueryPlan, + Stage1ToStage3Pass, Stage1ToStage3Run, }; -use crate::stage2_physical_asap_optimization::PhysicalCandidate; -use crate::PlanningModels; mod measured_operation_costs; use measured_operation_costs::measured_cpu_ns; @@ -684,7 +698,7 @@ pub enum SelectionError { entries: usize, }, /// Bad input: the deployment's capabilities contradict the data - /// workload ([`crate::DeploymentCapabilities::validate`]). + /// workload ([`asap_logical_asap_optimization::DeploymentCapabilities::validate`]). #[error("deployment capabilities: {0}")] InvalidDeploymentCapabilities(DeploymentCapabilitiesError), #[error("candidate {candidate}, node {node:?}: {error}")] @@ -1205,7 +1219,7 @@ fn values_per_result(root: &QueryRoot, series_read: Option) -> Option /// above one only operators that keep or drop its rows, or join two such /// inputs without adding unmatched rows of both sides. fn at_most_one_row(node: &OperatorNode) -> bool { - let global = |reduction: &asap_types::pre_asap::Reduction| { + let global = |reduction: &asap_types::ir::operator::Reduction| { reduction .group_keys() .is_some_and(|by| !by.is_without() && by.keys().is_empty()) @@ -1247,7 +1261,7 @@ fn at_most_one_row(node: &OperatorNode) -> bool { Operator::NonASAP(NonASAPOp::Join { kind, left, right, .. }) => { - use asap_types::ir::operator_properties::JoinKind; + use asap_types::ir::operator::operator_properties::JoinKind; match kind { JoinKind::Semi | JoinKind::Anti => at_most_one_row(left), JoinKind::Full => false, @@ -1369,7 +1383,7 @@ pub fn resize_for_failure_probability( /// first query whose output is not shown to meet its target, as a reason. An output meets its target when its theoretical root guarantee /// (composed in Stage 1 with the built-in model) satisfies it, or when the /// deployment's empirical evidence shows that this candidate's output does -/// ([`crate::accuracy::AccuracyEvidenceProvider::output_guarantee`]). A +/// ([`asap_logical_asap_optimization::accuracy::AccuracyEvidenceProvider::output_guarantee`]). A /// certified estimate inside the plan is not enough on its own: a difference /// or ratio of certified estimates can be unbounded. The root guarantee's /// symbolic statistics are bound first ([`bound_root_guarantee`]); one still @@ -1379,7 +1393,7 @@ pub fn resize_for_failure_probability( /// the metric a counter ([`confirmed_weights`]), and unmet otherwise. /// /// A target that sets no metric accepts a guarantee in any metric that meets -/// its ε, δ and scope ([`crate::accuracy::AccuracyModel::satisfies`]); one that sets a metric +/// its ε, δ and scope ([`asap_logical_asap_optimization::accuracy::AccuracyModel::satisfies`]); one that sets a metric /// accepts only guarantees in it (or convertible to it). fn accuracy_check( candidate: &PhysicalCandidate, @@ -1589,7 +1603,7 @@ fn bound_root_guarantee( } /// `guarantee` with the statistics `value` supplies bound. A sign gate -/// ([`samples_sign_statistic`](crate::accuracy::composition::samples_sign_statistic)) +/// ([`samples_sign_statistic`](asap_logical_asap_optimization::accuracy::composition::samples_sign_statistic)) /// is an indicator that only 1 (proven) opens: any other value leaves it /// unbound, so it can never read as a perfect bound. fn bind_statistics( @@ -3091,16 +3105,15 @@ mod tests { settings, ) } - use crate::accuracy::AccuracyEvidenceProvider; - use crate::deployment_capabilities::DeploymentCapabilities; - use crate::stage2_physical_asap_optimization::optimize_physical_asap; - use crate::test_support::{entry, entry_with, lower_promql}; + use asap_logical_asap_optimization::accuracy::AccuracyEvidenceProvider; + use asap_logical_asap_optimization::deployment_capabilities::DeploymentCapabilities; + use asap_logical_asap_optimization::test_support::{entry, entry_with, lower_promql}; + use asap_physical_asap_optimization::optimize_physical_asap; + use asap_types::ir::properties::AccuracyTarget; + use asap_types::ir::properties::{AccuracyMetric, BoundExpr, ProbabilityExpr}; + use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; + use asap_types::ir::schema::{GroupingStrategy, SketchKind}; use asap_types::ir::{ExprSemantics, ScalarExpr}; - use asap_types::post_asap::{ - AccuracyMetric, BoundExpr, GroupingStrategy, ProbabilityExpr, SketchKind, - }; - use asap_types::pre_asap::expr_ir::ArithmeticOpKind; - use asap_types::types::AccuracyTarget; use asap_types::workload::{DataArrival, MetricType, SourceFacts}; /// `G` of `root` over `data` ([`values_per_result`]). @@ -3197,7 +3210,7 @@ mod tests { "topk by (job) (10, sum_over_time(m[1m]))", AccuracyTarget::max_error(0.01).failure_probability(0.001), ); - let inventory = crate::per_computation_candidates::generate_per_computation_candidates( + let inventory = asap_logical_asap_optimization::per_computation_candidates::generate_per_computation_candidates( vec![(0, QueryRoot::Operator(root))], ) .unwrap(); @@ -3211,7 +3224,7 @@ mod tests { .map(|index| { let mut choice = vec![0; inventory.targets.len()]; choice[topk] = index; - let roots: Vec<_> = crate::per_computation_candidates::compose_logical_candidate( + let roots: Vec<_> = asap_logical_asap_optimization::per_computation_candidates::compose_logical_candidate( &inventory, &choice, ) .unwrap() @@ -3356,7 +3369,7 @@ mod tests { /// reading as a perfect bound. Other statistics pass through. #[test] fn the_sign_gate_opens_only_at_one() { - use crate::accuracy::composition::samples_sign_statistic; + use asap_logical_asap_optimization::accuracy::composition::samples_sign_statistic; let gated = AccuracyGuarantee::new( AccuracyMetric::Relative, BoundExpr::Product { @@ -3403,7 +3416,7 @@ mod tests { /// name. #[test] fn sign_of_counter_samples_is_bound_from_the_declared_type() { - use crate::accuracy::composition::samples_sign_statistic; + use asap_logical_asap_optimization::accuracy::composition::samples_sign_statistic; let gated = AccuracyGuarantee::new( AccuracyMetric::Relative, BoundExpr::Product { @@ -3446,7 +3459,8 @@ mod tests { use asap_types::workload::MetricType; let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); let root = lower_promql("topk by (job) (10, sum_over_time(m[1m]))", target.clone()); - let root = asap_types::ir::schema_support::with_promql_series_identity(&root).unwrap(); + let root = + asap_types::ir::schema::schema_support::with_promql_series_identity(&root).unwrap(); let data = with_metric_types(with_series(Some(10_000_000)), &[("m", MetricType::Counter)]); let run = crate::pass::run_stage1_to_stage3( vec![(0, QueryRoot::Operator(root))], @@ -3473,7 +3487,7 @@ mod tests { .collect() }; let proven = [WeightDomain::NonNegative { - proof: asap_types::post_asap::NonNegativeWeightProof::CounterSamples, + proof: asap_types::ir::schema::NonNegativeWeightProof::CounterSamples, }]; let selection = run.stage3_plan_selection.as_ref().unwrap(); assert_eq!(domains(&selection.plan), proven, "{selection:?}"); @@ -3484,7 +3498,7 @@ mod tests { /// and stays unproven otherwise; the executor reads no data workload. #[test] fn confirmed_weights_rewrite_only_the_summary_update() { - use asap_types::post_asap::NonNegativeWeightProof; + use asap_types::ir::schema::NonNegativeWeightProof; use asap_types::workload::MetricType; let p2 = &candidates()[1]; let domains = |roots: &[QueryRoot]| -> Vec { @@ -3613,7 +3627,7 @@ mod tests { .iter_mut() .find(|n| n.id == inner) .unwrap(); - node.materialization = Materialization::MatAtQuery(asap_types::ir::Medium::Memory); + node.materialization = Materialization::MatAtQuery(asap_types::physical::Medium::Memory); let stored = price(&candidate.dag, &recurrences(2), &data(), &settings).unwrap(); for (id, c) in &stored.per_node { assert_eq!(c.multiplier, 1, "node {id} is computed once"); @@ -3630,7 +3644,7 @@ mod tests { /// candidates `P1`, `P2`, ... fn all_candidates(query: &str, target: &AccuracyTarget) -> Vec { let root = lower_promql(query, target.clone()); - let inventory = crate::per_computation_candidates::generate_per_computation_candidates( + let inventory = asap_logical_asap_optimization::per_computation_candidates::generate_per_computation_candidates( vec![(0, QueryRoot::Operator(root))], ) .unwrap(); @@ -3651,7 +3665,7 @@ mod tests { .iter() .enumerate() .map(|(index, choice)| { - let roots: Vec<_> = crate::per_computation_candidates::compose_logical_candidate( + let roots: Vec<_> = asap_logical_asap_optimization::per_computation_candidates::compose_logical_candidate( &inventory, choice, ) .unwrap() @@ -3893,11 +3907,11 @@ mod tests { "{}", rejected[0].reason ); - let table = crate::test_support::scan_from( - asap_types::pre_asap::Source::Table { + let table = asap_logical_asap_optimization::test_support::scan_from( + asap_types::ir::operator::Source::Table { table_ref: "t".into(), }, - Schema::new(vec![asap_types::pre_asap::schema::Field::plain( + Schema::new(vec![asap_types::ir::schema::Field::plain( "v", DataType::Float64, false, @@ -3931,28 +3945,28 @@ mod tests { Some(10_000) ); let table = || { - crate::test_support::scan_from( - asap_types::pre_asap::Source::Table { + asap_logical_asap_optimization::test_support::scan_from( + asap_types::ir::operator::Source::Table { table_ref: "t".into(), }, Schema::new(vec![ - asap_types::pre_asap::schema::Field::plain("k", DataType::Utf8, false), - asap_types::pre_asap::schema::Field::plain("v", DataType::Float64, false), + asap_types::ir::schema::Field::plain("k", DataType::Utf8, false), + asap_types::ir::schema::Field::plain("v", DataType::Float64, false), ]), ) }; let count = || AggIntent::Count { accuracy: AccuracyTarget::exact(), }; - let global = crate::test_support::agg(vec![], count(), table()); - let grouped = crate::test_support::agg(vec![0], count(), table()); + let global = asap_logical_asap_optimization::test_support::agg(vec![], count(), table()); + let grouped = asap_logical_asap_optimization::test_support::agg(vec![0], count(), table()); assert_eq!( values_of(&QueryRoot::Operator(global.clone()), &none), Some(1) ); assert_eq!(values_of(&QueryRoot::Operator(grouped), &data()), None); let cross = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Join { - kind: asap_types::ir::operator_properties::JoinKind::Cross, + kind: asap_types::ir::operator::operator_properties::JoinKind::Cross, pred: asap_types::ir::Predicate(ScalarExpr::Literal(ScalarValue::Boolean(true))), left: global.clone(), right: global, @@ -4180,7 +4194,7 @@ mod tests { | Payload::NonASAP(NonASAPOp::Aggregate { .. }) ) { node.materialization = - Materialization::MatAtIngestion(asap_types::ir::Medium::Memory); + Materialization::MatAtIngestion(asap_types::physical::Medium::Memory); } } } @@ -4779,8 +4793,8 @@ mod tests { fn operand_domain( &self, _operand: &OperatorNode, - ) -> Option { - Some(crate::accuracy::OperandDomain { + ) -> Option { + Some(asap_logical_asap_optimization::accuracy::OperandDomain { lower: 1.0, upper: 1_000.0, max_samples: 1_000_000, @@ -4861,15 +4875,17 @@ mod tests { fn stage3_binds_each_metric_stream_norm() { // Count-Min summaries of each metric's raw value column. let raw = |query: &str, g: AccuracyGuarantee| { - ScalarExpr::PromqlScalarFromVector(crate::test_support::estimate_over( - crate::test_support::weighted( - crate::test_support::cms_summary(query), - asap_types::post_asap::SummaryInputExpr::Column( - asap_types::pre_asap::ColumnRef::SampleValue, + ScalarExpr::PromqlScalarFromVector( + asap_logical_asap_optimization::test_support::estimate_over( + asap_logical_asap_optimization::test_support::weighted( + asap_logical_asap_optimization::test_support::cms_summary(query), + asap_types::ir::schema::SummaryInputExpr::Column( + asap_types::ir::scalar::ColumnRef::SampleValue, + ), ), + g, ), - g, - )) + ) }; let root = scalar_root( ArithmeticOpKind::Sub, @@ -4898,24 +4914,26 @@ mod tests { /// value (`m * 1000`) needs a norm nothing binds; both stay unmet. #[test] fn stage3_binds_a_stream_norm_only_for_the_raw_value_column() { - use asap_types::post_asap::SummaryInputExpr; - use asap_types::pre_asap::ColumnRef; + use asap_types::ir::scalar::ColumnRef; + use asap_types::ir::schema::SummaryInputExpr; let frequency = guarantee(AccuracyMetric::ABSOLUTE_STREAM_L1, 0.01, 0.001); let doubled = |summary: Rc| { scalar_root( ArithmeticOpKind::Mul, ScalarExpr::literal_f64(2.0), - ScalarExpr::PromqlScalarFromVector(crate::test_support::estimate_over( - summary, - frequency.clone(), - )), + ScalarExpr::PromqlScalarFromVector( + asap_logical_asap_optimization::test_support::estimate_over( + summary, + frequency.clone(), + ), + ), ) }; let models = PlanningModels::builtin(); let data = norms(&[("m", 100.0)]); let column = || SummaryInputExpr::Column(ColumnRef::SampleValue); - let raw = doubled(crate::test_support::weighted( - crate::test_support::cms_summary("count by (job) (m)"), + let raw = doubled(asap_logical_asap_optimization::test_support::weighted( + asap_logical_asap_optimization::test_support::cms_summary("count by (job) (m)"), column(), )); let g = bound_of(&raw, &data, &models).unwrap(); @@ -4924,11 +4942,13 @@ mod tests { "{g:?}" ); for unbound in [ - doubled(Rc::new(crate::test_support::cms_summary( - "count by (job) (m)", - ))), - doubled(crate::test_support::weighted( - crate::test_support::cms_summary("count by (job) (m * 1000)"), + doubled(Rc::new( + asap_logical_asap_optimization::test_support::cms_summary("count by (job) (m)"), + )), + doubled(asap_logical_asap_optimization::test_support::weighted( + asap_logical_asap_optimization::test_support::cms_summary( + "count by (job) (m * 1000)", + ), column(), )), ] { @@ -5032,7 +5052,7 @@ mod tests { "quantile_over_time(0.99, m[1m])", AccuracyTarget::max_error(0.01).failure_probability(0.01), ); - let inventory = crate::per_computation_candidates::generate_per_computation_candidates( + let inventory = asap_logical_asap_optimization::per_computation_candidates::generate_per_computation_candidates( vec![(0, QueryRoot::Operator(root))], ) .unwrap(); @@ -5044,7 +5064,7 @@ mod tests { .candidates .iter() .position(|c| { - *c == crate::per_computation_candidates::ComputationCandidate::Summary( + *c == asap_logical_asap_optimization::per_computation_candidates::ComputationCandidate::Summary( SketchAlgorithm::Kll, ) }) @@ -5052,8 +5072,10 @@ mod tests { }) .collect(); let roots = - crate::per_computation_candidates::compose_logical_candidate(&inventory, &choice) - .unwrap(); + asap_logical_asap_optimization::per_computation_candidates::compose_logical_candidate( + &inventory, &choice, + ) + .unwrap(); let QueryRoot::Operator(root) = &roots[0].1 else { panic!("operator root") }; @@ -5077,17 +5099,19 @@ mod tests { /// with nothing materialized. fn first_candidate_dag(query: &str) -> PhysicalASAPDAG { let root = lower_promql(query, AccuracyTarget::exact()); - let inventory = crate::per_computation_candidates::generate_per_computation_candidates( + let inventory = asap_logical_asap_optimization::per_computation_candidates::generate_per_computation_candidates( vec![(0, QueryRoot::Operator(root))], ) .unwrap(); let choice = vec![0; inventory.targets.len()]; let roots: Vec<_> = - crate::per_computation_candidates::compose_logical_candidate(&inventory, &choice) - .unwrap() - .into_iter() - .map(|(_, root)| root) - .collect(); + asap_logical_asap_optimization::per_computation_candidates::compose_logical_candidate( + &inventory, &choice, + ) + .unwrap() + .into_iter() + .map(|(_, root)| root) + .collect(); optimize_physical_asap( "L1", &roots, @@ -5131,8 +5155,9 @@ mod tests { } const AT_INGESTION: Materialization = - Materialization::MatAtIngestion(asap_types::ir::Medium::Memory); - const AT_QUERY: Materialization = Materialization::MatAtQuery(asap_types::ir::Medium::Memory); + Materialization::MatAtIngestion(asap_types::physical::Medium::Memory); + const AT_QUERY: Materialization = + Materialization::MatAtQuery(asap_types::physical::Medium::Memory); fn node_of(dag: &PhysicalASAPDAG, pick: impl Fn(&Payload) -> bool) -> PhysicalASAPNodeId { dag.nodes.iter().find(|n| pick(&n.payload)).unwrap().id @@ -5508,7 +5533,7 @@ mod tests { deployment: &DeploymentCapabilities, ) -> Vec { let root = lower_promql("quantile_over_time(0.99, m[5m])", p99_target()); - let inventory = crate::per_computation_candidates::generate_per_computation_candidates( + let inventory = asap_logical_asap_optimization::per_computation_candidates::generate_per_computation_candidates( vec![(0, QueryRoot::Operator(root))], ) .unwrap(); @@ -5519,27 +5544,35 @@ mod tests { .position(|c| { matches!( c, - crate::per_computation_candidates::ComputationCandidate::Summary( + asap_logical_asap_optimization::per_computation_candidates::ComputationCandidate::Summary( SketchAlgorithm::Kll ) ) }) .unwrap(); let roots: Vec<_> = - crate::per_computation_candidates::compose_logical_candidate(&inventory, &[kll]) - .unwrap() - .into_iter() - .map(|(_, root)| root) - .collect(); - let rule_candidates = - crate::asap_aware_cse::pane_candidates(&roots, &[Some(u64::from(MINUTE))]); + asap_logical_asap_optimization::per_computation_candidates::compose_logical_candidate( + &inventory, + &[kll], + ) + .unwrap() + .into_iter() + .map(|(_, root)| root) + .collect(); + let rule_candidates = asap_logical_asap_optimization::asap_aware_cse::pane_candidates( + &roots, + &[Some(u64::from(MINUTE))], + ); let roots = - crate::asap_aware_cse::window_composition_pass2_candidates(&roots, &rule_candidates) - .unwrap() - .pop() - .unwrap() - .roots - .unwrap(); + asap_logical_asap_optimization::asap_aware_cse::window_composition_pass2_candidates( + &roots, + &rule_candidates, + ) + .unwrap() + .pop() + .unwrap() + .roots + .unwrap(); let mut candidates = optimize_physical_asap( "L1", &roots, @@ -5708,14 +5741,16 @@ mod tests { .collect(); // The exact candidate of each query. let inventory = - crate::per_computation_candidates::generate_per_computation_candidates(roots).unwrap(); + asap_logical_asap_optimization::per_computation_candidates::generate_per_computation_candidates(roots).unwrap(); let exact = vec![0; inventory.targets.len()]; let roots: Vec = - crate::per_computation_candidates::compose_logical_candidate(&inventory, &exact) - .unwrap() - .into_iter() - .map(|(_, root)| root) - .collect(); + asap_logical_asap_optimization::per_computation_candidates::compose_logical_candidate( + &inventory, &exact, + ) + .unwrap() + .into_iter() + .map(|(_, root)| root) + .collect(); let mut candidate = optimize_physical_asap( "L1", &roots, @@ -6207,7 +6242,7 @@ mod tests { /// disk, nothing to average memory, and costs only its disk price. #[test] fn state_on_disk_is_counted_as_disk_not_memory() { - use asap_types::ir::Medium; + use asap_types::physical::Medium; let settings = unit_costs(|c| c.cost_per_disk_byte_second = 1e-9); let work_only = unit_costs(|c| c.cost_per_memory_byte_second = 0.0); let state = |dag: &PhysicalASAPDAG| node_of(dag, is_build); diff --git a/crates/asap-aware-mapping/src/stage3_plan_selection/measured_operation_costs.rs b/crates/plan-selection/src/measured_operation_costs.rs similarity index 97% rename from crates/asap-aware-mapping/src/stage3_plan_selection/measured_operation_costs.rs rename to crates/plan-selection/src/measured_operation_costs.rs index 1af6491fc..2d8054bc9 100644 --- a/crates/asap-aware-mapping/src/stage3_plan_selection/measured_operation_costs.rs +++ b/crates/plan-selection/src/measured_operation_costs.rs @@ -21,10 +21,10 @@ //! [`ResourceUsage`]: super::ResourceUsage //! [`ResourceSpeed`]: super::ResourceSpeed -use asap_types::post_asap::SketchParams; +use asap_types::ir::schema::SketchParams; use serde::{Deserialize, Serialize}; -use crate::analytical_cost::AnalyticalCostError; +use crate::cost::analytical_cost::AnalyticalCostError; /// One operation on a sketch's state, as plan selection prices it. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] diff --git a/crates/asap-aware-mapping/src/pane_sharing.rs b/crates/plan-selection/src/pane_sharing.rs similarity index 100% rename from crates/asap-aware-mapping/src/pane_sharing.rs rename to crates/plan-selection/src/pane_sharing.rs diff --git a/crates/asap-aware-mapping/src/pass/mod.rs b/crates/plan-selection/src/pass/mod.rs similarity index 96% rename from crates/asap-aware-mapping/src/pass/mod.rs rename to crates/plan-selection/src/pass/mod.rs index 25b3929c9..97d6983d7 100644 --- a/crates/asap-aware-mapping/src/pass/mod.rs +++ b/crates/plan-selection/src/pass/mod.rs @@ -18,25 +18,24 @@ use std::collections::BTreeMap; use std::rc::Rc; use asap_types::ir::physical_export::compile_physical_asap_workload; -use asap_types::ir::timing::{ +use asap_types::ir::properties::timing::{ apply_materialization_timings, MaterializationAssignment, TimingMemo, }; +use asap_types::ir::properties::ExecutionDataStateError; use asap_types::ir::{OperatorNode, QueryRoot}; -use asap_types::parsed_workload::ParsedWorkload; -use asap_types::post_asap::ExecutionDataStateError; +use asap_types::workload::parsed_workload::ParsedWorkload; use asap_types::workload::WorkloadError; -use crate::accuracy::{ +use crate::{PlanSelectionSettings, Selection, SelectionError}; +use asap_logical_asap_optimization::accuracy::{ AccuracyEvidenceProvider, AccuracyModel, DefaultAccuracyModel, NoAccuracyEvidence, }; -use crate::deployment_capabilities::DeploymentCapabilities; -use crate::per_computation_candidates::PerComputationCandidateError; -use crate::stage3_plan_selection::{PlanSelectionSettings, Selection, SelectionError}; +use asap_logical_asap_optimization::deployment_capabilities::DeploymentCapabilities; +use asap_logical_asap_optimization::per_computation_candidates::PerComputationCandidateError; pub use stage1_to_stage3::{ - run_stage1_to_stage3, run_stage1_to_stage3_from_pass1, run_stage2_and_stage3, - stage1_logical_asap_optimization, LogicalCandidate, Pass2Candidate, PrunedCandidate, - Stage1Candidate, Stage1ToStage3Pass, Stage1ToStage3Run, + run_stage1_to_stage3, run_stage1_to_stage3_from_pass1, run_stage2_and_stage3, LogicalCandidate, + PrunedCandidate, Stage1ToStage3Pass, Stage1ToStage3Run, }; static DEFAULT_ACCURACY_MODEL: DefaultAccuracyModel = DefaultAccuracyModel; @@ -506,10 +505,10 @@ mod tests { /// plan; a scalar root without one is rejected. #[test] fn contract_rejects_a_scalar_reference_without_its_plan() { - use asap_types::ir::non_asap::NonASAPOp; - use asap_types::ir::operator_properties::Source; + use asap_types::ir::operator::non_asap::NonASAPOp; + use asap_types::ir::operator::operator_properties::Source; + use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::{Operator, ScalarExpr}; - use asap_types::pre_asap::schema::{DataType, Field, Schema}; let scan = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { source: Source::TimeSeries { metric: "m".into() }, predicates: vec![], @@ -544,11 +543,11 @@ mod tests { /// under another entry, or read by no scalar root is rejected. #[test] fn contract_checks_scalar_plan_order_entry_and_count() { - use asap_types::ir::non_asap::NonASAPOp; - use asap_types::ir::operator_properties::Source; + use asap_types::ir::operator::non_asap::NonASAPOp; + use asap_types::ir::operator::operator_properties::Source; + use asap_types::ir::scalar::ArithmeticOpKind; + use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::{ExprSemantics, Operator, ScalarExpr}; - use asap_types::pre_asap::schema::{DataType, Field, Schema}; - use asap_types::pre_asap::ArithmeticOpKind; let series = |metric: &str| { OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { source: Source::TimeSeries { diff --git a/crates/plan-selection/src/pass/stage1_to_stage3.rs b/crates/plan-selection/src/pass/stage1_to_stage3.rs new file mode 100644 index 000000000..139823342 --- /dev/null +++ b/crates/plan-selection/src/pass/stage1_to_stage3.rs @@ -0,0 +1,322 @@ +//! Stage 1 (logical ASAP-aware optimization) → Stage 2 (physical ASAP-aware +//! optimization) → Stage 3 (plan selection), starting from the query-language +//! frontends' roots; Stage 0 runs before it in the facade. +//! +//! This is the planner pipeline (#509 MVP). [`Stage1ToStage3Pass`] is the +//! [`OptimizationPass`](super::OptimizationPass) that runs it. +//! +//! [`run_stage1_to_stage3`] runs, for one workload: +//! 1. Stage 1, logical ASAP-aware optimization: Pass 1 per-computation candidate +//! generation, then composition of one whole-workload candidate per choice +//! of a per-computation candidate for every target (the Cartesian product, +//! in mixed-radix order, the last target varying fastest), each followed +//! by its Pass 2 candidates: those of the identical-expression rule +//! ([`identical_expression_pass2_candidates`]: one per subset of its +//! sharing candidates), then those of the summary-capability rule +//! ([`summary_capability_pass2_candidates`]: one per choice of shared +//! summary builds), each followed by its own identical-expression +//! candidates, and, for the composed candidate and each +//! summary-capability candidate, those of the window-composition rule +//! ([`window_composition_pass2_candidates`]: one per non-empty subset of +//! the builds a repeating query could build as tumbling panes), each +//! followed by its own identical-expression candidates, and, for the +//! composed candidate, those of the reuse rules (exact top-k reuse, then +//! each rollup Pass 2 candidate followed by its own exact top-k reuse +//! Pass 2 candidates: [`topk_reuse_pass2_candidates`], +//! [`rollup_pass2_candidates`]), each followed by its own +//! identical-expression candidates; every distinct candidate once; the +//! independent candidate is kept; +//! 2. Stage 2, physical ASAP-aware optimization of every Stage 1 candidate +//! ([`optimize_physical_asap`]): one physical candidate per +//! materialization assignment of its summary builds; +//! 3. Stage 3, plan selection over the physical candidates ([`select_plan`]). +//! +//! Stage 0, the query-language frontends, runs before it (the `asap-planner` +//! facade lowers the workload). The window-composition rule offers +//! per-query tumbling panes only; sliding windows, Exponential Histograms +//! and a pane width shared across queries are not planned yet. +//! +//! Every combination is enumerated and plan selection prices every +//! candidate; nothing is capped. A search that avoids building every +//! candidate is future work (issue #671 surveys the options). + +use std::rc::Rc; + +use asap_types::ir::schema::schema_support::with_promql_series_identity; +use asap_types::ir::{OperatorNode, QueryRoot}; +use asap_types::workload::{DataWorkload, QueryLanguage, QueryWorkloadEntry}; + +use super::{OptimizationInput, OptimizationPass, OptimizeError, PlanOutput, QueryPlan}; +use crate::PlanningModels; +use crate::{select_plan, Selection, SelectionError}; +use asap_logical_asap_optimization::per_computation_candidates::{ + generate_per_computation_candidates, PerComputationCandidateError, PerComputationCandidates, +}; +use asap_logical_asap_optimization::workload_candidates::{ + stage1_logical_asap_optimization, Pass2Candidate, Stage1Candidate, +}; +use asap_physical_asap_optimization::{optimize_physical_asap, PhysicalCandidate}; + +/// One whole-workload CandidateLogicalASAPDAG: `choice[t]` picks target +/// `t`'s candidate, or its fused computation's +/// (`targets[t].computation_for(choice[t])`), of the Pass 1 inventory. `roots` are the +/// composed query roots in input order, or why the choice did not compose. +#[derive(Debug)] +pub struct LogicalCandidate { + /// `L`, numbered in enumeration order. + pub id: String, + pub choice: Vec, + pub roots: Result, PerComputationCandidateError>, + /// For a Pass 2 candidate: what it changed of its Pass 1 candidate. + pub pass2: Option, +} + +/// A composed logical candidate that physical ASAP-aware optimization could +/// not implement, with the reason. +#[derive(Debug, Clone, PartialEq)] +pub struct PrunedCandidate { + pub from_logical: String, + pub reason: String, +} + +/// The output of Stage 1, Stage 2 and Stage 3 for one workload +/// ([`run_stage1_to_stage3`]). +#[derive(Debug)] +pub struct Stage1ToStage3Run { + /// Stage 1, Pass 1: the per-computation candidates, with the roots they + /// read. + pub stage1_pass1_per_computation: PerComputationCandidates, + /// Stage 1, Pass 1: the number of composed whole-workload candidates, + /// the size of the Cartesian product of the targets' candidates, + /// saturating at `usize::MAX`. Pass 2 adds to them. + pub combinations: usize, + /// Stage 1: every whole-workload logical candidate, in enumeration + /// order: each composed Pass 1 candidate, followed by its Pass 2 + /// candidates (those with [`LogicalCandidate::pass2`]). + pub stage1_logical_asap_optimization: Vec, + /// Stage 2: the physical candidates of each Stage 1 candidate that + /// composed and was implemented, one per materialization assignment: + /// `P` (nothing materialized), then `P-m`, from `L`. + pub stage2_physical_asap_optimization: Vec, + /// Stage 2: the Stage 1 candidates Stage 2 could not implement, with the + /// reasons. + pub stage2_pruned: Vec, + /// Stage 3: plan selection over the Stage 2 candidates. + pub stage3_plan_selection: Result, + /// Stage 3: the selected candidate's logical roots, with the weight + /// requirements it confirmed made proofs ([`Self::selected_logical`]). + pub stage3_selected_logical: Option>, +} + +impl Stage1ToStage3Run { + /// The selected physical candidate, if selection succeeded. + pub fn selected(&self) -> Option<&PhysicalCandidate> { + let selected = &self.stage3_plan_selection.as_ref().ok()?.selected; + self.stage2_physical_asap_optimization + .iter() + .find(|p| &p.id == selected) + } + + /// The composed roots of the logical candidate the selected physical + /// candidate implements: the plan before physical operator + /// implementation and materialization, with the weight requirements + /// Stage 3 confirmed made proofs, as in [`Selection::plan`]. + pub fn selected_logical(&self) -> Option<&[QueryRoot]> { + self.stage3_selected_logical.as_deref() + } +} + +/// Run Stage 1 → Stage 2 → Stage 3 over the frontends' `roots`, enumerating every +/// whole-workload candidate. `entries[i]` is the workload entry of +/// `roots[i]`: its accuracy target, latency bound, recurrence and +/// predictability. Only a Pass 1 failure is +/// an error; a candidate that does not compose or cannot be implemented is +/// listed with its reason, and a failed selection is reported in the run. +pub fn run_stage1_to_stage3( + roots: Vec<(Id, QueryRoot)>, + entries: &[QueryWorkloadEntry], + data: &DataWorkload, + models: PlanningModels<'_>, +) -> Result, PerComputationCandidateError> { + let per_computation = generate_per_computation_candidates(roots)?; + Ok(run_stage1_to_stage3_from_pass1( + per_computation, + entries, + data, + models, + )) +} + +/// [`run_stage1_to_stage3`] after Pass 1: enumerates every Stage 1 +/// candidate of `per_computation` (the Pass 1 inventory of the frontends' +/// roots, [`stage1_logical_asap_optimization`]) without the summaries the +/// deployment does not support +/// ([`PerComputationCandidates::without_unsupported_summaries`]), so a +/// caller can read the inventory, e.g. its +/// [`combinations`](PerComputationCandidates::combinations), before +/// enumerating. +pub fn run_stage1_to_stage3_from_pass1( + per_computation: PerComputationCandidates, + entries: &[QueryWorkloadEntry], + data: &DataWorkload, + models: PlanningModels<'_>, +) -> Stage1ToStage3Run { + let per_computation = + per_computation.without_unsupported_summaries(models.deployment_capabilities); + let recurrence: Vec<_> = entries.iter().map(|e| e.recurrence.clone()).collect(); + let stage1 = stage1_logical_asap_optimization(&per_computation, &recurrence).collect(); + run_stage2_and_stage3(per_computation, stage1, entries, data, models) +} + +/// Stage 2 and Stage 3 over `stage1`, the Stage 1 candidates of +/// `per_computation` in enumeration order (all of them, or the ones a caller +/// enumerated): candidate `n` (from 1) is `L`, implemented as `P` +/// with nothing materialized and as `P-m` (`k` from 1) for each other +/// materialization assignment. +/// +/// It does not prune: `per_computation` must already be pruned for +/// `models.deployment_capabilities` +/// ([`PerComputationCandidates::without_unsupported_summaries`]), with +/// `stage1` enumerated from that pruned inventory, as +/// [`run_stage1_to_stage3_from_pass1`] does. Pruning here would come too +/// late: each of `stage1`'s choices indexes the inventory it was enumerated +/// from, so a candidate with an unsupported summary would already be in +/// `stage1`. +pub fn run_stage2_and_stage3( + per_computation: PerComputationCandidates, + stage1: Vec, + entries: &[QueryWorkloadEntry], + data: &DataWorkload, + models: PlanningModels<'_>, +) -> Stage1ToStage3Run { + let combinations = per_computation.combinations(); + let mut logical: Vec = Vec::new(); + let mut physical = Vec::new(); + let mut pruned = Vec::new(); + for (index, candidate) in stage1.into_iter().enumerate() { + let number = index + 1; + let id = format!("L{number}"); + if let Ok(roots) = &candidate.roots { + match optimize_physical_asap(&id, roots, entries, data, models.deployment_capabilities) + { + Ok(implemented) => { + for (k, mut p) in implemented.into_iter().enumerate() { + p.id = match k { + 0 => format!("P{number}"), + k => format!("P{number}-m{k}"), + }; + physical.push(p); + } + } + Err(error) => pruned.push(PrunedCandidate { + from_logical: id.clone(), + reason: error.to_string(), + }), + } + } + logical.push(LogicalCandidate { + id, + choice: candidate.choice, + roots: candidate.roots, + pass2: candidate.pass2, + }); + } + let selection = select_plan(&physical, entries, data, models); + let targets = &crate::accuracy_targets(entries); + let selected_logical = selection.as_ref().ok().and_then(|selection| { + let from = &physical + .iter() + .find(|p| p.id == selection.selected)? + .from_logical; + let roots = logical + .iter() + .find(|l| &l.id == from)? + .roots + .as_ref() + .ok()?; + // As Stage 3 sized it for its queries' failure probabilities. + let sized = crate::resize_for_failure_probability(roots, targets, data).ok()?; + Some(crate::with_confirmed_weights(&sized, data)) + }); + Stage1ToStage3Run { + stage3_selected_logical: selected_logical, + stage1_pass1_per_computation: per_computation, + combinations, + stage1_logical_asap_optimization: logical, + stage2_physical_asap_optimization: physical, + stage2_pruned: pruned, + stage3_plan_selection: selection, + } +} + +/// The planner pipeline behind the optimization-pass interface: the +/// workload's roots in entry order, PromQL roots with each series' full +/// identity as a column (the row representation per-series state needs at +/// runtime; a root with no such representation is planned over its labels +/// alone), through [`run_stage1_to_stage3`] with every entry's accuracy target. The +/// output is the selected physical candidate's roots, with the open weight +/// requirements plan selection confirmed made proofs. +#[derive(Debug, Default, Clone, Copy)] +pub struct Stage1ToStage3Pass; + +impl OptimizationPass for Stage1ToStage3Pass { + fn name(&self) -> &'static str { + "stage1-to-stage3" + } + + fn optimize(&self, input: OptimizationInput<'_>) -> Result { + let workload = input.workload; + let promql = workload.query_workload().language == QueryLanguage::PromQL; + let mut roots: Vec<(usize, QueryRoot)> = workload + .operator_indices() + .iter() + .zip(workload.exprs()) + .map(|(&index, node)| { + let node = match promql { + true => with_promql_series_identity(node).unwrap_or_else(|_| Rc::clone(node)), + false => Rc::clone(node), + }; + (index, QueryRoot::Operator(node)) + }) + .chain( + workload + .scalar_roots() + .iter() + .map(|(index, expr)| (*index, QueryRoot::Scalar(expr.clone()))), + ) + .collect(); + roots.sort_by_key(|(index, _)| *index); + let entries: Vec<_> = workload.query_workload().entries().collect(); + let data = workload.data_workload().cloned().unwrap_or_default(); + let run = run_stage1_to_stage3(roots, &entries, &data, input.models)?; + let selection = run.stage3_plan_selection?; + let roots = selection.plan.clone(); + let mut output = PlanOutput::new(Vec::new()); + for (entry_index, root) in roots.iter().enumerate() { + match root { + QueryRoot::Operator(node) => output.plans.push(QueryPlan { + entry_index, + root: Rc::clone(node), + }), + QueryRoot::Scalar(expr) => { + // One plan per distinct operator the scalar reads. + let mut refs: Vec<&Rc> = Vec::new(); + for node in expr.operator_refs() { + if !refs.iter().any(|r| Rc::ptr_eq(r, node)) { + refs.push(node); + } + } + output + .scalar_plans + .extend(refs.into_iter().map(|node| QueryPlan { + entry_index, + root: Rc::clone(node), + })); + output.scalar_roots.push((entry_index, expr.clone())); + } + } + } + output.selection = Some(selection); + Ok(output) + } +} diff --git a/crates/asap-aware-mapping/tests/physical_handoff_cost.rs b/crates/plan-selection/tests/physical_handoff_cost.rs similarity index 97% rename from crates/asap-aware-mapping/tests/physical_handoff_cost.rs rename to crates/plan-selection/tests/physical_handoff_cost.rs index 0c267a36c..1b9de583b 100644 --- a/crates/asap-aware-mapping/tests/physical_handoff_cost.rs +++ b/crates/plan-selection/tests/physical_handoff_cost.rs @@ -1,11 +1,11 @@ -use asap_aware_mapping::analytical_cost::{ +use asap_plan_selection::cost::analytical_cost::{ EvidenceBackedPhysicalDAG, ExecutionMultiplicity, PhysicalDAGNode, PhysicalNodeEvidence, PhysicalOperator, }; -use asap_aware_mapping::physical_operator_statistics::{ +use asap_plan_selection::cost::physical_operator_statistics::{ ComparisonScope, EdgeStatistics, OperatorStatistics, ScanSelection, UnaryEdgeStatistics, }; -use asap_types::ir::operator_properties::Source; +use asap_types::ir::operator::operator_properties::Source; use asap_types::workload::{DataArrival, DurationMs, QueryRecurrence, TimestampMs}; use std::collections::HashMap; @@ -129,12 +129,12 @@ fn fixture() -> (EvidenceBackedPhysicalDAG, ComparisonScope) { ) } -use asap_aware_mapping::physical_handoff_cost::*; +use asap_plan_selection::cost::physical_handoff_cost::*; // Legacy mapping imports and the shared resource API are the very same Rust types. #[test] fn mapping_resource_reexports_are_wire_compatible_shared_types() { - let shared = asap_types::resources::PhysicalHandoffBytes { + let shared = asap_types::workload::resources::PhysicalHandoffBytes { network_bytes: 480, materialization_bytes: 40, }; @@ -143,7 +143,7 @@ fn mapping_resource_reexports_are_wire_compatible_shared_types() { serde_json::to_value(legacy).unwrap(), serde_json::json!({"network_bytes": 480, "materialization_bytes": 40}) ); - let shared_kind = asap_types::resources::PhysicalHandoffKind::Materialization; + let shared_kind = asap_types::workload::resources::PhysicalHandoffKind::Materialization; let mut handoff = transfer("persist", None); handoff.kind = shared_kind; let json = serde_json::to_value(&handoff).unwrap(); @@ -380,7 +380,7 @@ fn missing_stale_and_non_finite_evidence_is_rejected() { // Individual actions may fit while accumulation across actions or nodes overflows. #[test] fn handoff_accumulation_and_calibrated_cost_overflow_are_rejected() { - use asap_aware_mapping::analytical_cost::AnalyticalCostError; + use asap_plan_selection::cost::analytical_cost::AnalyticalCostError; let (mut dag, mut scope) = fixture(); scope.recurrence = QueryRecurrence::OneTime { invocations: 1, diff --git a/crates/asap-aware-mapping/tests/storage_io.rs b/crates/plan-selection/tests/storage_io.rs similarity index 95% rename from crates/asap-aware-mapping/tests/storage_io.rs rename to crates/plan-selection/tests/storage_io.rs index a9e749fd1..3925303f7 100644 --- a/crates/asap-aware-mapping/tests/storage_io.rs +++ b/crates/plan-selection/tests/storage_io.rs @@ -1,11 +1,11 @@ -use asap_aware_mapping::analytical_cost::{ +use asap_plan_selection::cost::analytical_cost::{ EvidenceBackedPhysicalDAG, ExecutionMultiplicity, PhysicalDAGNode, PhysicalNodeEvidence, PhysicalOperator, }; -use asap_aware_mapping::physical_operator_statistics::{ +use asap_plan_selection::cost::physical_operator_statistics::{ ComparisonScope, EdgeStatistics, OperatorStatistics, ScanSelection, UnaryEdgeStatistics, }; -use asap_types::ir::operator_properties::Source; +use asap_types::ir::operator::operator_properties::Source; use asap_types::workload::{DataArrival, DurationMs, QueryRecurrence, TimestampMs}; use std::collections::HashMap; @@ -129,15 +129,15 @@ fn fixture() -> (EvidenceBackedPhysicalDAG, ComparisonScope) { ) } -use asap_aware_mapping::storage_io::*; +use asap_plan_selection::cost::storage_io::*; // The compatibility import and shared resource namespace expose one Rust type. #[test] fn storage_estimates_use_the_shared_resource_type_without_wire_changes() { let (dag, scope) = fixture(); let estimate = estimate_storage_io(&dag, &scope, &profile(&dag), "evidence-v1").unwrap(); - let shared: asap_types::resources::StorageResources = estimate.total; - let legacy: asap_aware_mapping::storage_io::StorageResources = shared; + let shared: asap_types::workload::resources::StorageResources = estimate.total; + let legacy: asap_plan_selection::cost::storage_io::StorageResources = shared; assert_eq!(shared, legacy); let wire = serde_json::to_value(&estimate).unwrap(); assert_eq!( @@ -312,7 +312,7 @@ fn multiplicity_and_cross_node_overflow_are_unavailable() { .push(large.clone()); assert_eq!( estimate_storage_io(&dag, &scope, &profile, "evidence-v1"), - Err(asap_aware_mapping::analytical_cost::AnalyticalCostError::Overflow) + Err(asap_plan_selection::cost::analytical_cost::AnalyticalCostError::Overflow) ); scope.recurrence = QueryRecurrence::OneTime { invocations: 1, @@ -329,7 +329,7 @@ fn multiplicity_and_cross_node_overflow_are_unavailable() { }); assert_eq!( estimate_storage_io(&dag, &scope, &profile, "evidence-v1"), - Err(asap_aware_mapping::analytical_cost::AnalyticalCostError::Overflow) + Err(asap_plan_selection::cost::analytical_cost::AnalyticalCostError::Overflow) ); } diff --git a/crates/planner/Cargo.toml b/crates/planner/Cargo.toml index 59825a3fb..9a6c00610 100644 --- a/crates/planner/Cargo.toml +++ b/crates/planner/Cargo.toml @@ -6,12 +6,13 @@ edition = "2021" # The library facade (issue #429): one entry point that takes prepared input and # returns the selected DAG. It is the only crate that depends on every frontend # — before it, the sole facade re-exporting more than one was `asap-devtools`, -# a developer-tools crate. Everything below the frontend boundary lives in -# `asap-aware-mapping`, so a third party writing an optimization pass (issue -# #430) depends on that crate alone and never pulls in DataFusion. +# a developer-tools crate. Everything below the frontend boundary lives in the +# stage crates (`asap-logical-asap-optimization`, `asap-physical-asap-optimization`, +# `asap-plan-selection`), so a third party writing an optimization pass (issue +# #430) depends on `asap-plan-selection` alone and never pulls in DataFusion. [dependencies] asap-types = { path = "../types" } -asap-aware-mapping = { path = "../asap-aware-mapping" } +asap-plan-selection = { path = "../plan-selection" } asap-frontend-sql = { path = "../frontend-sql" } asap-frontend-promql = { path = "../frontend-promql" } asap-frontend-metricsql = { path = "../frontend-metricsql" } @@ -21,3 +22,4 @@ thiserror = "2" # The SQL frontend plans through DataFusion, which is async; `e2e_plan` is # therefore async and its tests need a runtime. tokio = { version = "1", features = ["rt", "macros", "rt-multi-thread"] } +asap-logical-asap-optimization = { path = "../logical-asap-optimization" } diff --git a/crates/planner/src/lib.rs b/crates/planner/src/lib.rs index 025d2c64b..7d4a683cf 100644 --- a/crates/planner/src/lib.rs +++ b/crates/planner/src/lib.rs @@ -3,7 +3,7 @@ //! //! ```text //! PlanningWorkload ──frontends──▶ ParsedWorkload ──optimization pass──▶ PlanOutput -//! (this crate) (asap-aware-mapping) +//! (this crate) (asap-plan-selection) //! ``` //! //! The default pass, [`Stage1ToStage3Pass`], runs Stage 1 (logical ASAP-aware @@ -13,9 +13,9 @@ //! [`e2e_plan`] runs both steps. A caller who already holds pre-ASAP IR — a //! new frontend, a deserialized plan, a test that does not want to build SQL //! and a catalog — skips this crate and calls -//! [`asap_aware_mapping::optimize`] directly. +//! [`asap_plan_selection::optimize`] directly. -use asap_types::parsed_workload::{ParsedWorkload, ParsedWorkloadError}; +use asap_types::workload::parsed_workload::{ParsedWorkload, ParsedWorkloadError}; use asap_types::workload::{PlanningWorkload, QueryLanguage, SqlDialect, WorkloadError}; use asap_frontend_metricsql::{lower_metricsql_query_workload, MetricsqlError}; @@ -28,7 +28,7 @@ use asap_frontend_sql::{lower_sql_dialect, SqlCatalog, SqlError}; // The optimization stage's vocabulary is this facade's vocabulary too: a caller // configures the same models and reads the same output whether it goes through // `e2e_plan` or straight to `optimize`. -pub use asap_aware_mapping::pass::{ +pub use asap_plan_selection::pass::{ optimize, run_stage1_to_stage3, OptimizationInput, OptimizationPass, OptimizeError, PassRegistry, PlanOutput, PlanningModels, QueryPlan, Stage1ToStage3Pass, Stage1ToStage3Run, }; diff --git a/crates/planner/tests/e2e_plan.rs b/crates/planner/tests/e2e_plan.rs index 7dff4f324..c0b6a5104 100644 --- a/crates/planner/tests/e2e_plan.rs +++ b/crates/planner/tests/e2e_plan.rs @@ -3,14 +3,14 @@ use std::rc::Rc; -use asap_aware_mapping::pass::{ +use asap_frontend_sql::{lower_sql_dialect, SqlCatalog}; +use asap_plan_selection::pass::{ run_stage1_to_stage3, OptimizationInput, OptimizationPass, OptimizeError, PlanOutput, PlanningModels, Stage1ToStage3Pass, }; -use asap_frontend_sql::{lower_sql_dialect, SqlCatalog}; use asap_planner::{e2e_plan, FrontendInput, PlanError, UserInput, UserInputError}; -use asap_types::pre_asap::schema::{DataType, Field, Schema}; -use asap_types::types::{AccuracyMetric, AccuracyTarget}; +use asap_types::ir::properties::{AccuracyMetric, AccuracyTarget}; +use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::workload::{ BatchEntry, DataArrival, DataWorkload, DurationMs, PlanningWorkload, Predictability, Query, QueryLanguage, QueryRequirements, QueryWorkload, RepeatedDemand, RepeatingEntry, diff --git a/crates/planner/tests/plan_stages.rs b/crates/planner/tests/plan_stages.rs index 3eaa119bc..4d22c85d8 100644 --- a/crates/planner/tests/plan_stages.rs +++ b/crates/planner/tests/plan_stages.rs @@ -1,9 +1,9 @@ //! The planner pipeline ([`run_stage1_to_stage3`]): the planning stages in order over //! the frontends' roots, enumerating every whole-workload candidate. -use asap_aware_mapping::pass::{run_stage1_to_stage3, PlanningModels, Stage1ToStage3Run}; use asap_frontend_promql::lower_promql_query_workload; -use asap_types::types::AccuracyTarget; +use asap_plan_selection::pass::{run_stage1_to_stage3, PlanningModels, Stage1ToStage3Run}; +use asap_types::ir::properties::AccuracyTarget; use asap_types::workload::{ BatchEntry, DataWorkload, DurationMs, PlanningWorkload, Query, QueryLanguage, QueryRequirements, QueryWorkload, diff --git a/crates/planner/tests/summary_sharing.rs b/crates/planner/tests/summary_sharing.rs index c901eabd4..f7dbb4864 100644 --- a/crates/planner/tests/summary_sharing.rs +++ b/crates/planner/tests/summary_sharing.rs @@ -7,18 +7,18 @@ use asap_types::ir::{ASAPOp, OperatorNode, QueryRoot}; use std::rc::Rc; -use asap_aware_mapping::pass::{ - run_stage1_to_stage3, PlanOutput, PlanningModels, QueryPlan, Stage1ToStage3Run, -}; -use asap_aware_mapping::realization::{default_size_params, DEFAULT_DELTA}; -use asap_aware_mapping::stage3_plan_selection::RejectionCause; use asap_frontend_promql::{lower_promql_query_workload, promql_metric_names}; use asap_frontend_sql::SqlCatalog; +use asap_logical_asap_optimization::realization::{default_size_params, DEFAULT_DELTA}; +use asap_plan_selection::pass::{ + run_stage1_to_stage3, PlanOutput, PlanningModels, QueryPlan, Stage1ToStage3Run, +}; +use asap_plan_selection::RejectionCause; use asap_planner::{e2e_plan, FrontendInput, UserInput}; -use asap_types::post_asap::{FieldDataType, SketchAlgorithm, SketchParams}; -use asap_types::pre_asap::agg_intent::default_quantile; -use asap_types::pre_asap::schema::{DataType, Field, Schema}; -use asap_types::types::{AccuracyMetric, AccuracyTarget}; +use asap_types::ir::operator::agg_intent::default_quantile; +use asap_types::ir::properties::{AccuracyMetric, AccuracyTarget}; +use asap_types::ir::schema::{DataType, Field, Schema}; +use asap_types::ir::schema::{FieldDataType, SketchAlgorithm, SketchParams}; use asap_types::workload::{ DataArrival, DataWorkload, DurationMs, PlanningWorkload, Predictability, Query, QueryLanguage, QueryRequirements, QueryWorkload, RepeatedDemand, RepeatingEntry, RepetitionInterval, @@ -554,7 +554,7 @@ async fn exact_state_roots_are_finalized() { ); // With nothing materialized, the root and the exact state it reads // run at query time. - use asap_types::post_asap::{DataPrimitive, ExecutionDataState, ExecutionTiming}; + use asap_types::ir::properties::{DataPrimitive, ExecutionDataState, ExecutionTiming}; assert_eq!(root.output_state, ExecutionDataState::QUERY_ROWS, "{query}"); let state = dag .edges diff --git a/crates/sql-function-catalog/src/lib.rs b/crates/sql-function-catalog/src/lib.rs index f2aedbaa9..6b2a0ceb5 100644 --- a/crates/sql-function-catalog/src/lib.rs +++ b/crates/sql-function-catalog/src/lib.rs @@ -74,7 +74,7 @@ pub enum Arity { /// classification [`lower_agg_intent`](../asap_frontend_sql/index.html) /// switches on to build the real `AggIntent`. /// -/// Deliberately *not* `asap_types::pre_asap::agg_intent::AggIntent` itself: +/// Deliberately *not* `asap_types::ir::operator::agg_intent::AggIntent` itself: /// most `AggIntent` variants carry call-site-only state that isn't a /// function of the name alone -- the ambient `AccuracyTarget` (thread-local, /// not catalog data), φ pulled from a call's literal 2nd argument, whether diff --git a/crates/types/src/ir/canonicalize.rs b/crates/types/src/ir/canonicalize.rs index a5d64511d..abfcf7dc5 100644 --- a/crates/types/src/ir/canonicalize.rs +++ b/crates/types/src/ir/canonicalize.rs @@ -30,14 +30,14 @@ use std::collections::HashMap; use std::rc::Rc; -use super::node::{Operator, OperatorNode}; -use super::non_asap::NonASAPOp; use super::scalar::{ExprSemantics, Predicate, ProjectItem, ScalarExpr, SortKey}; -use crate::ir::operator_properties::{JoinKind, Reduction}; +use crate::ir::operator::agg_intent::{topk, AggIntent}; +use crate::ir::operator::node::{Operator, OperatorNode}; +use crate::ir::operator::non_asap::NonASAPOp; +use crate::ir::operator::operator_properties::{JoinKind, Reduction}; +use crate::ir::properties::AccuracyTarget; +use crate::ir::scalar::expr_ir::{CompareOpKind, ScalarValue}; use crate::ir::SchemaDerivationError; -use crate::pre_asap::agg_intent::{topk, AggIntent}; -use crate::pre_asap::expr_ir::{CompareOpKind, ScalarValue}; -use crate::types::AccuracyTarget; /// Rewrite the DAG under `root` into its canonical form (bottom-up). /// Idempotent: an already-canonical DAG comes back as the same `Rc`. Only @@ -220,7 +220,7 @@ fn try_promote_additive_top_ranking( }; // A heavy-hitter sketch ranks the raw update stream; a filtered measure // counts only part of it, and no binding rule applies the filter (#466). - if crate::ir::non_asap::any_measure_filtered(filters) { + if crate::ir::operator::non_asap::any_measure_filtered(filters) { return Ok(None); } let [ranked_agg] = measures.as_slice() else { @@ -374,12 +374,12 @@ fn contains_scalar_subquery(expr: &ScalarExpr) -> bool { #[cfg(test)] mod tests { use super::*; - use crate::ir::operator_properties::WindowFuncKind; - use crate::ir::operator_properties::{ + use crate::ir::operator::operator_properties::WindowFuncKind; + use crate::ir::operator::operator_properties::{ ConcatDiscriminatorKey, GroupKeys, Source, WindowFrame, WindowFrameBound, WindowFrameOffset, WindowFrameUnits, }; - use crate::pre_asap::schema::{DataType, Field, Schema}; + use crate::ir::schema::{DataType, Field, Schema}; fn node(op: NonASAPOp) -> Rc { Rc::new(OperatorNode::new(Operator::NonASAP(op)).expect("fixture derives a schema")) @@ -514,8 +514,8 @@ mod tests { // `COUNT(col)` over a nullable column. #[test] fn does_not_promote_a_filtered_count_ranking() { + use crate::ir::scalar::{CompareOpKind, ScalarValue}; use crate::ir::Predicate; - use crate::pre_asap::{CompareOpKind, ScalarValue}; let greater = Predicate(ScalarExpr::Compare { left: Box::new(ScalarExpr::Column(2)), op: CompareOpKind::Gt, diff --git a/crates/types/src/ir/cse.rs b/crates/types/src/ir/cse.rs index 3305468bd..8ab2541bb 100644 --- a/crates/types/src/ir/cse.rs +++ b/crates/types/src/ir/cse.rs @@ -55,7 +55,7 @@ use std::collections::HashMap; use std::hash::{Hash, Hasher}; use std::rc::Rc; -use super::node::{Operator, OperatorNode}; +use crate::ir::operator::node::{Operator, OperatorNode}; /// [`structural_hash`]'s memoization cache: an already-hashed node's `Rc` /// pointer to its hash. A fresh cache is always correct; what matters is @@ -281,19 +281,19 @@ pub fn share_common_sub_dags( #[cfg(test)] mod tests { use super::*; - use crate::ir::asap::ASAPOp; - use crate::ir::operator_properties::{BinaryOpKind, GroupKeys, Reduction, Source}; - use crate::ir::ScalarExpr; - use crate::ir::{BinaryOperator, NonASAPOp}; - use crate::post_asap::guarantee::AccuracyGuarantee; - use crate::post_asap::sketch::{ + use crate::ir::operator::agg_intent::AggIntent; + use crate::ir::operator::asap::ASAPOp; + use crate::ir::operator::operator_properties::{BinaryOpKind, GroupKeys, Reduction, Source}; + use crate::ir::properties::guarantee::AccuracyGuarantee; + use crate::ir::scalar::expr_ir::{ColumnRef, CompareOpKind}; + use crate::ir::schema::sketch::{ GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, SummaryUpdate, }; - use crate::pre_asap::agg_intent::AggIntent; - use crate::pre_asap::expr_ir::{ColumnRef, CompareOpKind}; - use crate::pre_asap::schema::{DataType, Field, FieldDataType, Schema}; + use crate::ir::schema::{DataType, Field, FieldDataType, Schema}; + use crate::ir::ScalarExpr; + use crate::ir::{BinaryOperator, NonASAPOp}; - use crate::types::AccuracyTarget; + use crate::ir::properties::AccuracyTarget; fn node(op: NonASAPOp) -> Rc { OperatorNode::new_shared(crate::ir::Operator::NonASAP(op)).unwrap() @@ -369,8 +369,8 @@ mod tests { // compute different values, so structural sharing keeps them apart. #[test] fn filtered_and_unfiltered_aggregates_do_not_merge() { + use crate::ir::scalar::ScalarValue; use crate::ir::Predicate; - use crate::pre_asap::ScalarValue; let unfiltered = quantile_agg(vec![1], Some(2), 0.5); let filtered = node(NonASAPOp::Aggregate { reduction: Reduction::by(vec![1]), @@ -601,7 +601,7 @@ mod tests { #[test] fn evaluations_share_their_producer_but_not_each_other() { - use crate::post_asap::sketch::SketchStatistic; + use crate::ir::schema::sketch::SketchStatistic; let evaluation = |q: f64| { Rc::new(OperatorNode::with_schema( Operator::ASAP(ASAPOp::SummaryEstimate { diff --git a/crates/types/src/ir/flat.rs b/crates/types/src/ir/flat.rs index 4c80a9c05..2b651d7a3 100644 --- a/crates/types/src/ir/flat.rs +++ b/crates/types/src/ir/flat.rs @@ -13,11 +13,12 @@ use std::rc::Rc; use serde::{Deserialize, Serialize}; -use super::node::{Operator, OperatorNode, OperatorResultKind}; -use super::query::QueryRoot; -use crate::post_asap::execution_data_state::{ExecutionTiming, Materialization}; -use crate::post_asap::guarantee::AccuracyGuarantee; -use crate::pre_asap::schema::Schema; +use crate::ir::operator::node::{Operator, OperatorNode, OperatorResultKind}; +use crate::ir::operator::query::QueryRoot; +use crate::ir::properties::execution_data_state::ExecutionTiming; +use crate::ir::properties::guarantee::AccuracyGuarantee; +use crate::ir::schema::Schema; +use crate::physical::materialization::Materialization; /// Index of a node in [`FlatDag::nodes`]. pub type NodeId = usize; @@ -88,9 +89,9 @@ fn visit( #[cfg(test)] mod tests { use super::*; + use crate::ir::scalar::expr_ir::ScalarValue; + use crate::ir::schema::{DataType, Field}; use crate::ir::{NonASAPOp, Predicate, ProjectItem, ScalarExpr}; - use crate::pre_asap::expr_ir::ScalarValue; - use crate::pre_asap::schema::{DataType, Field}; fn values() -> Rc { OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Values { diff --git a/crates/types/src/ir/mod.rs b/crates/types/src/ir/mod.rs index 262aaf903..fefa42f62 100644 --- a/crates/types/src/ir/mod.rs +++ b/crates/types/src/ir/mod.rs @@ -1,32 +1,22 @@ -//! The operator IR from #511: one operator DAG for every planning stage. -pub mod aggregate_schema; -pub mod asap; -pub mod error; -pub mod node; -pub mod non_asap; -pub mod operator_properties; -pub mod pane_chain; -pub mod query; +//! The operator IR from #511: one operator DAG for every planning stage, +//! arranged by IR section. +pub mod operator; +pub mod properties; pub mod scalar; -pub use asap::{ASAPOp, UNIMPLEMENTED_ASAP_OP}; -pub use error::SchemaDerivationError; -pub use node::{Operator, OperatorNode, OperatorResultKind}; -pub use non_asap::{BinaryOperator, NonASAPOp, TimeRangeKind}; -pub use query::QueryRoot; +pub mod schema; +pub use operator::asap::{ASAPOp, UNIMPLEMENTED_ASAP_OP}; +pub use operator::node::{Operator, OperatorNode, OperatorResultKind}; +pub use operator::non_asap::{BinaryOperator, NonASAPOp, TimeRangeKind}; +pub use operator::query::QueryRoot; pub use scalar::{ExprSemantics, Predicate, ProjectItem, ScalarExpr, SortKey}; +pub use schema::error::SchemaDerivationError; pub mod canonicalize; pub mod cse; pub mod flat; /// Physical ASAP DAG: the flattened operators plus execution timing. pub mod physical_export; -/// Execution timing for physical plans: a materialization assignment expanded onto every node. -pub mod timing; -pub use crate::post_asap::execution_data_state::{Materialization, Medium}; -pub use timing::{ +pub use properties::timing::{ apply_materialization_timings, data_state, is_filtered_build, planned_data_state, split_shared_by_timing, validate_at_ingestion, MaterializationAssignment, TimingMemo, }; -pub mod schema_support; -/// Semantic observation coverage, separate from field layout and physical timing. -pub mod summary_coverage; diff --git a/crates/types/src/pre_asap/agg_intent.rs b/crates/types/src/ir/operator/agg_intent.rs similarity index 98% rename from crates/types/src/pre_asap/agg_intent.rs rename to crates/types/src/ir/operator/agg_intent.rs index ca74e8879..c79cc697d 100644 --- a/crates/types/src/pre_asap/agg_intent.rs +++ b/crates/types/src/ir/operator/agg_intent.rs @@ -15,9 +15,9 @@ use serde::{Deserialize, Serialize}; -use crate::ir::operator_properties::DataModel; -use crate::pre_asap::schema::{ColumnId, DataType, Field, FieldDataType}; -use crate::types::AccuracyTarget; +use crate::ir::operator::operator_properties::DataModel; +use crate::ir::properties::AccuracyTarget; +use crate::ir::schema::{ColumnId, DataType, Field, FieldDataType}; /// "What to compute" — the vocabulary the planner pivots on. /// @@ -392,7 +392,7 @@ impl AggIntent { /// malformed selectors fail instead of acquiring a fabricated output type. pub fn arg_selector_columns( &self, - schema: &super::schema::Schema, + schema: &crate::ir::schema::Schema, ) -> Result, String> { let Self::Extension { ext_kind, payload } = self else { return Ok(None); @@ -407,14 +407,14 @@ impl AggIntent { return Err("arg selector requires arg_col and val_col only".into()); } let resolve = |field: &str| -> Result { - let reference: super::expr_ir::ColumnRef = serde_json::from_value( + let reference: crate::ir::scalar::expr_ir::ColumnRef = serde_json::from_value( fields .get(field) .ok_or_else(|| format!("missing arg selector {field}"))? .clone(), ) .map_err(|e| e.to_string())?; - super::column_resolution::resolve_column_ref(&reference, schema) + crate::ir::scalar::column_resolution::resolve_column_ref(&reference, schema) .map_err(|e| e.to_string()) }; Ok(Some((resolve("arg_col")?, resolve("val_col")?))) @@ -734,7 +734,7 @@ pub fn default_quantile(q: f64) -> AggIntent { #[cfg(test)] mod tests { use super::*; - use crate::pre_asap::schema::{DataType, Field}; + use crate::ir::schema::{DataType, Field}; fn c(name: &str, dtype: DataType) -> Field { Field::plain(name, dtype, false) @@ -962,7 +962,8 @@ mod tests { #[cfg(test)] mod arg_selector_contract_tests { use super::*; - use crate::pre_asap::{ColumnRef, Schema}; + use crate::ir::scalar::ColumnRef; + use crate::ir::schema::Schema; #[test] fn arg_selector_rejects_missing_or_unresolved_arguments() { let schema = Schema::new(vec![Field::plain("value", DataType::Float64, false)]); diff --git a/crates/types/src/ir/asap.rs b/crates/types/src/ir/operator/asap.rs similarity index 92% rename from crates/types/src/ir/asap.rs rename to crates/types/src/ir/operator/asap.rs index 7ddd2ff15..7cf6f5dd7 100644 --- a/crates/types/src/ir/asap.rs +++ b/crates/types/src/ir/operator/asap.rs @@ -6,11 +6,11 @@ use std::rc::Rc; use serde::{Deserialize, Serialize}; use super::node::{OperatorNode, OperatorResultKind}; -use crate::ir::operator_properties::Reduction; +use crate::ir::operator::maintained_population::{MaintainedPopulation, PopulationStatistic}; +use crate::ir::operator::operator_properties::Reduction; +use crate::ir::schema::sketch::{GroupingStrategy, SketchStatistic, SummaryUpdate}; +use crate::ir::schema::{ColumnId, DataType, Field, FieldDataType, Schema}; use crate::ir::SchemaDerivationError; -use crate::post_asap::maintained_population::{MaintainedPopulation, PopulationStatistic}; -use crate::post_asap::sketch::{GroupingStrategy, SketchStatistic, SummaryUpdate}; -use crate::pre_asap::schema::{ColumnId, DataType, Field, FieldDataType, Schema}; /// Why an ASAP operator cannot be used yet. pub const UNIMPLEMENTED_ASAP_OP: &str = @@ -31,7 +31,7 @@ pub enum ASAPOp> { reduction: Reduction, grouping: GroupingStrategy, #[serde(default)] - filter: Option>, + filter: Option>, }, /// Evaluate a query result from built summary state. Output is a /// row-shaped schema. @@ -69,10 +69,10 @@ pub enum ASAPOp> { /// the `SummaryAgg` schema, one state per group and pane; a /// `SummaryMerge` over the chain gives the state of one build over the /// whole `pane_count · pane_width_ms` window - /// ([`whole_window_build`](crate::ir::pane_chain::whole_window_build)). + /// ([`whole_window_build`](crate::ir::operator::pane_chain::whole_window_build)). /// One compact node whatever the pane count: costs and guarantees come /// from formulas, and the executor expands the panes - /// ([`expand_pane_chains`](crate::ir::pane_chain::expand_pane_chains)). + /// ([`expand_pane_chains`](crate::ir::operator::pane_chain::expand_pane_chains)). PaneChain { child: C, family: FieldDataType, @@ -80,7 +80,7 @@ pub enum ASAPOp> { reduction: Reduction, grouping: GroupingStrategy, #[serde(default)] - filter: Option>, + filter: Option>, pane_width_ms: u64, pane_count: u32, }, @@ -149,7 +149,7 @@ impl ASAPOp { grouping: grouping.clone(), filter: filter .as_ref() - .map(|p| super::scalar::Predicate(p.0.map_operator_refs(&mut f))), + .map(|p| crate::ir::scalar::Predicate(p.0.map_operator_refs(&mut f))), }, PaneChain { child, @@ -168,7 +168,7 @@ impl ASAPOp { grouping: grouping.clone(), filter: filter .as_ref() - .map(|p| super::scalar::Predicate(p.0.map_operator_refs(&mut f))), + .map(|p| crate::ir::scalar::Predicate(p.0.map_operator_refs(&mut f))), pane_width_ms: *pane_width_ms, pane_count: *pane_count, }, @@ -286,15 +286,15 @@ impl ASAPOp { reduction, .. } => { - let mut schema = crate::ir::aggregate_schema::aggregate_output_schema( + let mut schema = crate::ir::schema::aggregate_schema::aggregate_output_schema( &child.schema, reduction, - &[crate::pre_asap::AggIntent::Sum { col: None }], + &[crate::ir::operator::AggIntent::Sum { col: None }], &[], )?; let index = match reduction { - Reduction::PerEntity => crate::pre_asap::resolve_column_ref( - &crate::pre_asap::ColumnRef::SampleValue, + Reduction::PerEntity => crate::ir::scalar::resolve_column_ref( + &crate::ir::scalar::ColumnRef::SampleValue, &schema, ) .map_err(|e| SchemaDerivationError::InvalidScalarSignature(e.to_string()))?, @@ -359,11 +359,11 @@ impl ASAPOp { }) = child.summary_build() { { - use crate::post_asap::{ExactKind, SummaryInputExpr}; - use crate::pre_asap::AggIntent; + use crate::ir::operator::AggIntent; + use crate::ir::schema::{ExactKind, SummaryInputExpr}; let column = match &input.weight { SummaryInputExpr::Column(col) => Some( - crate::pre_asap::column_resolution::resolve_column_ref( + crate::ir::scalar::column_resolution::resolve_column_ref( col, &source.schema, ) @@ -378,7 +378,7 @@ impl ASAPOp { ExactKind::Min => Some(AggIntent::Min { col: column }), ExactKind::Max => Some(AggIntent::Max { col: column }), ExactKind::Count => Some(AggIntent::Count { - accuracy: crate::types::AccuracyTarget::exact(), + accuracy: crate::ir::properties::AccuracyTarget::exact(), }), ExactKind::Rate => Some(AggIntent::Rate), ExactKind::IRate => Some(AggIntent::IRate), @@ -386,7 +386,7 @@ impl ASAPOp { }; measure .map(|measure| { - super::NonASAPOp::Aggregate { + crate::ir::NonASAPOp::Aggregate { child: Rc::clone(source), reduction: reduction.clone(), measures: vec![measure], @@ -427,8 +427,8 @@ impl ASAPOp { } MaintainPopulation { child, .. } => child.schema.clone(), EvaluatePopulation { child, evaluation } => { - use crate::post_asap::maintained_population::PopulationInput; - use crate::pre_asap::{AggIntent, GroupKeys}; + use crate::ir::operator::maintained_population::PopulationInput; + use crate::ir::operator::{AggIntent, GroupKeys}; let Some(MaintainPopulation { child: source, population, @@ -469,7 +469,7 @@ impl ASAPOp { ) } }; - let accuracy = crate::types::AccuracyTarget::exact(); + let accuracy = crate::ir::properties::AccuracyTarget::exact(); let measure = match evaluation { PopulationStatistic::Quantile { q } => AggIntent::Quantile { q: *q, @@ -481,7 +481,7 @@ impl ASAPOp { PopulationStatistic::Average => AggIntent::Avg { col: column }, PopulationStatistic::TopK { .. } => unreachable!(), }; - super::NonASAPOp::Aggregate { + crate::ir::NonASAPOp::Aggregate { child: source.clone(), reduction: Reduction::Reduce(keys), measures: vec![measure], @@ -574,7 +574,7 @@ impl ASAPOp { query, } => { needs_state(summary_input, "SummaryEstimate")?; - use crate::post_asap::sketch::SketchCategory as C; + use crate::ir::schema::sketch::SketchCategory as C; let states: Vec<_> = summary_input .schema .fields @@ -692,13 +692,13 @@ impl ASAPOp { )); } fn check( - expr: &crate::post_asap::SummaryInputExpr, + expr: &crate::ir::schema::SummaryInputExpr, schema: &Schema, ) -> Result<(), SchemaDerivationError> { - use crate::post_asap::SummaryInputExpr; + use crate::ir::schema::SummaryInputExpr; match expr { SummaryInputExpr::Column(col) => { - crate::pre_asap::resolve_column_ref(col, schema).map_err(|e| { + crate::ir::scalar::resolve_column_ref(col, schema).map_err(|e| { SchemaDerivationError::InvalidScalarSignature(e.to_string()) })?; } @@ -727,8 +727,8 @@ impl ASAPOp { } /// The plain value an exact accumulator finalizes to. -fn finalized_data_type(kind: &crate::post_asap::sketch::ExactKind) -> DataType { - use crate::post_asap::sketch::ExactKind; +fn finalized_data_type(kind: &crate::ir::schema::sketch::ExactKind) -> DataType { + use crate::ir::schema::sketch::ExactKind; match kind { ExactKind::Count => DataType::Int64, _ => DataType::Float64, @@ -785,8 +785,8 @@ fn quantile_may_be_empty(state: &OperatorNode) -> bool { } let global = matches!(reduction, Reduction::Reduce(by) if by.keys().is_empty()); let nullable = match &input.weight { - crate::post_asap::sketch::SummaryInputExpr::Column(column) => { - crate::pre_asap::resolve_column_ref(column, &child.schema) + crate::ir::schema::sketch::SummaryInputExpr::Column(column) => { + crate::ir::scalar::resolve_column_ref(column, &child.schema) .ok() .and_then(|index| child.schema.fields.get(index)) .is_some_and(|field| field.nullable) diff --git a/crates/types/src/post_asap/maintained_population.rs b/crates/types/src/ir/operator/maintained_population.rs similarity index 95% rename from crates/types/src/post_asap/maintained_population.rs rename to crates/types/src/ir/operator/maintained_population.rs index 5e64b5436..2d2eebc30 100644 --- a/crates/types/src/post_asap/maintained_population.rs +++ b/crates/types/src/ir/operator/maintained_population.rs @@ -43,7 +43,7 @@ pub enum PopulationInput { Rows { input: std::rc::Rc, value_column: usize, - grouping: crate::pre_asap::GroupKeys, + grouping: crate::ir::operator::GroupKeys, }, } @@ -57,8 +57,10 @@ pub struct MaintainedPopulation { impl CurrentSeriesInput { /// Verify the named contract against the canonical maintenance input. pub fn matches_node(&self, input: &crate::ir::OperatorNode) -> bool { + use crate::ir::operator::Source; + use crate::ir::scalar::{CompareOpKind, ScalarValue}; + use crate::ir::schema::DataType; use crate::ir::{NonASAPOp, Operator, ScalarExpr, TimeRangeKind}; - use crate::pre_asap::{CompareOpKind, DataType, ScalarValue, Source}; let input = match &input.operator { Operator::NonASAP(NonASAPOp::TimeRange { range, @@ -137,8 +139,9 @@ impl CurrentSeriesInput { impl MaintainedPopulation { /// Whether `input` is the maintenance input this population declares. pub fn matches_node(&self, input: &crate::ir::OperatorNode) -> bool { + use crate::ir::operator::Source; + use crate::ir::schema::DataType; use crate::ir::{NonASAPOp, Operator}; - use crate::pre_asap::{DataType, Source}; match &self.input { PopulationInput::CurrentSeries(spec) => spec.matches_node(input), PopulationInput::Rows { diff --git a/crates/types/src/ir/operator/mod.rs b/crates/types/src/ir/operator/mod.rs new file mode 100644 index 000000000..f1198045a --- /dev/null +++ b/crates/types/src/ir/operator/mod.rs @@ -0,0 +1,31 @@ +//! #511 §1: the operators of the unified DAG and the parameter types their +//! payloads are built from. +//! +//! - [`node`] — [`OperatorNode`](node::OperatorNode): an operator plus the +//! planning properties every traversal needs. +//! - [`non_asap`] / [`asap`] — ordinary query operators and ASAP operators. +//! - [`operator_properties`] — operator parameter types ([`GroupKeys`], +//! [`Reduction`], [`Source`], …). +//! - [`agg_intent`] — the aggregation-intent vocabulary ([`AggIntent`]). +//! - [`maintained_population`] — maintained populations and their evaluations. +//! - [`pane_chain`] — what a tumbling-pane chain computes. +//! - [`query`] — a query root: an operator result or a standalone scalar. +pub mod agg_intent; +pub mod asap; +pub mod maintained_population; +pub mod node; +pub mod non_asap; +pub mod operator_properties; +pub mod pane_chain; +pub mod query; + +pub use agg_intent::{ + agg_accuracy, agg_is_exact, agg_is_mergeable, default_cardinality, default_quantile, AggIntent, + MathFunc, TimeFunc, +}; +pub use operator_properties::{ + AtModifier, BinaryOpKind, ColState, ConcatDiscriminatorKey, DataModel, GroupKeys, GroupSide, + InfoMatcher, JoinKind, PromQLVectorSetOpKind, Reduction, RelationalSetOpKind, SampleKind, + Source, TimeShift, VectorGrouping, VectorMatch, VectorMatchKind, WindowFrame, WindowFrameBound, + WindowFrameOffset, WindowFrameUnits, WindowFuncKind, +}; diff --git a/crates/types/src/ir/node.rs b/crates/types/src/ir/operator/node.rs similarity index 95% rename from crates/types/src/ir/node.rs rename to crates/types/src/ir/operator/node.rs index 622bf80e9..dd851ed2b 100644 --- a/crates/types/src/ir/node.rs +++ b/crates/types/src/ir/operator/node.rs @@ -10,11 +10,12 @@ use serde::{Deserialize, Serialize}; use super::asap::ASAPOp; use super::non_asap::NonASAPOp; -use super::summary_coverage::{CoverageMemo, SummaryCoverage}; +use crate::ir::properties::execution_data_state::ExecutionTiming; +use crate::ir::properties::guarantee::AccuracyGuarantee; +use crate::ir::properties::summary_coverage::{CoverageMemo, SummaryCoverage}; +use crate::ir::schema::Schema; use crate::ir::SchemaDerivationError; -use crate::post_asap::execution_data_state::{ExecutionTiming, Materialization}; -use crate::post_asap::guarantee::AccuracyGuarantee; -use crate::pre_asap::schema::Schema; +use crate::physical::materialization::Materialization; /// The output category of an operator, derived from the operation and its /// inputs. Matching column schemas do not make categories interchangeable. @@ -297,9 +298,9 @@ impl OperatorNode { "execution timing is unassigned".into(), ) })?; - if timing == crate::post_asap::ExecutionTiming::IngestionTime + if timing == crate::ir::properties::ExecutionTiming::IngestionTime && node.children().iter().any(|child| { - child.timing != Some(crate::post_asap::ExecutionTiming::IngestionTime) + child.timing != Some(crate::ir::properties::ExecutionTiming::IngestionTime) }) { return Err(SchemaDerivationError::InvalidScalarSignature( @@ -322,10 +323,9 @@ impl OperatorNode { let mut coverage = CoverageMemo::new(); for node in Self::reachable(self) { if node.schema.time_index.is_some_and(|i| { - node.schema - .fields - .get(i) - .is_none_or(|f| f.plain_dtype() != Some(&crate::pre_asap::DataType::Timestamp)) + node.schema.fields.get(i).is_none_or(|f| { + f.plain_dtype() != Some(&crate::ir::schema::DataType::Timestamp) + }) }) || node .schema .unique_keys diff --git a/crates/types/src/ir/non_asap.rs b/crates/types/src/ir/operator/non_asap.rs similarity index 99% rename from crates/types/src/ir/non_asap.rs rename to crates/types/src/ir/operator/non_asap.rs index bb05197e8..99ecfeb5a 100644 --- a/crates/types/src/ir/non_asap.rs +++ b/crates/types/src/ir/operator/non_asap.rs @@ -7,15 +7,15 @@ use std::time::Duration; use serde::{Deserialize, Serialize}; use super::node::{OperatorNode, OperatorResultKind}; -use super::scalar::{Predicate, ProjectItem, ScalarExpr, SortKey}; -use crate::ir::aggregate_schema::aggregate_output_schema; -use crate::ir::operator_properties::{ +use crate::ir::operator::agg_intent::AggIntent; +use crate::ir::operator::operator_properties::{ BinaryOpKind, ConcatDiscriminatorKey, GroupKeys, InfoMatcher, JoinKind, Reduction, RelationalSetOpKind, SampleKind, Source, TimeShift, VectorMatch, WindowFrame, WindowFuncKind, }; +use crate::ir::scalar::{Predicate, ProjectItem, ScalarExpr, SortKey}; +use crate::ir::schema::aggregate_schema::aggregate_output_schema; +use crate::ir::schema::{ColumnId, DataType, Field, FieldDataType, Schema}; use crate::ir::SchemaDerivationError; -use crate::pre_asap::agg_intent::AggIntent; -use crate::pre_asap::schema::{ColumnId, DataType, Field, FieldDataType, Schema}; /// All semantics owned by a binary operator. #[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] @@ -785,8 +785,8 @@ impl NonASAPOp { .and_then(|m| m.grouping.as_ref()); let right_rows = matches!( operator.kind, - BinaryOpKind::Set(crate::pre_asap::PromQLVectorSetOpKind::Or) - ) || matches!(grouping, Some(g) if g.side == crate::pre_asap::GroupSide::Right); + BinaryOpKind::Set(crate::ir::operator::PromQLVectorSetOpKind::Or) + ) || matches!(grouping, Some(g) if g.side == crate::ir::operator::GroupSide::Right); let mut additions = Vec::new(); if right_rows { additions.extend( @@ -1111,11 +1111,11 @@ pub fn any_measure_filtered(filters: &[Option]) -> bool { #[cfg(test)] mod tests { use super::*; - use crate::ir::operator_properties::{ + use crate::ir::operator::operator_properties::{ AtModifier, VectorMatchKind, WindowFrameBound, WindowFrameOffset, WindowFrameUnits, }; + use crate::ir::scalar::expr_ir::{ArithmeticOpKind, CompareOpKind, ScalarValue}; use crate::ir::scalar::ExprSemantics; - use crate::pre_asap::expr_ir::{ArithmeticOpKind, CompareOpKind, ScalarValue}; fn col(name: &str, dtype: DataType, nullable: bool) -> Field { Field::plain(name, dtype, nullable) diff --git a/crates/types/src/ir/operator_properties.rs b/crates/types/src/ir/operator/operator_properties.rs similarity index 99% rename from crates/types/src/ir/operator_properties.rs rename to crates/types/src/ir/operator/operator_properties.rs index 7a42aad90..1403ca553 100644 --- a/crates/types/src/ir/operator_properties.rs +++ b/crates/types/src/ir/operator/operator_properties.rs @@ -3,8 +3,9 @@ //! For example, `Aggregate.by` uses [`GroupKeys`], a join chooses [`JoinKind`], //! and a SQL window carries [`WindowFrame`]. These types describe what an //! operator does. Derived node metadata (schema, guarantee, timing) lives on -//! [`super::OperatorNode`], not in this module. -use crate::pre_asap::{ArithmeticOpKind, ColumnId, ColumnRef, CompareOpKind, ScalarValue}; +//! [`crate::ir::OperatorNode`], not in this module. +use crate::ir::scalar::{ArithmeticOpKind, ColumnRef, CompareOpKind, ScalarValue}; +use crate::ir::schema::ColumnId; use serde::{Deserialize, Serialize}; /// The column-reference type an operator parameter is generic over: /// positional [`ColumnId`] once bound, name-based [`ColumnRef`] before. diff --git a/crates/types/src/ir/pane_chain.rs b/crates/types/src/ir/operator/pane_chain.rs similarity index 99% rename from crates/types/src/ir/pane_chain.rs rename to crates/types/src/ir/operator/pane_chain.rs index 6b9c93dc0..7fb0ff61d 100644 --- a/crates/types/src/ir/pane_chain.rs +++ b/crates/types/src/ir/operator/pane_chain.rs @@ -15,8 +15,8 @@ use super::asap::ASAPOp; use super::node::{Operator, OperatorNode}; use super::non_asap::{NonASAPOp, TimeRangeKind}; use super::operator_properties::TimeShift; -use super::SchemaDerivationError; -use crate::post_asap::ExecutionTiming; +use crate::ir::properties::ExecutionTiming; +use crate::ir::SchemaDerivationError; fn invalid(message: &str) -> SchemaDerivationError { SchemaDerivationError::InvalidScalarSignature(message.into()) diff --git a/crates/types/src/ir/query.rs b/crates/types/src/ir/operator/query.rs similarity index 90% rename from crates/types/src/ir/query.rs rename to crates/types/src/ir/operator/query.rs index 67ec50359..542936334 100644 --- a/crates/types/src/ir/query.rs +++ b/crates/types/src/ir/operator/query.rs @@ -1,6 +1,6 @@ //! Query results are either an operator result or a standalone scalar expression. //! The root discriminator is not an operator and never creates a dag node. -use super::{OperatorNode, ScalarExpr}; +use crate::ir::{OperatorNode, ScalarExpr}; use serde::{Deserialize, Serialize}; use std::rc::Rc; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -18,7 +18,7 @@ impl QueryRoot { match self { Self::Operator(node) => node.validate_structure(), Self::Scalar(expr) => { - expr.scalar_type(&crate::pre_asap::Schema::default())?; + expr.scalar_type(&crate::ir::schema::Schema::default())?; for node in expr.operator_refs() { node.validate_structure()?; } diff --git a/crates/types/src/ir/physical_export.rs b/crates/types/src/ir/physical_export.rs index 562399f7f..3e27090c2 100644 --- a/crates/types/src/ir/physical_export.rs +++ b/crates/types/src/ir/physical_export.rs @@ -2,7 +2,7 @@ //! //! The flattened operators of [`super::flat`], plus the materialization and //! execution timing (data state) of every node and edge. The input must -//! already be timed ([`super::timing::apply_materialization_timings`]); export +//! already be timed ([`crate::ir::properties::timing::apply_materialization_timings`]); export //! reads each node's timing and materialization and does not re-run //! data-state validation. The deployment executes the plan as given //! (#509 Assumption 8): it does not change materialization or timing. @@ -13,19 +13,20 @@ use std::rc::Rc; use serde::{Deserialize, Serialize}; use thiserror::Error; -use super::asap::ASAPOp; use super::flat::{flatten, NodeId}; -use super::node::{Operator, OperatorNode}; -use super::non_asap::NonASAPOp; -use super::operator_properties::Reduction; -use super::query::QueryRoot; use super::scalar::ScalarExpr; -use super::timing::{data_state, is_materialization_point}; -use crate::post_asap::execution_data_state::{ - ExecutionDataState, ExecutionDataStateError, ExecutionTiming, Materialization, +use crate::ir::operator::asap::ASAPOp; +use crate::ir::operator::node::{Operator, OperatorNode}; +use crate::ir::operator::non_asap::NonASAPOp; +use crate::ir::operator::operator_properties::Reduction; +use crate::ir::operator::query::QueryRoot; +use crate::ir::properties::execution_data_state::{ + ExecutionDataState, ExecutionDataStateError, ExecutionTiming, }; -use crate::post_asap::guarantee::AccuracyGuarantee; -use crate::pre_asap::schema::{FieldDataType, Schema}; +use crate::ir::properties::guarantee::AccuracyGuarantee; +use crate::ir::properties::timing::{data_state, is_materialization_point}; +use crate::ir::schema::{FieldDataType, Schema}; +use crate::physical::materialization::Materialization; pub const PHYSICAL_ASAP_DAG_WIRE_VERSION: u32 = 10; @@ -409,7 +410,7 @@ pub fn compile_physical_asap_dag( } /// Export the timed DAG below `root`. Every reachable node must carry a -/// timing (see [`super::timing::apply_materialization_timings`]); the data-state +/// timing (see [`crate::ir::properties::timing::apply_materialization_timings`]); the data-state /// rules were checked by that pass and are not re-run here. pub fn compile_physical_asap_dag_with_node_ids( root: &Rc, diff --git a/crates/types/src/types.rs b/crates/types/src/ir/properties/accuracy.rs similarity index 97% rename from crates/types/src/types.rs rename to crates/types/src/ir/properties/accuracy.rs index 1e56792e4..7eba92182 100644 --- a/crates/types/src/types.rs +++ b/crates/types/src/ir/properties/accuracy.rs @@ -3,15 +3,15 @@ //! A requirement and a guarantee share one outer shape, [`AccuracySpec`]: //! an error metric, an error ε, a failure probability δ and a scope. A //! requirement ([`AccuracyTarget`]) holds concrete numbers and may leave its -//! metric unset; a guarantee ([`crate::post_asap::AccuracyGuarantee`]) +//! metric unset; a guarantee ([`crate::ir::properties::AccuracyGuarantee`]) //! always states its metric and holds symbolic expressions that plan //! selection binds. A target without a metric accepts a guarantee in any //! metric that meets its ε, δ and scope. use serde::{Deserialize, Serialize}; +use crate::ir::operator::AggIntent; use crate::ir::{NonASAPOp, Operator, OperatorNode, QueryRoot, ScalarExpr}; -use crate::pre_asap::AggIntent; /// The stream norm an [`AccuracyMetric::Absolute`] error is stated in /// multiples of. A scaled error `ε` means `|estimate − truth| ≤ ε · ‖f‖`, @@ -116,7 +116,7 @@ pub enum AccuracyScope { } /// The outer shape of an accuracy guarantee -/// ([`crate::post_asap::AccuracyGuarantee`]): +/// ([`crate::ir::properties::AccuracyGuarantee`]): /// `Pr[err_metric(estimate, truth) > error] ≤ failure_probability` over the /// values `accuracy_scope` names. An [`AccuracyTarget`] has the same fields, with /// concrete numbers, an optional metric and `max_error` for `error`: a target @@ -311,7 +311,7 @@ fn node_metric(node: &OperatorNode) -> Option { /// select, order or rename rows, down to an aggregate's measure. A group /// key or any other expression is absolute. fn column_metric(node: &OperatorNode, column: usize) -> Option { - use crate::ir::operator_properties::JoinKind; + use crate::ir::operator::operator_properties::JoinKind; let Operator::NonASAP(op) = &node.operator else { return Some(AccuracyMetric::ABSOLUTE); }; @@ -379,7 +379,7 @@ fn column_metric(node: &OperatorNode, column: usize) -> Option { /// The measure the aggregate `node`'s output `column` holds, or `None` for /// a group key, a kept label or `count_values`' synthesized label, as -/// [`aggregate_output_schema`](crate::ir::aggregate_schema::aggregate_output_schema) +/// [`aggregate_output_schema`](crate::ir::schema::aggregate_schema::aggregate_output_schema) /// lays the columns out. A per-entity reduction replaces its one measure's /// input column in place and keeps every other column; a cross-series one /// emits its kept labels first, then one column per measure, with a @@ -387,13 +387,13 @@ fn column_metric(node: &OperatorNode, column: usize) -> Option { /// count. fn aggregate_measure_at<'a>( node: &OperatorNode, - reduction: &crate::ir::operator_properties::Reduction, + reduction: &crate::ir::operator::operator_properties::Reduction, measures: &'a [AggIntent], child: &OperatorNode, column: usize, ) -> Option<&'a AggIntent> { - use crate::ir::operator_properties::Reduction; - use crate::pre_asap::{resolve_column_ref, ColumnRef}; + use crate::ir::operator::operator_properties::Reduction; + use crate::ir::scalar::{resolve_column_ref, ColumnRef}; let fields = &node.schema.fields; match reduction { Reduction::PerEntity => { @@ -433,8 +433,8 @@ fn aggregate_measure_at<'a>( #[cfg(test)] mod tests { use super::*; - use crate::pre_asap::schema::{DataType, Field, Schema}; - use crate::pre_asap::{Reduction, Source}; + use crate::ir::operator::{Reduction, Source}; + use crate::ir::schema::{DataType, Field, Schema}; use std::rc::Rc; fn node(op: NonASAPOp) -> Rc { @@ -635,9 +635,9 @@ mod tests { /// metrics, or any other expression, is absolute. #[test] fn root_metric_looks_through_a_null_guard_to_its_statistic() { - use crate::ir::operator_properties::JoinKind; + use crate::ir::operator::operator_properties::JoinKind; + use crate::ir::scalar::{CompareOpKind, ScalarValue}; use crate::ir::{Predicate, ProjectItem}; - use crate::pre_asap::{CompareOpKind, ScalarValue}; let exact = AccuracyTarget::exact(); let QueryRoot::Operator(l2) = aggregate(vec![AggIntent::FrequencyL2 { col: None, diff --git a/crates/types/src/post_asap/execution_data_state.rs b/crates/types/src/ir/properties/execution_data_state.rs similarity index 83% rename from crates/types/src/post_asap/execution_data_state.rs rename to crates/types/src/ir/properties/execution_data_state.rs index 5f4e65543..72a0fe91d 100644 --- a/crates/types/src/post_asap/execution_data_state.rs +++ b/crates/types/src/ir/properties/execution_data_state.rs @@ -4,11 +4,13 @@ //! query time); [`ExecutionDataState`] pairs it with the [`DataPrimitive`] //! the edge carries (raw values vs. summary state). [`Materialization`] is //! the per-sub-DAG choice a node's timing is derived from. The rules that assign -//! and check them over a DAG live in [`crate::ir::timing`], which reports +//! and check them over a DAG live in [`crate::ir::properties::timing`], which reports //! violations as [`ExecutionDataStateError`]. use thiserror::Error; +use crate::physical::Materialization; + /// When a post-ASAP value is produced. #[derive( Default, Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, @@ -29,48 +31,6 @@ impl ExecutionTiming { } } -/// Where a materialized output is stored (#509 §Materialization). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] -pub enum Medium { - Memory, - Disk, -} - -/// The materialization option of one sub-DAG (#509 §Materialization, -/// Algorithm 3): whether its output is stored across query executions, and -/// whether it is computed at ingestion time or at query time. -#[derive( - Default, Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, -)] -pub enum Materialization { - /// Runs at query time for each execution; its output is discarded. - #[default] - NotMaterialized, - /// Runs as data arrives; its output is stored until a query reads it. - MatAtIngestion(Medium), - /// Runs when a query first needs it; its output is reused by the other - /// queries of the same evaluation and discarded after it. - MatAtQuery(Medium), - /// Runs at query time and keeps its output across evaluations: a - /// tumbling-pane chain whose readers evaluate together every pane - /// width builds its newest pane at each evaluation and keeps the older - /// ones for the next (#509 Example 4's B3). Offered only when the - /// deployment keeps query-time results. - MatAtQueryKeptAcrossEvaluations(Medium), -} - -impl Materialization { - /// When the sub-DAG runs: only `MatAtIngestion` runs at ingestion time. - pub fn timing(self) -> ExecutionTiming { - match self { - Self::MatAtIngestion(_) => ExecutionTiming::IngestionTime, - Self::NotMaterialized - | Self::MatAtQuery(_) - | Self::MatAtQueryKeptAcrossEvaluations(_) => ExecutionTiming::QueryTime, - } - } -} - /// The primitive representation carried by a post-ASAP edge. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] pub enum DataPrimitive { diff --git a/crates/types/src/post_asap/guarantee.rs b/crates/types/src/ir/properties/guarantee.rs similarity index 99% rename from crates/types/src/post_asap/guarantee.rs rename to crates/types/src/ir/properties/guarantee.rs index f1cc7c4a4..2e88a7616 100644 --- a/crates/types/src/post_asap/guarantee.rs +++ b/crates/types/src/ir/properties/guarantee.rs @@ -9,7 +9,7 @@ //! failure-probability expressions, the provenance trail, and the typed //! rejection reasons. The *algebra* that composes these (the `AccuracyModel` //! trait, its default conservative rules, and budget allocation) lives one -//! layer up in `asap_aware_mapping::accuracy`: this crate defines the +//! layer up in `asap_logical_asap_optimization::accuracy`: this crate defines the //! shapes, the planning crate decides. //! //! ## What a guarantee says @@ -52,8 +52,8 @@ use serde::{Deserialize, Serialize}; -use crate::types::AccuracyTarget; -pub use crate::types::{AccuracyMetric, AccuracyScope, AccuracySpec, NormScale}; +use crate::ir::properties::AccuracyTarget; +pub use crate::ir::properties::{AccuracyMetric, AccuracyScope, AccuracySpec, NormScale}; /// A symbolic, serializable error-bound expression. Non-negative real /// arithmetic only — there is no subtraction, so a bound can never be diff --git a/crates/types/src/ir/properties/mod.rs b/crates/types/src/ir/properties/mod.rs new file mode 100644 index 000000000..455a489f2 --- /dev/null +++ b/crates/types/src/ir/properties/mod.rs @@ -0,0 +1,28 @@ +//! #511 §2.2–2.3: the planning properties a node carries besides its +//! operator and schema. +//! +//! - [`accuracy`] — accuracy requirements and the error metrics they are +//! stated in ([`AccuracyTarget`], [`AccuracyMetric`]). +//! - [`guarantee`] — machine-readable accuracy guarantees +//! ([`AccuracyGuarantee`]). +//! - [`execution_data_state`] — when a node's value is produced +//! ([`ExecutionTiming`]) and what its edge carries ([`ExecutionDataState`]). +//! - [`timing`] — execution timing written into every node from a +//! materialization assignment, and its validation. +//! - [`summary_coverage`] — what a summary state covers. +pub mod accuracy; +pub mod execution_data_state; +pub mod guarantee; +pub mod summary_coverage; +pub mod timing; + +pub use accuracy::{ + default_metric_for_root, AccuracyMetric, AccuracyScope, AccuracySpec, AccuracyTarget, NormScale, +}; +pub use execution_data_state::{ + DataPrimitive, ExecutionDataState, ExecutionDataStateError, ExecutionTiming, +}; +pub use guarantee::{ + AccuracyError, AccuracyGuarantee, BoundExpr, CompositionOperator, GuaranteeSource, + ProbabilityExpr, VALUES_PER_RESULT, +}; diff --git a/crates/types/src/ir/summary_coverage.rs b/crates/types/src/ir/properties/summary_coverage.rs similarity index 98% rename from crates/types/src/ir/summary_coverage.rs rename to crates/types/src/ir/properties/summary_coverage.rs index fb99b2c45..98d4f6386 100644 --- a/crates/types/src/ir/summary_coverage.rs +++ b/crates/types/src/ir/properties/summary_coverage.rs @@ -17,13 +17,13 @@ use std::rc::Rc; use thiserror::Error; -use super::asap::ASAPOp; -use super::node::{Operator, OperatorNode}; -use super::non_asap::{NonASAPOp, TimeRangeKind}; -use super::scalar::{Predicate, ScalarExpr}; -use crate::post_asap::family_capability::{FamilyCapability, MergeInputRule}; -use crate::pre_asap::expr_ir::{CompareOpKind, ScalarValue}; -use crate::pre_asap::schema::{ColumnId, Schema}; +use crate::ir::operator::asap::ASAPOp; +use crate::ir::operator::node::{Operator, OperatorNode}; +use crate::ir::operator::non_asap::{NonASAPOp, TimeRangeKind}; +use crate::ir::scalar::expr_ir::{CompareOpKind, ScalarValue}; +use crate::ir::scalar::{Predicate, ScalarExpr}; +use crate::ir::schema::family_capability::{FamilyCapability, MergeInputRule}; +use crate::ir::schema::{ColumnId, Schema}; #[derive(Debug, Clone, PartialEq)] pub struct SummaryCoverage { @@ -127,7 +127,7 @@ impl SummaryCoverage { // Back-to-back panes cover the whole window one build over it // would. Some(ASAPOp::PaneChain { .. }) => of_summary_agg( - &super::pane_chain::whole_window_build(node).map_err(|error| { + &crate::ir::operator::pane_chain::whole_window_build(node).map_err(|error| { CoverageError::PaneChainExpansion { reason: error.to_string(), } diff --git a/crates/types/src/ir/timing.rs b/crates/types/src/ir/properties/timing.rs similarity index 97% rename from crates/types/src/ir/timing.rs rename to crates/types/src/ir/properties/timing.rs index 07115b11a..57ddc29d0 100644 --- a/crates/types/src/ir/timing.rs +++ b/crates/types/src/ir/properties/timing.rs @@ -55,15 +55,15 @@ use std::collections::HashMap; use std::rc::Rc; -use super::asap::ASAPOp; -use super::node::{Operator, OperatorNode}; -use super::non_asap::NonASAPOp; -use crate::ir::operator_properties::BinaryOpKind; -use crate::post_asap::execution_data_state::{ - DataPrimitive, ExecutionDataState, ExecutionDataStateError, ExecutionTiming, Materialization, - Medium, +use crate::ir::operator::asap::ASAPOp; +use crate::ir::operator::node::{Operator, OperatorNode}; +use crate::ir::operator::non_asap::NonASAPOp; +use crate::ir::operator::operator_properties::BinaryOpKind; +use crate::ir::properties::execution_data_state::{ + DataPrimitive, ExecutionDataState, ExecutionDataStateError, ExecutionTiming, }; -use crate::pre_asap::schema::{DataType, FieldDataType, Schema}; +use crate::ir::schema::{DataType, FieldDataType, Schema}; +use crate::physical::materialization::{Materialization, Medium}; /// The materialization option of each sub-DAG (#509 §Materialization: /// "Materialization decides, for each sub-DAG, whether its output is @@ -261,7 +261,7 @@ fn forces_query_time( .schema .fields .iter() - .any(|f| f.name == crate::pre_asap::schema::PROMQL_SERIES_IDENTITY) + .any(|f| f.name == crate::ir::schema::PROMQL_SERIES_IDENTITY) && per_series_rows(lhs).is_none_or(|rows| per_series_rows(rhs) != Some(rows)) => { true @@ -711,7 +711,7 @@ fn validate_non_asap( } => { let is_div = matches!( operator.kind, - BinaryOpKind::Arithmetic(crate::pre_asap::ArithmeticOpKind::Div) + BinaryOpKind::Arithmetic(crate::ir::scalar::ArithmeticOpKind::Div) ); if (operator.checked_relative_division && operator.checked_finite_division) || ((operator.checked_relative_division || operator.checked_finite_division) @@ -725,7 +725,7 @@ fn validate_non_asap( let plain_float_or_ts = |schema: &Schema| { schema.fields.iter().all(|field| { !field.nullable - && if field.name == crate::pre_asap::schema::PROMQL_SERIES_IDENTITY { + && if field.name == crate::ir::schema::PROMQL_SERIES_IDENTITY { field.dtype == FieldDataType::Plain(DataType::Utf8) } else { matches!( @@ -749,14 +749,14 @@ fn validate_non_asap( .schema .fields .iter() - .filter(|f| f.name == crate::pre_asap::schema::PROMQL_SERIES_IDENTITY) + .filter(|f| f.name == crate::ir::schema::PROMQL_SERIES_IDENTITY) .count() > 1 || (node .schema .fields .iter() - .any(|f| f.name == crate::pre_asap::schema::PROMQL_SERIES_IDENTITY) + .any(|f| f.name == crate::ir::schema::PROMQL_SERIES_IDENTITY) && per_series_rows(lhs) .is_none_or(|rows| per_series_rows(rhs) != Some(rows))) || !plain_float_or_ts(&node.schema) @@ -882,13 +882,13 @@ pub fn split_shared_by_timing( /// Ingestion-time arithmetic needs the same per-series population on both sides. fn per_series_rows(node: &OperatorNode) -> Option<&OperatorNode> { - use crate::post_asap::ExactKind; + use crate::ir::schema::ExactKind; match &node.operator { Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child }) => match &child.operator { Operator::ASAP(ASAPOp::SummaryAgg { child, family: FieldDataType::ExactAggregate(ExactKind::Sum | ExactKind::Count, _), - reduction: crate::pre_asap::Reduction::PerEntity, + reduction: crate::ir::operator::Reduction::PerEntity, filter: None, .. }) => Some(child), @@ -905,15 +905,15 @@ fn per_series_rows(node: &OperatorNode) -> Option<&OperatorNode> { #[cfg(test)] mod tests { use super::*; - use crate::ir::non_asap::NonASAPOp; - use crate::ir::operator_properties::{Reduction, Source}; - use crate::post_asap::sketch::{ + use crate::ir::operator::agg_intent::AggIntent; + use crate::ir::operator::non_asap::NonASAPOp; + use crate::ir::operator::operator_properties::{Reduction, Source}; + use crate::ir::scalar::expr_ir::ColumnRef; + use crate::ir::schema::sketch::{ ExactKind, ExactParams, GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, SketchStatistic, SummaryUpdate, }; - use crate::pre_asap::agg_intent::AggIntent; - use crate::pre_asap::expr_ir::ColumnRef; - use crate::pre_asap::schema::Field; + use crate::ir::schema::Field; fn scan_with(fields: Vec) -> Rc { OperatorNode::new_shared(crate::ir::Operator::NonASAP(NonASAPOp::Scan { @@ -1302,8 +1302,8 @@ mod tests { /// error; the all-at-ingestion default leaves it at query time. #[test] fn a_filtered_summary_build_never_runs_at_ingestion_time() { + use crate::ir::scalar::expr_ir::ScalarValue; use crate::ir::{Predicate, ScalarExpr}; - use crate::pre_asap::expr_ir::ScalarValue; let plain = agg(scan(), kll()); let Operator::ASAP(ASAPOp::SummaryAgg { child: rows, @@ -1613,9 +1613,10 @@ mod tests { // operands cover the same series; arbitrary labels are never admitted. #[test] fn ingestion_time_binary_accepts_only_well_typed_series_identity() { - use crate::ir::non_asap::BinaryOperator; - use crate::pre_asap::{ArithmeticOpKind, BinaryOpKind}; - let identity = crate::pre_asap::schema::PROMQL_SERIES_IDENTITY; + use crate::ir::operator::non_asap::BinaryOperator; + use crate::ir::operator::BinaryOpKind; + use crate::ir::scalar::ArithmeticOpKind; + let identity = crate::ir::schema::PROMQL_SERIES_IDENTITY; // A finalized per-series Sum of `metric`'s rows. let operand = |metric: &str, schema: &Schema| { let rows = OperatorNode::new_shared(crate::ir::Operator::NonASAP(NonASAPOp::Scan { diff --git a/crates/types/src/pre_asap/column_resolution.rs b/crates/types/src/ir/scalar/column_resolution.rs similarity index 98% rename from crates/types/src/pre_asap/column_resolution.rs rename to crates/types/src/ir/scalar/column_resolution.rs index 30411f6a7..94bef1401 100644 --- a/crates/types/src/pre_asap/column_resolution.rs +++ b/crates/types/src/ir/scalar/column_resolution.rs @@ -8,7 +8,7 @@ use thiserror::Error; use super::expr_ir::ColumnRef; -use super::schema::{ColumnId, DataType, FieldDataType, Schema}; +use crate::ir::schema::{ColumnId, DataType, FieldDataType, Schema}; /// Errors returned by the resolution helpers. #[derive(Debug, Error, PartialEq, Eq)] @@ -123,7 +123,7 @@ pub fn resolve_group_keys_promql( #[cfg(test)] mod tests { use super::*; - use crate::pre_asap::schema::Field; + use crate::ir::schema::Field; /// The conventional PromQL leaf shape: `(ts: Timestamp, value: Float64)`. fn ts_value_schema() -> Schema { diff --git a/crates/types/src/pre_asap/expr_ir.rs b/crates/types/src/ir/scalar/expr_ir.rs similarity index 96% rename from crates/types/src/pre_asap/expr_ir.rs rename to crates/types/src/ir/scalar/expr_ir.rs index fe373e58f..50606f185 100644 --- a/crates/types/src/pre_asap/expr_ir.rs +++ b/crates/types/src/ir/scalar/expr_ir.rs @@ -1,7 +1,7 @@ //! Column-reference and scalar-operator vocabulary shared by the IR's scalar //! expressions ([`crate::ir::ScalarExpr`]) and the front ends' unresolved //! form: [`ColumnRef`] (name-based, front-end-emitted; positional -//! [`ColumnId`](super::schema::ColumnId) once bound), and [`ScalarValue`], +//! [`ColumnId`](crate::ir::schema::ColumnId) once bound), and [`ScalarValue`], //! [`CompareOpKind`], [`ArithmeticOpKind`] — the **union** of what the two //! front ends need: PromQL contributes `Regex` / `NotRegex` (`=~` / `!~`); SQL //! contributes arithmetic, `CASE`, `IN`, `CAST`, `IS [NOT] NULL`, scalar @@ -11,7 +11,7 @@ use serde::{Deserialize, Serialize}; /// A name-based column reference — the front-end-emitted, unresolved form of /// [`ScalarExpr::Column`](crate::ir::ScalarExpr::Column); front-end name -/// resolution turns it into a positional [`ColumnId`](super::schema::ColumnId). +/// resolution turns it into a positional [`ColumnId`](crate::ir::schema::ColumnId). /// This is a logical reference, not schema metadata or a runtime data array. /// `SampleValue` names the implicit PromQL sample column; `Wildcard` represents /// an all-columns/rows request. diff --git a/crates/types/src/ir/scalar.rs b/crates/types/src/ir/scalar/mod.rs similarity index 98% rename from crates/types/src/ir/scalar.rs rename to crates/types/src/ir/scalar/mod.rs index eec779f41..042735629 100644 --- a/crates/types/src/ir/scalar.rs +++ b/crates/types/src/ir/scalar/mod.rs @@ -8,15 +8,21 @@ //! plan-reading variants (`PromqlScalarFromVector`, `ScalarSubquery`, `Exists`, //! `InSubquery`); every traversal of the operator DAG follows them. +pub mod column_resolution; +pub mod expr_ir; +pub mod scalar_type_rules; + +pub use column_resolution::{resolve_column_ref, resolve_column_refs, ResolveError}; +pub use expr_ir::{ArithmeticOpKind, ColumnRef, CompareOpKind, ScalarValue}; + use std::rc::Rc; use serde::{Deserialize, Serialize}; -use super::node::OperatorNode; +use crate::ir::operator::node::OperatorNode; +use crate::ir::scalar::scalar_type_rules::MapScalarFunction; +use crate::ir::schema::{ColumnId, DataType, Schema}; use crate::ir::SchemaDerivationError; -use crate::pre_asap::expr_ir::{ArithmeticOpKind, CompareOpKind, ScalarValue}; -use crate::pre_asap::scalar_type_rules::MapScalarFunction; -use crate::pre_asap::schema::{ColumnId, DataType, Schema}; /// Which language's numeric and comparison rules an expression follows. /// Both languages use `Float64`, so a result type alone does not preserve @@ -509,7 +515,8 @@ impl ScalarExpr { (to.clone(), *try_cast || nullable) } ScalarExpr::FunctionCall { name, args } => { - if let Some(arity) = crate::pre_asap::scalar_type_rules::promql_function_arity(name) + if let Some(arity) = + crate::ir::scalar::scalar_type_rules::promql_function_arity(name) { if args.len() != arity || args @@ -631,7 +638,7 @@ fn relation(node: &OperatorNode) -> Result<(), SchemaDerivationError> { } fn scalar_subquery_field( node: &OperatorNode, -) -> Result<&crate::pre_asap::Field, SchemaDerivationError> { +) -> Result<&crate::ir::schema::Field, SchemaDerivationError> { relation(node)?; match node.schema.fields.as_slice() { [field] => Ok(field), @@ -767,9 +774,9 @@ pub fn element_access_type( #[cfg(test)] mod tests { use super::*; - use crate::ir::operator_properties::Source; + use crate::ir::operator::operator_properties::Source; + use crate::ir::schema::{Field, FieldDataType}; use crate::ir::{NonASAPOp, OperatorNode}; - use crate::pre_asap::schema::{Field, FieldDataType}; fn call(name: &str, args: Vec) -> ScalarExpr { ScalarExpr::FunctionCall { diff --git a/crates/types/src/pre_asap/scalar_type_rules.rs b/crates/types/src/ir/scalar/scalar_type_rules.rs similarity index 99% rename from crates/types/src/pre_asap/scalar_type_rules.rs rename to crates/types/src/ir/scalar/scalar_type_rules.rs index 3efad3f7e..81be9e466 100644 --- a/crates/types/src/pre_asap/scalar_type_rules.rs +++ b/crates/types/src/ir/scalar/scalar_type_rules.rs @@ -1,6 +1,6 @@ //! Shared type and nullability rules used to validate scalar expressions. //! These rules do not evaluate expressions or define physical representations. -use super::schema::DataType; +use crate::ir::schema::DataType; /// Names are resolved once against this closed builtin set; unknown functions /// remain outside these type rules. Map access keeps the first duplicate key diff --git a/crates/types/src/ir/aggregate_schema.rs b/crates/types/src/ir/schema/aggregate_schema.rs similarity index 95% rename from crates/types/src/ir/aggregate_schema.rs rename to crates/types/src/ir/schema/aggregate_schema.rs index f7476047b..2ce6ac2ab 100644 --- a/crates/types/src/ir/aggregate_schema.rs +++ b/crates/types/src/ir/schema/aggregate_schema.rs @@ -5,9 +5,11 @@ //! aggregate result; a PromQL per-series range reduction preserves labels //! and produces a Float64 sample value. This module derives schemas, not //! aggregate values or summary candidates. -use super::operator_properties::*; -use super::SchemaDerivationError; -use crate::pre_asap::{AggIntent, ColumnId, ColumnRef, DataType, Field, FieldDataType, Schema}; +use crate::ir::operator::operator_properties::*; +use crate::ir::operator::AggIntent; +use crate::ir::scalar::ColumnRef; +use crate::ir::schema::{ColumnId, DataType, Field, FieldDataType, Schema}; +use crate::ir::SchemaDerivationError; /// Output schema of a *per-series* window/range reduction (`rate`/`increase`, /// or an `*_over_time` reducer under a time `Window`). Such a reduction emits /// one value per series, so every label column of `input` is preserved and only @@ -20,7 +22,7 @@ fn per_series_reduction_schema( let vi = if let Some(index) = agg.input_cols().first() { *index } else { - crate::pre_asap::column_resolution::resolve_column_ref(&ColumnRef::SampleValue, input) + crate::ir::scalar::column_resolution::resolve_column_ref(&ColumnRef::SampleValue, input) .map_err(|error| SchemaDerivationError::InvalidSampleColumn(error.to_string()))? }; if !matches!( @@ -101,10 +103,12 @@ pub fn aggregate_output_schema( ))?; out_cols.push(c.clone()); } - let value_col_idx = - crate::pre_asap::column_resolution::resolve_column_ref(&ColumnRef::SampleValue, in_schema) - .ok() - .or_else(|| (0..in_schema.fields.len()).find(|i| !by.contains(i))); + let value_col_idx = crate::ir::scalar::column_resolution::resolve_column_ref( + &ColumnRef::SampleValue, + in_schema, + ) + .ok() + .or_else(|| (0..in_schema.fields.len()).find(|i| !by.contains(i))); let probe = value_col_idx .and_then(|i| in_schema.fields.get(i)) .cloned() @@ -200,7 +204,7 @@ fn without_output_schema( let mut out_cols: Vec = Vec::new(); for (i, col) in in_schema.fields.iter().enumerate() { let is_time = in_schema.time_index == Some(i); - let is_value = crate::pre_asap::column_resolution::resolve_column_ref( + let is_value = crate::ir::scalar::column_resolution::resolve_column_ref( &ColumnRef::SampleValue, in_schema, ) diff --git a/crates/types/src/ir/error.rs b/crates/types/src/ir/schema/error.rs similarity index 88% rename from crates/types/src/ir/error.rs rename to crates/types/src/ir/schema/error.rs index a0f9e2271..d4f3b05b5 100644 --- a/crates/types/src/ir/error.rs +++ b/crates/types/src/ir/schema/error.rs @@ -3,7 +3,7 @@ //! [`SchemaDerivationError`] distinguishes invalid scalar signatures, out-of-range //! grouping columns, empty concatenations, and invalid sample columns. //! Structural DAG and execution-timing validation have separate error types. -use crate::pre_asap::ColumnId; +use crate::ir::schema::ColumnId; use thiserror::Error; /// Errors from schema and type derivation over an operator DAG. #[derive(Debug, Error)] @@ -17,5 +17,5 @@ pub enum SchemaDerivationError { #[error("invalid per-series sample column: {0}")] InvalidSampleColumn(String), #[error("invalid summary coverage: {0}")] - Coverage(#[from] super::summary_coverage::CoverageError), + Coverage(#[from] crate::ir::properties::summary_coverage::CoverageError), } diff --git a/crates/types/src/post_asap/family_capability.rs b/crates/types/src/ir/schema/family_capability.rs similarity index 98% rename from crates/types/src/post_asap/family_capability.rs rename to crates/types/src/ir/schema/family_capability.rs index 2309367cf..b8d07c153 100644 --- a/crates/types/src/post_asap/family_capability.rs +++ b/crates/types/src/ir/schema/family_capability.rs @@ -10,7 +10,7 @@ //! serve every consumer whose statistic it answers. use super::sketch::{ExactKind, GroupingStrategy, HydraKind, SketchAlgorithm, SketchStatistic}; -use crate::pre_asap::schema::FieldDataType; +use crate::ir::schema::FieldDataType; /// What one summary family can do: its row of the family-capability table. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -230,8 +230,8 @@ impl FamilyCapability { #[cfg(test)] mod tests { use super::*; - use crate::post_asap::{ExactParams, SketchKind, SketchParams}; - use crate::pre_asap::expr_ir::ColumnRef; + use crate::ir::scalar::expr_ir::ColumnRef; + use crate::ir::schema::{ExactParams, SketchKind, SketchParams}; fn sketch(algorithm: SketchAlgorithm, params: SketchParams) -> FieldDataType { FieldDataType::Sketch( diff --git a/crates/types/src/pre_asap/schema.rs b/crates/types/src/ir/schema/mod.rs similarity index 96% rename from crates/types/src/pre_asap/schema.rs rename to crates/types/src/ir/schema/mod.rs index 16b1cb17a..3fd7f3e5e 100644 --- a/crates/types/src/pre_asap/schema.rs +++ b/crates/types/src/ir/schema/mod.rs @@ -13,13 +13,23 @@ #![allow(dead_code)] -use serde::{Deserialize, Serialize}; - -use crate::post_asap::sketch::{ - ExactKind, ExactParams, GroupingStrategy, SamplingKind, SamplingParams, SketchKind, - StatModelKind, StatModelParams, WaveletKind, WaveletParams, +pub mod aggregate_schema; +pub mod error; +pub mod family_capability; +pub mod schema_support; +pub mod sketch; + +pub use aggregate_schema::aggregate_output_schema; +pub use family_capability::{FamilyCapability, MergeInputRule, MergedError, StatisticKind}; +pub use sketch::{ + default_hydra_params, hydra_kind_for, EntityIdentity, ExactKind, ExactParams, GroupingStrategy, + HydraKind, HydraParams, NonNegativeWeightProof, SamplingKind, SamplingParams, SketchAlgorithm, + SketchCategory, SketchKind, SketchParams, SketchStatistic, StatModelKind, StatModelParams, + SummaryInputExpr, SummaryUpdate, WaveletKind, WaveletParams, WeightDomain, }; +use serde::{Deserialize, Serialize}; + /// Zero-based position of a column in a particular operator's input or output. /// /// During planning, the position indexes [`Schema::fields`] to obtain metadata; diff --git a/crates/types/src/ir/schema_support.rs b/crates/types/src/ir/schema/schema_support.rs similarity index 97% rename from crates/types/src/ir/schema_support.rs rename to crates/types/src/ir/schema/schema_support.rs index 8717c828b..3a1054f3d 100644 --- a/crates/types/src/ir/schema_support.rs +++ b/crates/types/src/ir/schema/schema_support.rs @@ -1,5 +1,5 @@ //! Series-identity realization for the unified dag. -use crate::pre_asap::schema::*; +use crate::ir::schema::*; /// Resolve a PromQL root to rows carrying [`PROMQL_SERIES_IDENTITY`] before /// candidate search. `closed` describes physical columns here: the final /// column contains every dynamic source label. It does not assert that the @@ -12,8 +12,8 @@ use crate::pre_asap::schema::*; pub fn with_promql_series_identity( root: &std::rc::Rc, ) -> Result, String> { + use crate::ir::operator::Source; use crate::ir::{NonASAPOp, Operator, OperatorNode}; - use crate::pre_asap::Source; use std::{collections::HashMap, rc::Rc}; fn visit( node: &Rc, @@ -104,8 +104,8 @@ pub fn with_promql_series_identity( #[cfg(test)] mod tests { use super::with_promql_series_identity; + use crate::ir::scalar::ScalarValue; use crate::ir::{NonASAPOp, Operator, OperatorNode, ScalarExpr}; - use crate::pre_asap::ScalarValue; // A direct scalar literal (`vector(1)`) stays a valid vector input when // series identity is realized. diff --git a/crates/types/src/post_asap/sketch.rs b/crates/types/src/ir/schema/sketch.rs similarity index 99% rename from crates/types/src/post_asap/sketch.rs rename to crates/types/src/ir/schema/sketch.rs index 8b9c180c6..8900e7953 100644 --- a/crates/types/src/post_asap/sketch.rs +++ b/crates/types/src/ir/schema/sketch.rs @@ -1,6 +1,6 @@ use serde::{Deserialize, Serialize}; -use crate::pre_asap::ColumnRef; +use crate::ir::scalar::ColumnRef; // ── Exact accumulators ────────────────────────────────────────────────────── @@ -255,7 +255,7 @@ pub enum SketchCategory { /// quantile-style, cardinality-style, frequency-style, or heavy-hitter/ /// top-k-style estimation — together with the concrete [`SketchAlgorithm`] /// and [`SketchParams`] realizing it. Sits between -/// [`FieldDataType::Sketch`](super::schema::FieldDataType::Sketch) +/// [`FieldDataType::Sketch`](crate::ir::schema::FieldDataType::Sketch) /// (the `Sketch` family as a whole, sibling to `Sample`/`Wavelet`/ /// `StatModel`) and the bare algorithm: `Kll` vs. `DDSketch` is a choice /// *within* `Quantile`, not a choice *of* `SketchKind` — every `Quantile` @@ -264,7 +264,7 @@ pub enum SketchCategory { /// [`SketchKind::new`] is the one place `(SketchAlgorithm, SketchParams)` /// pairs get classified into a category; construct through it rather than /// naming a variant directly, so a new algorithm can't drift out of sync -/// with its category. See `asap_aware_mapping::summary_candidates` for the +/// with its category. See `asap_logical_asap_optimization::summary_candidates` for the /// `AggIntent -> [SketchAlgorithm]` candidate list this ultimately groups. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct SketchKind { @@ -840,7 +840,7 @@ mod tests { } ); let family = |params| { - crate::post_asap::FieldDataType::Sketch( + crate::ir::schema::FieldDataType::Sketch( SketchKind::new(SketchAlgorithm::Kll, params), GroupingStrategy::default(), ) diff --git a/crates/types/src/lib.rs b/crates/types/src/lib.rs index c0c42efaf..5b153b02f 100644 --- a/crates/types/src/lib.rs +++ b/crates/types/src/lib.rs @@ -1,25 +1,23 @@ -//! `asap-types` — shared vocabulary for the whole workspace. +//! `asap-types` — shared vocabulary for the whole workspace. Every planning +//! stage depends on it. //! -//! - [`ir`] — the unified operator IR: one operator language before and -//! after ASAP optimization ([`ir::OperatorNode`]), plus its passes -//! (canonicalize, CSE, timing), the flat DAG form ([`ir::flat`]) and the -//! physical ASAP DAG ([`ir::physical_export`]). -//! - [`pre_asap`] — the shared field vocabulary the IR's operators are -//! built from (grouping keys, reductions, sources, aggregation intents, -//! scalar literal / operator kinds, [`pre_asap::Schema`]). -//! - [`post_asap`] — summary-state types (families, kinds, parameters, -//! grouping strategy), accuracy guarantees, and the execution-timing -//! vocabulary. No execution logic lives in this workspace (issue #190). -//! [`post_asap::query_time`] holds pure posterior error-bound math -//! (issue #239) a future sketch runtime's evaluation path can call; see its -//! docs for why it is unwired today. -//! - [`types`] / [`workload`] / [`parsed_workload`] / [`resources`] — -//! workload, batch and resource types. +//! - [`ir`] — the unified operator IR (#511): one operator DAG for logical +//! and physical plans ([`ir::OperatorNode`]), arranged by IR section: +//! [`ir::operator`] (§1 operators and their parameter types), +//! [`ir::scalar`] (§2.2 scalar expressions), [`ir::schema`] (§2.1 the +//! per-edge schema and the summary-state types it carries) and +//! [`ir::properties`] (§2.2–2.3 accuracy, guarantee, execution timing and +//! summary coverage), plus its passes (canonicalize, CSE), the flat DAG +//! form ([`ir::flat`]) and the physical ASAP DAG ([`ir::physical_export`]). +//! - [`workload`] — the planner inputs (#509): the query and data +//! workloads, the parsed workload the query-language frontends produce +//! ([`workload::parsed_workload`]) and resource dimensions +//! ([`workload::resources`]). +//! - [`physical`] — Stage 2 decision data: the materialization of a +//! sub-DAG ([`physical::Materialization`]) and window-summary panes. +//! +//! No execution logic lives in this crate (issue #190). pub mod ir; -pub mod parsed_workload; -pub mod post_asap; -pub mod pre_asap; -pub mod resources; +pub mod physical; pub mod serde_f64; -pub mod types; pub mod workload; diff --git a/crates/types/src/physical/materialization.rs b/crates/types/src/physical/materialization.rs new file mode 100644 index 000000000..b50fbd2e7 --- /dev/null +++ b/crates/types/src/physical/materialization.rs @@ -0,0 +1,48 @@ +//! The materialization option of each sub-DAG (#509 §Materialization), the +//! Stage 2 decision a node's execution timing +//! ([`ExecutionTiming`](crate::ir::properties::ExecutionTiming)) is derived +//! from. + +use crate::ir::properties::ExecutionTiming; + +/// Where a materialized output is stored (#509 §Materialization). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +pub enum Medium { + Memory, + Disk, +} + +/// The materialization option of one sub-DAG (#509 §Materialization, +/// Algorithm 3): whether its output is stored across query executions, and +/// whether it is computed at ingestion time or at query time. +#[derive( + Default, Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, +)] +pub enum Materialization { + /// Runs at query time for each execution; its output is discarded. + #[default] + NotMaterialized, + /// Runs as data arrives; its output is stored until a query reads it. + MatAtIngestion(Medium), + /// Runs when a query first needs it; its output is reused by the other + /// queries of the same evaluation and discarded after it. + MatAtQuery(Medium), + /// Runs at query time and keeps its output across evaluations: a + /// tumbling-pane chain whose readers evaluate together every pane + /// width builds its newest pane at each evaluation and keeps the older + /// ones for the next (#509 Example 4's B3). Offered only when the + /// deployment keeps query-time results. + MatAtQueryKeptAcrossEvaluations(Medium), +} + +impl Materialization { + /// When the sub-DAG runs: only `MatAtIngestion` runs at ingestion time. + pub fn timing(self) -> ExecutionTiming { + match self { + Self::MatAtIngestion(_) => ExecutionTiming::IngestionTime, + Self::NotMaterialized + | Self::MatAtQuery(_) + | Self::MatAtQueryKeptAcrossEvaluations(_) => ExecutionTiming::QueryTime, + } + } +} diff --git a/crates/types/src/physical/mod.rs b/crates/types/src/physical/mod.rs new file mode 100644 index 000000000..5ba0bea9d --- /dev/null +++ b/crates/types/src/physical/mod.rs @@ -0,0 +1,13 @@ +//! #509 Stage 2 decision data: the materialization of each sub-DAG and the +//! window-summary panes it may build. +//! +//! - [`materialization`] — whether a sub-DAG's output is stored and when it +//! is computed ([`Materialization`], [`Medium`]). +//! - [`summary_window`] — summary-window pane primitives. +pub mod materialization; +pub mod summary_window; + +pub use materialization::{Materialization, Medium}; +pub use summary_window::{ + plan_pane_phase, validate_pane_coverage, PaneCoverageError, PaneLayout, WindowEdgeCoverage, +}; diff --git a/crates/types/src/post_asap/summary_window.rs b/crates/types/src/physical/summary_window.rs similarity index 100% rename from crates/types/src/post_asap/summary_window.rs rename to crates/types/src/physical/summary_window.rs diff --git a/crates/types/src/post_asap/mod.rs b/crates/types/src/post_asap/mod.rs deleted file mode 100644 index bc0d26e7b..000000000 --- a/crates/types/src/post_asap/mod.rs +++ /dev/null @@ -1,57 +0,0 @@ -//! Summary-state types and the execution-timing vocabulary of the operator -//! IR ([`crate::ir`]). -//! -//! Where an intent ([`crate::pre_asap::AggIntent`]) says *what* to compute -//! ("a quantile to ε accuracy"), these types say *how* a summary realizes it: -//! the family, kind/algorithm and parameters are committed — one -//! `(Kind, Params)` pair per family ([`sketch::ExactKind`]/[`sketch::ExactParams`], -//! [`sketch::SamplingKind`]/[`sketch::SamplingParams`], -//! [`sketch::WaveletKind`]/[`sketch::WaveletParams`], -//! [`sketch::StatModelKind`]/[`sketch::StatModelParams`]). The `Sketch` family -//! nests a third level, [`sketch::SketchKind`] (quantile/cardinality/ -//! frequency/top-k), carrying the committed [`sketch::SketchAlgorithm`] and -//! [`sketch::SketchParams`], because it is the one family with more than one -//! algorithm per purpose. -//! -//! [`sketch::GroupingStrategy`] is a second, orthogonal axis: how many -//! physical instances of a summary exist across a grouped aggregate's `by` -//! subpopulations (per-subpopulation vs. one shared Hydra instance; the -//! planner does not offer Hydra yet, #703). It rides on `ASAPOp::SummaryAgg` and on -//! sketch-valued edge types. -//! -//! The rest: what each family can do ([`family_capability`]), accuracy -//! guarantees ([`guarantee`]), maintained populations, -//! summary-window panes, and the execution timing / -//! data-state vocabulary ([`execution_data_state`]). - -pub mod execution_data_state; -pub mod family_capability; -pub mod guarantee; -pub mod maintained_population; -pub mod query_time; -pub mod sketch; -pub mod summary_window; - -pub use crate::pre_asap::schema::{Field, FieldDataType, Schema}; -pub use execution_data_state::{ - DataPrimitive, ExecutionDataState, ExecutionDataStateError, ExecutionTiming, Materialization, - Medium, -}; -pub use family_capability::{FamilyCapability, MergeInputRule, MergedError, StatisticKind}; -pub use guarantee::{ - AccuracyError, AccuracyGuarantee, AccuracyMetric, BoundExpr, CompositionOperator, - GuaranteeSource, ProbabilityExpr, VALUES_PER_RESULT, -}; -pub use query_time::{ - classic_cms_sizing, cms_posterior_error_bound, count_sketch_posterior_error_bound, - cu_sketch_posterior_error_bound, traditional_a_priori_bound, -}; -pub use sketch::{ - default_hydra_params, hydra_kind_for, EntityIdentity, ExactKind, ExactParams, GroupingStrategy, - HydraKind, HydraParams, NonNegativeWeightProof, SamplingKind, SamplingParams, SketchAlgorithm, - SketchCategory, SketchKind, SketchParams, SketchStatistic, StatModelKind, StatModelParams, - SummaryInputExpr, SummaryUpdate, WaveletKind, WaveletParams, WeightDomain, -}; -pub use summary_window::{ - plan_pane_phase, validate_pane_coverage, PaneCoverageError, PaneLayout, WindowEdgeCoverage, -}; diff --git a/crates/types/src/post_asap/query_time/mod.rs b/crates/types/src/post_asap/query_time/mod.rs deleted file mode 100644 index 25de41bcd..000000000 --- a/crates/types/src/post_asap/query_time/mod.rs +++ /dev/null @@ -1,34 +0,0 @@ -//! Code meant to run at **query execution time**, not planning time — -//! folder-separated from `post_asap`'s other modules ([`super::sketch`], -//! [`super::guarantee`]) on purpose. -//! -//! `asap-types`' crate doc states the invariant this workspace otherwise -//! holds to without exception: "no execution logic lives in this -//! workspace (issue #190) — a downstream deployment crate is expected to -//! supply that." Everything else under [`super`] is *planning*-time IR — -//! types the planner (`asap-aware-mapping`) constructs and commits to -//! before a query ever runs, describing *what* summary will be built, not -//! *computing over* one that has run. -//! -//! [`error_estimation`] is the one deliberate exception, and this -//! submodule exists so that exception is visible in the directory listing -//! itself, not just in prose: it holds pure, sketch-object-agnostic *math* -//! (given a sketch's real counter values, compute a tighter posterior -//! error bound) that only makes sense to invoke *after* a real sketch has -//! ingested data and is being evaluated — a downstream runtime's job, not -//! this crate's. Nothing in `asap-types` or `asap-aware-mapping` calls -//! into this module today; it is a library waiting for a runtime that -//! doesn't exist yet in this workspace (see [`error_estimation`]'s own -//! docs for exactly what's blocked and why). See issue #250 for the -//! still-unimplemented idea of recording this module's query-time -//! observations over time and feeding them into a future replan. - -pub mod error_estimation; - -// Glob, not a named list: this submodule exists only to hold -// `error_estimation` today, so re-exporting everything it makes public -// keeps this hop in sync automatically — a new `pub fn` there needs no -// matching edit here, only in `post_asap::mod`'s own list below (the -// actual curated short-path public surface, kept explicit like `expr`'s -// and `sketch`'s re-exports). -pub use error_estimation::*; diff --git a/crates/types/src/pre_asap/mod.rs b/crates/types/src/pre_asap/mod.rs deleted file mode 100644 index c309c7b30..000000000 --- a/crates/types/src/pre_asap/mod.rs +++ /dev/null @@ -1,34 +0,0 @@ -//! Shared vocabulary of the operator IR. The operators themselves live in -//! [`crate::ir`]; this module holds the field types they are built from. -//! -//! Operator parameters and schema derivation live in [`crate::ir`]. -//! - [`agg_intent`] — the aggregation-intent vocabulary ([`AggIntent`]). -//! - [`expr_ir`] — [`ColumnRef`] and the scalar literal / operator kinds -//! ([`ScalarValue`], [`CompareOpKind`], [`ArithmeticOpKind`]). -//! - [`schema`] — the per-edge [`Schema`] every node carries. -//! - [`column_resolution`] — turn a name-based `ColumnRef` into a positional -//! `ColumnId` against a [`Schema`] (used by front-end name resolution). -//! - [`scalar_type_rules`] — shared scalar type and nullability rules. - -pub mod agg_intent; -pub mod column_resolution; -pub mod expr_ir; -pub mod scalar_type_rules; -pub mod schema; - -pub use crate::ir::operator_properties::{ - AtModifier, BinaryOpKind, ColState, ConcatDiscriminatorKey, DataModel, GroupKeys, GroupSide, - InfoMatcher, JoinKind, PromQLVectorSetOpKind, Reduction, RelationalSetOpKind, SampleKind, - Source, TimeShift, VectorGrouping, VectorMatch, VectorMatchKind, WindowFrame, WindowFrameBound, - WindowFrameOffset, WindowFrameUnits, WindowFuncKind, -}; -pub use agg_intent::{ - agg_accuracy, agg_is_exact, agg_is_mergeable, default_cardinality, default_quantile, AggIntent, - MathFunc, TimeFunc, -}; -pub use column_resolution::{resolve_column_ref, resolve_column_refs, ResolveError}; -pub use expr_ir::{ArithmeticOpKind, ColumnRef, CompareOpKind, ScalarValue}; -pub use schema::{ColumnId, DataType, Field, FieldDataType, Schema}; - -pub use crate::ir::aggregate_schema::aggregate_output_schema; -pub use crate::ir::SchemaDerivationError; diff --git a/crates/types/src/workload.rs b/crates/types/src/workload/mod.rs similarity index 99% rename from crates/types/src/workload.rs rename to crates/types/src/workload/mod.rs index 40a107b3b..7a072e84d 100644 --- a/crates/types/src/workload.rs +++ b/crates/types/src/workload/mod.rs @@ -1,6 +1,9 @@ +pub mod parsed_workload; +pub mod resources; + use std::collections::BTreeMap; -use crate::types::AccuracyTarget; +use crate::ir::properties::AccuracyTarget; use serde::{Deserialize, Serialize}; // ── Query surface ───────────────────────────────────────────────────────────── diff --git a/crates/types/src/parsed_workload.rs b/crates/types/src/workload/parsed_workload.rs similarity index 100% rename from crates/types/src/parsed_workload.rs rename to crates/types/src/workload/parsed_workload.rs diff --git a/crates/types/src/resources.rs b/crates/types/src/workload/resources.rs similarity index 100% rename from crates/types/src/resources.rs rename to crates/types/src/workload/resources.rs diff --git a/crates/types/src/resources/cache.rs b/crates/types/src/workload/resources/cache.rs similarity index 100% rename from crates/types/src/resources/cache.rs rename to crates/types/src/workload/resources/cache.rs diff --git a/crates/types/src/resources/cpu.rs b/crates/types/src/workload/resources/cpu.rs similarity index 100% rename from crates/types/src/resources/cpu.rs rename to crates/types/src/workload/resources/cpu.rs diff --git a/crates/types/src/resources/measurement.rs b/crates/types/src/workload/resources/measurement.rs similarity index 100% rename from crates/types/src/resources/measurement.rs rename to crates/types/src/workload/resources/measurement.rs diff --git a/crates/types/src/resources/physical.rs b/crates/types/src/workload/resources/physical.rs similarity index 100% rename from crates/types/src/resources/physical.rs rename to crates/types/src/workload/resources/physical.rs diff --git a/crates/types/src/resources/physical_handoff.rs b/crates/types/src/workload/resources/physical_handoff.rs similarity index 100% rename from crates/types/src/resources/physical_handoff.rs rename to crates/types/src/workload/resources/physical_handoff.rs diff --git a/crates/types/src/resources/storage.rs b/crates/types/src/workload/resources/storage.rs similarity index 100% rename from crates/types/src/resources/storage.rs rename to crates/types/src/workload/resources/storage.rs diff --git a/crates/types/tests/pane_chain.rs b/crates/types/tests/pane_chain.rs index a5894e584..eac2b2bfe 100644 --- a/crates/types/tests/pane_chain.rs +++ b/crates/types/tests/pane_chain.rs @@ -5,13 +5,16 @@ use std::ops::Bound; use std::rc::Rc; use std::time::Duration; -use asap_types::ir::operator_properties::{Reduction, Source, TimeShift}; -use asap_types::ir::pane_chain::{expand_pane_chains, merged_chain, pane, whole_window_build}; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, TimeRangeKind}; -use asap_types::post_asap::{ +use asap_types::ir::operator::operator_properties::{Reduction, Source, TimeShift}; +use asap_types::ir::operator::pane_chain::{ + expand_pane_chains, merged_chain, pane, whole_window_build, +}; +use asap_types::ir::scalar::ColumnRef; +use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; +use asap_types::ir::schema::{ GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, SketchStatistic, SummaryUpdate, }; -use asap_types::pre_asap::{ColumnRef, DataType, Field, FieldDataType, Schema}; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, TimeRangeKind}; const MINUTE: u64 = 60_000; @@ -168,11 +171,9 @@ fn expansion_merges_the_panes() { /// An unmaterialized chain expands into query-time panes. #[test] fn expansion_carries_the_chains_timing_onto_its_panes() { - use asap_types::ir::{ - apply_materialization_timings, Materialization, MaterializationAssignment, Medium, - TimingMemo, - }; - use asap_types::post_asap::ExecutionTiming; + use asap_types::ir::properties::ExecutionTiming; + use asap_types::ir::{apply_materialization_timings, MaterializationAssignment, TimingMemo}; + use asap_types::physical::{Materialization, Medium}; let at_ingestion = Materialization::MatAtIngestion(Medium::Memory); for (materialization, timing) in [ (at_ingestion, ExecutionTiming::IngestionTime), diff --git a/crates/types/tests/physical_export.rs b/crates/types/tests/physical_export.rs index 5d1f2aa42..b52f08bfa 100644 --- a/crates/types/tests/physical_export.rs +++ b/crates/types/tests/physical_export.rs @@ -1,16 +1,18 @@ //! A timed plan exports as a PhysicalASAPDAG that keeps timing. +use asap_types::ir::operator::{Reduction, Source}; use asap_types::ir::physical_export::{ compile_physical_asap_dag, compile_physical_asap_workload, PhysicalASAPDAGDocument, PhysicalASAPDAGValidationError, }; +use asap_types::ir::properties::{ExecutionDataStateError, ExecutionTiming}; +use asap_types::ir::scalar::ColumnRef; +use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; +use asap_types::ir::schema::{ExactKind, ExactParams, SummaryUpdate}; use asap_types::ir::{ - apply_materialization_timings, ASAPOp, Materialization, MaterializationAssignment, Medium, - NonASAPOp, Operator, OperatorNode, QueryRoot, TimingMemo, + apply_materialization_timings, ASAPOp, MaterializationAssignment, NonASAPOp, Operator, + OperatorNode, QueryRoot, TimingMemo, }; -use asap_types::post_asap::{ - ExactKind, ExactParams, ExecutionDataStateError, ExecutionTiming, SummaryUpdate, -}; -use asap_types::pre_asap::{ColumnRef, DataType, Field, FieldDataType, Reduction, Schema, Source}; +use asap_types::physical::{Materialization, Medium}; use std::rc::Rc; /// Scan(t) → SummaryAgg(sum by key) → FinalizeExactAccumulator, untimed. @@ -152,8 +154,8 @@ fn batch_shares_the_summary_and_keeps_one_root_per_query() { /// in the batch, reading the shared nodes by id; operator queries keep `roots`. #[test] fn scalar_roots_export_in_batch_order_and_reference_nodes() { + use asap_types::ir::scalar::ArithmeticOpKind; use asap_types::ir::{ExprSemantics, ScalarExpr}; - use asap_types::pre_asap::ArithmeticOpKind; let timed = apply_materialization_timings( &plan(), &MaterializationAssignment::default(), @@ -227,7 +229,7 @@ fn scalar_roots_export_in_batch_order_and_reference_nodes() { // Keys must strictly ascend: a duplicate or a descending key is rejected. let literal: ScalarExpr = - ScalarExpr::Literal(asap_types::pre_asap::ScalarValue::Float64(2.0)); + ScalarExpr::Literal(asap_types::ir::scalar::ScalarValue::Float64(2.0)); let mut duplicate = valid_scalar_only.clone(); duplicate.scalar_roots.push((0, literal.clone())); assert_eq!( @@ -248,8 +250,8 @@ fn scalar_roots_export_in_batch_order_and_reference_nodes() { #[test] fn a_reference_read_twice_is_one_scalar_ref_edge() { use asap_types::ir::physical_export::EdgeRole; + use asap_types::ir::scalar::ArithmeticOpKind; use asap_types::ir::{ExprSemantics, ProjectItem, ScalarExpr}; - use asap_types::pre_asap::ArithmeticOpKind; let series = |metric: &str| { OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { source: Source::TimeSeries { @@ -297,9 +299,10 @@ fn a_reference_read_twice_is_one_scalar_ref_edge() { /// `Left` and `Right` would reorder the operands of `a - b`. #[test] fn swapped_edge_roles_are_rejected() { + use asap_types::ir::operator::JoinKind; use asap_types::ir::physical_export::EdgeRole; + use asap_types::ir::scalar::ScalarValue; use asap_types::ir::{Predicate, ScalarExpr}; - use asap_types::pre_asap::{JoinKind, ScalarValue}; let scan = |table: &str| { OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { source: Source::Table { @@ -358,7 +361,7 @@ fn exports_summary_over_summary_as_typed_precompute_edges() { use asap_types::ir::physical_export::{ compile_physical_asap_dag_with_node_ids, GroupingEdgeCompatibility, WindowEdgeCompatibility, }; - use asap_types::post_asap::ExecutionDataState; + use asap_types::ir::properties::ExecutionDataState; let scan = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { source: Source::TimeSeries { metric: "m".into() }, predicates: vec![], diff --git a/crates/types/tests/planner_vocabulary.rs b/crates/types/tests/planner_vocabulary.rs index 82f1cbc1a..bb04225d2 100644 --- a/crates/types/tests/planner_vocabulary.rs +++ b/crates/types/tests/planner_vocabulary.rs @@ -1,6 +1,6 @@ use asap_types::ir::physical_export::WindowEdgeCompatibility; -use asap_types::post_asap::{validate_pane_coverage, PaneLayout, WindowEdgeCoverage}; -use asap_types::resources::{PhysicalHandoffBytes, PhysicalHandoffKind}; +use asap_types::physical::{validate_pane_coverage, PaneLayout, WindowEdgeCoverage}; +use asap_types::workload::resources::{PhysicalHandoffBytes, PhysicalHandoffKind}; // Renamed pane APIs still read and emit the deployed wire contract. #[test] diff --git a/crates/types/tests/schema_rebuilding.rs b/crates/types/tests/schema_rebuilding.rs index 2b7daf049..ef66b648d 100644 --- a/crates/types/tests/schema_rebuilding.rs +++ b/crates/types/tests/schema_rebuilding.rs @@ -1,10 +1,9 @@ +use asap_types::ir::operator::{AggIntent, Reduction, Source}; +use asap_types::ir::properties::{AccuracyGuarantee, ExecutionTiming}; +use asap_types::ir::scalar::ColumnRef; +use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; +use asap_types::ir::schema::{ExactKind, ExactParams, GroupingStrategy, SummaryUpdate}; use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode}; -use asap_types::post_asap::{ - AccuracyGuarantee, ExactKind, ExactParams, ExecutionTiming, GroupingStrategy, SummaryUpdate, -}; -use asap_types::pre_asap::{ - AggIntent, ColumnRef, DataType, Field, FieldDataType, Reduction, Schema, Source, -}; use std::rc::Rc; fn scan(key_type: DataType, name: &str) -> Rc { @@ -122,7 +121,7 @@ fn validation_rejects_structural_overrides_for_both_categories() { /// Passthrough rewrites derive metadata and arity, but cannot guess alias positions. #[test] fn rebuilding_updates_metadata_and_requires_new_aliases_after_arity_changes() { - use asap_types::pre_asap::GroupKeys; + use asap_types::ir::operator::GroupKeys; let input = scan(DataType::Timestamp, "key"); let original = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Limit { n: Some(10), @@ -159,8 +158,8 @@ fn rebuilding_updates_metadata_and_requires_new_aliases_after_arity_changes() { /// Maintaining membership and finalizing values preserve identity/time metadata. #[test] fn summary_transitions_preserve_structural_metadata() { - use asap_types::post_asap::maintained_population::{MaintainedPopulation, PopulationInput}; - use asap_types::pre_asap::GroupKeys; + use asap_types::ir::operator::maintained_population::{MaintainedPopulation, PopulationInput}; + use asap_types::ir::operator::GroupKeys; let mut schema = scan(DataType::Timestamp, "key").schema.clone(); schema.closed = true; schema.time_index = Some(0); diff --git a/crates/types/tests/structure_contract.rs b/crates/types/tests/structure_contract.rs index ee502421e..b0697c2d8 100644 --- a/crates/types/tests/structure_contract.rs +++ b/crates/types/tests/structure_contract.rs @@ -1,7 +1,9 @@ +use asap_types::ir::operator::Source; +use asap_types::ir::scalar::ScalarValue; +use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::{ ExprSemantics, NonASAPOp, OperatorNode, OperatorResultKind, Predicate, ScalarExpr, }; -use asap_types::pre_asap::{DataType, Field, ScalarValue, Schema, Source}; use std::rc::Rc; fn scan() -> Rc { OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Scan { @@ -84,12 +86,13 @@ fn scalar_type_rules_fail_closed() { /// A state family is not interchangeable with another sketch or a scalar field. #[test] fn state_evaluations_and_passthrough_keep_their_contracts() { - use asap_types::ir::{ASAPOp, Operator, ProjectItem}; - use asap_types::post_asap::{ + use asap_types::ir::operator::Reduction; + use asap_types::ir::scalar::ColumnRef; + use asap_types::ir::schema::{ FieldDataType, GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, SketchStatistic, SummaryUpdate, }; - use asap_types::pre_asap::{ColumnRef, Reduction}; + use asap_types::ir::{ASAPOp, Operator, ProjectItem}; let family = FieldDataType::Sketch( SketchKind::new( SketchAlgorithm::Kll, @@ -144,8 +147,8 @@ fn state_evaluations_and_passthrough_keep_their_contracts() { /// Phase validation checks dependencies, without declaring a computation query-only. #[test] fn execution_timing_checks_edges_not_function_names() { + use asap_types::ir::properties::ExecutionTiming::{IngestionTime, QueryTime}; use asap_types::ir::ProjectItem; - use asap_types::post_asap::ExecutionTiming::{IngestionTime, QueryTime}; let input = Rc::new((*scan()).clone().with_timing(Some(IngestionTime))); let mut project = OperatorNode::new(asap_types::ir::Operator::NonASAP(NonASAPOp::Project { child: input, @@ -174,9 +177,11 @@ fn execution_timing_checks_edges_not_function_names() { /// Both operator categories use the same fallible schema-deriving constructor. #[test] fn shared_construction_derives_both_operator_categories() { + use asap_types::ir::operator::Reduction; + use asap_types::ir::scalar::ColumnRef; + use asap_types::ir::schema::FieldDataType; + use asap_types::ir::schema::{ExactKind, ExactParams, GroupingStrategy, SummaryUpdate}; use asap_types::ir::{ASAPOp, Operator, ProjectItem}; - use asap_types::post_asap::{ExactKind, ExactParams, GroupingStrategy, SummaryUpdate}; - use asap_types::pre_asap::{ColumnRef, FieldDataType, Reduction}; let input = scan(); let state = Rc::new( diff --git a/crates/types/tests/summary_coverage.rs b/crates/types/tests/summary_coverage.rs index b2f0bc84e..16330393d 100644 --- a/crates/types/tests/summary_coverage.rs +++ b/crates/types/tests/summary_coverage.rs @@ -5,18 +5,23 @@ use std::ops::Bound; use std::rc::Rc; use std::time::Duration; -use asap_types::ir::operator_properties::{Reduction, Source}; -use asap_types::ir::summary_coverage::{ColumnIdentity, Constraint, CoverageError, SelectionBox}; +use asap_types::ir::operator::operator_properties::{Reduction, Source}; +use asap_types::ir::operator::TimeShift; +use asap_types::ir::properties::summary_coverage::{ + ColumnIdentity, Constraint, CoverageError, SelectionBox, +}; +use asap_types::ir::properties::ExecutionTiming; +use asap_types::ir::scalar::expr_ir::{ArithmeticOpKind, CompareOpKind, ScalarValue}; +use asap_types::ir::scalar::ColumnRef; +use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; +use asap_types::ir::schema::{ + ExactKind, ExactParams, GroupingStrategy, MergeInputRule, SketchAlgorithm, SketchKind, + SketchParams, SummaryUpdate, +}; use asap_types::ir::{ ASAPOp, ExprSemantics, NonASAPOp, Operator, OperatorNode, Predicate, ProjectItem, ScalarExpr, SchemaDerivationError, TimeRangeKind, }; -use asap_types::post_asap::{ - ExactKind, ExactParams, ExecutionTiming, GroupingStrategy, MergeInputRule, SketchAlgorithm, - SketchKind, SketchParams, SummaryUpdate, -}; -use asap_types::pre_asap::expr_ir::{ArithmeticOpKind, CompareOpKind, ScalarValue}; -use asap_types::pre_asap::{ColumnRef, DataType, Field, FieldDataType, Schema, TimeShift}; const JOB: usize = 0; const REGION: usize = 1; diff --git a/crates/types/tests/summary_merge_structure.rs b/crates/types/tests/summary_merge_structure.rs index 956e717b9..03f01eb4e 100644 --- a/crates/types/tests/summary_merge_structure.rs +++ b/crates/types/tests/summary_merge_structure.rs @@ -1,17 +1,19 @@ //! Window composition merges compatible summary states without consuming raw rows. -use asap_types::{ - ir::operator_properties::{Reduction, Source}, - ir::{apply_materialization_timings, MaterializationAssignment, TimingMemo}, - ir::{ASAPOp, ExprSemantics, NonASAPOp, Operator, OperatorNode, Predicate, ScalarExpr}, - post_asap::{ - ExactKind, ExactParams, ExecutionTiming, FamilyCapability, GroupingStrategy, HydraKind, - HydraParams, Materialization, Medium, MergeInputRule, MergedError, SamplingKind, - SamplingParams, SketchAlgorithm, SketchKind, SketchParams, SketchStatistic, StatModelKind, - StatModelParams, StatisticKind, SummaryUpdate, WaveletKind, WaveletParams, - }, - pre_asap::expr_ir::{CompareOpKind, ScalarValue}, - pre_asap::{ColumnRef, DataType, Field, FieldDataType, Schema}, +use asap_types::ir::operator::operator_properties::{Reduction, Source}; +use asap_types::ir::properties::ExecutionTiming; +use asap_types::ir::scalar::expr_ir::{CompareOpKind, ScalarValue}; +use asap_types::ir::scalar::ColumnRef; +use asap_types::ir::schema::{ + DataType, ExactKind, ExactParams, FamilyCapability, Field, FieldDataType, GroupingStrategy, + HydraKind, HydraParams, MergeInputRule, MergedError, SamplingKind, SamplingParams, Schema, + SketchAlgorithm, SketchKind, SketchParams, SketchStatistic, StatModelKind, StatModelParams, + StatisticKind, SummaryUpdate, WaveletKind, WaveletParams, }; +use asap_types::ir::{ + apply_materialization_timings, ASAPOp, ExprSemantics, MaterializationAssignment, NonASAPOp, + Operator, OperatorNode, Predicate, ScalarExpr, TimingMemo, +}; +use asap_types::physical::{Materialization, Medium}; use std::rc::Rc; /// KLL over `value` for one `region`, so states of different regions are /// disjoint. diff --git a/docs/design_docs/architecture/README.md b/docs/design_docs/architecture/README.md index 86346e156..7069c23c5 100644 --- a/docs/design_docs/architecture/README.md +++ b/docs/design_docs/architecture/README.md @@ -44,9 +44,11 @@ target. | Shared IR | `asap-types` | The unified operator IR (`ir`: one `OperatorNode` before and after ASAP optimization), schemas, workloads, guarantees, and exported plan data | | Front-end common | `frontend-common` | Name-based `UnresolvedOp` tree shared by the front ends, and `resolve_root` into the operator IR | | Query frontends | `frontend-sql`, `frontend-promql`, `frontend-metricsql` | Stage 0: parse source languages and produce canonical Pre-ASAP queries | -| ASAP-aware mapping | `asap-aware-mapping` | Stages 1–3: candidate generation, ASAP-aware CSE, materialization, accuracy checks, costing and selection | +| Logical ASAP-aware optimization | `logical-asap-optimization` | Stage 1: per-computation candidates (Pass 1), ASAP-aware CSE (Pass 2), the analytical accuracy model | +| Physical ASAP-aware optimization | `physical-asap-optimization` | Stage 2: materialization and physical operator implementation | +| Plan selection | `plan-selection` | Stage 3: accuracy and latency checks, costing and selection; the planning models and the Stage 1 to Stage 3 driver | | Planner facade | `planner` | `e2e_plan`: Stage 0 followed by Stages 1–3 | -| Executor | `asap-physical-operators` | Runs a `PhysicalASAPDAG` over `asap_sketchlib` summaries | +| Executor | `executor` | Stage 4: runs a `PhysicalASAPDAG` over `asap_sketchlib` summaries; no stage crate depends on it | | Developer inspection | `devtools` | `stage_pipeline` and the Stage Viewer documents, corpus coverage tools | | End-to-end validation | `integration-tests` | Verify behavior across frontends, stages and execution | diff --git a/docs/design_docs/architecture/input-output-workflow.md b/docs/design_docs/architecture/input-output-workflow.md index 98b3bdc70..f2785ca72 100644 --- a/docs/design_docs/architecture/input-output-workflow.md +++ b/docs/design_docs/architecture/input-output-workflow.md @@ -294,7 +294,7 @@ PlanningWorkload + frontend dependencies + PlanningModels | Purpose | Operation | Result | |---|---|---| | Plan a workload end to end | `asap_planner::e2e_plan` | `PlanOutput` | -| Plan already-lowered roots and inspect every stage | `asap_aware_mapping::run_stage1_to_stage3` | `Stage1ToStage3Run`, whose `stage3_plan_selection` is the `Selection` | +| Plan already-lowered roots and inspect every stage | `asap_plan_selection::run_stage1_to_stage3` | `Stage1ToStage3Run`, whose `stage3_plan_selection` is the `Selection` | | Inspect a workload's stages as a document | `stage_pipeline` (devtools) | The stage document the [Stage Viewer](../../user_guide_docs/stage-viewer.md) shows | The integration associates each lowered root with a caller-owned `Id` and its diff --git a/docs/design_docs/architecture/updated_interface_with_pluggable_optimization.md b/docs/design_docs/architecture/updated_interface_with_pluggable_optimization.md index 946aed86e..2eda09d8b 100644 --- a/docs/design_docs/architecture/updated_interface_with_pluggable_optimization.md +++ b/docs/design_docs/architecture/updated_interface_with_pluggable_optimization.md @@ -235,20 +235,21 @@ pass no longer runs these steps; another pass need not run any of them. | Crate | What it holds | |---|---| | `asap-types` | `ParsedWorkload` | -| `asap-aware-mapping` | `OptimizationPass`, `OptimizationInput`, `PlanOutput`, `PlanningModels`, `optimize`, `PassRegistry`, `Stage1ToStage3Pass`, `run_stage1_to_stage3` | +| `asap-plan-selection` | `OptimizationPass`, `OptimizationInput`, `PlanOutput`, `PlanningModels`, `optimize`, `PassRegistry`, `Stage1ToStage3Pass`, `run_stage1_to_stage3` | | `asap-planner` *(new)* | `e2e_plan`, `UserInput`, `FrontendInput`, lowering dispatch | ```text asap-planner ──┬──> asap-frontend-{sql, promql, metricsql} - └──> asap-aware-mapping ──> asap-types - ▲ + └──> asap-plan-selection ──> asap-physical-asap-optimization + ▲ ──> asap-logical-asap-optimization ──> asap-types a pass depends only this far ``` `asap-planner` is separate because it is the only crate depending on every frontend; before it, the sole facade re-exporting more than one was `asap-devtools`, a developer-tools crate. `PlanningModels` lives in -`asap-aware-mapping` because both inputs use it, and `asap-planner` re-exports +`asap-plan-selection` (Stage 3, the only stage that uses the deployment's +models) because both inputs use it, and `asap-planner` re-exports it. --- diff --git a/docs/design_docs/concepts/accuracy-models.md b/docs/design_docs/concepts/accuracy-models.md index 4bf309b1f..e78af1fb2 100644 --- a/docs/design_docs/concepts/accuracy-models.md +++ b/docs/design_docs/concepts/accuracy-models.md @@ -284,7 +284,7 @@ rule or a different estimator configuration would be needed in those cases. ## Organization and extension contract -The `asap-aware-mapping::accuracy` module separates these responsibilities: +The `asap_logical_asap_optimization::accuracy` module separates these responsibilities: ```text accuracy/ @@ -323,6 +323,6 @@ not require a new selection rule for every sketch. Deployment extensions to `AccuracyModel` remain possible, but carry the same obligation to justify metrics, assumptions and propagation. -For implementation details, see the [accuracy module](../../../crates/asap-aware-mapping/src/accuracy/mod.rs), -[guarantee representation](../../../crates/types/src/post_asap/guarantee.rs), and +For implementation details, see the [accuracy module](../../../crates/logical-asap-optimization/src/accuracy/mod.rs), +[guarantee representation](../../../crates/types/src/ir/properties/guarantee.rs), and [accuracy propagation companion](../../develop_docs/end-to-end-accuracy-guarantees.md). diff --git a/docs/design_docs/concepts/post-asap-ir.md b/docs/design_docs/concepts/post-asap-ir.md index 20abf5a8f..69ba50585 100644 --- a/docs/design_docs/concepts/post-asap-ir.md +++ b/docs/design_docs/concepts/post-asap-ir.md @@ -22,7 +22,7 @@ model or downstream runtime supports it. ## ASAP operators -Every variant of [`ASAPOp`](../../../crates/types/src/ir/asap.rs) operates +Every variant of [`ASAPOp`](../../../crates/types/src/ir/operator/asap.rs) operates over summary state rather than raw data. The summary family, kind/algorithm and parameters are committed in the node; the state itself is typed by the `FieldDataType` of the output field that carries it (`ExactAggregate`, @@ -111,7 +111,7 @@ Materialization ([#509 §Materialization](../proposals/planner-layering.md#mater gives each sub-DAG one of three options: `NotMaterialized`, `MatAtIngestion(medium)` or `MatAtQuery(medium)`, with `Medium::Memory` or `Medium::Disk`. A -[`MaterializationAssignment`](../../../crates/types/src/ir/timing.rs) records +[`MaterializationAssignment`](../../../crates/types/src/ir/properties/timing.rs) records the option per sub-DAG, named by the node whose output would be stored. Any node may be named: a summary state, a population, or, as in #509 Example 4, the tumbling-window panes a query reads. `set(node, materialization)` assigns diff --git a/docs/design_docs/decisions/concat-unique-keys.md b/docs/design_docs/decisions/concat-unique-keys.md index dad5ae692..11a46ee68 100644 --- a/docs/design_docs/decisions/concat-unique-keys.md +++ b/docs/design_docs/decisions/concat-unique-keys.md @@ -4,7 +4,7 @@ > legacy `replacement.rs` and `explanation.rs` it mentions have been removed. > Historical: `crates/types/src/pre_asap/query_expr.rs` was removed in #543; -> `Concat` is now `NonASAPOp::Concat` (`crates/types/src/ir/non_asap.rs`). +> `Concat` is now `NonASAPOp::Concat` (`crates/types/src/ir/operator/non_asap.rs`). > **Update**: the investigation below found no current call site paying for a > redundant `Dedup` that this override would remove, and the original version @@ -18,7 +18,7 @@ ## Context -[`QueryExpr::Concat`](../../../crates/types/src/ir/non_asap.rs) (the +[`QueryExpr::Concat`](../../../crates/types/src/ir/operator/non_asap.rs) (the n-ary exact `UNION ALL` node, renamed from `Merge` in #226) always drops `unique_keys` on its output — `merge_drops_the_branches_unique_keys` and `merge_and_setop_agree_on_unique_keys` pin this down. Issue #228 asks whether diff --git a/docs/design_docs/physical-planning-and-deployment.md b/docs/design_docs/physical-planning-and-deployment.md index 22d06f1f3..768760ed6 100644 --- a/docs/design_docs/physical-planning-and-deployment.md +++ b/docs/design_docs/physical-planning-and-deployment.md @@ -303,7 +303,7 @@ and `query`. *Maintenance* stays materialization's word (section 2): it covers how state is built, retained, reused and scheduled. A precompute DAG is the physical object that maintenance compiles to, so reusing *maintenance* for it collapses two layers: Stage 2 materialization (#509) owns maintenance, and -`asap-physical-operators::physical_planner` owns the DAGs. +`asap_executor::physical_planner` owns the DAGs. ### Precompute Physical DAG @@ -514,7 +514,7 @@ Deployment: ``` The deployment engine executes the bound Physical DAGs through ASAPPlanner's -shared physical operator implementation library, `asap-physical-operators`, and +shared physical operator implementation library, `asap-executor`, and its DAG runtime. The merge executes once per run for both consumers. Execution does not introduce additional planning decisions. diff --git a/docs/design_docs/proposals/asap-aware-mapping/analytical-resource-cost.md b/docs/design_docs/proposals/asap-aware-mapping/analytical-resource-cost.md index e53862bda..0970b306e 100644 --- a/docs/design_docs/proposals/asap-aware-mapping/analytical-resource-cost.md +++ b/docs/design_docs/proposals/asap-aware-mapping/analytical-resource-cost.md @@ -1,7 +1,7 @@ # Analytical resource cost model > Status: implemented model with explicit support limits. The -> [analytical estimator](../../../../crates/asap-aware-mapping/src/analytical_cost.rs) +> [analytical estimator](../../../../crates/plan-selection/src/cost/analytical_cost.rs) > is what Stage 3 measures resource usage with. The physical-plan adapter and > the legacy `CostModel` described below have been removed; Stage 3's cost > models 1–3 replace them. @@ -123,7 +123,7 @@ required: the result cache is filled by evaluations in this horizon. Buffer residency is a steady-state capacity/working-set model; it does not model cold-start warming or access order. -`asap_types::resources` is the single definition site for `CacheProfile`, +`asap_types::workload::resources` is the single definition site for `CacheProfile`, `CacheEvidence`, and `CacheCapacityEvidence` (implemented in `resources/cache.rs`). These schemas describe cache assumptions, not additive CPU or byte consumption. The mapping module re-exports the same types for existing import paths, while diff --git a/docs/design_docs/proposals/asap-aware-mapping/maintained-populations.md b/docs/design_docs/proposals/asap-aware-mapping/maintained-populations.md index 78edc6f8d..99ec9ae49 100644 --- a/docs/design_docs/proposals/asap-aware-mapping/maintained-populations.md +++ b/docs/design_docs/proposals/asap-aware-mapping/maintained-populations.md @@ -1,7 +1,7 @@ # Shared maintained population rule > Status: planner rule implemented; deployment support is conditional. See -> [MaintainedPopulationStrategy](../../../../crates/asap-aware-mapping/src/maintained_population.rs). +> [MaintainedPopulationStrategy](../../../../crates/logical-asap-optimization/src/maintained_population.rs). > A deployment must provide the membership, freshness, state and operation > capabilities described below. Planner representation alone does not implement > population maintenance in a runtime. @@ -51,7 +51,7 @@ column and grouping determine whether consumers refer to the same population. ## Rule: share one population across compatible readouts **Realization:** `MaintainedPopulationStrategy::candidate` builds the sub-DAG; -Pass 1 does not offer it yet. Code: [maintained_population.rs](../../../../crates/asap-aware-mapping/src/maintained_population.rs). +Pass 1 does not offer it yet. Code: [maintained_population.rs](../../../../crates/logical-asap-optimization/src/maintained_population.rs). **Target sub-DAGs:** diff --git a/docs/design_docs/proposals/asap-primitive-schema.md b/docs/design_docs/proposals/asap-primitive-schema.md index 0031c0aa3..4f809ab99 100644 --- a/docs/design_docs/proposals/asap-primitive-schema.md +++ b/docs/design_docs/proposals/asap-primitive-schema.md @@ -7,7 +7,7 @@ This document is the single source of truth for the schema, and column design fo Unlike existing Database engines, which work on raw data or explicitly defined materialized tables with schema and column names provided by the users, ASAPPlanner is designed for querying and execution over the mix of raw data and ASAP Primitives. ASAP primitives are usually compact summaries over raw data. Therefore, it introduces new requirement when we design the schema and node definitions for LogicalASAPDAG and PhysicalASAPDAG. Assuming we have the Logical DAG defined for a canonicalized representation for a batch of queries (the `LogicalDAG` produced by the frontends, [planning stages §0](planner-layering.md#0-language-specific-frontends); the stages that follow are in [planner-layering.md](planner-layering.md#stages)). -The LogicalASAPDAG will share/reuse the NonASAP operator and ScalarExpr nodes in LogicalDAG ([decoupling operators from scalar expressions](decoupling_op_and_expr.md), with the unified operator type in [operator sharing §1.1](operator-sharing.md#11-unified-operator-type)), but replacing some operators in LogicalDAG with the operators operated with ASAP Primitives. The complete list is `ASAPOp` in `crates/types/src/ir/asap.rs`: +The LogicalASAPDAG will share/reuse the NonASAP operator and ScalarExpr nodes in LogicalDAG ([decoupling operators from scalar expressions](decoupling_op_and_expr.md), with the unified operator type in [operator sharing §1.1](operator-sharing.md#11-unified-operator-type)), but replacing some operators in LogicalDAG with the operators operated with ASAP Primitives. The complete list is `ASAPOp` in `crates/types/src/ir/operator/asap.rs`: | Operator | Input → output | Status | |---|---|---| @@ -49,7 +49,7 @@ This section fixes the words used below. They follow relational databases and Ap | **Relation / table** | A set (bag) of rows with the same columns. A base table is stored; a derived relation is the output of a query operator. | Every edge in the DAG carries a relation. A `Scan` reads a base table (SQL table or PromQL metric); every other operator outputs a derived relation. | | **Row / tuple** | One element of a relation: one value per column. | One output row of a node. For PromQL, one sample of one series at one time. | | **Column** | One position in every row, with a name and a type. Qualified as `table.column` when names can collide (DataFusion `Column { relation, name }`). | `ColumnId` refers to a column of the input schema; `(table, name)` identifies it across nodes (§4.2.2). | -| **Schema** | The ordered list of columns of a relation: name, data type, nullability (Arrow `Schema` of `Field { name, data_type, nullable }`; DataFusion `DFSchema` adds the table qualifier). The schema is *metadata*: it describes rows, it contains none. | `Schema` of `Field { name, dtype, nullable, table }` in `crates/types/src/pre_asap/schema.rs`. Unlike Arrow, `dtype` can be a summary state type (§3). | +| **Schema** | The ordered list of columns of a relation: name, data type, nullability (Arrow `Schema` of `Field { name, data_type, nullable }`; DataFusion `DFSchema` adds the table qualifier). The schema is *metadata*: it describes rows, it contains none. | `Schema` of `Field { name, dtype, nullable, table }` in `crates/types/src/ir/schema/mod.rs`. Unlike Arrow, `dtype` can be a summary state type (§3). | | **Data type** | The type of a column's values (`Int64`, `Utf8`, `Timestamp`, …). | `DataType`, wrapped as `FieldDataType::Plain`. | | **Aggregate state** | The intermediate value of an aggregate function before its final result, e.g. `(sum, count)` for `AVG` (DataFusion `Accumulator::state`, partial/final aggregation). It is never exposed as a column type to users. | Summary state *is* a column type here (`FieldDataType::Sketch`, `ExactAggregate`, …), so state can flow along edges and be merged, stored and read by later operators. | | **View / materialized view** | A view is a named query (its *definition*). A materialized view also stores the query's result rows; a query can then be answered from it when its definition matches (view matching, §4.1). | A built summary state is a materialized aggregation view whose aggregate is a summary family. Its definition and which rows it took are its coverage (§4). | @@ -65,7 +65,7 @@ Schema represents the **metadata** of information flow along an **edge** between Schema definition here is shared between LogicalDAG, LogicalASAPDAG, and PhysicalASAPDAG. The schema contain fields, and each field is mapping to a column in the physical data representation. Based on our requirement, each field should contain the following information. -1. **What type of the ASAP Primitive is.** The field's type is a [`FieldDataType`](#63-schema-and-field-types-cratestypessrcpre_asapschemars) (`crates/types/src/pre_asap/schema.rs`). A column is either a raw value or a summary state: +1. **What type of the ASAP Primitive is.** The field's type is a [`FieldDataType`](#63-schema-and-field-types-cratestypessrcirschemamodrs) (`crates/types/src/ir/schema/mod.rs`). A column is either a raw value or a summary state: - **Raw value**: `Plain(DataType)`, e.g., a number or a string. - **Summary state**: described from coarse to fine by four levels: @@ -77,12 +77,12 @@ Based on our requirement, each field should contain the following information. | Algorithm | `SketchAlgorithm` (other families: `ExactKind`, `SamplingKind`, …) | `Kll`, `Cms`, `Hll`, `DDSketch`, … | `Kll` | | Parameters | `SketchParams` (other families: `ExactParams`, `SamplingParams`, …) | per algorithm | `Kll { k: 200 }` | - - For a sketch, category, algorithm and parameters are bundled as one `SketchKind` ([§6.4](#64-summary-family-parameters-cratestypessrcpost_asapsketchrs)). A sketch also records its `GroupingStrategy`: one instance per group, or one shared structure (Hydra) for all groups. + - For a sketch, category, algorithm and parameters are bundled as one `SketchKind` ([§6.4](#64-summary-family-parameters-cratestypessrcirschemasketchrs)). A sketch also records its `GroupingStrategy`: one instance per group, or one shared structure (Hydra) for all groups. - So a quantile KLL sketch with `k = 200`, one instance per group, has the type `Sketch(SketchKind { Quantile, Kll, Kll { k: 200 } }, PerSubpopulationInstance)`. 2. **What query intent the summarized ASAP Primitive can support.** This is not stored in the field: it follows from the type in item 1. There are two kinds of intent: - - **Statistical aggregation intents** (`AggIntent`, `crates/types/src/pre_asap/agg_intent.rs`): which aggregate the state can answer, and how it is read out. + - **Statistical aggregation intents** (`AggIntent`, `crates/types/src/ir/operator/agg_intent.rs`): which aggregate the state can answer, and how it is read out. | Intent | Candidate types | Evaluation | |---|---|---| @@ -93,7 +93,7 @@ Based on our requirement, each field should contain the following information. | `FrequencyL2`, `FrequencyEntropy` | `Sketch`: `UnivMon` | `SummaryEstimate(FrequencyL2 \| FrequencyEntropy)` | | `Sum`, `Count`, `Min`, `Max`, `Rate`, `IRate`, `Increase` (exact) | `ExactAggregate(ExactKind, …)` | `FinalizeExactAccumulator` | - The sketch candidates are `summary_candidates(intent)` in `crates/asap-aware-mapping/src/realization.rs`; exact accumulators come from Pass 1's `ExactAccumulator` candidates in `per_computation_candidates.rs`; the evaluations are `SketchStatistic` ([§6.5](#65-update-input-and-evaluations-post_asapsketchrs-post_asapmaintained_populationrs)). + The sketch candidates are `summary_candidates(intent)` in `crates/logical-asap-optimization/src/realization.rs`; exact accumulators come from Pass 1's `ExactAccumulator` candidates in `per_computation_candidates.rs`; the evaluations are `SketchStatistic` ([§6.5](#65-update-input-and-evaluations-irschemasketchrs-iroperatormaintained_populationrs)). - **Time window aggregation intents**: whether states built over smaller windows can answer a larger one. This depends on how the family combines states: - **Merge** (`SummaryMerge`, §5.6): states over disjoint panes combine into the state of their union, e.g. two 1-minute KLL states answer a 2-minute quantile. Requires a mergeable family. @@ -886,7 +886,7 @@ OperatorNode §6.1 **Example: the p99 latency state of §5.1.** The `SummaryAgg` node (§6.1) holds `ASAPOp::SummaryAgg` (§6.2) with `family = Sketch(SketchKind::new(Kll, Kll { k: 200 }), PerSubpopulationInstance)` (§6.3, §6.4) and `input = SummaryUpdate::column(latency)` (§6.5). Its `schema` (§6.3) is `(job Utf8, state Sketch(KLL k=200))`: the `state` field has the same type as `family`. A `SummaryEstimate` above it holds `query = Quantile { q: 0.99 }` (§6.5), and its node carries the `guarantee` of the evaluation. `coverage()` on the `SummaryAgg` node returns a `SummaryCoverage` (§6.6). -### 6.1 DAG node (`crates/types/src/ir/node.rs`) +### 6.1 DAG node (`crates/types/src/ir/operator/node.rs`) Every operator in a DAG is wrapped in an `OperatorNode`. `coverage()` is explained in §6.6. @@ -922,7 +922,7 @@ impl OperatorNode { } ``` -### 6.2 ASAP operators (`crates/types/src/ir/asap.rs`) +### 6.2 ASAP operators (`crates/types/src/ir/operator/asap.rs`) `ASAPOp` is generic over how it refers to its inputs: `C` is `Rc` by default, and can also be a node id (e.g. in a flat DAG). @@ -1024,7 +1024,7 @@ impl ASAPOp { } ``` -### 6.3 Schema and field types (`crates/types/src/pre_asap/schema.rs`) +### 6.3 Schema and field types (`crates/types/src/ir/schema/mod.rs`) ```rust /// Position of a column in one schema (0-based). Local to that schema: @@ -1125,7 +1125,7 @@ pub enum DataType { } ``` -### 6.4 Summary-family parameters (`crates/types/src/post_asap/sketch.rs`) +### 6.4 Summary-family parameters (`crates/types/src/ir/schema/sketch.rs`) ```rust /// Which exact accumulator. None of them has parameters, so `ExactParams` @@ -1221,14 +1221,14 @@ pub enum WaveletKind { Haar } pub enum WaveletParams { Haar { coeffi pub enum StatModelKind { Parametric } pub enum StatModelParams { Parametric { family: String } } ``` -### 6.5 Update input and evaluations (`post_asap/sketch.rs`, `post_asap/maintained_population.rs`) +### 6.5 Update input and evaluations (`ir/schema/sketch.rs`, `ir/operator/maintained_population.rs`) This part of the code answers two questions about summary state: 1. **Update input** (`SummaryUpdate`): when a `SummaryAgg` reads one input row, what does it add to the state? It adds a **weight**, optionally under an **item** key. 2. **Evaluations**: once the state is built, what can be asked of it? `SketchStatistic` is what `SummaryEstimate` asks a sketch (§5.2). `MaintainedPopulation` and `PopulationStatistic` are the exact-population counterpart, used by `MaintainPopulation` and `EvaluatePopulation` (§5.4, §5.5). -**Update input examples** (as the planner builds them in `asap-aware-mapping`): +**Update input examples** (as the planner builds them in Stage 1, `asap-logical-asap-optimization`): | Query | Sketch | `item` | `weight` | `weight_domain` | |---|---|---|---|---| @@ -1325,9 +1325,9 @@ pub struct CurrentSeriesInput { } ``` -A finalized value's accuracy statement is `AccuracyGuarantee` (`post_asap/guarantee.rs`): a `spec` (`AccuracySpec`) with `accuracy_metric` (which error is measured, e.g. rank error), `error` (the error bound), `failure_probability` (the chance the bound does not hold) and `accuracy_scope` (which output values one statement covers), plus `provenance` (which estimates the bound came from). It is attached to evaluation and finalized nodes, and to `MaintainPopulation` nodes (whose population is exact); never to sketch or accumulator state. +A finalized value's accuracy statement is `AccuracyGuarantee` (`ir/properties/guarantee.rs`): a `spec` (`AccuracySpec`) with `accuracy_metric` (which error is measured, e.g. rank error), `error` (the error bound), `failure_probability` (the chance the bound does not hold) and `accuracy_scope` (which output values one statement covers), plus `provenance` (which estimates the bound came from). It is attached to evaluation and finalized nodes, and to `MaintainPopulation` nodes (whose population is exact); never to sketch or accumulator state. -### 6.6 Summary coverage (`crates/types/src/ir/summary_coverage.rs`) +### 6.6 Summary coverage (`crates/types/src/ir/properties/summary_coverage.rs`) The code for §4, as of #646. On `main` today `SummaryCoverage` is still the older declared form (`source` plus `regions`). diff --git a/docs/design_docs/proposals/planner-layering.md b/docs/design_docs/proposals/planner-layering.md index 6d94576a7..e2206f8fa 100644 --- a/docs/design_docs/proposals/planner-layering.md +++ b/docs/design_docs/proposals/planner-layering.md @@ -149,7 +149,7 @@ ASAPPlanner relies on the assumptions below. ### ASAPPlanner System Overview: Planning Stages -ASAPPlanner takes a [query workload](https://github.com/ProjectASAP/ASAPPlanner/blob/main/crates/types/src/workload.rs), a [data workload](https://github.com/ProjectASAP/ASAPPlanner/blob/main/crates/types/src/workload.rs#L384) and the deployment's +ASAPPlanner takes a [query workload](https://github.com/ProjectASAP/ASAPPlanner/blob/main/crates/types/src/workload/mod.rs), a [data workload](https://github.com/ProjectASAP/ASAPPlanner/blob/main/crates/types/src/workload/mod.rs#L384) and the deployment's inputs (TODO: define this data structure, issue [#525](https://github.com/ProjectASAP/ASAPPlanner/issues/525)), and returns one optimal physical plan. It decides what is computed, how it is computed, and which plan is best. The deployment only supplies data and query inputs and executes the plan: it provides its empirical cost model, empirical accuracy model and capabilities, but never optimize queries. @@ -920,7 +920,7 @@ To see examples in DAG Viewer, following the instructions [here](TODO: write an Each example's workload is shown as tables. Field names in code font are the fields of -[`workload.rs`](https://github.com/ProjectASAP/ASAPPlanner/blob/main/crates/types/src/workload.rs), +[`workload.rs`](https://github.com/ProjectASAP/ASAPPlanner/blob/main/crates/types/src/workload/mod.rs), except `lookback` and `as_of`, which the query text states (its range and offset). Each query reads the event-time window [`as_of` − `lookback`, `as_of`]. `as_of` is the window's end; `lookback` is its length. An diff --git a/docs/develop_docs/asap-aware-mapping-architecture.md b/docs/develop_docs/asap-aware-mapping-architecture.md index 45efadf4b..9acc848e7 100644 --- a/docs/develop_docs/asap-aware-mapping-architecture.md +++ b/docs/develop_docs/asap-aware-mapping-architecture.md @@ -8,59 +8,72 @@ code. For the public types and their rules, read the ## Crates +One crate per planning stage, so Cargo enforces the one-way stage flow: a +stage crate depends only on `asap-types` and the stages before it, and none +depends on the executor. + | Crate | Role | | --- | --- | -| `asap-types` (`crates/types`) | The IR (`ir`: `OperatorNode`, `NonASAPOp`, `ASAPOp`, timing, export), workload types (`workload`), accuracy types (`types`: `AccuracyTarget`, `AccuracyMetric`), summary types (`post_asap`: `SketchAlgorithm`, `FamilyCapability`, `AccuracyGuarantee`) | +| `asap-types` (`crates/types`) | Shared vocabulary, by IR section: `ir::operator` (`OperatorNode`, `NonASAPOp`, `ASAPOp`, operator parameters, `AggIntent`), `ir::scalar` (`ScalarExpr`, `ColumnRef`), `ir::schema` (`Schema`, `FieldDataType`, summary-state types such as `SketchAlgorithm` and `FamilyCapability`), `ir::properties` (`AccuracyTarget`, `AccuracyMetric`, `AccuracyGuarantee`, execution timing, summary coverage), plus `ir::flat` and `ir::physical_export`; the planner inputs (`workload`); Stage 2 decision data (`physical`: `Materialization`, panes) | | `asap-frontend-sql`, `asap-frontend-promql`, `asap-frontend-metricsql` (with `asap-frontend-common`) | Stage 0, the query-language frontends: lower each query into the logical DAG (`QueryRoot`s of `NonASAPOp` nodes, no summaries, no timing) | -| `asap-aware-mapping` (`crates/asap-aware-mapping`) | Stages 1–3 | +| `asap-logical-asap-optimization` (`crates/logical-asap-optimization`) | Stage 1: Pass 1, Pass 2, the whole-workload Stage 1 candidates and the analytical accuracy model | +| `asap-physical-asap-optimization` (`crates/physical-asap-optimization`) | Stage 2: materialization and physical operator implementation | +| `asap-plan-selection` (`crates/plan-selection`) | Stage 3: accuracy and latency checks, resource usage and cost models; the planning models (`PlanningModels`) and the Stage 1 to Stage 3 driver (`pass`) | | `asap-planner` (`crates/planner`) | The facade: `e2e_plan` runs Stage 0, then the optimization pass (`Stage1ToStage3Pass` by default) | -| `asap-physical-operators` | The executor's operators and summary kernels over `asap_sketchlib`; it runs a `PhysicalASAPDAG` | +| `asap-executor` (`crates/executor`) | Stage 4, the reference executor: operators and summary kernels over `asap_sketchlib`; it runs a `PhysicalASAPDAG` | | `asap-devtools` | `stage_pipeline` and the other inspection tools, the design-doc example workloads (`planner_layering_examples`) and the candidate labels | -Dependencies point one way: the frontends and `asap-aware-mapping` depend on -`asap-types`, never on each other; `asap-planner` depends on all of them. +```text +asap-types ◄── frontends (Stage 0) +asap-types ◄── asap-logical-asap-optimization (Stage 1) + ◄── asap-physical-asap-optimization (Stage 2) + ◄── asap-plan-selection (Stage 3, also depends on Stage 1) + ◄── asap-planner (also depends on the frontends) +asap-types ◄── asap-executor (Stage 4; no stage crate depends on it) +``` -## The stages in `asap-aware-mapping` +## The stages ```text QueryRoots (Stage 0) │ per_computation_candidates::generate_per_computation_candidates Stage 1, Pass 1 ▼ PerComputationCandidates ──without_unsupported_summaries(deployment)──┐ - │ pass::stage1_to_stage3::stage1_logical_asap_optimization │ Stage 1, enumeration + │ workload_candidates::stage1_logical_asap_optimization │ Stage 1, enumeration │ compose_logical_candidate per choice, then asap_aware_cse rules │ + Pass 2 ▼ │ Stage1Candidate / LogicalCandidate (each with its Pass2Candidate) ◄──┘ - │ stage2_physical_asap_optimization::optimize_physical_asap Stage 2 + │ asap_physical_asap_optimization::optimize_physical_asap Stage 2 ▼ PhysicalCandidate (one per materialization assignment) - │ stage3_plan_selection::select_plan Stage 3 + │ asap_plan_selection::select_plan Stage 3 ▼ Selection (selected id and plan, costs, accepted, rejected, front) ``` -`pass::run_stage1_to_stage3` runs all of it over lowered roots and returns +`asap_plan_selection::pass::run_stage1_to_stage3` runs all of it over lowered roots and returns every stage's candidates (`Stage1ToStage3Run`); `Stage1ToStage3Pass` is the same pipeline behind the `OptimizationPass` trait that `asap-planner` calls. | Concern | Where it lives | | --- | --- | -| Pass 1: which computations a query has (`ComputationTarget`), how each can be computed (`ComputationCandidate`: `Exact`, `ExactAccumulator`, `Summary`), fused computations (`FusedComputation`), pruning (`PruneReason`) and composing one choice into a logical DAG (`compose_logical_candidate`) | `per_computation_candidates.rs` | -| Which summary families serve an intent (`summary_candidates`), the intent's accuracy target and (ε, δ) budget, default sizing | `realization.rs` | -| Pass 2 rules: identical-expression, summary-capability, window-composition (tumbling panes), rollup and exact top-k reuse; each produces `*Candidate` parts and `*Pass2Candidate` whole-workload plans | `asap_aware_cse.rs` | -| Enumeration of Stage 1 candidates (the Cartesian product of Pass 1 choices, each followed by its Pass 2 candidates) | `pass/stage1_to_stage3.rs` | -| What each summary family can merge and answer (`FamilyCapability::of`: `merge_inputs`, `readouts`) | `asap-types`, `post_asap/family_capability.rs` | -| Stage 2: materialization options per summary build (`NotMaterialized`, `MatAtIngestion`, `MatAtQuery`, `MatAtQueryKeptAcrossEvaluations`), ingestion-time eligibility, physical operator implementation | `stage2_physical_asap_optimization.rs` | -| Timing and data-state edge rules applied from a `MaterializationAssignment` | `asap-types`, `ir/timing.rs` and `post_asap/execution_data_state.rs` | -| Stage 3: accuracy check, latency check, deployment-capability checks, resource usage (`ResourceUsage`), cost models 1–3 (`CostModel`), measured operation costs | `stage3_plan_selection.rs`, `stage3_plan_selection/measured_operation_costs.rs` | -| The analytical per-operator resource model Stage 3 measures with | `analytical_cost.rs`, `physical_operator_statistics.rs` | -| The accuracy algebra: per-family guarantees, composition, budget split | `accuracy/` (`AccuracyModel`, `DefaultAccuracyModel`, `estimators/`, `composition.rs`, `budget.rs`) | -| What the executor supports (`DeploymentCapabilities`) | `deployment_capabilities.rs` | -| The inputs every stage reads (`PlanningModels`: accuracy model, evidence, plan selection settings, deployment capabilities) and the `OptimizationPass` trait | `pass/mod.rs` | +| Pass 1: which computations a query has (`ComputationTarget`), how each can be computed (`ComputationCandidate`: `Exact`, `ExactAccumulator`, `Summary`), fused computations (`FusedComputation`), pruning (`PruneReason`) and composing one choice into a logical DAG (`compose_logical_candidate`) | `asap-logical-asap-optimization`, `per_computation_candidates.rs` | +| Which summary families serve an intent (`summary_candidates`), the intent's accuracy target and (ε, δ) budget, default sizing | `asap-logical-asap-optimization`, `realization.rs` | +| Pass 2 rules: identical-expression, summary-capability, window-composition (tumbling panes), rollup and exact top-k reuse; each produces `*Candidate` parts and `*Pass2Candidate` whole-workload plans | `asap-logical-asap-optimization`, `asap_aware_cse.rs` | +| Enumeration of Stage 1 candidates (the Cartesian product of Pass 1 choices, each followed by its Pass 2 candidates) | `asap-logical-asap-optimization`, `workload_candidates.rs` | +| What each summary family can merge and answer (`FamilyCapability::of`: `merge_inputs`, `readouts`) | `asap-types`, `ir/schema/family_capability.rs` | +| Stage 2: materialization options per summary build (`NotMaterialized`, `MatAtIngestion`, `MatAtQuery`, `MatAtQueryKeptAcrossEvaluations`), ingestion-time eligibility, physical operator implementation | `asap-physical-asap-optimization` | +| Timing and data-state edge rules applied from a `MaterializationAssignment` | `asap-types`, `ir/properties/timing.rs` and `ir/properties/execution_data_state.rs`; `Materialization` in `physical/materialization.rs` | +| Stage 3: accuracy check, latency check, deployment-capability checks, resource usage (`ResourceUsage`), cost models 1–3 (`CostModel`), measured operation costs | `asap-plan-selection`, `lib.rs` and `measured_operation_costs.rs` | +| The analytical per-operator resource model Stage 3 measures with | `asap-plan-selection`, `cost/analytical_cost.rs` and `cost/physical_operator_statistics.rs` | +| The accuracy algebra: per-family guarantees, composition, budget split | `asap-logical-asap-optimization`, `accuracy/` (`AccuracyModel`, `DefaultAccuracyModel`, `estimators/`, `composition.rs`, `budget.rs`) | +| What the executor supports (`DeploymentCapabilities`) | `asap-logical-asap-optimization`, `deployment_capabilities.rs` | +| The inputs every stage reads (`PlanningModels`: accuracy model, evidence, plan selection settings, deployment capabilities) and the `OptimizationPass` trait | `asap-plan-selection`, `pass/mod.rs` | Not yet wired into the stages: `maintained_population.rs` (maintained -populations for instant top-k, quantile and sum), `pane_sharing.rs`, -`storage_io.rs`, `physical_handoff_cost.rs` and `query_physical_lowering.rs`. +populations for instant top-k, quantile and sum) in Stage 1, and +`pane_sharing.rs`, `cost/storage_io.rs`, `cost/physical_handoff_cost.rs` and +`cost/query_physical_lowering.rs` in plan selection. ## Algorithm in one pass @@ -89,9 +102,10 @@ displays and refuses workloads over `--max-combinations`. ## Verification -- Unit tests sit beside each module (`cargo nextest run -p asap-aware-mapping - -E 'test()'`). -- `crates/asap-aware-mapping/tests/per_computation_candidates.rs` covers +- Unit tests sit beside each module (`cargo nextest run -p + asap-logical-asap-optimization -p asap-physical-asap-optimization -p + asap-plan-selection -E 'test()'`). +- `crates/logical-asap-optimization/tests/per_computation_candidates.rs` covers Pass 1. - `crates/integration-tests/tests/planner_layering_example{1,2,3,4}.rs` pin the design document's examples through every stage; diff --git a/docs/develop_docs/asap-aware-mapping-contracts.md b/docs/develop_docs/asap-aware-mapping-contracts.md index 1c06ff62f..c7e6ce164 100644 --- a/docs/develop_docs/asap-aware-mapping-contracts.md +++ b/docs/develop_docs/asap-aware-mapping-contracts.md @@ -5,7 +5,8 @@ the stage code relies on: what each stage's input and output may contain, and which component owns each decision. Read the [architecture](asap-aware-mapping-architecture.md) first; use the [extension guide](extend-asap-aware-mapping.md) when changing one. Field-level -details are in the rustdoc (`cargo doc -p asap-aware-mapping -p asap-types +details are in the rustdoc (`cargo doc -p asap-logical-asap-optimization -p +asap-physical-asap-optimization -p asap-plan-selection -p asap-types --no-deps`). ## 1. IR at each stage @@ -116,4 +117,4 @@ details are in the rustdoc (`cargo doc -p asap-aware-mapping -p asap-types | When each build runs and whether it is kept | Stage 2 | | Which plan is valid and which is chosen | Stage 3 | | What the executor can build, retain and how it is priced | the deployment, through `PlanningModels` (`DeploymentCapabilities`, `PlanSelectionSettings`, `AccuracyModel`) | -| Running the plan | the executor (`asap-physical-operators`), outside the planner | +| Running the plan | the executor (`asap-executor`), outside the planner | diff --git a/docs/develop_docs/end-to-end-accuracy-guarantees.md b/docs/develop_docs/end-to-end-accuracy-guarantees.md index 405f2a80e..9f4000aaa 100644 --- a/docs/develop_docs/end-to-end-accuracy-guarantees.md +++ b/docs/develop_docs/end-to-end-accuracy-guarantees.md @@ -40,8 +40,8 @@ The main implementation locations are: | Concern | Location | | --- | --- | -| Guarantee and error vocabulary | `asap_types::post_asap::guarantee` | -| Accuracy model and built-in propagation | `asap_aware_mapping::accuracy` | +| Guarantee and error vocabulary | `asap_types::ir::properties::guarantee` | +| Accuracy model and built-in propagation | `asap_logical_asap_optimization::accuracy` | | Candidate construction and legality filtering | `asap_aware_mapping::replacement` | | Parameter sizing hooks | `asap_aware_mapping::cost_model` | | Guarantee and rejection export | The `stage_pipeline` devtool's document (`crates/devtools/src/bin/stage_pipeline.rs`) | diff --git a/docs/develop_docs/extend-asap-aware-mapping.md b/docs/develop_docs/extend-asap-aware-mapping.md index b27f4a1ca..a02d844a4 100644 --- a/docs/develop_docs/extend-asap-aware-mapping.md +++ b/docs/develop_docs/extend-asap-aware-mapping.md @@ -11,30 +11,32 @@ owns the decision. A new sketch that Pass 1 can offer for an intent. 1. **Type.** Add the variant to `SketchAlgorithm` and its parameters to - `SketchParams` (`crates/types/src/post_asap/sketch.rs`). + `SketchParams` (`crates/types/src/ir/schema/sketch.rs`). 2. **Capability.** Add its row to `FamilyCapability::of` - (`crates/types/src/post_asap/family_capability.rs`): which inputs it can + (`crates/types/src/ir/schema/family_capability.rs`): which inputs it can merge (`MergeInputRule`) and which statistics it answers (`StatisticKind`, with the merged error). Pass 2's sharing, pane and rollup rules read only this table. 3. **Offer and size.** List it for its intents in - `realization::summary_candidates`, most preferred first, and size it in + `realization::summary_candidates` (`crates/logical-asap-optimization`), + most preferred first, and size it in `realization::default_size_params`. -4. **Accuracy.** Add its error model under `accuracy/estimators/` and return +4. **Accuracy.** Add its error model under + `crates/logical-asap-optimization/src/accuracy/estimators/` and return it from `DefaultAccuracyModel`'s `local_guarantee`, by inverting the same sizing formulas. Without a guarantee, every candidate that uses it is rejected. 5. **Cost.** Give it a state size and per-operation CPU in - `stage3_plan_selection.rs` (and accept its parameters in - `stage3_plan_selection/measured_operation_costs.rs` if deployments measure - it). + `crates/plan-selection/src/lib.rs` (and accept its parameters in + `crates/plan-selection/src/measured_operation_costs.rs` if deployments + measure it). 6. **Executor.** Wrap the `asap_sketchlib` implementation in - `crates/asap-physical-operators/src/summary_kernels/` and register it in + `crates/executor/src/summary_kernels/` and register it in `factory.rs` and `capability.rs`. Gaps in the library are fixed upstream, not with a local copy. 7. **Labels.** Name it in `asap_devtools::candidate_labels`. -Tests: a Pass 1 test in `crates/asap-aware-mapping/tests/per_computation_candidates.rs` +Tests: a Pass 1 test in `crates/logical-asap-optimization/tests/per_computation_candidates.rs` that the intent gets the new candidate (and that a deployment without it prunes it with `UnsupportedByDeployment`); an estimator test beside the estimator; an execution test in `crates/integration-tests/tests/` such as @@ -45,12 +47,13 @@ estimator; an execution test in `crates/integration-tests/tests/` such as A new way for computations of one Pass 1 candidate to share, split or reuse work. -1. In `asap_aware_cse.rs`, add the part it can choose as a `*Candidate` type +1. In `crates/logical-asap-optimization/src/asap_aware_cse.rs`, add the part it can choose as a `*Candidate` type with a finder (for example `rollup_candidates(roots)`), and the function that applies a set of them and returns `*Pass2Candidate`s (for example `rollup_pass2_candidates`). Read summary families only through `FamilyCapability`. -2. In `pass/stage1_to_stage3.rs`, call it from Stage 1 enumeration after the +2. In `crates/logical-asap-optimization/src/workload_candidates.rs`, call it + from Stage 1 enumeration after the existing rules and record what it applied in a new `Pass2Candidate` field. 3. Rebuild its label from that field in `asap_devtools::candidate_labels`. 4. Add its row to the Pass 2 rule table of the @@ -67,13 +70,13 @@ contains the candidate and that Stage 3 prices it. ## 3. Add a Stage 2 materialization option 1. Add the `Materialization` variant - (`crates/types/src/post_asap/execution_data_state.rs`) and its timing in - `crates/types/src/ir/timing.rs`, with its data-state edge rules. -2. Offer it in `stage2_physical_asap_optimization.rs` (the + (`crates/types/src/physical/materialization.rs`) and its timing in + `crates/types/src/ir/properties/timing.rs`, with its data-state edge rules. +2. Offer it in `crates/physical-asap-optimization/src/lib.rs` (the `MaterializationCandidate` options), after the existing options so default candidate ids stay the same, and gate it on the deployment input that allows it. -3. Price it in `stage3_plan_selection.rs`: what it keeps in memory or on +3. Price it in `crates/plan-selection/src/lib.rs`: what it keeps in memory or on disk, and what each evaluation rebuilds. Tests: Stage 2 unit tests for when the option is offered; diff --git a/docs/develop_docs/library-api.md b/docs/develop_docs/library-api.md index bc97bd7a0..d8a12f20f 100644 --- a/docs/develop_docs/library-api.md +++ b/docs/develop_docs/library-api.md @@ -15,7 +15,7 @@ plan, and a serializable DAG is not evidence of runtime readiness. | --- | --- | --- | | Pre-ASAP IR | Frontend `lower_*` | [Lower a query](#lower-a-query-into-pre-asap-ir) | | Selected plan for a workload (Stages 0–3) | `asap_planner::e2e_plan` | [Plan a workload](#plan-a-workload-through-the-stages) | -| Every stage's candidates for already lowered roots | `asap_aware_mapping::run_stage1_to_stage3` | [Plan a workload](#plan-a-workload-through-the-stages) | +| Every stage's candidates for already lowered roots | `asap_plan_selection::run_stage1_to_stage3` | [Plan a workload](#plan-a-workload-through-the-stages) | | Physical DAG for an executor | `PlanOutput::physical_asap_dag`, or timing and export functions | [Export](#export) | Each recipe ends at a different artifact. Use only the stages needed for that @@ -23,19 +23,22 @@ artifact, while preserving the checks required by its intended consumer. ## Dependencies -Inside this workspace, depend on the frontend you need, `asap-aware-mapping`, -and `asap-types`, or on `asap-planner`, which wraps every frontend and the -planning stages. External users can use Git dependencies pinned to a compatible -revision; use the same revision across these crates, and pick a revision at -least as new as the APIs you call (this guide describes `main` at `775e21ee`). +Inside this workspace, depend on the frontend you need, the stage crates +(`asap-logical-asap-optimization` for Stage 1, `asap-physical-asap-optimization` +for Stage 2, `asap-plan-selection` for Stage 3 and the Stage 1 to Stage 3 +driver), and `asap-types`, or on `asap-planner`, which wraps every frontend and +the planning stages. External users can use Git dependencies pinned to a +compatible revision; use the same revision across these crates, and pick a +revision at least as new as the APIs you call. Replace `REV` below with a +`main` revision that has the stage crates (`crates/plan-selection`). For the examples below: ```toml [dependencies] -asap-planner = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "775e21ee38879631fe9752c105d50bea890d3d4a" } -asap-frontend-promql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "775e21ee38879631fe9752c105d50bea890d3d4a" } -asap-aware-mapping = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "775e21ee38879631fe9752c105d50bea890d3d4a" } -asap-types = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "775e21ee38879631fe9752c105d50bea890d3d4a" } +asap-planner = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "REV" } +asap-frontend-promql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "REV" } +asap-plan-selection = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "REV" } +asap-types = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "REV" } # For the async e2e_plan example: tokio = { version = "1", features = ["rt", "macros", "rt-multi-thread"] } ``` @@ -101,7 +104,7 @@ use asap_types::workload::{ BatchEntry, DataWorkload, DurationMs, Query, PlanningWorkload, QueryLanguage, QueryRequirements, QueryWorkload, }; -use asap_types::types::AccuracyTarget; +use asap_types::ir::properties::AccuracyTarget; fn main() -> Result<(), Box> { let workload = PlanningWorkload { @@ -163,7 +166,7 @@ UserInput::new(workload: &PlanningWorkload, frontend_specific: FrontendInput<'_> FrontendInput::{Sql { catalog: &SqlCatalog }, Promql { histograms: Option }, Metricsql} // A caller that already holds lowered roots calls the stages directly: -asap_aware_mapping::run_stage1_to_stage3( +asap_plan_selection::run_stage1_to_stage3( roots: Vec<(Id, QueryRoot)>, entries: &[QueryWorkloadEntry], // one per root: accuracy, latency bound, recurrence data: &DataWorkload, @@ -175,8 +178,8 @@ asap_aware_mapping::run_stage1_to_stage3( | --- | --- | | `PlanningModels::new(accuracy, evidence)` | Your `&dyn AccuracyModel` and `&dyn AccuracyEvidenceProvider`, with default plan selection settings and deployment capabilities; `with_accuracy` and `with_evidence` replace one of them | | `PlanningModels::builtin()` | Built-in accuracy model (`DefaultAccuracyModel`) and no extra evidence (`NoAccuracyEvidence`). `with_plan_selection(&settings)` sets the deployment's `PlanSelectionSettings`; `with_deployment_capabilities(&capabilities)` sets its `DeploymentCapabilities` (field `deployment_capabilities`) | -| `DeploymentCapabilities` (`asap_aware_mapping::deployment_capabilities`) | What the executor supports: `supported_summaries` (`None`, the default, supports every summary the planner builds; otherwise `SupportedSummary { family: SketchAlgorithm, readouts: Vec }` entries; Pass 1 prunes others with `PruneReason::UnsupportedByDeployment`), `raw_data_retained` (default `true`; when `false`, a candidate that scans raw data at query time is invalid, and stored data is refused with `SelectionError::InvalidDeploymentCapabilities`) and `query_time_results_retained` (default `false`; when `true`, Stage 2 offers `MatAtQueryKeptAcrossEvaluations` for tumbling panes) | -| `PlanSelectionSettings` (`asap_aware_mapping::stage3_plan_selection`) | `cost_model`, `resource_speed` (`ms_per_cpu_op`, `ms_per_scanned_byte`), `non_repeating_query_horizon_seconds` (default 3,600) and `measured_operation_costs` (default empty: one `MeasuredOperationCost { sketch, operation, cpu_ns_per_operation }` per measured sketch configuration and `SummaryOperation` (`Update`, `Merge`, `Estimate`); a sketch node with an exactly matching entry is priced from it, every other node from the formulas, and each node's `NodeCost::price_source` says which) | +| `DeploymentCapabilities` (`asap_logical_asap_optimization::deployment_capabilities`) | What the executor supports: `supported_summaries` (`None`, the default, supports every summary the planner builds; otherwise `SupportedSummary { family: SketchAlgorithm, readouts: Vec }` entries; Pass 1 prunes others with `PruneReason::UnsupportedByDeployment`), `raw_data_retained` (default `true`; when `false`, a candidate that scans raw data at query time is invalid, and stored data is refused with `SelectionError::InvalidDeploymentCapabilities`) and `query_time_results_retained` (default `false`; when `true`, Stage 2 offers `MatAtQueryKeptAcrossEvaluations` for tumbling panes) | +| `PlanSelectionSettings` (`asap_plan_selection`) | `cost_model`, `resource_speed` (`ms_per_cpu_op`, `ms_per_scanned_byte`), `non_repeating_query_horizon_seconds` (default 3,600) and `measured_operation_costs` (default empty: one `MeasuredOperationCost { sketch, operation, cpu_ns_per_operation }` per measured sketch configuration and `SummaryOperation` (`Update`, `Merge`, `Estimate`); a sketch node with an exactly matching entry is priced from it, every other node from the formulas, and each node's `NodeCost::price_source` says which) | | `CostModel::UsageOverTime(ResourceUnitCost)` | Cost model 1 (default, `ResourceUnitCost::illustrative()`): unit cost × usage per second; the cheapest valid candidate wins | | `CostModel::ProvisionedCapacity(ResourceUnitCost)` | Cost model 2: unit cost × peak memory, peak CPU cores and disk; the cheapest valid candidate wins | | `CostModel::ClusterCapacity(ClusterLimits)` | Cost model 3: memory, disk and optional CPU cores are hard limits; `ClusterObjective::{LowestLatency, LeastMemory, ParetoFront}` chooses among the candidates that fit | @@ -194,7 +197,7 @@ dependencies): ```rust use asap_planner::{e2e_plan, FrontendInput, PlanningModels, UserInput}; -use asap_types::types::AccuracyTarget; +use asap_types::ir::properties::AccuracyTarget; use asap_types::workload::{ BatchEntry, DataArrival, DataWorkload, DurationMs, MetricType, PlanningWorkload, Query, QueryLanguage, QueryRequirements, QueryWorkload, SourceFacts, @@ -258,7 +261,7 @@ rejected heap top-k candidates, `P2` and `P3`, each with the reason model 3 instead: ```rust -use asap_aware_mapping::stage3_plan_selection::{ +use asap_plan_selection::{ ClusterLimits, ClusterObjective, CostModel, PlanSelectionSettings, }; @@ -329,13 +332,14 @@ cost/guarantee evidence needed downstream instead of exporting only a bare DAG. For public symbol details, build local API documentation with: ```sh -cargo doc -p asap-aware-mapping -p asap-types --no-deps +cargo doc -p asap-logical-asap-optimization -p asap-physical-asap-optimization \ + -p asap-plan-selection -p asap-types --no-deps ``` ## Source references - [Frontend PromQL](../../crates/frontend-promql/src/lib.rs), [SQL](../../crates/frontend-sql/src/lib.rs), [MetricsQL](../../crates/frontend-metricsql/src/lib.rs) -- [Planner facade](../../crates/planner/src/lib.rs), [planning stages](../../crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs), [plan selection and cost models 1–3](../../crates/asap-aware-mapping/src/stage3_plan_selection.rs) -- [Deployment capabilities](../../crates/asap-aware-mapping/src/deployment_capabilities.rs) -- [Workload types](../../crates/types/src/workload.rs) +- [Planner facade](../../crates/planner/src/lib.rs), [planning stages](../../crates/plan-selection/src/pass/stage1_to_stage3.rs), [plan selection and cost models 1–3](../../crates/plan-selection/src/lib.rs) +- [Deployment capabilities](../../crates/logical-asap-optimization/src/deployment_capabilities.rs) +- [Workload types](../../crates/types/src/workload/mod.rs) - [Planner-runtime contract](../design_docs/architecture/planner-runtime-contract.md) diff --git a/docs/develop_docs/offline-sketch-evidence.md b/docs/develop_docs/offline-sketch-evidence.md index d3dd03cdb..aa341c59f 100644 --- a/docs/develop_docs/offline-sketch-evidence.md +++ b/docs/develop_docs/offline-sketch-evidence.md @@ -20,7 +20,7 @@ wire format. `crates/asap-aware-mapping/tests/data/offline-evidence-synthetic.js is an explicitly fabricated format fixture, never benchmark evidence. Runtime validation additionally checks cross-field constraints and matching context. -Resource values share the internal `asap_types::resources::PhysicalResources` container. Its CPU payload preserves units: `ModeledCpu { cpu_ops: f64 }` represents modeled operations, while `MeasuredCpu` contains optional measured `build_cpu_ns`, `update_cpu_ns`, `merge_cpu_ns`, `prepare_cpu_ns`, and `read_cpu_ns` diff --git a/docs/develop_docs/physical-compile-coverage.md b/docs/develop_docs/physical-compile-coverage.md index 3f39a094b..1d48d2e64 100644 --- a/docs/develop_docs/physical-compile-coverage.md +++ b/docs/develop_docs/physical-compile-coverage.md @@ -1,7 +1,7 @@ # Physical compile coverage for deployment computation Audience: developers moving computation from ASAPQuery-backend into -`asap_physical_operators::physical_planner`. +`asap_executor::physical_planner`. ## Contract diff --git a/docs/develop_docs/physical-handoff-costs.md b/docs/develop_docs/physical-handoff-costs.md index 037b08900..4d56ad6c1 100644 --- a/docs/develop_docs/physical-handoff-costs.md +++ b/docs/develop_docs/physical-handoff-costs.md @@ -1,6 +1,6 @@ # Physical handoff byte estimates -`asap_types::resources` owns the canonical `PhysicalHandoffBytes` and +`asap_types::workload::resources` owns the canonical `PhysicalHandoffBytes` and `PhysicalHandoffKind` definitions in `resources/physical_handoff.rs`. The mapping crate re-exports those same types from `physical_handoff_cost` for import compatibility; all estimator and export consumers therefore use shared definitions, not copies. @@ -95,5 +95,5 @@ a separate follow-up integration point. Verification: ```sh -cargo test -p asap-aware-mapping --test physical_handoff_cost +cargo test -p asap-plan-selection --test physical_handoff_cost ``` diff --git a/docs/develop_docs/planner-vocabulary-migration.md b/docs/develop_docs/planner-vocabulary-migration.md index c29982a95..7a6604474 100644 --- a/docs/develop_docs/planner-vocabulary-migration.md +++ b/docs/develop_docs/planner-vocabulary-migration.md @@ -28,7 +28,7 @@ names. | `resources::boundary` | `resources::physical_handoff` | | `BoundaryKind` | `PhysicalHandoffKind` | | `BoundaryResources` | `PhysicalHandoffBytes` | -| `asap_aware_mapping::boundary_cost` | `asap_aware_mapping::physical_handoff_cost` | +| `asap_aware_mapping::boundary_cost` | `asap_plan_selection::cost::physical_handoff_cost` | | `PhysicalBoundary` | `PhysicalHandoff` | | `BoundaryNodeEvidence`, `BoundaryPlanEvidence` | `PhysicalHandoffNodeEvidence`, `PhysicalHandoffPlanEvidence` | | `BoundaryProfile`, `BoundaryCalibration`, `BoundaryEstimate` | `PhysicalHandoffProfile`, `PhysicalHandoffCalibration`, `PhysicalHandoffEstimate` | diff --git a/docs/develop_docs/pre-asap-ir.md b/docs/develop_docs/pre-asap-ir.md index 35ce46c3c..355aae925 100644 --- a/docs/develop_docs/pre-asap-ir.md +++ b/docs/develop_docs/pre-asap-ir.md @@ -23,7 +23,7 @@ Only operations that are semantically relevant to answering the query and select ## The node -A plan is a DAG of `Rc` (`crates/types/src/ir/node.rs`). Nodes are immutable +A plan is a DAG of `Rc` (`crates/types/src/ir/operator/node.rs`). Nodes are immutable and shared through `Rc`: a structurally identical sub-DAG referenced from several parents is one node, and that pointer identity is what CSE, target discovery and plan assembly key on. @@ -63,7 +63,7 @@ derived one. ## Schema -One `Schema` type (`crates/types/src/pre_asap/schema.rs`) describes every edge, whether it +One `Schema` type (`crates/types/src/ir/schema/mod.rs`) describes every edge, whether it carries rows or summary state: ```rust @@ -92,7 +92,7 @@ across consumers when its row identity is provable. ## Scalar expressions -Value computation lives in `ScalarExpr` (`crates/types/src/ir/scalar.rs`), owned **by value** +Value computation lives in `ScalarExpr` (`crates/types/src/ir/scalar/mod.rs`), owned **by value** by an operator field: `Scan.predicates`, `Filter.pred`, `Join.pred`, `Project.cols[i].expr`, `Aggregate.having`, `Sort.keys[i].expr`, `SQLWindowFunc.args`/`order_by`, `PromqlRelabel.value`, `Values.rows`, and `QueryRoot::Scalar` and `PromqlVectorFromScalar`. A scalar expression never diff --git a/docs/develop_docs/storage-operation-costs.md b/docs/develop_docs/storage-operation-costs.md index c86900ca6..b77bab46a 100644 --- a/docs/develop_docs/storage-operation-costs.md +++ b/docs/develop_docs/storage-operation-costs.md @@ -43,9 +43,9 @@ profile and coefficients before ranking. Scan read extents must add up to the scan's authoritative `source_read_bytes`; explicit additional storage actions can be bound to other physical nodes. -Their sole data type is `asap_types::resources::StorageResources`, defined in +Their sole data type is `asap_types::workload::resources::StorageResources`, defined in the shared resources module alongside CPU and byte dimensions. The mapping -crate re-exports it at `asap_aware_mapping::storage_io::StorageResources` for +crate re-exports it at `asap_plan_selection::cost::storage_io::StorageResources` for source compatibility; the four integer JSON fields are unchanged. Pure term enumeration and checked addition live with the shared type. Access profiles, request-count estimation, calibration, and ranking remain in the mapping @@ -82,6 +82,6 @@ cache-adjusted bytes and CPU with uncached operation counts. Verification: ```sh -cargo test -p asap-aware-mapping --test storage_io -cargo test -p asap-aware-mapping --lib storage_io +cargo test -p asap-plan-selection --test storage_io +cargo test -p asap-plan-selection --lib storage_io ``` diff --git a/tools/verify_metricsql_external_consumer.sh b/tools/verify_metricsql_external_consumer.sh index f4e9d89ac..0fa137494 100644 --- a/tools/verify_metricsql_external_consumer.sh +++ b/tools/verify_metricsql_external_consumer.sh @@ -21,7 +21,7 @@ cat > "$consumer_dir/src/main.rs" <<'EOF' fn main() { asap_frontend_metricsql::lower_metricsql( "sum(rate(requests_total[5m]))", - asap_types::types::AccuracyTarget::exact(), + asap_types::ir::properties::AccuracyTarget::exact(), std::time::Duration::from_secs(15), ) .expect("a downstream stable crate can parse and lower MetricsQL");