From e29f1abdb6fbf799ea6752b02e055bebf0a3d0b0 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sun, 18 Oct 2026 14:46:48 +0000 Subject: [PATCH 1/6] refactor!: move AccuracyModel to plan-selection Stage 3 is the only stage that uses the deployment's accuracy model (#509, #572 Q22). The AccuracyModel trait and DefaultAccuracyModel move from asap-logical-asap-optimization to asap_plan_selection::accuracy. Stage 1 calls the built-in analytical rules directly: the free functions accuracy::{local_guarantee, propagate, satisfies}. BREAKING CHANGE: asap_logical_asap_optimization::{AccuracyModel, DefaultAccuracyModel} are removed; use asap_plan_selection::{AccuracyModel, DefaultAccuracyModel}. AccuracyModel::local_guarantee and DefaultAccuracyModel::sketch_guarantee are removed (no caller). --- .../tests/univmon_candidates.rs | 12 +- .../src/accuracy/composition.rs | 1261 ++++++++--------- .../src/accuracy/estimators/cms.rs | 36 +- .../src/accuracy/estimators/count_sketch.rs | 25 +- .../src/accuracy/estimators/hll.rs | 21 +- .../src/accuracy/estimators/kll.rs | 23 +- .../src/accuracy/estimators/mod.rs | 2 +- .../src/accuracy/estimators/univmon.rs | 2 +- .../src/accuracy/mod.rs | 157 +- crates/logical-asap-optimization/src/lib.rs | 22 +- .../src/per_computation_candidates.rs | 58 +- crates/plan-selection/src/accuracy.rs | 65 + crates/plan-selection/src/lib.rs | 4 +- crates/plan-selection/src/pass/mod.rs | 5 +- crates/types/src/ir/properties/guarantee.rs | 11 +- 15 files changed, 824 insertions(+), 880 deletions(-) create mode 100644 crates/plan-selection/src/accuracy.rs diff --git a/crates/frontend-promql/tests/univmon_candidates.rs b/crates/frontend-promql/tests/univmon_candidates.rs index a84cc3ad..b283cd1c 100644 --- a/crates/frontend-promql/tests/univmon_candidates.rs +++ b/crates/frontend-promql/tests/univmon_candidates.rs @@ -1,6 +1,6 @@ use std::rc::Rc; -use asap_logical_asap_optimization::accuracy::{AccuracyModel, DefaultAccuracyModel}; +use asap_logical_asap_optimization::accuracy::{local_guarantee, satisfies}; mod support; use asap_types::ir::cse::share_common_sub_dags; use asap_types::ir::properties::AccuracyMetric; @@ -88,9 +88,7 @@ fn four_evaluations_size_their_value_frequency_states_and_keep_honest_guarantees Some(true) ); assert_eq!(guarantee.spec.failure_probability.evaluate(), None); - assert!(DefaultAccuracyModel - .local_guarantee(family, query) - .is_none()); + assert!(local_guarantee(family, query).is_none()); } 1 => { assert!(matches!( @@ -102,7 +100,7 @@ fn four_evaluations_size_their_value_frequency_states_and_keep_honest_guarantees 2 => { let guarantee = guarantee.expect("an L2 guarantee"); assert_eq!(guarantee.spec.accuracy_metric, AccuracyMetric::Relative); - assert!(DefaultAccuracyModel.satisfies( + assert!(satisfies( guarantee, &AccuracyTarget::max_error(0.02).accuracy_metric(AccuracyMetric::Relative) )); @@ -111,9 +109,7 @@ fn four_evaluations_size_their_value_frequency_states_and_keep_honest_guarantees // Entropy has no certified bound, so composition certifies nothing. _ => { assert!(guarantee.is_none()); - assert!(DefaultAccuracyModel - .local_guarantee(family, query) - .is_none()); + assert!(local_guarantee(family, query).is_none()); } } post_asap_dag(root); diff --git a/crates/logical-asap-optimization/src/accuracy/composition.rs b/crates/logical-asap-optimization/src/accuracy/composition.rs index dcac1f39..a731f788 100644 --- a/crates/logical-asap-optimization/src/accuracy/composition.rs +++ b/crates/logical-asap-optimization/src/accuracy/composition.rs @@ -78,471 +78,468 @@ pub fn parse_source_norm_statistic(statistic: &str) -> Option<(&'static str, &st }) } -impl DefaultAccuracyModel { - fn additive( - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - local: &AccuracyGuarantee, - rule: &str, - ) -> AccuracyGuarantee { - let mut terms: Vec = inputs.iter().map(|g| g.spec.error.clone()).collect(); - terms.push(local.spec.error.clone()); - let mut deltas: Vec = inputs - .iter() - .map(|g| g.spec.failure_probability.clone()) - .collect(); - deltas.push(local.spec.failure_probability.clone()); - AccuracyGuarantee::new( - local.spec.accuracy_metric, - BoundExpr::Sum { terms }, - ProbabilityExpr::UnionBound { terms: deltas }, - composed_provenance(op, inputs, local, rule), - ) - } +fn additive( + op: &CompositionOperator, + inputs: &[AccuracyGuarantee], + local: &AccuracyGuarantee, + rule: &str, +) -> AccuracyGuarantee { + let mut terms: Vec = inputs.iter().map(|g| g.spec.error.clone()).collect(); + terms.push(local.spec.error.clone()); + let mut deltas: Vec = inputs + .iter() + .map(|g| g.spec.failure_probability.clone()) + .collect(); + deltas.push(local.spec.failure_probability.clone()); + AccuracyGuarantee::new( + local.spec.accuracy_metric, + BoundExpr::Sum { terms }, + ProbabilityExpr::UnionBound { terms: deltas }, + composed_provenance(op, inputs, local, rule), + ) +} - /// `(1 + ε_total) = Π (1 + ε_i)` ⇒ for two factors - /// `ε_in + ε_out + ε_in·ε_out`; written out as the sum of all - /// cross-products so the expression DAG is exact for any input count. - fn multiplicative( - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - local: &AccuracyGuarantee, - ) -> AccuracyGuarantee { - let factors: Vec<&BoundExpr> = inputs - .iter() - .map(|g| &g.spec.error) - .chain(std::iter::once(&local.spec.error)) - .collect(); - // Every non-empty subset's product: Π(1+ε_i) − 1 = Σ_{S≠∅} Π_{i∈S} ε_i. - let mut terms = Vec::new(); - for mask in 1..(1u32 << factors.len()) { - let subset: Vec = factors - .iter() - .enumerate() - .filter(|(i, _)| mask & (1 << i) != 0) - .map(|(_, b)| (*b).clone()) - .collect(); - terms.push(if subset.len() == 1 { - subset.into_iter().next().expect("one element") - } else { - BoundExpr::Product { factors: subset } - }); - } - let mut deltas: Vec = inputs +/// `(1 + ε_total) = Π (1 + ε_i)` ⇒ for two factors +/// `ε_in + ε_out + ε_in·ε_out`; written out as the sum of all +/// cross-products so the expression DAG is exact for any input count. +fn multiplicative( + op: &CompositionOperator, + inputs: &[AccuracyGuarantee], + local: &AccuracyGuarantee, +) -> AccuracyGuarantee { + let factors: Vec<&BoundExpr> = inputs + .iter() + .map(|g| &g.spec.error) + .chain(std::iter::once(&local.spec.error)) + .collect(); + // Every non-empty subset's product: Π(1+ε_i) − 1 = Σ_{S≠∅} Π_{i∈S} ε_i. + let mut terms = Vec::new(); + for mask in 1..(1u32 << factors.len()) { + let subset: Vec = factors .iter() - .map(|g| g.spec.failure_probability.clone()) + .enumerate() + .filter(|(i, _)| mask & (1 << i) != 0) + .map(|(_, b)| (*b).clone()) .collect(); - deltas.push(local.spec.failure_probability.clone()); - AccuracyGuarantee::new( - AccuracyMetric::Relative, - BoundExpr::Sum { terms }, - ProbabilityExpr::UnionBound { terms: deltas }, - composed_provenance(op, inputs, local, "relative_cross_term_union_bound"), - ) - } - - fn lipschitz( - op: &CompositionOperator, - constant: f64, - inputs: &[AccuracyGuarantee], - local: Option<&AccuracyGuarantee>, - ) -> AccuracyGuarantee { - let input = &inputs[0]; - let scaled = BoundExpr::Scaled { - factor: constant, - inner: Box::new(input.spec.error.clone()), - }; - let (bound, delta) = match local { - Some(local) => ( - BoundExpr::Sum { - terms: vec![scaled, local.spec.error.clone()], - }, - ProbabilityExpr::UnionBound { - terms: vec![ - input.spec.failure_probability.clone(), - local.spec.failure_probability.clone(), - ], - }, - ), - None => (scaled, input.spec.failure_probability.clone()), - }; - let exact_local = AccuracyGuarantee::exact("deterministic Lipschitz transformation"); - AccuracyGuarantee::new( - AccuracyMetric::ABSOLUTE, - bound, - delta, - composed_provenance( - op, - inputs, - local.unwrap_or(&exact_local), - "lipschitz_union_bound", - ), - ) + terms.push(if subset.len() == 1 { + subset.into_iter().next().expect("one element") + } else { + BoundExpr::Product { factors: subset } + }); } + let mut deltas: Vec = inputs + .iter() + .map(|g| g.spec.failure_probability.clone()) + .collect(); + deltas.push(local.spec.failure_probability.clone()); + AccuracyGuarantee::new( + AccuracyMetric::Relative, + BoundExpr::Sum { terms }, + ProbabilityExpr::UnionBound { terms: deltas }, + composed_provenance(op, inputs, local, "relative_cross_term_union_bound"), + ) +} - /// Exact `sum` over approximate inputs: `B ≤ Σ B_i`, `δ ≤ Σ δ_i`. The - /// planner composes one *per-value* child guarantee over an unknown - /// number of input rows, so both the bound and the union bound scale by - /// `stats.input_row_count` — an [`BoundExpr::Unknown`] leaf when it is - /// not supplied. Each input's normalized bound is first converted to - /// absolute units via the statistic its metric is normalized by (also - /// unknown unless supplied); a `Rank` input has no such conversion. - fn exact_sum( - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - stats: &PropagationStats, - ) -> Result { - if all_relative(inputs) { - return Self::relative_fold(op, inputs, stats, true, "exact_sum_relative"); - } - let mut terms = Vec::with_capacity(inputs.len()); - let mut deltas = Vec::with_capacity(inputs.len()); - let mut provenance = Vec::new(); - for (i, input) in inputs.iter().enumerate() { - let absolute = - absolute_bound(input).ok_or_else(|| AccuracyError::UnsupportedComposition { - operator: op.clone(), - input_metrics: inputs.iter().map(|g| g.spec.accuracy_metric).collect(), - local_metric: None, - reason: format!( - "input {i} carries a {:?} guarantee, which has no registered \ - conversion to an absolute value error", - input.spec.accuracy_metric - ), - })?; - if let BoundExpr::Product { factors } = &absolute { - for f in factors { - if let BoundExpr::Unknown { statistic } = f { - provenance.push(GuaranteeSource::UnavailableStatistic { - statistic: statistic.clone(), - }); - } - } - } - terms.push(absolute); - deltas.push(input.spec.failure_probability.clone()); - } - let count = row_count(stats, &mut provenance); - let exact_local = AccuracyGuarantee::exact("ExactAggregate(Sum)"); - provenance.extend(composed_provenance( - op, - inputs, - &exact_local, - "exact_sum_union_bound", - )); - Ok(AccuracyGuarantee::new( - AccuracyMetric::ABSOLUTE, - BoundExpr::Product { - factors: vec![count.clone(), BoundExpr::Sum { terms }], +fn lipschitz( + op: &CompositionOperator, + constant: f64, + inputs: &[AccuracyGuarantee], + local: Option<&AccuracyGuarantee>, +) -> AccuracyGuarantee { + let input = &inputs[0]; + let scaled = BoundExpr::Scaled { + factor: constant, + inner: Box::new(input.spec.error.clone()), + }; + let (bound, delta) = match local { + Some(local) => ( + BoundExpr::Sum { + terms: vec![scaled, local.spec.error.clone()], }, - ProbabilityExpr::Scaled { - count, - inner: Box::new(ProbabilityExpr::UnionBound { terms: deltas }), + ProbabilityExpr::UnionBound { + terms: vec![ + input.spec.failure_probability.clone(), + local.spec.failure_probability.clone(), + ], }, - provenance, - )) - } - - /// Exact arithmetic mean over values with absolute-error guarantees. - /// Averaging cannot amplify the largest absolute input error. The event - /// that every row respects its bound is still protected conservatively - /// by a union bound over the input row count. - fn exact_average( - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - stats: &PropagationStats, - ) -> Result { - if all_relative(inputs) { - return Self::relative_fold(op, inputs, stats, true, "exact_average_relative"); - } - if inputs - .iter() - .any(|input| input.spec.accuracy_metric != AccuracyMetric::ABSOLUTE) - { - return Err(AccuracyError::UnsupportedComposition { - operator: op.clone(), - input_metrics: inputs.iter().map(|g| g.spec.accuracy_metric).collect(), - local_metric: None, - reason: "exact average requires absolute-error input guarantees".into(), - }); - } - let mut provenance = Vec::new(); - let count = row_count(stats, &mut provenance); - let exact_local = AccuracyGuarantee::exact("ExactAggregate(Average)"); - provenance.extend(composed_provenance( + ), + None => (scaled, input.spec.failure_probability.clone()), + }; + let exact_local = AccuracyGuarantee::exact("deterministic Lipschitz transformation"); + AccuracyGuarantee::new( + AccuracyMetric::ABSOLUTE, + bound, + delta, + composed_provenance( op, inputs, - &exact_local, - "exact_average_union_bound", - )); - Ok(AccuracyGuarantee::new( - AccuracyMetric::ABSOLUTE, - BoundExpr::Max { - terms: inputs.iter().map(|g| g.spec.error.clone()).collect(), - }, - ProbabilityExpr::Scaled { - count, - inner: Box::new(ProbabilityExpr::UnionBound { - terms: inputs - .iter() - .map(|g| g.spec.failure_probability.clone()) - .collect(), - }), - }, - provenance, - )) - } + local.unwrap_or(&exact_local), + "lipschitz_union_bound", + ), + ) +} - /// Exact `max`/`min` over approximate inputs of one shared metric: the - /// returned value's error is at most the largest input bound (order - /// statistics are monotone under a uniform perturbation), with - /// probability by the union bound over every input row. This bounds the - /// returned *value*; it does not identify the true winning key. - fn exact_extremum( - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - stats: &PropagationStats, - ) -> Result { - if all_relative(inputs) { - return Self::relative_fold(op, inputs, stats, false, "exact_extremum_relative"); - } - let metric = inputs[0].spec.accuracy_metric; - if inputs - .iter() - .any(|g| g.spec.accuracy_metric != AccuracyMetric::ABSOLUTE) - { - return Err(AccuracyError::UnsupportedComposition { - operator: op.clone(), - input_metrics: inputs.iter().map(|g| g.spec.accuracy_metric).collect(), - local_metric: None, - reason: "exact max/min requires absolute-error input guarantees".into(), - }); - } - let mut provenance = Vec::new(); - let count = row_count(stats, &mut provenance); - let exact_local = AccuracyGuarantee::exact("ExactAggregate(Max)"); - provenance.extend(composed_provenance( - op, - inputs, - &exact_local, - "exact_extremum_union_bound", - )); - Ok(AccuracyGuarantee::new( - metric, - BoundExpr::Max { - terms: inputs.iter().map(|g| g.spec.error.clone()).collect(), - }, - ProbabilityExpr::Scaled { - count, - inner: Box::new(ProbabilityExpr::UnionBound { - terms: inputs - .iter() - .map(|g| g.spec.failure_probability.clone()) - .collect(), - }), - }, - provenance, - )) +/// Exact `sum` over approximate inputs: `B ≤ Σ B_i`, `δ ≤ Σ δ_i`. The +/// planner composes one *per-value* child guarantee over an unknown +/// number of input rows, so both the bound and the union bound scale by +/// `stats.input_row_count` — an [`BoundExpr::Unknown`] leaf when it is +/// not supplied. Each input's normalized bound is first converted to +/// absolute units via the statistic its metric is normalized by (also +/// unknown unless supplied); a `Rank` input has no such conversion. +fn exact_sum( + op: &CompositionOperator, + inputs: &[AccuracyGuarantee], + stats: &PropagationStats, +) -> Result { + if all_relative(inputs) { + return relative_fold(op, inputs, stats, true, "exact_sum_relative"); } - - /// An exact sum, mean, max or min of values each within relative `α_i` - /// is within relative `max α_i`. A max or min holds whatever the signs; - /// a sum or mean only over non-negative values (no cancellation), so - /// with `needs_sign` and the sign unknown the bound is the symbolic - /// [`VALUES_NON_NEGATIVE`]. The failure probability is the union bound - /// over every input row. - fn relative_fold( - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - stats: &PropagationStats, - needs_sign: bool, - rule: &str, - ) -> Result { - if needs_sign && stats.values_non_negative == Some(false) { - return Err(AccuracyError::UnsupportedComposition { + let mut terms = Vec::with_capacity(inputs.len()); + let mut deltas = Vec::with_capacity(inputs.len()); + let mut provenance = Vec::new(); + for (i, input) in inputs.iter().enumerate() { + let absolute = + absolute_bound(input).ok_or_else(|| AccuracyError::UnsupportedComposition { operator: op.clone(), input_metrics: inputs.iter().map(|g| g.spec.accuracy_metric).collect(), local_metric: None, - reason: "relative error of a sum over signed values is unbounded".into(), - }); - } - let mut provenance = Vec::new(); - let count = row_count(stats, &mut provenance); - let bound = if needs_sign && stats.values_non_negative.is_none() { - match &stats.sign_statistic { - // The sign of one metric's samples: the bound, gated on it. - Some(statistic) => { + reason: format!( + "input {i} carries a {:?} guarantee, which has no registered \ + conversion to an absolute value error", + input.spec.accuracy_metric + ), + })?; + if let BoundExpr::Product { factors } = &absolute { + for f in factors { + if let BoundExpr::Unknown { statistic } = f { provenance.push(GuaranteeSource::UnavailableStatistic { statistic: statistic.clone(), }); - BoundExpr::Product { - factors: vec![ - BoundExpr::Max { - terms: inputs.iter().map(|g| g.spec.error.clone()).collect(), - }, - BoundExpr::Unknown { - statistic: statistic.clone(), - }, - ], - } - } - None => { - provenance.push(GuaranteeSource::UnavailableStatistic { - statistic: VALUES_NON_NEGATIVE.into(), - }); - BoundExpr::Unknown { - statistic: VALUES_NON_NEGATIVE.into(), - } } } - } else { - BoundExpr::Max { - terms: inputs.iter().map(|g| g.spec.error.clone()).collect(), - } - }; - provenance.extend(composed_provenance( - op, - inputs, - &AccuracyGuarantee::exact("exact aggregation"), - rule, - )); - Ok(AccuracyGuarantee::new( - AccuracyMetric::Relative, - bound, - ProbabilityExpr::Scaled { - count, - inner: Box::new(ProbabilityExpr::UnionBound { - terms: inputs - .iter() - .map(|g| g.spec.failure_probability.clone()) - .collect(), - }), - }, - provenance, - )) + } + terms.push(absolute); + deltas.push(input.spec.failure_probability.clone()); } + let count = row_count(stats, &mut provenance); + let exact_local = AccuracyGuarantee::exact("ExactAggregate(Sum)"); + provenance.extend(composed_provenance( + op, + inputs, + &exact_local, + "exact_sum_union_bound", + )); + Ok(AccuracyGuarantee::new( + AccuracyMetric::ABSOLUTE, + BoundExpr::Product { + factors: vec![count.clone(), BoundExpr::Sum { terms }], + }, + ProbabilityExpr::Scaled { + count, + inner: Box::new(ProbabilityExpr::UnionBound { terms: deltas }), + }, + provenance, + )) +} - /// Exact division of two relative-value estimates. If the numerator is - /// within `a` and the denominator within `b`, their ratio is within - /// `(a + b) / (1 - b)`. DDSketch supplies those deterministic bounds. - fn exact_division( - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - stats: &PropagationStats, - ) -> Result { - let unsupported = |reason: String| AccuracyError::UnsupportedComposition { +/// Exact arithmetic mean over values with absolute-error guarantees. +/// Averaging cannot amplify the largest absolute input error. The event +/// that every row respects its bound is still protected conservatively +/// by a union bound over the input row count. +fn exact_average( + op: &CompositionOperator, + inputs: &[AccuracyGuarantee], + stats: &PropagationStats, +) -> Result { + if all_relative(inputs) { + return relative_fold(op, inputs, stats, true, "exact_average_relative"); + } + if inputs + .iter() + .any(|input| input.spec.accuracy_metric != AccuracyMetric::ABSOLUTE) + { + return Err(AccuracyError::UnsupportedComposition { operator: op.clone(), input_metrics: inputs.iter().map(|g| g.spec.accuracy_metric).collect(), local_metric: None, - reason, - }; - if inputs.len() != 2 - || inputs.iter().any(|input| { - input.spec.accuracy_metric != AccuracyMetric::Relative && !input.is_exact() - }) - { - return Err(unsupported( - "division needs two relative-value or exact guarantees".into(), - )); - } - let Some(numerator) = inputs[0].spec.error.evaluate() else { - return Err(unsupported( - "numerator relative bound is unavailable".into(), - )); - }; - let Some(denominator) = inputs[1].spec.error.evaluate() else { - return Err(unsupported( - "denominator relative bound is unavailable".into(), - )); - }; - if !(numerator.is_finite() - && denominator.is_finite() - && numerator >= 0.0 - && (0.0..1.0).contains(&denominator)) - { - return Err(unsupported( - "division needs finite non-negative bounds and a denominator bound below one" - .into(), - )); - } - let Some(domains) = &stats.division_operand_domains else { - return Err(unsupported( - "division needs finite operand domains and a nonzero denominator proof".into(), - )); - }; - for domain in domains { - if !domain.lower.is_finite() - || !domain.upper.is_finite() - || domain.lower > domain.upper - || domain.contract.trim().is_empty() - { - return Err(unsupported("invalid division operand domain".into())); + reason: "exact average requires absolute-error input guarantees".into(), + }); + } + let mut provenance = Vec::new(); + let count = row_count(stats, &mut provenance); + let exact_local = AccuracyGuarantee::exact("ExactAggregate(Average)"); + provenance.extend(composed_provenance( + op, + inputs, + &exact_local, + "exact_average_union_bound", + )); + Ok(AccuracyGuarantee::new( + AccuracyMetric::ABSOLUTE, + BoundExpr::Max { + terms: inputs.iter().map(|g| g.spec.error.clone()).collect(), + }, + ProbabilityExpr::Scaled { + count, + inner: Box::new(ProbabilityExpr::UnionBound { + terms: inputs + .iter() + .map(|g| g.spec.failure_probability.clone()) + .collect(), + }), + }, + provenance, + )) +} + +/// Exact `max`/`min` over approximate inputs of one shared metric: the +/// returned value's error is at most the largest input bound (order +/// statistics are monotone under a uniform perturbation), with +/// probability by the union bound over every input row. This bounds the +/// returned *value*; it does not identify the true winning key. +fn exact_extremum( + op: &CompositionOperator, + inputs: &[AccuracyGuarantee], + stats: &PropagationStats, +) -> Result { + if all_relative(inputs) { + return relative_fold(op, inputs, stats, false, "exact_extremum_relative"); + } + let metric = inputs[0].spec.accuracy_metric; + if inputs + .iter() + .any(|g| g.spec.accuracy_metric != AccuracyMetric::ABSOLUTE) + { + return Err(AccuracyError::UnsupportedComposition { + operator: op.clone(), + input_metrics: inputs.iter().map(|g| g.spec.accuracy_metric).collect(), + local_metric: None, + reason: "exact max/min requires absolute-error input guarantees".into(), + }); + } + let mut provenance = Vec::new(); + let count = row_count(stats, &mut provenance); + let exact_local = AccuracyGuarantee::exact("ExactAggregate(Max)"); + provenance.extend(composed_provenance( + op, + inputs, + &exact_local, + "exact_extremum_union_bound", + )); + Ok(AccuracyGuarantee::new( + metric, + BoundExpr::Max { + terms: inputs.iter().map(|g| g.spec.error.clone()).collect(), + }, + ProbabilityExpr::Scaled { + count, + inner: Box::new(ProbabilityExpr::UnionBound { + terms: inputs + .iter() + .map(|g| g.spec.failure_probability.clone()) + .collect(), + }), + }, + provenance, + )) +} + +/// An exact sum, mean, max or min of values each within relative `α_i` +/// is within relative `max α_i`. A max or min holds whatever the signs; +/// a sum or mean only over non-negative values (no cancellation), so +/// with `needs_sign` and the sign unknown the bound is the symbolic +/// [`VALUES_NON_NEGATIVE`]. The failure probability is the union bound +/// over every input row. +fn relative_fold( + op: &CompositionOperator, + inputs: &[AccuracyGuarantee], + stats: &PropagationStats, + needs_sign: bool, + rule: &str, +) -> Result { + if needs_sign && stats.values_non_negative == Some(false) { + return Err(AccuracyError::UnsupportedComposition { + operator: op.clone(), + input_metrics: inputs.iter().map(|g| g.spec.accuracy_metric).collect(), + local_metric: None, + reason: "relative error of a sum over signed values is unbounded".into(), + }); + } + let mut provenance = Vec::new(); + let count = row_count(stats, &mut provenance); + let bound = if needs_sign && stats.values_non_negative.is_none() { + match &stats.sign_statistic { + // The sign of one metric's samples: the bound, gated on it. + Some(statistic) => { + provenance.push(GuaranteeSource::UnavailableStatistic { + statistic: statistic.clone(), + }); + BoundExpr::Product { + factors: vec![ + BoundExpr::Max { + terms: inputs.iter().map(|g| g.spec.error.clone()).collect(), + }, + BoundExpr::Unknown { + statistic: statistic.clone(), + }, + ], + } + } + None => { + provenance.push(GuaranteeSource::UnavailableStatistic { + statistic: VALUES_NON_NEGATIVE.into(), + }); + BoundExpr::Unknown { + statistic: VALUES_NON_NEGATIVE.into(), + } } } - if domains[1].lower <= 0.0 && domains[1].upper >= 0.0 { - return Err(unsupported("denominator domain includes zero".into())); + } else { + BoundExpr::Max { + terms: inputs.iter().map(|g| g.spec.error.clone()).collect(), } - // Keep the true and perturbed quotients finite and out of the - // subnormal range, where Float64 division loses relative accuracy. - // A nonzero numerator interval touching zero cannot prove this. - let num = &domains[0]; - if num.lower <= 0.0 && num.upper >= 0.0 && (num.lower != 0.0 || num.upper != 0.0) { - return Err(unsupported( - "numerator domain cannot exclude underflow near zero".into(), - )); + }; + provenance.extend(composed_provenance( + op, + inputs, + &AccuracyGuarantee::exact("exact aggregation"), + rule, + )); + Ok(AccuracyGuarantee::new( + AccuracyMetric::Relative, + bound, + ProbabilityExpr::Scaled { + count, + inner: Box::new(ProbabilityExpr::UnionBound { + terms: inputs + .iter() + .map(|g| g.spec.failure_probability.clone()) + .collect(), + }), + }, + provenance, + )) +} + +/// Exact division of two relative-value estimates. If the numerator is +/// within `a` and the denominator within `b`, their ratio is within +/// `(a + b) / (1 - b)`. DDSketch supplies those deterministic bounds. +fn exact_division( + op: &CompositionOperator, + inputs: &[AccuracyGuarantee], + stats: &PropagationStats, +) -> Result { + let unsupported = |reason: String| AccuracyError::UnsupportedComposition { + operator: op.clone(), + input_metrics: inputs.iter().map(|g| g.spec.accuracy_metric).collect(), + local_metric: None, + reason, + }; + if inputs.len() != 2 + || inputs.iter().any(|input| { + input.spec.accuracy_metric != AccuracyMetric::Relative && !input.is_exact() + }) + { + return Err(unsupported( + "division needs two relative-value or exact guarantees".into(), + )); + } + let Some(numerator) = inputs[0].spec.error.evaluate() else { + return Err(unsupported( + "numerator relative bound is unavailable".into(), + )); + }; + let Some(denominator) = inputs[1].spec.error.evaluate() else { + return Err(unsupported( + "denominator relative bound is unavailable".into(), + )); + }; + if !(numerator.is_finite() + && denominator.is_finite() + && numerator >= 0.0 + && (0.0..1.0).contains(&denominator)) + { + return Err(unsupported( + "division needs finite non-negative bounds and a denominator bound below one".into(), + )); + } + let Some(domains) = &stats.division_operand_domains else { + return Err(unsupported( + "division needs finite operand domains and a nonzero denominator proof".into(), + )); + }; + for domain in domains { + if !domain.lower.is_finite() + || !domain.upper.is_finite() + || domain.lower > domain.upper + || domain.contract.trim().is_empty() + { + return Err(unsupported("invalid division operand domain".into())); } - for n in [num.lower, num.upper] { - for d in [domains[1].lower, domains[1].upper] { - for nf in [1.0 - numerator, 1.0 + numerator] { - for df in [1.0 - denominator, 1.0 + denominator] { - let quotient = (n * nf) / (d * df); - if !(n * nf).is_finite() - || !(d * df).is_finite() - || !quotient.is_finite() - || (n != 0.0 && quotient.abs() < f64::MIN_POSITIVE) - { - return Err(unsupported( - "division may overflow or underflow Float64".into(), - )); - } + } + if domains[1].lower <= 0.0 && domains[1].upper >= 0.0 { + return Err(unsupported("denominator domain includes zero".into())); + } + // Keep the true and perturbed quotients finite and out of the + // subnormal range, where Float64 division loses relative accuracy. + // A nonzero numerator interval touching zero cannot prove this. + let num = &domains[0]; + if num.lower <= 0.0 && num.upper >= 0.0 && (num.lower != 0.0 || num.upper != 0.0) { + return Err(unsupported( + "numerator domain cannot exclude underflow near zero".into(), + )); + } + for n in [num.lower, num.upper] { + for d in [domains[1].lower, domains[1].upper] { + for nf in [1.0 - numerator, 1.0 + numerator] { + for df in [1.0 - denominator, 1.0 + denominator] { + let quotient = (n * nf) / (d * df); + if !(n * nf).is_finite() + || !(d * df).is_finite() + || !quotient.is_finite() + || (n != 0.0 && quotient.abs() < f64::MIN_POSITIVE) + { + return Err(unsupported( + "division may overflow or underflow Float64".into(), + )); } } } } - Ok(AccuracyGuarantee::new( - AccuracyMetric::Relative, - BoundExpr::Constant { - value: (numerator + denominator) / (1.0 - denominator), - }, - ProbabilityExpr::UnionBound { - terms: inputs - .iter() - .map(|input| input.spec.failure_probability.clone()) - .collect(), - }, - inputs + } + Ok(AccuracyGuarantee::new( + AccuracyMetric::Relative, + BoundExpr::Constant { + value: (numerator + denominator) / (1.0 - denominator), + }, + ProbabilityExpr::UnionBound { + terms: inputs .iter() - .enumerate() - .map(|(input_index, guarantee)| GuaranteeSource::ChildGuarantee { - input_index, - guarantee: Box::new(guarantee.clone()), - }) - .chain(domains.iter().enumerate().map(|(input_index, domain)| { - GuaranteeSource::InputValueDomain { - input_index, - lower: domain.lower, - upper: domain.upper, - max_samples: domain.max_samples, - contract: domain.contract.clone(), - } - })) - .chain(std::iter::once(GuaranteeSource::CompositionStep { - operator: op.clone(), - rule: "relative_division".into(), - })) + .map(|input| input.spec.failure_probability.clone()) .collect(), - )) - } + }, + inputs + .iter() + .enumerate() + .map(|(input_index, guarantee)| GuaranteeSource::ChildGuarantee { + input_index, + guarantee: Box::new(guarantee.clone()), + }) + .chain(domains.iter().enumerate().map(|(input_index, domain)| { + GuaranteeSource::InputValueDomain { + input_index, + lower: domain.lower, + upper: domain.upper, + max_samples: domain.max_samples, + contract: domain.contract.clone(), + } + })) + .chain(std::iter::once(GuaranteeSource::CompositionStep { + operator: op.clone(), + rule: "relative_division".into(), + })) + .collect(), + )) } /// Whether `guarantee` is a sketch's distinct-count estimate. @@ -697,7 +694,7 @@ fn composed_provenance( provenance } -pub(super) fn propagate( +pub fn propagate( op: &CompositionOperator, inputs: &[AccuracyGuarantee], local: Option<&AccuracyGuarantee>, @@ -806,12 +803,7 @@ pub(super) fn propagate( )); } match local.spec.accuracy_metric { - AccuracyMetric::ABSOLUTE => Ok(DefaultAccuracyModel::additive( - op, - inputs, - local, - "additive_union_bound", - )), + AccuracyMetric::ABSOLUTE => Ok(additive(op, inputs, local, "additive_union_bound")), AccuracyMetric::Relative => { if stats.values_non_negative == Some(false) { return Err(unsupported( @@ -840,7 +832,7 @@ pub(super) fn propagate( provenance, )); } - Ok(DefaultAccuracyModel::multiplicative(op, inputs, local)) + Ok(multiplicative(op, inputs, local)) } AccuracyMetric::Rank | AccuracyMetric::ABSOLUTE_STREAM_L1 @@ -867,18 +859,12 @@ pub(super) fn propagate( "Lipschitz rule needs an absolute-error local guarantee".into(), )); } - Ok(DefaultAccuracyModel::lipschitz( - op, *constant, inputs, local, - )) - } - CompositionOperator::ExactSum => DefaultAccuracyModel::exact_sum(op, inputs, stats), - CompositionOperator::ExactAverage => DefaultAccuracyModel::exact_average(op, inputs, stats), - CompositionOperator::ExactExtremum => { - DefaultAccuracyModel::exact_extremum(op, inputs, stats) - } - CompositionOperator::ExactDivision => { - DefaultAccuracyModel::exact_division(op, inputs, stats) + Ok(lipschitz(op, *constant, inputs, local)) } + CompositionOperator::ExactSum => exact_sum(op, inputs, stats), + CompositionOperator::ExactAverage => exact_average(op, inputs, stats), + CompositionOperator::ExactExtremum => exact_extremum(op, inputs, stats), + CompositionOperator::ExactDivision => exact_division(op, inputs, stats), CompositionOperator::CounterRate | CompositionOperator::InstantCounterRate | CompositionOperator::CounterIncrease => Err(unsupported( @@ -1025,29 +1011,22 @@ mod tests { fn checked_division_propagates_value_bounds_and_rejects_rank_bounds() { let op = CompositionOperator::CheckedRelativeDivision; let inputs = [rel(0.01), rel(0.01)]; - let g = DefaultAccuracyModel - .propagate(&op, &inputs, None, &Default::default()) - .unwrap(); + let g = propagate(&op, &inputs, None, &Default::default()).unwrap(); assert!((g.spec.error.evaluate().unwrap() - 0.02 / 0.99).abs() < 1e-14); let mut rank = inputs[0].clone(); rank.spec.accuracy_metric = AccuracyMetric::Rank; - assert!(DefaultAccuracyModel - .propagate(&op, &[rank.clone(), rank], None, &Default::default()) - .is_err()); - assert!(DefaultAccuracyModel - .propagate(&op, &[rel(0.01), rel(1.0)], None, &Default::default()) - .is_err()); + assert!(propagate(&op, &[rank.clone(), rank], None, &Default::default()).is_err()); + assert!(propagate(&op, &[rel(0.01), rel(1.0)], None, &Default::default()).is_err()); } #[test] fn relative_division_requires_operand_domains() { - assert!(DefaultAccuracyModel - .propagate( - &CompositionOperator::ExactDivision, - &[rel(0.01), rel(0.01)], - None, - &PropagationStats::default() - ) - .is_err()); + assert!(propagate( + &CompositionOperator::ExactDivision, + &[rel(0.01), rel(0.01)], + None, + &PropagationStats::default() + ) + .is_err()); } #[test] fn relative_division_preserves_asymmetric_bound_and_domain_provenance() { @@ -1056,14 +1035,13 @@ mod tests { division_operand_domains: Some([domain(-20., -2.), denominator]), ..Default::default() }; - let got = DefaultAccuracyModel - .propagate( - &CompositionOperator::ExactDivision, - &[rel(0.02), rel(0.03)], - None, - &stats, - ) - .unwrap(); + let got = propagate( + &CompositionOperator::ExactDivision, + &[rel(0.02), rel(0.03)], + None, + &stats, + ) + .unwrap(); assert!((got.spec.error.evaluate().unwrap() - 0.05 / 0.97).abs() < 1e-14); assert_eq!(got.spec.failure_probability.evaluate(), Some(0.)); assert_eq!( @@ -1089,55 +1067,51 @@ mod tests { division_operand_domains: Some(domains), ..Default::default() }; - assert!(DefaultAccuracyModel - .propagate( - &CompositionOperator::ExactDivision, - &[rel(0.01), rel(0.01)], - None, - &stats - ) - .is_err()); + assert!(propagate( + &CompositionOperator::ExactDivision, + &[rel(0.01), rel(0.01)], + None, + &stats + ) + .is_err()); } let stats = PropagationStats { division_operand_domains: Some([domain(1., 2.), domain(1., 2.)]), ..Default::default() }; for bound in [1., f64::NAN, f64::INFINITY, -0.1] { - assert!(DefaultAccuracyModel - .propagate( - &CompositionOperator::ExactDivision, - &[rel(0.01), rel(bound)], - None, - &stats - ) - .is_err()); + assert!(propagate( + &CompositionOperator::ExactDivision, + &[rel(0.01), rel(bound)], + None, + &stats + ) + .is_err()); } } #[test] fn exact_child_contributes_zero_error() { let local = abs(0.05, 0.01); - let out = DefaultAccuracyModel - .propagate( - &CompositionOperator::ApproximateAggregate, - &[AccuracyGuarantee::exact("sum")], - Some(&local), - &PropagationStats::default(), - ) - .unwrap(); + let out = propagate( + &CompositionOperator::ApproximateAggregate, + &[AccuracyGuarantee::exact("sum")], + Some(&local), + &PropagationStats::default(), + ) + .unwrap(); assert_eq!(out.spec.error.evaluate(), Some(0.05)); assert_eq!(out.spec.failure_probability.evaluate(), Some(0.01)); assert_eq!(out.spec.accuracy_metric, AccuracyMetric::ABSOLUTE); } #[test] fn additive_bounds_and_delta_union_bound_compose() { - let out = DefaultAccuracyModel - .propagate( - &CompositionOperator::ApproximateAggregate, - &[abs(0.02, 0.01)], - Some(&abs(0.03, 0.02)), - &PropagationStats::default(), - ) - .unwrap(); + let out = propagate( + &CompositionOperator::ApproximateAggregate, + &[abs(0.02, 0.01)], + Some(&abs(0.03, 0.02)), + &PropagationStats::default(), + ) + .unwrap(); assert!((out.spec.error.evaluate().unwrap() - 0.05).abs() < 1e-12); // Union bound, not 1 − (1−0.01)(1−0.02) = 0.0298. assert!((out.spec.failure_probability.evaluate().unwrap() - 0.03).abs() < 1e-12); @@ -1152,30 +1126,28 @@ mod tests { values_non_negative: Some(true), ..Default::default() }; - let out = DefaultAccuracyModel - .propagate( - &CompositionOperator::ApproximateAggregate, - &[rel(0.1)], - Some(&rel(0.2)), - &stats, - ) - .unwrap(); + let out = propagate( + &CompositionOperator::ApproximateAggregate, + &[rel(0.1)], + Some(&rel(0.2)), + &stats, + ) + .unwrap(); // 0.1 + 0.2 + 0.1·0.2 = 0.32, not 0.3. assert!((out.spec.error.evaluate().unwrap() - 0.32).abs() < 1e-12); assert_eq!(out.spec.accuracy_metric, AccuracyMetric::Relative); } #[test] fn relative_error_without_sign_knowledge_remains_symbolic() { - let unknown = DefaultAccuracyModel - .propagate( - &CompositionOperator::ApproximateAggregate, - &[rel(0.1)], - Some(&rel(0.2)), - &PropagationStats::default(), - ) - .unwrap(); + let unknown = propagate( + &CompositionOperator::ApproximateAggregate, + &[rel(0.1)], + Some(&rel(0.2)), + &PropagationStats::default(), + ) + .unwrap(); assert!(unknown.has_unknown()); - let signed = DefaultAccuracyModel.propagate( + let signed = propagate( &CompositionOperator::ApproximateAggregate, &[rel(0.1)], Some(&rel(0.2)), @@ -1189,14 +1161,13 @@ mod tests { #[test] fn incompatible_metrics_are_rejected_not_treated_as_exact() { // HLL cardinality error under a CMS frequency guarantee. - let err = DefaultAccuracyModel - .propagate( - &CompositionOperator::ApproximateAggregate, - &[with_metric(AccuracyMetric::Relative, 0.01)], - Some(&with_metric(AccuracyMetric::ABSOLUTE_STREAM_L1, 0.01)), - &PropagationStats::default(), - ) - .unwrap_err(); + let err = propagate( + &CompositionOperator::ApproximateAggregate, + &[with_metric(AccuracyMetric::Relative, 0.01)], + Some(&with_metric(AccuracyMetric::ABSOLUTE_STREAM_L1, 0.01)), + &PropagationStats::default(), + ) + .unwrap_err(); assert!(matches!( err, AccuracyError::UnsupportedComposition { @@ -1206,51 +1177,47 @@ mod tests { } if input_metrics == vec![AccuracyMetric::Relative] )); // Quantile rank error under value-additive logic. - let err = DefaultAccuracyModel - .propagate( - &CompositionOperator::ApproximateAggregate, - &[with_metric(AccuracyMetric::Rank, 0.01)], - Some(&abs(0.01, 0.0)), - &PropagationStats::default(), - ) - .unwrap_err(); + let err = propagate( + &CompositionOperator::ApproximateAggregate, + &[with_metric(AccuracyMetric::Rank, 0.01)], + Some(&abs(0.01, 0.0)), + &PropagationStats::default(), + ) + .unwrap_err(); assert!(matches!(err, AccuracyError::UnsupportedComposition { .. })); } #[test] fn same_metric_rank_over_rank_has_no_registered_rule() { - let err = DefaultAccuracyModel - .propagate( - &CompositionOperator::ApproximateAggregate, - &[with_metric(AccuracyMetric::Rank, 0.01)], - Some(&with_metric(AccuracyMetric::Rank, 0.01)), - &PropagationStats::default(), - ) - .unwrap_err(); + let err = propagate( + &CompositionOperator::ApproximateAggregate, + &[with_metric(AccuracyMetric::Rank, 0.01)], + Some(&with_metric(AccuracyMetric::Rank, 0.01)), + &PropagationStats::default(), + ) + .unwrap_err(); assert!(matches!(err, AccuracyError::UnsupportedComposition { .. })); } #[test] fn lipschitz_scales_the_input_bound() { - let out = DefaultAccuracyModel - .propagate( - &CompositionOperator::Lipschitz { constant: 3.0 }, - &[abs(0.1, 0.01)], - Some(&abs(0.05, 0.02)), - &PropagationStats::default(), - ) - .unwrap(); + let out = propagate( + &CompositionOperator::Lipschitz { constant: 3.0 }, + &[abs(0.1, 0.01)], + Some(&abs(0.05, 0.02)), + &PropagationStats::default(), + ) + .unwrap(); assert!((out.spec.error.evaluate().unwrap() - 0.35).abs() < 1e-12); assert!((out.spec.failure_probability.evaluate().unwrap() - 0.03).abs() < 1e-12); } #[test] fn exact_sum_over_approximate_sums_bounds_and_keeps_unknown_row_count_unknown() { - let out = DefaultAccuracyModel - .propagate( - &CompositionOperator::ExactSum, - &[abs(0.1, 0.01)], - None, - &PropagationStats::default(), - ) - .unwrap(); + let out = propagate( + &CompositionOperator::ExactSum, + &[abs(0.1, 0.01)], + None, + &PropagationStats::default(), + ) + .unwrap(); assert_eq!(out.spec.accuracy_metric, AccuracyMetric::ABSOLUTE); assert_eq!( out.spec.error.evaluate(), @@ -1261,20 +1228,19 @@ mod tests { s, GuaranteeSource::UnavailableStatistic { statistic } if statistic == "input_row_count" ))); - assert!(!DefaultAccuracyModel.satisfies(&out, &AccuracyTarget::max_error(1.0))); + assert!(!satisfies(&out, &AccuracyTarget::max_error(1.0))); let known = PropagationStats { input_row_count: Some(4), ..Default::default() }; - let out = DefaultAccuracyModel - .propagate( - &CompositionOperator::ExactSum, - &[abs(0.1, 0.01)], - None, - &known, - ) - .unwrap(); + let out = propagate( + &CompositionOperator::ExactSum, + &[abs(0.1, 0.01)], + None, + &known, + ) + .unwrap(); assert!((out.spec.error.evaluate().unwrap() - 0.4).abs() < 1e-12); assert!((out.spec.failure_probability.evaluate().unwrap() - 0.04).abs() < 1e-12); } @@ -1284,14 +1250,13 @@ mod tests { input_row_count: Some(2), ..Default::default() }; - let out = DefaultAccuracyModel - .propagate( - &CompositionOperator::ExactExtremum, - &[abs(0.1, 0.01), abs(0.3, 0.01)], - None, - &known, - ) - .unwrap(); + let out = propagate( + &CompositionOperator::ExactExtremum, + &[abs(0.1, 0.01), abs(0.3, 0.01)], + None, + &known, + ) + .unwrap(); assert!((out.spec.error.evaluate().unwrap() - 0.3).abs() < 1e-12); assert!((out.spec.failure_probability.evaluate().unwrap() - 0.04).abs() < 1e-12); } @@ -1299,17 +1264,16 @@ mod tests { /// `max α_i` of the true extremum, whatever the signs. #[test] fn exact_extremum_keeps_a_relative_value_bound() { - let out = DefaultAccuracyModel - .propagate( - &CompositionOperator::ExactExtremum, - &[rel(0.01), rel(0.02)], - None, - &PropagationStats { - input_row_count: Some(3), - ..Default::default() - }, - ) - .unwrap(); + let out = propagate( + &CompositionOperator::ExactExtremum, + &[rel(0.01), rel(0.02)], + None, + &PropagationStats { + input_row_count: Some(3), + ..Default::default() + }, + ) + .unwrap(); assert_eq!(out.spec.accuracy_metric, AccuracyMetric::Relative); assert_eq!(out.spec.error.evaluate(), Some(0.02)); assert_eq!(out.spec.failure_probability.evaluate(), Some(0.0)); @@ -1324,7 +1288,7 @@ mod tests { CompositionOperator::ExactAverage, ] { let with = |values_non_negative| { - DefaultAccuracyModel.propagate( + propagate( &op, &[rel(0.01)], None, @@ -1362,18 +1326,17 @@ mod tests { CompositionOperator::ExactSum, CompositionOperator::ExactAverage, ] { - let out = DefaultAccuracyModel - .propagate( - &op, - &[rel(0.01)], - None, - &PropagationStats { - sign_statistic: Some(statistic.clone()), - input_row_count: Some(3), - ..Default::default() - }, - ) - .unwrap(); + let out = propagate( + &op, + &[rel(0.01)], + None, + &PropagationStats { + sign_statistic: Some(statistic.clone()), + input_row_count: Some(3), + ..Default::default() + }, + ) + .unwrap(); assert_eq!(out.spec.error.evaluate(), None, "{op:?}"); let proven = out.bind(&|s| (s == statistic).then_some(1.0)); assert_eq!(proven.spec.error.evaluate(), Some(0.01), "{op:?}"); @@ -1384,51 +1347,48 @@ mod tests { #[test] fn exact_average_has_its_own_absolute_error_rule() { - let out = DefaultAccuracyModel - .propagate( - &CompositionOperator::ExactAverage, - &[abs(0.25, 0.01)], - None, - &PropagationStats { - input_row_count: Some(4), - ..PropagationStats::default() - }, - ) - .unwrap(); + let out = propagate( + &CompositionOperator::ExactAverage, + &[abs(0.25, 0.01)], + None, + &PropagationStats { + input_row_count: Some(4), + ..PropagationStats::default() + }, + ) + .unwrap(); assert_eq!(out.spec.accuracy_metric, AccuracyMetric::ABSOLUTE); assert_eq!(out.spec.error.evaluate(), Some(0.25)); assert_eq!(out.spec.failure_probability.evaluate(), Some(0.04)); } #[test] fn topk_selection_requires_a_separated_margin_certificate() { - let unknown = DefaultAccuracyModel - .propagate( - &CompositionOperator::TopKSelection, - &[abs(0.1, 0.01)], - None, - &PropagationStats::default(), - ) - .unwrap(); + let unknown = propagate( + &CompositionOperator::TopKSelection, + &[abs(0.1, 0.01)], + None, + &PropagationStats::default(), + ) + .unwrap(); assert!(unknown.has_unknown()); - let certified = DefaultAccuracyModel - .propagate( - &CompositionOperator::TopKSelection, - &[abs(0.1, 0.01)], - None, - &PropagationStats { - topk_selected_lower_bound: Some(101.0), - topk_excluded_upper_bound: Some(100.0), - topk_interval_failure_probability: Some(0.005), - ..Default::default() - }, - ) - .unwrap(); + let certified = propagate( + &CompositionOperator::TopKSelection, + &[abs(0.1, 0.01)], + None, + &PropagationStats { + topk_selected_lower_bound: Some(101.0), + topk_excluded_upper_bound: Some(100.0), + topk_interval_failure_probability: Some(0.005), + ..Default::default() + }, + ) + .unwrap(); assert_eq!(certified.spec.accuracy_metric, AccuracyMetric::ABSOLUTE); assert_eq!(certified.spec.error.evaluate(), Some(0.1)); assert_eq!(certified.spec.failure_probability.evaluate(), Some(0.005)); - let overlapping = DefaultAccuracyModel.propagate( + let overlapping = propagate( &CompositionOperator::TopKSelection, &[abs(0.1, 0.01)], None, @@ -1441,22 +1401,21 @@ mod tests { ); assert!(overlapping.is_err()); - let partial = DefaultAccuracyModel - .propagate( - &CompositionOperator::TopKSelection, - &[abs(0.1, 0.01)], - None, - &PropagationStats { - topk_selected_lower_bound: Some(101.0), - topk_interval_failure_probability: Some(0.005), - ..Default::default() - }, - ) - .unwrap(); + let partial = propagate( + &CompositionOperator::TopKSelection, + &[abs(0.1, 0.01)], + None, + &PropagationStats { + topk_selected_lower_bound: Some(101.0), + topk_interval_failure_probability: Some(0.005), + ..Default::default() + }, + ) + .unwrap(); assert!(partial.has_unknown()); assert_eq!(partial.spec.failure_probability.evaluate(), Some(0.005)); - let invalid_partial = DefaultAccuracyModel.propagate( + let invalid_partial = propagate( &CompositionOperator::TopKSelection, &[abs(0.1, 0.01)], None, diff --git a/crates/logical-asap-optimization/src/accuracy/estimators/cms.rs b/crates/logical-asap-optimization/src/accuracy/estimators/cms.rs index cbf1b5a6..e7a85f36 100644 --- a/crates/logical-asap-optimization/src/accuracy/estimators/cms.rs +++ b/crates/logical-asap-optimization/src/accuracy/estimators/cms.rs @@ -45,17 +45,16 @@ mod tests { use asap_types::ir::schema::{GroupingStrategy, SketchKind}; let c = asap_types::ir::operator::agg_intent::default_cardinality(); let params = default_size_params(SketchAlgorithm::Cms, &c, 0.01, 0.001); - let g = DefaultAccuracyModel - .local_guarantee( - &FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::Cms, params), - GroupingStrategy::default(), - ), - &SketchStatistic::Cardinality, - ) - .unwrap(); + let g = local_guarantee( + &FieldDataType::Sketch( + SketchKind::new(SketchAlgorithm::Cms, params), + GroupingStrategy::default(), + ), + &SketchStatistic::Cardinality, + ) + .unwrap(); assert_eq!(g.spec.accuracy_metric, AccuracyMetric::ABSOLUTE_STREAM_L1); - assert!(DefaultAccuracyModel.satisfies( + assert!(satisfies( &g, &AccuracyTarget::max_error(0.01) .failure_probability(0.001) @@ -72,15 +71,14 @@ mod tests { depth: 5, heap_size: 10, }; - let topk_frequency = DefaultAccuracyModel - .local_guarantee( - &FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::CmsWithHeap, cms_heap), - GroupingStrategy::default(), - ), - &SketchStatistic::TopK { k: 10 }, - ) - .expect("heap sketch still provides per-key frequency intervals"); + let topk_frequency = local_guarantee( + &FieldDataType::Sketch( + SketchKind::new(SketchAlgorithm::CmsWithHeap, cms_heap), + GroupingStrategy::default(), + ), + &SketchStatistic::TopK { k: 10 }, + ) + .expect("heap sketch still provides per-key frequency intervals"); assert_eq!( topk_frequency.spec.accuracy_metric, AccuracyMetric::ABSOLUTE_STREAM_L1 diff --git a/crates/logical-asap-optimization/src/accuracy/estimators/count_sketch.rs b/crates/logical-asap-optimization/src/accuracy/estimators/count_sketch.rs index e5bf5c44..8e6ed51b 100644 --- a/crates/logical-asap-optimization/src/accuracy/estimators/count_sketch.rs +++ b/crates/logical-asap-optimization/src/accuracy/estimators/count_sketch.rs @@ -75,23 +75,22 @@ mod tests { use asap_types::ir::schema::{GroupingStrategy, SketchKind}; let intent = default_cardinality(); let count_sketch = default_size_params(SketchAlgorithm::CountSketch, &intent, 0.01, 0.01); - let guarantee = DefaultAccuracyModel - .local_guarantee( - &FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::CountSketch, count_sketch), - GroupingStrategy::default(), - ), - &SketchStatistic::PointCount { - key: asap_types::ir::scalar::expr_ir::ColumnRef::SampleValue, - value: None, - }, - ) - .expect("CountSketch has a parameter-derived L2 guarantee"); + let guarantee = local_guarantee( + &FieldDataType::Sketch( + SketchKind::new(SketchAlgorithm::CountSketch, count_sketch), + GroupingStrategy::default(), + ), + &SketchStatistic::PointCount { + key: asap_types::ir::scalar::expr_ir::ColumnRef::SampleValue, + value: None, + }, + ) + .expect("CountSketch has a parameter-derived L2 guarantee"); assert_eq!( guarantee.spec.accuracy_metric, AccuracyMetric::ABSOLUTE_STREAM_L2 ); - assert!(DefaultAccuracyModel.satisfies( + assert!(satisfies( &guarantee, &AccuracyTarget::max_error(0.01) .failure_probability(0.01) diff --git a/crates/logical-asap-optimization/src/accuracy/estimators/hll.rs b/crates/logical-asap-optimization/src/accuracy/estimators/hll.rs index 083e43fc..38bf9c77 100644 --- a/crates/logical-asap-optimization/src/accuracy/estimators/hll.rs +++ b/crates/logical-asap-optimization/src/accuracy/estimators/hll.rs @@ -481,22 +481,21 @@ mod tests { use asap_types::ir::schema::{GroupingStrategy, SketchKind}; let c = default_cardinality(); let params = default_size_params(SketchAlgorithm::Hll, &c, 0.01, 0.01); - let g = DefaultAccuracyModel - .local_guarantee( - &FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::Hll, params), - GroupingStrategy::default(), - ), - &SketchStatistic::Cardinality, - ) - .unwrap(); + let g = local_guarantee( + &FieldDataType::Sketch( + SketchKind::new(SketchAlgorithm::Hll, params), + GroupingStrategy::default(), + ), + &SketchStatistic::Cardinality, + ) + .unwrap(); assert_eq!(g.spec.accuracy_metric, AccuracyMetric::Relative); assert_eq!(g.spec.failure_probability.evaluate(), None); - assert!(DefaultAccuracyModel.satisfies( + assert!(satisfies( &g, &AccuracyTarget::max_error(0.01).accuracy_metric(AccuracyMetric::Relative) )); - assert!(!DefaultAccuracyModel.satisfies( + assert!(!satisfies( &g, &AccuracyTarget::max_error(0.01) .failure_probability(0.01) diff --git a/crates/logical-asap-optimization/src/accuracy/estimators/kll.rs b/crates/logical-asap-optimization/src/accuracy/estimators/kll.rs index b9b19e2b..a39cecc0 100644 --- a/crates/logical-asap-optimization/src/accuracy/estimators/kll.rs +++ b/crates/logical-asap-optimization/src/accuracy/estimators/kll.rs @@ -191,22 +191,21 @@ mod tests { use asap_types::ir::schema::{GroupingStrategy, SketchKind}; let q = default_quantile(0.99); let params = default_size_params(SketchAlgorithm::Kll, &q, 0.01, 0.01); - let g = DefaultAccuracyModel - .local_guarantee( - &FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::Kll, params), - GroupingStrategy::default(), - ), - &SketchStatistic::Quantile { q: 0.99 }, - ) - .unwrap(); + let g = local_guarantee( + &FieldDataType::Sketch( + SketchKind::new(SketchAlgorithm::Kll, params), + GroupingStrategy::default(), + ), + &SketchStatistic::Quantile { q: 0.99 }, + ) + .unwrap(); assert_eq!(g.spec.accuracy_metric, AccuracyMetric::Rank); - assert!(DefaultAccuracyModel.satisfies( + assert!(satisfies( &g, &AccuracyTarget::max_error(0.01).accuracy_metric(AccuracyMetric::Rank) )); assert_eq!(g.spec.failure_probability.evaluate(), Some(0.01)); - assert!(DefaultAccuracyModel.satisfies( + assert!(satisfies( &g, &AccuracyTarget::max_error(0.01) .failure_probability(0.01) @@ -240,7 +239,7 @@ mod tests { &SketchStatistic::Quantile { q: 0.99 }, ) .unwrap(); - assert!(DefaultAccuracyModel.satisfies( + assert!(satisfies( &g, &AccuracyTarget::max_error(0.01) .failure_probability(0.001) diff --git a/crates/logical-asap-optimization/src/accuracy/estimators/mod.rs b/crates/logical-asap-optimization/src/accuracy/estimators/mod.rs index 052180d1..a20d01c5 100644 --- a/crates/logical-asap-optimization/src/accuracy/estimators/mod.rs +++ b/crates/logical-asap-optimization/src/accuracy/estimators/mod.rs @@ -54,7 +54,7 @@ fn bounded_guarantee( ) } -pub(super) fn local_guarantee( +pub fn local_guarantee( family: &FieldDataType, query: &SketchStatistic, ) -> Option { diff --git a/crates/logical-asap-optimization/src/accuracy/estimators/univmon.rs b/crates/logical-asap-optimization/src/accuracy/estimators/univmon.rs index 82592d91..78e4027a 100644 --- a/crates/logical-asap-optimization/src/accuracy/estimators/univmon.rs +++ b/crates/logical-asap-optimization/src/accuracy/estimators/univmon.rs @@ -378,7 +378,7 @@ mod tests { } fn meets(guarantee: &AccuracyGuarantee, epsilon: f64, delta: f64) -> bool { - DefaultAccuracyModel.satisfies( + satisfies( guarantee, &crate::accuracy::native_target( &AccuracyTarget::max_error(epsilon).failure_probability(delta), diff --git a/crates/logical-asap-optimization/src/accuracy/mod.rs b/crates/logical-asap-optimization/src/accuracy/mod.rs index 3866a4c0..85ac3940 100644 --- a/crates/logical-asap-optimization/src/accuracy/mod.rs +++ b/crates/logical-asap-optimization/src/accuracy/mod.rs @@ -22,99 +22,38 @@ use asap_types::ir::properties::{AccuracyScope, AccuracyTarget}; use asap_types::ir::schema::{FieldDataType, SketchAlgorithm, SketchParams, SketchStatistic}; use asap_types::ir::OperatorNode; -/// The deployment-extensible accuracy algebra. This crate ships -/// [`DefaultAccuracyModel`]; a deployment with a proof for a composition the -/// default rejects (a registered cross-metric conversion, say) implements -/// this trait and passes it to plan selection in -/// `asap_plan_selection::PlanningModels::accuracy`. -pub trait AccuracyModel { - /// The guarantee of reading `query` out of a summary of family `family` - /// built over an **exact** input — derived from the family's committed - /// parameters by inverting the same sizing formulas - /// [`crate::realization::default_size_params`] uses. `None` when this - /// model has no error model for the family (the default has none for - /// `Sample`/`Wavelet`/`StatModel`). - fn local_guarantee( - &self, - family: &FieldDataType, - query: &SketchStatistic, - ) -> Option; - - /// Compose `inputs`' guarantees (in the parent's child order) with the - /// parent's own `local` guarantee under `op`. `Err` is the fail-closed - /// answer: no registered rule, or a missing input guarantee. - fn propagate( - &self, - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - local: Option<&AccuracyGuarantee>, - stats: &PropagationStats, - ) -> Result; - - /// Whether `guarantee` meets `target`: a bound within its ε, and within - /// its δ when it states one, over the values the target's scope covers - /// (a per-value δ does not meet a whole-result target). When the target - /// sets a metric, the bound must be in that metric (or one with a sound - /// direct comparison): a bound in another metric does not satisfy, - /// however small. When it sets none, a bound in any metric does, each - /// compared in its own metric. Unknown required dimensions fail; - /// selection separately excludes missing accuracy evidence. - fn satisfies(&self, guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool; -} - -/// The built-in estimator and composition models, with conservative target checks. -#[derive(Debug, Default, Clone, Copy)] -pub struct DefaultAccuracyModel; +pub use composition::propagate; +pub use estimators::local_guarantee; /// Small relative tolerance for comparing an evaluated bound against a /// target, so a parameter sized by `⌈·⌉` to *exactly* meet ε is not rejected /// by floating-point noise. const SATISFACTION_TOLERANCE: f64 = 1e-9; -impl DefaultAccuracyModel { - /// Derive the guarantee for the committed estimator parameters and evaluation. - pub fn sketch_guarantee( - algorithm: &SketchAlgorithm, - params: &SketchParams, - query: &SketchStatistic, - ) -> Option { - estimators::sketch_guarantee(algorithm, params, query) - } -} - -impl AccuracyModel for DefaultAccuracyModel { - fn local_guarantee( - &self, - family: &FieldDataType, - query: &SketchStatistic, - ) -> Option { - estimators::local_guarantee(family, query) - } - fn propagate( - &self, - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - local: Option<&AccuracyGuarantee>, - stats: &PropagationStats, - ) -> Result { - composition::propagate(op, inputs, local, stats) - } - fn satisfies(&self, guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool { - let within = |value: Option, limit: f64| { - value.is_some_and(|v| v <= limit * (1.0 + SATISFACTION_TOLERANCE) + f64::EPSILON) - }; - if target.is_exact() || guarantee.is_exact() { - return guarantee.is_exact(); - } - target - .accuracy_metric - .is_none_or(|metric| metric_meets(guarantee.spec.accuracy_metric, metric)) - && scope_meets(guarantee, target) - && within(guarantee.spec.error.evaluate(), target.max_error) - && target - .failure_probability - .is_none_or(|delta| within(guarantee.spec.failure_probability.evaluate(), delta)) +/// The built-in conservative check of whether `guarantee` meets `target`: +/// a bound within its ε, and within its δ when it states one, over the +/// values the target's scope covers (a per-value δ does not meet a +/// whole-result target). When the target sets a metric, the bound must be +/// in that metric (or one with a sound direct comparison): a bound in +/// another metric does not satisfy, however small. When it sets none, a +/// bound in any metric does, each compared in its own metric. Unknown +/// required dimensions fail. Plan selection's default accuracy model +/// (`asap_plan_selection::DefaultAccuracyModel`) delegates to it. +pub fn satisfies(guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool { + let within = |value: Option, limit: f64| { + value.is_some_and(|v| v <= limit * (1.0 + SATISFACTION_TOLERANCE) + f64::EPSILON) + }; + if target.is_exact() || guarantee.is_exact() { + return guarantee.is_exact(); } + target + .accuracy_metric + .is_none_or(|metric| metric_meets(guarantee.spec.accuracy_metric, metric)) + && scope_meets(guarantee, target) + && within(guarantee.spec.error.evaluate(), target.max_error) + && target + .failure_probability + .is_none_or(|delta| within(guarantee.spec.failure_probability.evaluate(), delta)) } /// Whether an error bound stated in `guarantee` bounds the error `target` @@ -175,10 +114,12 @@ mod tests { unknown.spec.error = BoundExpr::Unknown { statistic: "x".into(), }; - assert!(!DefaultAccuracyModel.satisfies(&unknown, &AccuracyTarget::max_error(1.0))); - assert!(!DefaultAccuracyModel.satisfies(&abs(0.0, 0.01), &AccuracyTarget::exact())); - assert!(DefaultAccuracyModel - .satisfies(&AccuracyGuarantee::exact("x"), &AccuracyTarget::exact())); + assert!(!satisfies(&unknown, &AccuracyTarget::max_error(1.0))); + assert!(!satisfies(&abs(0.0, 0.01), &AccuracyTarget::exact())); + assert!(satisfies( + &AccuracyGuarantee::exact("x"), + &AccuracyTarget::exact() + )); } fn guarantee(metric: AccuracyMetric, bound: f64, delta: f64) -> AccuracyGuarantee { @@ -198,7 +139,6 @@ mod tests { /// meets every metric. #[test] fn satisfies_requires_the_targets_metric() { - let model = DefaultAccuracyModel; let target = |metric| { AccuracyTarget::max_error(0.5) .failure_probability(0.5) @@ -224,12 +164,12 @@ mod tests { (AccuracyMetric::Rank, AccuracyMetric::Rank, true), ] { assert_eq!( - model.satisfies(&guarantee(bound, 0.01, 0.01), &target(wanted)), + satisfies(&guarantee(bound, 0.01, 0.01), &target(wanted)), meets, "{bound:?} against {wanted:?}" ); } - assert!(model.satisfies( + assert!(satisfies( &AccuracyGuarantee::exact("x"), &target(AccuracyMetric::Rank) )); @@ -243,8 +183,8 @@ mod tests { g.spec.failure_probability = ProbabilityExpr::Unknown { statistic: "x".into(), }; - assert!(DefaultAccuracyModel.satisfies(&g, &AccuracyTarget::max_error(0.01))); - assert!(!DefaultAccuracyModel.satisfies( + assert!(satisfies(&g, &AccuracyTarget::max_error(0.01))); + assert!(!satisfies( &g, &AccuracyTarget::max_error(0.01).failure_probability(0.5) )); @@ -257,14 +197,13 @@ mod tests { /// directly. #[test] fn per_value_guarantees_meet_whole_result_targets_over_g_values() { - let model = DefaultAccuracyModel; let g = guarantee(AccuracyMetric::ABSOLUTE, 0.01, 0.001); assert_eq!(g.spec.accuracy_scope, AccuracyScope::PerValue); let whole = AccuracyTarget::max_error(0.01).failure_probability(0.01); assert_eq!(whole.accuracy_scope, AccuracyScope::WholeResult); - assert!(!model.satisfies(&g, &whole)); + assert!(!satisfies(&g, &whole)); let per_value = whole.clone().accuracy_scope(AccuracyScope::PerValue); - assert!(model.satisfies(&g, &per_value)); + assert!(satisfies(&g, &per_value)); let over = |values| g.over_whole_result(values); assert_eq!( over(Some(10)).spec.accuracy_scope, @@ -274,10 +213,10 @@ mod tests { over(Some(10)).spec.failure_probability.evaluate(), Some(0.01) ); - assert!(model.satisfies(&over(Some(10)), &whole)); - assert!(!model.satisfies(&over(Some(11)), &whole)); - assert!(!model.satisfies(&over(None), &whole)); - assert!(model.satisfies(&over(Some(10)), &per_value)); + assert!(satisfies(&over(Some(10)), &whole)); + assert!(!satisfies(&over(Some(11)), &whole)); + assert!(!satisfies(&over(None), &whole)); + assert!(satisfies(&over(Some(10)), &per_value)); } /// The scope bounds only the failure probability: a target without a δ @@ -285,9 +224,8 @@ mod tests { /// target whatever the number of values. #[test] fn scope_matters_only_with_a_failure_probability() { - let model = DefaultAccuracyModel; let g = guarantee(AccuracyMetric::ABSOLUTE, 0.01, 0.001); - assert!(model.satisfies(&g, &AccuracyTarget::max_error(0.01))); + assert!(satisfies(&g, &AccuracyTarget::max_error(0.01))); let never_fails = AccuracyGuarantee::new( AccuracyMetric::Relative, BoundExpr::Constant { value: 0.01 }, @@ -297,8 +235,8 @@ mod tests { let whole = AccuracyTarget::max_error(0.01) .failure_probability(0.01) .accuracy_metric(AccuracyMetric::Relative); - assert!(model.satisfies(&never_fails, &whole)); - assert!(model.satisfies(&never_fails.over_whole_result(None), &whole)); + assert!(satisfies(&never_fails, &whole)); + assert!(satisfies(&never_fails.over_whole_result(None), &whole)); } /// A target whose metric the user did not set accepts a guarantee in @@ -307,7 +245,6 @@ mod tests { /// or failure probability still misses it. #[test] fn a_target_without_a_metric_accepts_any_metric() { - let model = DefaultAccuracyModel; let target = AccuracyTarget::max_error(0.01) .failure_probability(0.01) .accuracy_scope(AccuracyScope::PerValue); @@ -320,15 +257,15 @@ mod tests { AccuracyMetric::Rank, ] { assert!( - model.satisfies(&guarantee(metric, 0.01, 0.01), &target), + satisfies(&guarantee(metric, 0.01, 0.01), &target), "{metric:?}" ); assert!( - !model.satisfies(&guarantee(metric, 0.02, 0.01), &target), + !satisfies(&guarantee(metric, 0.02, 0.01), &target), "{metric:?}" ); assert!( - !model.satisfies(&guarantee(metric, 0.01, 0.02), &target), + !satisfies(&guarantee(metric, 0.01, 0.02), &target), "{metric:?}" ); } diff --git a/crates/logical-asap-optimization/src/lib.rs b/crates/logical-asap-optimization/src/lib.rs index e8cece06..9608ad58 100644 --- a/crates/logical-asap-optimization/src/lib.rs +++ b/crates/logical-asap-optimization/src/lib.rs @@ -30,16 +30,17 @@ //! - [`realization`] — the summary families that can serve an aggregate //! intent, the intent's accuracy target and `(ε, δ)` budget, and default //! sketch sizing, shared by Pass 1 and Pass 2. -//! - [`accuracy`] — the analytical accuracy model: per-family estimators, -//! composition and budget allocation, and the -//! [`AccuracyModel`](accuracy::AccuracyModel) extension point (issue #172) -//! that derives a machine-readable +//! - [`accuracy`] — the built-in analytical accuracy rules: per-family +//! estimators, composition and budget allocation. They derive a +//! machine-readable //! [`AccuracyGuarantee`](asap_types::ir::properties::AccuracyGuarantee) -//! for every finalized value and propagates it through +//! for every finalized value and propagate it through //! approximate-over-approximate compositions under conservative rules -//! (no independence assumptions, unknown statistics stay unknown). Plan -//! selection checks each candidate's guarantee against its query's -//! `AccuracyTarget` before pricing it. +//! (no independence assumptions, unknown statistics stay unknown). A +//! composition with no rule leaves the guarantee `None` and keeps the +//! candidate. Stage 1 has no pluggable accuracy model: plan selection +//! (Stage 3) checks each candidate's guarantee against its query's +//! `AccuracyTarget` with the deployment's `AccuracyModel` before pricing it. //! - [`deployment_capabilities`] — the deployment's supported summaries. //! - [`maintained_population`] — builds a maintained-population sub-DAG for //! an instant top-k/quantile/sum query; Pass 1 does not offer it yet (#681). @@ -53,10 +54,7 @@ pub mod realization; #[cfg(any(test, feature = "test-support"))] pub mod test_support; -pub use accuracy::{ - AccuracyEvidenceProvider, AccuracyModel, DefaultAccuracyModel, NoAccuracyEvidence, - PropagationStats, -}; +pub use accuracy::{AccuracyEvidenceProvider, NoAccuracyEvidence, PropagationStats}; pub use deployment_capabilities::{ DeploymentCapabilities, DeploymentCapabilitiesError, SupportedSummary, }; diff --git a/crates/logical-asap-optimization/src/per_computation_candidates.rs b/crates/logical-asap-optimization/src/per_computation_candidates.rs index cc1d0236..0daa883d 100644 --- a/crates/logical-asap-optimization/src/per_computation_candidates.rs +++ b/crates/logical-asap-optimization/src/per_computation_candidates.rs @@ -119,7 +119,7 @@ use crate::accuracy::composition::{ PRODUCT_OPERAND_DOMAINS, QUOTIENT_OPERAND_DOMAINS, }; use crate::accuracy::estimators::{hll, univmon}; -use crate::accuracy::{per_input_target, AccuracyModel, DefaultAccuracyModel, PropagationStats}; +use crate::accuracy::{local_guarantee, per_input_target, propagate, PropagationStats}; use crate::deployment_capabilities::DeploymentCapabilities; use crate::function_rules::approximate_input_composition; use crate::realization::{accuracy_budget, default_size_params, sizing_target, summary_candidates}; @@ -1361,7 +1361,7 @@ fn with_guarantees( } /// The theoretical guarantee of a composed node, given its children's, -/// composed with the built-in accuracy model ([`DefaultAccuracyModel`]); +/// composed with the built-in analytical rules ([`propagate`]); /// Stage 1 reads no data statistics or deployment evidence. Exact over exact /// inputs; a summary estimate has its sized family's bound when /// [`estimate_has_bound`], composed with its input's when that is @@ -1428,9 +1428,7 @@ fn composed_guarantee(node: &OperatorNode) -> Option { ExactKind::IRate => CompositionOperator::InstantCounterRate, ExactKind::Increase => CompositionOperator::CounterIncrease, }; - DefaultAccuracyModel - .propagate(&op, &[input], None, &stats) - .ok() + propagate(&op, &[input], None, &stats).ok() } _ => None, }, @@ -1450,7 +1448,7 @@ fn composed_guarantee(node: &OperatorNode) -> Option { let local = match estimate_bound_over_samples(family, update, query) { Some(gated) => gated, None => source_distinct_guarantee(family, query, input) - .or_else(|| DefaultAccuracyModel.local_guarantee(family, query))?, + .or_else(|| local_guarantee(family, query))?, }; if exact_inputs(summary_input) { Some(local) @@ -1498,14 +1496,13 @@ fn composed_guarantee(node: &OperatorNode) -> Option { .. }) if filters.iter().all(Option::is_none) && !groups_by_estimate(reduction, child) => { match measures.as_slice() { - [intent] => DefaultAccuracyModel - .propagate( - &approximate_input_composition(intent)?, - &[operand_guarantee(child)?], - None, - &fold_stats(reduction, child)?, - ) - .ok(), + [intent] => propagate( + &approximate_input_composition(intent)?, + &[operand_guarantee(child)?], + None, + &fold_stats(reduction, child)?, + ) + .ok(), _ => None, } } @@ -1765,9 +1762,7 @@ fn compose( inputs: Vec, local: Option<&AccuracyGuarantee>, ) -> Option { - DefaultAccuracyModel - .propagate(op, &inputs, local, &PropagationStats::default()) - .ok() + propagate(op, &inputs, local, &PropagationStats::default()).ok() } /// A projection that computes over its child's sample value: the guarantee @@ -2215,17 +2210,16 @@ fn lipschitz_guarantee(args: Vec) -> Option Sketc let metric = statistic(intent) .ok() .and_then(|query| { - DefaultAccuracyModel.local_guarantee( + local_guarantee( &FieldDataType::Sketch(whole.clone(), GroupingStrategy::default()), &query, ) @@ -4528,9 +4522,7 @@ mod tests { }; let gated = estimate_bound_over_samples(&family, &update, &query).expect("gated"); assert_eq!(gated.spec.error.evaluate(), None); - let local = DefaultAccuracyModel - .local_guarantee(&family, &query) - .unwrap(); + let local = local_guarantee(&family, &query).unwrap(); let statistic = samples_sign_statistic("m"); // Every other statistic (a stream norm) bound alike. let others = |s: &str| Some(if s == statistic { 1.0 } else { 1_000.0 }); diff --git a/crates/plan-selection/src/accuracy.rs b/crates/plan-selection/src/accuracy.rs new file mode 100644 index 00000000..c76a330c --- /dev/null +++ b/crates/plan-selection/src/accuracy.rs @@ -0,0 +1,65 @@ +//! The deployment's accuracy model, read only by plan selection (#509 +//! Stage 3). +//! +//! Stage 1 composes each candidate's guarantee with the planner's built-in +//! analytical rules (`asap_logical_asap_optimization::accuracy`) and keeps a +//! candidate it has no rule for with `guarantee = None`. This stage decides, +//! with the model here, whether a guarantee meets its query's target and +//! how guarantees compose once the data workload and the deployment's +//! evidence bind the statistics Stage 1 left unknown. + +use asap_logical_asap_optimization::accuracy::{self as analytical, PropagationStats}; +use asap_types::ir::properties::{ + AccuracyError, AccuracyGuarantee, AccuracyTarget, CompositionOperator, +}; + +/// The deployment-extensible accuracy algebra plan selection uses. This +/// crate ships [`DefaultAccuracyModel`]; a deployment with a proof for a +/// composition the default rejects (a registered cross-metric conversion, +/// say) implements this trait and passes it in +/// [`PlanningModels::accuracy`](crate::PlanningModels::accuracy). +pub trait AccuracyModel { + /// Compose `inputs`' guarantees (in the parent's child order) with the + /// parent's own `local` guarantee under `op`, given the statistics + /// plan selection bound (`stats`). `Err` is the fail-closed answer: no + /// registered rule, or a missing input guarantee. + fn propagate( + &self, + op: &CompositionOperator, + inputs: &[AccuracyGuarantee], + local: Option<&AccuracyGuarantee>, + stats: &PropagationStats, + ) -> Result; + + /// Whether `guarantee` meets `target`: a bound within its ε, and within + /// its δ when it states one, over the values the target's scope covers + /// (a per-value δ does not meet a whole-result target). When the target + /// sets a metric, the bound must be in that metric (or one with a sound + /// direct comparison): a bound in another metric does not satisfy, + /// however small. When it sets none, a bound in any metric does, each + /// compared in its own metric. Unknown required dimensions fail; + /// selection separately excludes missing accuracy evidence. + fn satisfies(&self, guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool; +} + +/// The planner's built-in analytical rules as an [`AccuracyModel`]: the +/// same composition rules and conservative target check Stage 1 uses. The +/// default in [`PlanningModels::builtin`](crate::PlanningModels::builtin). +#[derive(Debug, Default, Clone, Copy)] +pub struct DefaultAccuracyModel; + +impl AccuracyModel for DefaultAccuracyModel { + fn propagate( + &self, + op: &CompositionOperator, + inputs: &[AccuracyGuarantee], + local: Option<&AccuracyGuarantee>, + stats: &PropagationStats, + ) -> Result { + analytical::propagate(op, inputs, local, stats) + } + + fn satisfies(&self, guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool { + analytical::satisfies(guarantee, target) + } +} diff --git a/crates/plan-selection/src/lib.rs b/crates/plan-selection/src/lib.rs index c9d4051e..fc4ee9ed 100644 --- a/crates/plan-selection/src/lib.rs +++ b/crates/plan-selection/src/lib.rs @@ -207,6 +207,8 @@ use asap_logical_asap_optimization::per_computation_candidates::{ }; use asap_physical_asap_optimization::PhysicalCandidate; +pub mod accuracy; +pub use accuracy::{AccuracyModel, DefaultAccuracyModel}; pub mod cost; /// Costed reuse of compatible physical pane producers. pub mod pane_sharing; @@ -1394,7 +1396,7 @@ pub fn resize_for_failure_probability( /// the metric a counter ([`confirmed_weights`]), and unmet otherwise. /// /// A target that sets no metric accepts a guarantee in any metric that meets -/// its ε, δ and scope ([`asap_logical_asap_optimization::accuracy::AccuracyModel::satisfies`]); one that sets a metric +/// its ε, δ and scope ([`AccuracyModel::satisfies`]); one that sets a metric /// accepts only guarantees in it (or convertible to it). fn accuracy_check( candidate: &PhysicalCandidate, diff --git a/crates/plan-selection/src/pass/mod.rs b/crates/plan-selection/src/pass/mod.rs index 97d6983d..8893d628 100644 --- a/crates/plan-selection/src/pass/mod.rs +++ b/crates/plan-selection/src/pass/mod.rs @@ -26,10 +26,9 @@ use asap_types::ir::{OperatorNode, QueryRoot}; use asap_types::workload::parsed_workload::ParsedWorkload; use asap_types::workload::WorkloadError; +use crate::accuracy::{AccuracyModel, DefaultAccuracyModel}; use crate::{PlanSelectionSettings, Selection, SelectionError}; -use asap_logical_asap_optimization::accuracy::{ - AccuracyEvidenceProvider, AccuracyModel, DefaultAccuracyModel, NoAccuracyEvidence, -}; +use asap_logical_asap_optimization::accuracy::{AccuracyEvidenceProvider, NoAccuracyEvidence}; use asap_logical_asap_optimization::deployment_capabilities::DeploymentCapabilities; use asap_logical_asap_optimization::per_computation_candidates::PerComputationCandidateError; diff --git a/crates/types/src/ir/properties/guarantee.rs b/crates/types/src/ir/properties/guarantee.rs index 2e88a761..45c38a1d 100644 --- a/crates/types/src/ir/properties/guarantee.rs +++ b/crates/types/src/ir/properties/guarantee.rs @@ -7,10 +7,11 @@ //! could silently consume an approximate child. This module is the //! *vocabulary* that fixes that — the typed metric, the symbolic bound and //! failure-probability expressions, the provenance trail, and the typed -//! rejection reasons. The *algebra* that composes these (the `AccuracyModel` -//! trait, its default conservative rules, and budget allocation) lives one -//! layer up in `asap_logical_asap_optimization::accuracy`: this crate defines the -//! shapes, the planning crate decides. +//! rejection reasons. The *algebra* that composes these (the built-in +//! conservative rules and budget allocation) lives one layer up in +//! `asap_logical_asap_optimization::accuracy`, and the deployment's +//! pluggable `AccuracyModel` trait in `asap_plan_selection::accuracy`: this +//! crate defines the shapes, the planning crates decide. //! //! ## What a guarantee says //! @@ -268,7 +269,7 @@ impl ProbabilityExpr { } /// How a parent operator consumes its inputs' values — the shape an -/// `AccuracyModel::propagate` rule is registered against. `#[non_exhaustive]` +/// `propagate` rule is registered against. `#[non_exhaustive]` /// for the same reason [`AccuracyMetric`] is. #[non_exhaustive] #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] From 48386a9d54493f62027c3999c78b2176bc701247 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sun, 18 Oct 2026 14:46:48 +0000 Subject: [PATCH 2/6] test: pin that the deployment accuracy model decides only in Stage 3 --- .../tests/deployment_accuracy_model.rs | 154 ++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 crates/planner/tests/deployment_accuracy_model.rs diff --git a/crates/planner/tests/deployment_accuracy_model.rs b/crates/planner/tests/deployment_accuracy_model.rs new file mode 100644 index 00000000..403245e2 --- /dev/null +++ b/crates/planner/tests/deployment_accuracy_model.rs @@ -0,0 +1,154 @@ +//! The deployment's accuracy model plugs into plan selection (#509 Stage 3) +//! only. Stage 1 composes guarantees with the built-in analytical rules, so +//! a different model leaves the candidates unchanged and changes only which +//! of them Stage 3 accepts. + +use std::collections::BTreeSet; +use std::rc::Rc; + +use asap_frontend_promql::promql_metric_names; +use asap_logical_asap_optimization::accuracy::PropagationStats; +use asap_plan_selection::{ + AccuracyModel, DefaultAccuracyModel, PlanOutput, PlanningModels, RejectionCause, +}; +use asap_planner::{e2e_plan, FrontendInput, UserInput}; +use asap_types::ir::properties::{ + AccuracyError, AccuracyGuarantee, AccuracyMetric, AccuracyTarget, CompositionOperator, +}; +use asap_types::ir::schema::FieldDataType; +use asap_types::ir::{ASAPOp, OperatorNode}; +use asap_types::workload::{ + DataArrival, DataWorkload, DurationMs, PlanningWorkload, Predictability, Query, QueryLanguage, + QueryRequirements, QueryWorkload, RepeatedDemand, RepeatingEntry, RepetitionInterval, +}; + +/// A deployment that trusts no approximate estimate: the built-in +/// composition rules, but only an exact guarantee meets a target. +struct ExactOnlyAccuracyModel; + +impl AccuracyModel for ExactOnlyAccuracyModel { + fn propagate( + &self, + op: &CompositionOperator, + inputs: &[AccuracyGuarantee], + local: Option<&AccuracyGuarantee>, + stats: &PropagationStats, + ) -> Result { + DefaultAccuracyModel.propagate(op, inputs, local, stats) + } + + fn satisfies(&self, guarantee: &AccuracyGuarantee, _target: &AccuracyTarget) -> bool { + guarantee.is_exact() + } +} + +/// `quantile_over_time(0.99, lat[5m])` with a 1% rank error, repeated every +/// ten minutes over a metric sampled every 15 s. +fn p99_workload() -> PlanningWorkload { + let query = "quantile_over_time(0.99, lat[5m])"; + PlanningWorkload { + query_workload: QueryWorkload { + language: QueryLanguage::PromQL, + query_batch: None, + repeating_queries: Some(vec![RepeatingEntry { + query: Query(query.into()), + demand: RepeatedDemand::FixedInterval(RepetitionInterval(600_000)), + requirements: QueryRequirements { + accuracy: AccuracyTarget::max_error(0.01).accuracy_metric(AccuracyMetric::Rank), + max_latency_ms: None, + }, + predictability: Predictability::Unknown, + }]), + }, + data_workload: Some(DataWorkload { + arrival: DataArrival::Ingesting { + rows_per_second: 1.0, + }, + ..DataWorkload::sampled_every(promql_metric_names(query).unwrap(), DurationMs(15_000)) + }), + } +} + +/// Whether `root` reads a sketch state. +fn reads_sketch(root: &Rc) -> bool { + OperatorNode::reachable(root).into_iter().any(|node| { + matches!( + node.asap(), + Some(ASAPOp::SummaryAgg { + family: FieldDataType::Sketch(..), + .. + }) + ) + }) +} + +/// Every candidate Stage 3 was given: the valid ones it priced and the +/// ones it rejected. +fn stage3_candidate_ids(output: &PlanOutput) -> BTreeSet { + let selection = output.selection.as_ref().expect("plan selection ran"); + selection + .costs + .keys() + .cloned() + .chain(selection.rejected.iter().map(|r| r.id.clone())) + .collect() +} + +/// The candidates for which Stage 3 accepted an approximate output for some +/// query. +fn approximate_accepted(output: &PlanOutput) -> BTreeSet { + let selection = output.selection.as_ref().expect("plan selection ran"); + selection + .accepted + .iter() + .filter(|(_, queries)| queries.iter().any(|q| q.accepted_metric.is_some())) + .map(|(id, _)| id.clone()) + .collect() +} + +#[tokio::test] +async fn the_deployment_accuracy_model_decides_only_in_plan_selection() { + // The user's workload: one p99 query with a 1% rank-error target. + let workload = p99_workload(); + + // The built-in models: Stage 3 checks guarantees with DefaultAccuracyModel. + let builtin = e2e_plan(UserInput::new( + &workload, + FrontendInput::Promql { histograms: None }, + PlanningModels::builtin(), + )) + .await + .expect("plans with the built-in models"); + // The candidates whose approximate output the built-in model accepts: + // the sketch candidates, sized in Stage 1 for the 1% target. + let builtin_approximate = approximate_accepted(&builtin); + assert!(!builtin_approximate.is_empty()); + + // The same workload with a deployment model that accepts only exact + // outputs, plugged into Stage 3 through `PlanningModels::with_accuracy`. + let exact_only_model = ExactOnlyAccuracyModel; + let exact_only = e2e_plan(UserInput::new( + &workload, + FrontendInput::Promql { histograms: None }, + PlanningModels::builtin().with_accuracy(&exact_only_model), + )) + .await + .expect("plans with the exact-only model"); + + // Stages 1 and 2 do not read the model: Stage 3 got the same candidates. + assert_eq!( + stage3_candidate_ids(&builtin), + stage3_candidate_ids(&exact_only) + ); + // Stage 3 now rejects each of those candidates for missing its target, + // accepts no approximate output and selects an exact plan. + let exact_only_selection = exact_only.selection.as_ref().unwrap(); + for id in &builtin_approximate { + assert!(exact_only_selection + .rejected + .iter() + .any(|r| &r.id == id && r.cause == RejectionCause::AccuracyTargetMissed)); + } + assert!(approximate_accepted(&exact_only).is_empty()); + assert!(!reads_sketch(&exact_only.plans[0].root)); +} From 49a92dbdd806e208818d3d8434f7d672700b158f Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sun, 18 Oct 2026 15:25:01 +0000 Subject: [PATCH 3/6] feat(plan-selection)!: let the deployment's accuracy model decide outputs without a guarantee - Move AccuracyEvidenceProvider and NoAccuracyEvidence to asap_plan_selection::accuracy, next to AccuracyModel; OperandDomain and PropagationStats stay in Stage 1. - Rename DefaultAccuracyModel to BuiltinAccuracyModel. AccuracyModel::propagate gets a default body (the built-in rules), so a deployment model implements only satisfies. - Add AccuracyModel::accepts_output_without_guarantee (default false). Stage 3 asks it only when neither Stage 1 nor the evidence supplies a guarantee; an output it accepts is recorded with AcceptedAccuracy::accepted_without_guarantee, and one it does not is rejected with the new RejectionCause::NoAccuracyGuarantee (was AccuracyTargetMissed). - Document accuracy::estimators::local_guarantee and accuracy::composition::propagate, and drop their duplicate re-exports. - stage_pipeline writes accepted_without_guarantee; the dag-viewer accepts the new cause; the example fixture is regenerated. BREAKING CHANGE: AccuracyEvidenceProvider/NoAccuracyEvidence move to asap_plan_selection; DefaultAccuracyModel is now BuiltinAccuracyModel; a candidate whose output has no guarantee is rejected as no_accuracy_guarantee instead of accuracy_target_missed; AcceptedAccuracy gains a field. --- crates/devtools/src/bin/stage_pipeline.rs | 7 +- .../tests/univmon_candidates.rs | 3 +- .../tests/promql_to_post_asap.rs | 3 +- .../src/accuracy/composition.rs | 6 + .../src/accuracy/estimators/mod.rs | 8 + .../src/accuracy/evidence.rs | 31 +-- .../src/accuracy/mod.rs | 15 +- crates/logical-asap-optimization/src/lib.rs | 2 +- .../src/per_computation_candidates.rs | 4 +- crates/plan-selection/src/accuracy.rs | 89 ++++++-- crates/plan-selection/src/lib.rs | 152 ++++++++++---- crates/plan-selection/src/pass/mod.rs | 12 +- .../tests/deployment_accuracy_model.rs | 155 +++++++++++--- crates/planner/tests/summary_sharing.rs | 2 +- .../examples/planner-layering-example1.json | 190 +++++++++++------- tools/dag-viewer/stages.js | 4 +- tools/dag-viewer/test_viewer.py | 3 +- 17 files changed, 466 insertions(+), 220 deletions(-) diff --git a/crates/devtools/src/bin/stage_pipeline.rs b/crates/devtools/src/bin/stage_pipeline.rs index e88b8157..9760d7ff 100644 --- a/crates/devtools/src/bin/stage_pipeline.rs +++ b/crates/devtools/src/bin/stage_pipeline.rs @@ -31,10 +31,12 @@ // - stage3_plan_selection: the selected candidate, chosen among all candidates; // the shown candidates' costs and accepted accuracy (per query: the // target's metric or null when not set, the accepted guarantee's -// metric or null when exact, the sketches read and a one-line +// metric or null when exact or accepted without a guarantee, whether +// the deployment's accuracy model accepted it without a guarantee +// (`accepted_without_guarantee`), the sketches read and a one-line // summary), and every other shown candidate under `rejected` with // its `cause`: a failed check (`failure_probability_resize_failed`, -// `accuracy_target_missed`, `raw_data_not_retained`, +// `accuracy_target_missed`, `no_accuracy_guarantee`, `raw_data_not_retained`, // `latency_bound_exceeded`, `over_cluster_capacity`) or a loss under // the cost model (`costlier`, `slower`, `more_memory`, // `not_lowest_latency_on_pareto_front`, `dominated`, @@ -617,6 +619,7 @@ fn plan_selection_json( json!({ "target_metric": accepted.target_metric, "accepted_metric": accepted.accepted_metric, + "accepted_without_guarantee": accepted.accepted_without_guarantee, "sketches": accepted.sketches, "summary": accepted.to_string(), }) diff --git a/crates/frontend-promql/tests/univmon_candidates.rs b/crates/frontend-promql/tests/univmon_candidates.rs index b283cd1c..56a22bdd 100644 --- a/crates/frontend-promql/tests/univmon_candidates.rs +++ b/crates/frontend-promql/tests/univmon_candidates.rs @@ -1,6 +1,7 @@ use std::rc::Rc; -use asap_logical_asap_optimization::accuracy::{local_guarantee, satisfies}; +use asap_logical_asap_optimization::accuracy::estimators::local_guarantee; +use asap_logical_asap_optimization::accuracy::satisfies; mod support; use asap_types::ir::cse::share_common_sub_dags; use asap_types::ir::properties::AccuracyMetric; diff --git a/crates/integration-tests/tests/promql_to_post_asap.rs b/crates/integration-tests/tests/promql_to_post_asap.rs index 3830f8a1..0a8f74c3 100644 --- a/crates/integration-tests/tests/promql_to_post_asap.rs +++ b/crates/integration-tests/tests/promql_to_post_asap.rs @@ -11,8 +11,9 @@ use std::rc::Rc; use asap_integration_tests::fixtures::lower_promql; use asap_integration_tests::planning::logical_candidates; use asap_integration_tests::post_asap::{at_ingestion, at_ingestion_post_asap_dag, post_asap_dag}; -use asap_logical_asap_optimization::accuracy::{AccuracyEvidenceProvider, OperandDomain}; +use asap_logical_asap_optimization::accuracy::OperandDomain; use asap_physical_asap_optimization::PhysicalCandidate; +use asap_plan_selection::AccuracyEvidenceProvider; use asap_plan_selection::PlanningModels; use asap_plan_selection::Selection; use asap_types::ir::operator::operator_properties::Reduction; diff --git a/crates/logical-asap-optimization/src/accuracy/composition.rs b/crates/logical-asap-optimization/src/accuracy/composition.rs index a731f788..d5787166 100644 --- a/crates/logical-asap-optimization/src/accuracy/composition.rs +++ b/crates/logical-asap-optimization/src/accuracy/composition.rs @@ -694,6 +694,12 @@ fn composed_provenance( provenance } +/// The built-in composition rule: `inputs`' guarantees (in the parent's +/// child order) composed with the parent's own `local` guarantee under `op`, +/// given the statistics in `stats` (all unknown in Stage 1; plan selection +/// binds some). `Err` is the fail-closed answer: no registered rule, or a +/// missing input guarantee. It means "no rule", not "infeasible": Pass 1 +/// keeps the candidate with `guarantee = None`. pub fn propagate( op: &CompositionOperator, inputs: &[AccuracyGuarantee], diff --git a/crates/logical-asap-optimization/src/accuracy/estimators/mod.rs b/crates/logical-asap-optimization/src/accuracy/estimators/mod.rs index a20d01c5..7d88fb7a 100644 --- a/crates/logical-asap-optimization/src/accuracy/estimators/mod.rs +++ b/crates/logical-asap-optimization/src/accuracy/estimators/mod.rs @@ -54,6 +54,14 @@ fn bounded_guarantee( ) } +/// The built-in guarantee of reading `query` out of a summary of family +/// `family` built over an **exact** input, derived from the family's +/// committed parameters by inverting the same sizing formulas +/// [`crate::realization::default_size_params`] uses. Exact for a plain value +/// or an exact aggregate. `None` when the family has no error model +/// (`Sample`, `Wavelet`, `StatModel`, and sketch evaluations without a +/// certified bound): Pass 1 then leaves the guarantee `None` and keeps the +/// candidate for plan selection to decide. pub fn local_guarantee( family: &FieldDataType, query: &SketchStatistic, diff --git a/crates/logical-asap-optimization/src/accuracy/evidence.rs b/crates/logical-asap-optimization/src/accuracy/evidence.rs index fce69d5d..211e8db3 100644 --- a/crates/logical-asap-optimization/src/accuracy/evidence.rs +++ b/crates/logical-asap-optimization/src/accuracy/evidence.rs @@ -1,6 +1,5 @@ -//! Scoped source contracts and evidence required by accuracy rules. +//! Scoped source contracts and statistics the accuracy rules consult. use super::*; -use asap_types::ir::QueryRoot; /// An enforced domain for the true values of one operand of an exact /// arithmetic operation, in every evaluation: they are finite and lie in @@ -59,31 +58,3 @@ pub struct PropagationStats { /// certificate. pub topk_interval_failure_probability: Option, } - -/// Supplies typed planning-time evidence required by propagation rules. -pub trait AccuracyEvidenceProvider { - /// Enforced domain of `operand`'s true values ([`OperandDomain`]), scoped - /// to this complete expression, including its source, filters, grouping - /// and window: a quantile ratio's operands, or the operands of a - /// product or quotient of estimates. `None` means unknown, including - /// emptiness. - fn operand_domain(&self, _operand: &OperatorNode) -> Option { - None - } - - /// The deployment's empirical accuracy for one candidate's whole query - /// output, as a guarantee. Plan selection (#509 Stage 3) accepts a - /// candidate whose output has no theoretical guarantee that meets the - /// target (e.g. a difference of two quantile estimates) only when this - /// shows the output meets it. Stage 1 composition never reads it: root - /// guarantees there come from the built-in theoretical model. `None` - /// means no evidence. - fn output_guarantee(&self, _output: &QueryRoot) -> Option { - None - } -} - -#[derive(Debug, Default, Clone, Copy)] -pub struct NoAccuracyEvidence; - -impl AccuracyEvidenceProvider for NoAccuracyEvidence {} diff --git a/crates/logical-asap-optimization/src/accuracy/mod.rs b/crates/logical-asap-optimization/src/accuracy/mod.rs index 85ac3940..93b9629d 100644 --- a/crates/logical-asap-optimization/src/accuracy/mod.rs +++ b/crates/logical-asap-optimization/src/accuracy/mod.rs @@ -1,9 +1,10 @@ -//! Planner accuracy interfaces and model dispatch. +//! The planner's built-in analytical accuracy rules: estimators, composition, budget split and target check. //! //! Estimator models derive local guarantees, composition propagates them, //! the budget split sizes each approximate input of a composition for an -//! equal share of its target, and evidence supplies scoped contracts. -//! Unknown evidence may retain a candidate but does not authorize selection. +//! equal share of its target, and `evidence` holds the scoped contracts and +//! statistics the rules consult. The deployment's evidence provider and +//! accuracy model live in plan selection (`asap_plan_selection::accuracy`). //! See `docs/design_docs/concepts/accuracy-models.md` for the design. pub mod budget; @@ -12,7 +13,7 @@ pub mod estimators; pub mod evidence; pub use budget::per_input_target; -pub use evidence::{AccuracyEvidenceProvider, NoAccuracyEvidence, OperandDomain, PropagationStats}; +pub use evidence::{OperandDomain, PropagationStats}; use asap_types::ir::properties::{ AccuracyError, AccuracyGuarantee, AccuracyMetric, BoundExpr, CompositionOperator, @@ -20,10 +21,6 @@ use asap_types::ir::properties::{ }; use asap_types::ir::properties::{AccuracyScope, AccuracyTarget}; use asap_types::ir::schema::{FieldDataType, SketchAlgorithm, SketchParams, SketchStatistic}; -use asap_types::ir::OperatorNode; - -pub use composition::propagate; -pub use estimators::local_guarantee; /// Small relative tolerance for comparing an evaluated bound against a /// target, so a parameter sized by `⌈·⌉` to *exactly* meet ε is not rejected @@ -38,7 +35,7 @@ const SATISFACTION_TOLERANCE: f64 = 1e-9; /// another metric does not satisfy, however small. When it sets none, a /// bound in any metric does, each compared in its own metric. Unknown /// required dimensions fail. Plan selection's default accuracy model -/// (`asap_plan_selection::DefaultAccuracyModel`) delegates to it. +/// (`asap_plan_selection::BuiltinAccuracyModel`) delegates to it. pub fn satisfies(guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool { let within = |value: Option, limit: f64| { value.is_some_and(|v| v <= limit * (1.0 + SATISFACTION_TOLERANCE) + f64::EPSILON) diff --git a/crates/logical-asap-optimization/src/lib.rs b/crates/logical-asap-optimization/src/lib.rs index 9608ad58..38468c54 100644 --- a/crates/logical-asap-optimization/src/lib.rs +++ b/crates/logical-asap-optimization/src/lib.rs @@ -54,7 +54,7 @@ pub mod realization; #[cfg(any(test, feature = "test-support"))] pub mod test_support; -pub use accuracy::{AccuracyEvidenceProvider, NoAccuracyEvidence, PropagationStats}; +pub use accuracy::PropagationStats; pub use deployment_capabilities::{ DeploymentCapabilities, DeploymentCapabilitiesError, SupportedSummary, }; diff --git a/crates/logical-asap-optimization/src/per_computation_candidates.rs b/crates/logical-asap-optimization/src/per_computation_candidates.rs index 0daa883d..75f09cb6 100644 --- a/crates/logical-asap-optimization/src/per_computation_candidates.rs +++ b/crates/logical-asap-optimization/src/per_computation_candidates.rs @@ -113,13 +113,15 @@ use asap_types::ir::{ use asap_types::physical::Materialization; use thiserror::Error; +use crate::accuracy::composition::propagate; use crate::accuracy::composition::samples_sign_statistic; use crate::accuracy::composition::{ additive_as_absolute, StreamNorm, DIVISION_OPERAND_DOMAINS, GROUP_ROW_COUNT, PRODUCT_OPERAND_DOMAINS, QUOTIENT_OPERAND_DOMAINS, }; +use crate::accuracy::estimators::local_guarantee; use crate::accuracy::estimators::{hll, univmon}; -use crate::accuracy::{local_guarantee, per_input_target, propagate, PropagationStats}; +use crate::accuracy::{per_input_target, PropagationStats}; use crate::deployment_capabilities::DeploymentCapabilities; use crate::function_rules::approximate_input_composition; use crate::realization::{accuracy_budget, default_size_params, sizing_target, summary_candidates}; diff --git a/crates/plan-selection/src/accuracy.rs b/crates/plan-selection/src/accuracy.rs index c76a330c..d9b76b05 100644 --- a/crates/plan-selection/src/accuracy.rs +++ b/crates/plan-selection/src/accuracy.rs @@ -4,17 +4,21 @@ //! Stage 1 composes each candidate's guarantee with the planner's built-in //! analytical rules (`asap_logical_asap_optimization::accuracy`) and keeps a //! candidate it has no rule for with `guarantee = None`. This stage decides, -//! with the model here, whether a guarantee meets its query's target and -//! how guarantees compose once the data workload and the deployment's -//! evidence bind the statistics Stage 1 left unknown. +//! with the model here, whether a guarantee meets its query's target, how +//! guarantees compose once the data workload and the deployment's evidence +//! ([`AccuracyEvidenceProvider`]) bind the statistics Stage 1 left unknown, +//! and whether an output with no guarantee at all is accepted. -use asap_logical_asap_optimization::accuracy::{self as analytical, PropagationStats}; +use asap_logical_asap_optimization::accuracy::{ + self as analytical, OperandDomain, PropagationStats, +}; use asap_types::ir::properties::{ AccuracyError, AccuracyGuarantee, AccuracyTarget, CompositionOperator, }; +use asap_types::ir::{OperatorNode, QueryRoot}; /// The deployment-extensible accuracy algebra plan selection uses. This -/// crate ships [`DefaultAccuracyModel`]; a deployment with a proof for a +/// crate ships [`BuiltinAccuracyModel`]; a deployment with a proof for a /// composition the default rejects (a registered cross-metric conversion, /// say) implements this trait and passes it in /// [`PlanningModels::accuracy`](crate::PlanningModels::accuracy). @@ -22,14 +26,18 @@ pub trait AccuracyModel { /// Compose `inputs`' guarantees (in the parent's child order) with the /// parent's own `local` guarantee under `op`, given the statistics /// plan selection bound (`stats`). `Err` is the fail-closed answer: no - /// registered rule, or a missing input guarantee. + /// registered rule, or a missing input guarantee. The default is the + /// built-in rules (`asap_logical_asap_optimization::accuracy::composition::propagate`); + /// a deployment overrides it only to add a rule. fn propagate( &self, op: &CompositionOperator, inputs: &[AccuracyGuarantee], local: Option<&AccuracyGuarantee>, stats: &PropagationStats, - ) -> Result; + ) -> Result { + analytical::composition::propagate(op, inputs, local, stats) + } /// Whether `guarantee` meets `target`: a bound within its ε, and within /// its δ when it states one, over the values the target's scope covers @@ -40,26 +48,65 @@ pub trait AccuracyModel { /// compared in its own metric. Unknown required dimensions fail; /// selection separately excludes missing accuracy evidence. fn satisfies(&self, guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool; + + /// Whether the deployment accepts `output`, one query's output in a + /// candidate, for `target` although the output has no guarantee: Stage 1 + /// had no analytical rule for it and the deployment's evidence supplies + /// none ([`AccuracyEvidenceProvider::output_guarantee`]). Stage 3 asks only + /// then; a guarantee from either source is checked with + /// [`Self::satisfies`] instead. The default accepts none, and Stage 3 + /// rejects such a candidate (`RejectionCause::NoAccuracyGuarantee`). + fn accepts_output_without_guarantee( + &self, + _output: &QueryRoot, + _target: &AccuracyTarget, + ) -> bool { + false + } } /// The planner's built-in analytical rules as an [`AccuracyModel`]: the -/// same composition rules and conservative target check Stage 1 uses. The -/// default in [`PlanningModels::builtin`](crate::PlanningModels::builtin). +/// trait's default composition, the conservative target check Stage 1's +/// `accuracy::satisfies` makes, and no output accepted without a guarantee. +/// The default in [`PlanningModels::builtin`](crate::PlanningModels::builtin). #[derive(Debug, Default, Clone, Copy)] -pub struct DefaultAccuracyModel; - -impl AccuracyModel for DefaultAccuracyModel { - fn propagate( - &self, - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - local: Option<&AccuracyGuarantee>, - stats: &PropagationStats, - ) -> Result { - analytical::propagate(op, inputs, local, stats) - } +pub struct BuiltinAccuracyModel; +impl AccuracyModel for BuiltinAccuracyModel { fn satisfies(&self, guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool { analytical::satisfies(guarantee, target) } } + +/// The deployment's planning-time accuracy evidence, read only by plan +/// selection: enforced operand domains the propagation rules need, and +/// empirical guarantees for whole query outputs. Passed in +/// [`PlanningModels::evidence`](crate::PlanningModels::evidence). The trait +/// moved here from `asap_logical_asap_optimization::accuracy`. +pub trait AccuracyEvidenceProvider { + /// Enforced domain of `operand`'s true values ([`OperandDomain`]), scoped + /// to this complete expression, including its source, filters, grouping + /// and window: a quantile ratio's operands, or the operands of a + /// product or quotient of estimates. `None` means unknown, including + /// emptiness. + fn operand_domain(&self, _operand: &OperatorNode) -> Option { + None + } + + /// The deployment's empirical accuracy for one candidate's whole query + /// output, as a guarantee. Plan selection (#509 Stage 3) accepts a + /// candidate whose output has no theoretical guarantee that meets the + /// target (e.g. a difference of two quantile estimates) when this shows + /// the output meets it. Stage 1 never reads it: root guarantees there + /// come from the built-in analytical rules. `None` means no evidence. + fn output_guarantee(&self, _output: &QueryRoot) -> Option { + None + } +} + +/// No deployment evidence: every operand domain and output guarantee is +/// unknown. The default in [`PlanningModels::builtin`](crate::PlanningModels::builtin). +#[derive(Debug, Default, Clone, Copy)] +pub struct NoAccuracyEvidence; + +impl AccuracyEvidenceProvider for NoAccuracyEvidence {} diff --git a/crates/plan-selection/src/lib.rs b/crates/plan-selection/src/lib.rs index fc4ee9ed..5425b251 100644 --- a/crates/plan-selection/src/lib.rs +++ b/crates/plan-selection/src/lib.rs @@ -3,11 +3,13 @@ //! Each CandidatePhysicalASAPDAG from Stage 2 is checked against every //! query's accuracy target: the query's output meets it when its root //! guarantee does (theoretical: Stage 1 composes it with the built-in -//! accuracy model, and this stage binds the statistics it left symbolic +//! analytical rules, and this stage binds the statistics it left symbolic //! from the data workload and the deployment's operand-domain contracts), //! or when the deployment's empirical evidence shows this //! candidate's output does (`AccuracyEvidenceProvider::output_guarantee`, -//! read only here). For a query with a target, Count-Min is admitted only +//! read only here). An output with no guarantee from either source is +//! accepted only when the deployment's accuracy model accepts it without one +//! (`AccuracyModel::accepts_output_without_guarantee`). For a query with a target, Count-Min is admitted only //! over weights proven non-negative; a query without one constrains nothing. //! A miss rejects the candidate as invalid, with a reason; only this stage //! discards valid candidates. @@ -208,7 +210,9 @@ use asap_logical_asap_optimization::per_computation_candidates::{ use asap_physical_asap_optimization::PhysicalCandidate; pub mod accuracy; -pub use accuracy::{AccuracyModel, DefaultAccuracyModel}; +pub use accuracy::{ + AccuracyEvidenceProvider, AccuracyModel, BuiltinAccuracyModel, NoAccuracyEvidence, +}; pub mod cost; /// Costed reuse of compatible physical pane producers. pub mod pane_sharing; @@ -570,7 +574,7 @@ pub struct Rejection { pub reason: String, } -/// Why Stage 3 did not select a candidate. The first five mean the +/// Why Stage 3 did not select a candidate. The first six mean the /// candidate failed a check and is invalid ([`Self::failed_a_check`]); the /// rest mean it was valid and lost under the cost model. JSON: the /// snake_case variant name, e.g. `"costlier"`. @@ -584,6 +588,11 @@ pub enum RejectionCause { /// it (or does not hold, as for a Count-Min sketch whose update weights /// are not proven non-negative) and no deployment evidence meets it. AccuracyTargetMissed, + /// A query's output has no accuracy guarantee (Stage 1 has no + /// analytical rule for it and the deployment's evidence supplies none), + /// and the deployment's accuracy model does not accept it without one + /// ([`AccuracyModel::accepts_output_without_guarantee`]). + NoAccuracyGuarantee, /// It scans raw data at query time in a deployment that does not /// retain raw data. RawDataNotRetained, @@ -618,6 +627,7 @@ impl RejectionCause { self, Self::FailureProbabilityResizeFailed | Self::AccuracyTargetMissed + | Self::NoAccuracyGuarantee | Self::RawDataNotRetained | Self::LatencyBoundExceeded | Self::OverClusterCapacity @@ -636,8 +646,14 @@ pub struct AcceptedAccuracy { /// The target's metric; `None` when the user did not set one. pub target_metric: Option, /// The metric the accepted guarantee is stated in; `None` for an exact - /// output, which meets every metric. + /// output, which meets every metric, and for one accepted without a + /// guarantee. pub accepted_metric: Option, + /// Whether the output has no guarantee and the deployment's accuracy + /// model accepted it anyway + /// ([`AccuracyModel::accepts_output_without_guarantee`]); `false` + /// otherwise. + pub accepted_without_guarantee: bool, /// The sketch algorithms the query's output reads, in plan order /// (e.g. `["Kll"]`); empty for an exact plan. pub sketches: Vec, @@ -645,7 +661,8 @@ pub struct AcceptedAccuracy { impl std::fmt::Display for AcceptedAccuracy { /// E.g. `metric not set; accepted rank (Kll)`, or - /// `metric relative; accepted relative (DDSketch)`. + /// `metric relative; accepted relative (DDSketch)`, or + /// `metric rank; accepted without a guarantee (Kll)`. fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let name = |metric: AccuracyMetric| { serde_json::to_value(metric) @@ -658,6 +675,7 @@ impl std::fmt::Display for AcceptedAccuracy { Some(metric) => write!(f, "metric {}; ", name(metric))?, } match self.accepted_metric { + _ if self.accepted_without_guarantee => write!(f, "accepted without a guarantee")?, None => write!(f, "accepted exact")?, Some(metric) => write!(f, "accepted {}", name(metric))?, } @@ -765,10 +783,10 @@ pub fn select_plan( }; let accepted_accuracy = match accuracy_check(&candidate, targets, data, &models) { Ok(accepted_accuracy) => accepted_accuracy, - Err(reason) => { + Err((cause, reason)) => { rejected.push(Rejection { id: candidate.id.clone(), - cause: RejectionCause::AccuracyTargetMissed, + cause, reason, }); continue; @@ -1386,7 +1404,7 @@ pub fn resize_for_failure_probability( /// first query whose output is not shown to meet its target, as a reason. An output meets its target when its theoretical root guarantee /// (composed in Stage 1 with the built-in model) satisfies it, or when the /// deployment's empirical evidence shows that this candidate's output does -/// ([`asap_logical_asap_optimization::accuracy::AccuracyEvidenceProvider::output_guarantee`]). A +/// ([`AccuracyEvidenceProvider::output_guarantee`]). A /// certified estimate inside the plan is not enough on its own: a difference /// or ratio of certified estimates can be unbounded. The root guarantee's /// symbolic statistics are bound first ([`bound_root_guarantee`]); one still @@ -1398,12 +1416,19 @@ pub fn resize_for_failure_probability( /// A target that sets no metric accepts a guarantee in any metric that meets /// its ε, δ and scope ([`AccuracyModel::satisfies`]); one that sets a metric /// accepts only guarantees in it (or convertible to it). +/// +/// An output with no guarantee from either source (Stage 1 had no rule, +/// and the evidence has none) is accepted only when the deployment's +/// accuracy model accepts it without one +/// ([`AccuracyModel::accepts_output_without_guarantee`]); otherwise the +/// candidate is rejected as [`RejectionCause::NoAccuracyGuarantee`]. Any +/// other miss is [`RejectionCause::AccuracyTargetMissed`]. fn accuracy_check( candidate: &PhysicalCandidate, targets: &[AccuracyTarget], data: &DataWorkload, models: &PlanningModels<'_>, -) -> Result, String> { +) -> Result, (RejectionCause, String)> { let mut accepted = Vec::with_capacity(targets.len()); for (query, (root, target)) in candidate.roots.iter().zip(targets).enumerate() { let query = query + 1; @@ -1437,10 +1462,13 @@ fn accuracy_check( } _ => String::new(), }; - return Err(format!( - "q{query}: {} needs non-negative update weights, and these are not proven \ - non-negative{unconfirmed}", - family_name(family) + return Err(( + RejectionCause::AccuracyTargetMissed, + format!( + "q{query}: {} needs non-negative update weights, and these are not \ + proven non-negative{unconfirmed}", + family_name(family) + ), )); } } @@ -1454,50 +1482,80 @@ fn accuracy_check( }; let guarantee = bound_root_guarantee(root, series, data, models).map(over_scope); let meets = |g: &AccuracyGuarantee| models.accuracy.satisfies(g, target); - let met = guarantee.clone().filter(meets).or_else(|| { - models - .evidence - .output_guarantee(root) - .map(over_scope) - .filter(meets) - }); + // The deployment's evidence is read only when the theoretical + // guarantee does not meet the target. + let evidence = || models.evidence.output_guarantee(root).map(over_scope); + let (met, evidence) = match guarantee.clone().filter(meets) { + Some(met) => (Some(met), None), + None => { + let evidence = evidence(); + (evidence.clone().filter(meets), evidence) + } + }; if let Some(met) = met { accepted.push(AcceptedAccuracy { target_metric: target.accuracy_metric, accepted_metric: (!met.is_exact()).then_some(met.spec.accuracy_metric), + accepted_without_guarantee: false, sketches, }); continue; } - return Err(match guarantee { - Some(g) - if values.is_none() - && target.accuracy_scope == AccuracyScope::WholeResult - && target.failure_probability.is_some() - && g.spec.failure_probability.evaluate().is_none() => + // No guarantee from either source: the deployment's model decides. + if guarantee.is_none() && evidence.is_none() { + if models + .accuracy + .accepts_output_without_guarantee(root, target) { + accepted.push(AcceptedAccuracy { + target_metric: target.accuracy_metric, + accepted_metric: None, + accepted_without_guarantee: true, + sketches, + }); + continue; + } + return Err(( + RejectionCause::NoAccuracyGuarantee, format!( - "q{query}: the output's guarantee holds for each value, and the number \ + "q{query}: the output has no guarantee, the deployment has no evidence \ + for it, and its accuracy model does not accept it for target {target:?} \ + without one" + ), + )); + } + return Err(( + RejectionCause::AccuracyTargetMissed, + match guarantee { + Some(g) + if values.is_none() + && target.accuracy_scope == AccuracyScope::WholeResult + && target.failure_probability.is_some() + && g.spec.failure_probability.evaluate().is_none() => + { + format!( + "q{query}: the output's guarantee holds for each value, and the number \ of values one evaluation returns is not bound (no declared series count \ for a source a PromQL vector reads; no bound on a SQL result's groups \ yet), \ so it does not meet whole-result target {target:?}, and the deployment \ has no evidence that it meets it" - ) - } - Some(g) => format!( - "q{query}: the output guarantees a {:?} bound {:?}, failure probability {:?}, \ + ) + } + Some(g) => format!( + "q{query}: the output guarantees a {:?} bound {:?}, failure probability {:?}, \ which misses target {target:?}, and the deployment has no evidence that it \ meets it", - g.spec.accuracy_metric, - g.spec.error.evaluate(), - g.spec.failure_probability.evaluate(), - ), - None => format!( - "q{query}: the output has no guarantee, and the deployment has no evidence \ + g.spec.accuracy_metric, + g.spec.error.evaluate(), + g.spec.failure_probability.evaluate(), + ), + None => format!( + "q{query}: the output has no guarantee, and the deployment has no evidence \ that it meets target {target:?}" - ), - }); + ), + }, + )); } Ok(accepted) } @@ -3106,7 +3164,7 @@ mod tests { settings, ) } - use asap_logical_asap_optimization::accuracy::AccuracyEvidenceProvider; + use crate::AccuracyEvidenceProvider; use asap_logical_asap_optimization::deployment_capabilities::DeploymentCapabilities; use asap_logical_asap_optimization::test_support::{entry, entry_with, lower_promql}; use asap_physical_asap_optimization::optimize_physical_asap; @@ -3348,7 +3406,15 @@ mod tests { ) .unwrap(); let p2 = selection.rejected.iter().find(|r| r.id == "P2").unwrap(); - assert_eq!(p2.cause, RejectionCause::AccuracyTargetMissed); + // Admitted weights leave the top-k output without a guarantee, + // which the built-in model does not accept; unproven weights + // miss the target outright. + let expected_cause = if admitted { + RejectionCause::NoAccuracyGuarantee + } else { + RejectionCause::AccuracyTargetMissed + }; + assert_eq!(p2.cause, expected_cause); assert_eq!( !p2.reason.contains("non-negative"), admitted, @@ -4160,7 +4226,7 @@ mod tests { .unwrap(); for id in &summaries { let rejection = builtin.rejected.iter().find(|r| r.id == *id).unwrap(); - assert_eq!(rejection.cause, RejectionCause::AccuracyTargetMissed); + assert_eq!(rejection.cause, RejectionCause::NoAccuracyGuarantee); assert!( rejection.reason.contains("no guarantee"), "{}", diff --git a/crates/plan-selection/src/pass/mod.rs b/crates/plan-selection/src/pass/mod.rs index 8893d628..5b1790e4 100644 --- a/crates/plan-selection/src/pass/mod.rs +++ b/crates/plan-selection/src/pass/mod.rs @@ -26,9 +26,10 @@ use asap_types::ir::{OperatorNode, QueryRoot}; use asap_types::workload::parsed_workload::ParsedWorkload; use asap_types::workload::WorkloadError; -use crate::accuracy::{AccuracyModel, DefaultAccuracyModel}; +use crate::accuracy::{ + AccuracyEvidenceProvider, AccuracyModel, BuiltinAccuracyModel, NoAccuracyEvidence, +}; use crate::{PlanSelectionSettings, Selection, SelectionError}; -use asap_logical_asap_optimization::accuracy::{AccuracyEvidenceProvider, NoAccuracyEvidence}; use asap_logical_asap_optimization::deployment_capabilities::DeploymentCapabilities; use asap_logical_asap_optimization::per_computation_candidates::PerComputationCandidateError; @@ -37,7 +38,7 @@ pub use stage1_to_stage3::{ PrunedCandidate, Stage1ToStage3Pass, Stage1ToStage3Run, }; -static DEFAULT_ACCURACY_MODEL: DefaultAccuracyModel = DefaultAccuracyModel; +static BUILTIN_ACCURACY_MODEL: BuiltinAccuracyModel = BuiltinAccuracyModel; static NO_ACCURACY_EVIDENCE: NoAccuracyEvidence = NoAccuracyEvidence; static DEFAULT_PLAN_SELECTION: std::sync::LazyLock = std::sync::LazyLock::new(PlanSelectionSettings::default); @@ -58,7 +59,8 @@ static DEFAULT_DEPLOYMENT: std::sync::LazyLock = #[non_exhaustive] pub struct PlanningModels<'a> { /// Decides whether a candidate's accuracy guarantee meets its query's - /// target; built-in [`DefaultAccuracyModel`] by default. + /// target, and whether an output with no guarantee is accepted; + /// [`BuiltinAccuracyModel`] by default. pub accuracy: &'a dyn AccuracyModel, /// Deployment facts the guarantees need (operand domains, observed /// guarantees); [`NoAccuracyEvidence`] by default. @@ -92,7 +94,7 @@ impl<'a> PlanningModels<'a> { /// The built-in models. pub fn builtin() -> PlanningModels<'static> { PlanningModels { - accuracy: &DEFAULT_ACCURACY_MODEL, + accuracy: &BUILTIN_ACCURACY_MODEL, evidence: &NO_ACCURACY_EVIDENCE, plan_selection: &DEFAULT_PLAN_SELECTION, deployment_capabilities: &DEFAULT_DEPLOYMENT, diff --git a/crates/planner/tests/deployment_accuracy_model.rs b/crates/planner/tests/deployment_accuracy_model.rs index 403245e2..2a68e89d 100644 --- a/crates/planner/tests/deployment_accuracy_model.rs +++ b/crates/planner/tests/deployment_accuracy_model.rs @@ -6,46 +6,33 @@ use std::collections::BTreeSet; use std::rc::Rc; -use asap_frontend_promql::promql_metric_names; -use asap_logical_asap_optimization::accuracy::PropagationStats; +use asap_frontend_promql::{lower_promql_query_workload, promql_metric_names}; use asap_plan_selection::{ - AccuracyModel, DefaultAccuracyModel, PlanOutput, PlanningModels, RejectionCause, + AccuracyModel, BuiltinAccuracyModel, PlanOutput, PlanningModels, RejectionCause, }; use asap_planner::{e2e_plan, FrontendInput, UserInput}; -use asap_types::ir::properties::{ - AccuracyError, AccuracyGuarantee, AccuracyMetric, AccuracyTarget, CompositionOperator, -}; +use asap_types::ir::properties::{AccuracyGuarantee, AccuracyMetric, AccuracyTarget}; use asap_types::ir::schema::FieldDataType; -use asap_types::ir::{ASAPOp, OperatorNode}; +use asap_types::ir::{ASAPOp, OperatorNode, QueryRoot}; use asap_types::workload::{ DataArrival, DataWorkload, DurationMs, PlanningWorkload, Predictability, Query, QueryLanguage, QueryRequirements, QueryWorkload, RepeatedDemand, RepeatingEntry, RepetitionInterval, }; -/// A deployment that trusts no approximate estimate: the built-in -/// composition rules, but only an exact guarantee meets a target. +/// A deployment that trusts no approximate estimate: only an exact +/// guarantee meets a target. Composition is the trait's default (the +/// built-in rules). struct ExactOnlyAccuracyModel; impl AccuracyModel for ExactOnlyAccuracyModel { - fn propagate( - &self, - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - local: Option<&AccuracyGuarantee>, - stats: &PropagationStats, - ) -> Result { - DefaultAccuracyModel.propagate(op, inputs, local, stats) - } - fn satisfies(&self, guarantee: &AccuracyGuarantee, _target: &AccuracyTarget) -> bool { guarantee.is_exact() } } -/// `quantile_over_time(0.99, lat[5m])` with a 1% rank error, repeated every -/// ten minutes over a metric sampled every 15 s. -fn p99_workload() -> PlanningWorkload { - let query = "quantile_over_time(0.99, lat[5m])"; +/// `query` with a 1% rank error, repeated every ten minutes over metrics +/// sampled every 15 s. +fn rank_target_workload(query: &str) -> PlanningWorkload { PlanningWorkload { query_workload: QueryWorkload { language: QueryLanguage::PromQL, @@ -82,6 +69,24 @@ fn reads_sketch(root: &Rc) -> bool { }) } +/// Every Stage 2 candidate for `workload` under `models`, rendered in full: +/// its id and its query roots with every node's guarantee and summary +/// parameters. Two runs that differ only in Stage 3's models must agree. +fn stage2_candidates(workload: &PlanningWorkload, models: PlanningModels<'_>) -> Vec { + let roots = lower_promql_query_workload(workload).expect("lowers"); + let run = asap_plan_selection::run_stage1_to_stage3( + roots.into_iter().enumerate().collect(), + &workload.query_workload.entries().collect::>(), + workload.data_workload.as_ref().unwrap(), + models, + ) + .expect("Stage 1 accepts the workload"); + run.stage2_physical_asap_optimization + .iter() + .map(|candidate| format!("{}: {:?}", candidate.id, candidate.roots)) + .collect() +} + /// Every candidate Stage 3 was given: the valid ones it priced and the /// ones it rejected. fn stage3_candidate_ids(output: &PlanOutput) -> BTreeSet { @@ -109,9 +114,9 @@ fn approximate_accepted(output: &PlanOutput) -> BTreeSet { #[tokio::test] async fn the_deployment_accuracy_model_decides_only_in_plan_selection() { // The user's workload: one p99 query with a 1% rank-error target. - let workload = p99_workload(); + let workload = rank_target_workload("quantile_over_time(0.99, lat[5m])"); - // The built-in models: Stage 3 checks guarantees with DefaultAccuracyModel. + // The built-in models: Stage 3 checks guarantees with BuiltinAccuracyModel. let builtin = e2e_plan(UserInput::new( &workload, FrontendInput::Promql { histograms: None }, @@ -135,11 +140,19 @@ async fn the_deployment_accuracy_model_decides_only_in_plan_selection() { .await .expect("plans with the exact-only model"); - // Stages 1 and 2 do not read the model: Stage 3 got the same candidates. + // Stages 1 and 2 do not read the model: Stage 3 got the same + // candidates, with the same guarantees and summary parameters. assert_eq!( stage3_candidate_ids(&builtin), stage3_candidate_ids(&exact_only) ); + assert_eq!( + stage2_candidates(&workload, PlanningModels::builtin()), + stage2_candidates( + &workload, + PlanningModels::builtin().with_accuracy(&exact_only_model) + ) + ); // Stage 3 now rejects each of those candidates for missing its target, // accepts no approximate output and selects an exact plan. let exact_only_selection = exact_only.selection.as_ref().unwrap(); @@ -152,3 +165,93 @@ async fn the_deployment_accuracy_model_decides_only_in_plan_selection() { assert!(approximate_accepted(&exact_only).is_empty()); assert!(!reads_sketch(&exact_only.plans[0].root)); } + +/// A deployment whose own measurements cover every output that has no +/// analytical guarantee: it accepts each such output for any target. +struct AcceptsOutputsWithoutGuarantee; + +impl AccuracyModel for AcceptsOutputsWithoutGuarantee { + fn satisfies(&self, guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool { + BuiltinAccuracyModel.satisfies(guarantee, target) + } + + fn accepts_output_without_guarantee( + &self, + _output: &QueryRoot, + _target: &AccuracyTarget, + ) -> bool { + true + } +} + +/// The candidates for which Stage 3 accepted some query's output that has +/// no guarantee. +fn accepted_without_guarantee(output: &PlanOutput) -> BTreeSet { + let selection = output.selection.as_ref().expect("plan selection ran"); + selection + .accepted + .iter() + .filter(|(_, queries)| queries.iter().any(|q| q.accepted_without_guarantee)) + .map(|(id, _)| id.clone()) + .collect() +} + +#[tokio::test] +async fn the_deployment_accuracy_model_decides_outputs_without_a_guarantee() { + // The user's workload: a difference of two quantile estimates, 1% rank + // error. Stage 1 has no analytical rule for a difference of rank + // estimates, so it keeps the summary candidates with no guarantee. + let workload = rank_target_workload("quantile(0.99, rate(x[5m])) - quantile(0.5, rate(x[5m]))"); + let promql = || FrontendInput::Promql { histograms: None }; + + // The built-in model accepts no output without a guarantee: Stage 3 + // rejects every summary candidate as having none and selects exact. + let builtin = e2e_plan(UserInput::new( + &workload, + promql(), + PlanningModels::builtin(), + )) + .await + .expect("plans with the built-in models"); + let builtin_selection = builtin.selection.as_ref().unwrap(); + let unguaranteed: BTreeSet = builtin_selection + .rejected + .iter() + .filter(|r| r.cause == RejectionCause::NoAccuracyGuarantee) + .map(|r| r.id.clone()) + .collect(); + assert!(!unguaranteed.is_empty()); + assert!(!reads_sketch(&builtin.plans[0].root)); + + // The deployment's model accepts those outputs through the hook. + let accepting_model = AcceptsOutputsWithoutGuarantee; + let accepting = e2e_plan(UserInput::new( + &workload, + promql(), + PlanningModels::builtin().with_accuracy(&accepting_model), + )) + .await + .expect("plans with the accepting model"); + let accepting_selection = accepting.selection.as_ref().unwrap(); + // Stages 1 and 2 produced the same candidates under both models. + assert_eq!( + stage2_candidates(&workload, PlanningModels::builtin()), + stage2_candidates( + &workload, + PlanningModels::builtin().with_accuracy(&accepting_model) + ) + ); + // Every candidate the built-in model rejected for having no guarantee + // is now valid and priced, accepted without a guarantee. + assert_eq!(accepted_without_guarantee(&accepting), unguaranteed); + for id in &unguaranteed { + assert!(accepting_selection.costs.contains_key(id), "{id}"); + } + assert!(accepting_selection + .rejected + .iter() + .all(|r| r.cause != RejectionCause::NoAccuracyGuarantee)); + // Stage 3 selects one of them, a plan that reads a sketch. + assert!(unguaranteed.contains(&accepting_selection.selected)); + assert!(reads_sketch(&accepting.plans[0].root)); +} diff --git a/crates/planner/tests/summary_sharing.rs b/crates/planner/tests/summary_sharing.rs index f7dbb486..c4c2317c 100644 --- a/crates/planner/tests/summary_sharing.rs +++ b/crates/planner/tests/summary_sharing.rs @@ -447,7 +447,7 @@ fn combined_scalar_references_keep_summary_candidates() { .expect("not selected"); assert_eq!( rejection.cause, - RejectionCause::AccuracyTargetMissed, + RejectionCause::NoAccuracyGuarantee, "{rejection:?}" ); assert!( diff --git a/tools/dag-viewer/examples/planner-layering-example1.json b/tools/dag-viewer/examples/planner-layering-example1.json index db38b862..5f67a4b3 100644 --- a/tools/dag-viewer/examples/planner-layering-example1.json +++ b/tools/dag-viewer/examples/planner-layering-example1.json @@ -103554,12 +103554,14 @@ "P1": [ { "accepted_metric": null, + "accepted_without_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, + "accepted_without_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103568,12 +103570,14 @@ "P2": [ { "accepted_metric": null, + "accepted_without_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, + "accepted_without_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103582,12 +103586,14 @@ "P3": [ { "accepted_metric": null, + "accepted_without_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, + "accepted_without_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103596,12 +103602,14 @@ "P4": [ { "accepted_metric": null, + "accepted_without_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, + "accepted_without_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103610,12 +103618,14 @@ "P4-m1": [ { "accepted_metric": null, + "accepted_without_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, + "accepted_without_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103624,12 +103634,14 @@ "P5": [ { "accepted_metric": null, + "accepted_without_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, + "accepted_without_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103638,12 +103650,14 @@ "P5-m1": [ { "accepted_metric": null, + "accepted_without_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, + "accepted_without_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103652,12 +103666,14 @@ "P6": [ { "accepted_metric": null, + "accepted_without_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, + "accepted_without_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103666,12 +103682,14 @@ "P6-m1": [ { "accepted_metric": null, + "accepted_without_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, + "accepted_without_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103680,12 +103698,14 @@ "P7": [ { "accepted_metric": null, + "accepted_without_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, + "accepted_without_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103694,12 +103714,14 @@ "P7-m1": [ { "accepted_metric": null, + "accepted_without_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, + "accepted_without_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103708,12 +103730,14 @@ "P8": [ { "accepted_metric": null, + "accepted_without_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, + "accepted_without_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103722,12 +103746,14 @@ "P8-m1": [ { "accepted_metric": null, + "accepted_without_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, + "accepted_without_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103736,12 +103762,14 @@ "P8-m2": [ { "accepted_metric": null, + "accepted_without_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, + "accepted_without_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103750,12 +103778,14 @@ "P8-m3": [ { "accepted_metric": null, + "accepted_without_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, + "accepted_without_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103764,12 +103794,14 @@ "P9": [ { "accepted_metric": null, + "accepted_without_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, + "accepted_without_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103778,12 +103810,14 @@ "P9-m1": [ { "accepted_metric": null, + "accepted_without_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, + "accepted_without_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103792,12 +103826,14 @@ "P9-m2": [ { "accepted_metric": null, + "accepted_without_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, + "accepted_without_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103806,12 +103842,14 @@ "P9-m3": [ { "accepted_metric": null, + "accepted_without_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, + "accepted_without_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -105738,194 +105776,194 @@ "front": [], "rejected": [ { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P10", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P11", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P12", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P13", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P13-m1", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P14", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P14-m1", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P15", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P15-m1", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P16", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P16-m1", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P17", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P17-m1", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P17-m2", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P17-m3", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P18", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P18-m1", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P18-m2", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P18-m3", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P19", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P20", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P21", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P22", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P22-m1", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P23", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P23-m1", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P24", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P24-m1", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P25", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P25-m1", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P26", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P26-m1", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P26-m2", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P26-m3", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P27", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P27-m1", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P27-m2", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P27-m3", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" }, { "cause": "tied_lost_on_enumeration_order", diff --git a/tools/dag-viewer/stages.js b/tools/dag-viewer/stages.js index d55e2305..e6cad9cd 100644 --- a/tools/dag-viewer/stages.js +++ b/tools/dag-viewer/stages.js @@ -168,8 +168,8 @@ function uniqueIds(list, where, errors) { // of a candidate that failed a check (`RejectionCause::failed_a_check`): // it is not priced. const FAILED_CHECK_CAUSES = [ - 'failure_probability_resize_failed', 'accuracy_target_missed', 'raw_data_not_retained', - 'latency_bound_exceeded', 'over_cluster_capacity', + 'failure_probability_resize_failed', 'accuracy_target_missed', 'no_accuracy_guarantee', + 'raw_data_not_retained', 'latency_bound_exceeded', 'over_cluster_capacity', ]; // Every `stage3_plan_selection.rejected[].cause` Stage 3 writes: the failed // checks, then the losses under the cost model (priced, not selected). diff --git a/tools/dag-viewer/test_viewer.py b/tools/dag-viewer/test_viewer.py index a87a00f9..424ae5d0 100644 --- a/tools/dag-viewer/test_viewer.py +++ b/tools/dag-viewer/test_viewer.py @@ -176,7 +176,8 @@ def test_status_follows_the_rejection_cause_not_the_cost(self): model and "rejected" when it failed a check, whether or not the document carries its cost.""" selection = self.doc["stage3_plan_selection"] - failed = next(r for r in selection["rejected"] if r["cause"] == "accuracy_target_missed") + failed_a_check = set(json.loads(self.js.eval("JSON.stringify(FAILED_CHECK_CAUSES)"))) + failed = next(r for r in selection["rejected"] if r["cause"] in failed_a_check) lost = next(r for r in selection["rejected"] if r["cause"] == "costlier") selection["costs"][failed["id"]] = selection["costs"][lost["id"]] del selection["costs"][lost["id"]] From ba9561d05ab9f71a2bee7ea7305fc73b590c656e Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sun, 18 Oct 2026 16:10:18 +0000 Subject: [PATCH 4/6] feat(plan-selection)!: let the deployment's accuracy model accept an output regardless of its guarantee The hook now has priority over every guarantee: Stage 3 asks AccuracyModel::accepts_output_regardless_of_guarantee first for each query output, and an output it accepts skips the guarantee, Count-Min weight and evidence checks. Otherwise the check is unchanged (Stage 1 guarantee, then evidence, then AccuracyTargetMissed, or NoAccuracyGuarantee when neither gives one). The default returns false, so the built-in models behave as before. Tests pin that the hook accepts an output whose guarantee misses and one with no guarantee, in Stage 3 and end to end. Also drop the unused crate-root re-export of PropagationStats and re-indent the accuracy rejection reasons. BREAKING CHANGE: AccuracyModel::accepts_output_without_guarantee is now accepts_output_regardless_of_guarantee, and AcceptedAccuracy::accepted_without_guarantee (stage_pipeline JSON key accepted_without_guarantee) is now accepted_regardless_of_guarantee; asap_logical_asap_optimization::PropagationStats is now only at accuracy::PropagationStats. --- crates/devtools/src/bin/stage_pipeline.rs | 9 +- crates/logical-asap-optimization/src/lib.rs | 1 - crates/plan-selection/src/accuracy.rs | 21 +- crates/plan-selection/src/lib.rs | 195 +++++++++++++----- .../tests/deployment_accuracy_model.rs | 107 +++++++--- .../examples/planner-layering-example1.json | 152 +++++++------- 6 files changed, 317 insertions(+), 168 deletions(-) diff --git a/crates/devtools/src/bin/stage_pipeline.rs b/crates/devtools/src/bin/stage_pipeline.rs index 9760d7ff..2e20af98 100644 --- a/crates/devtools/src/bin/stage_pipeline.rs +++ b/crates/devtools/src/bin/stage_pipeline.rs @@ -31,9 +31,10 @@ // - stage3_plan_selection: the selected candidate, chosen among all candidates; // the shown candidates' costs and accepted accuracy (per query: the // target's metric or null when not set, the accepted guarantee's -// metric or null when exact or accepted without a guarantee, whether -// the deployment's accuracy model accepted it without a guarantee -// (`accepted_without_guarantee`), the sketches read and a one-line +// metric or null when exact or accepted regardless of guarantee, +// whether the deployment's accuracy model accepted it directly, +// whatever its guarantee says (`accepted_regardless_of_guarantee`), +// the sketches read and a one-line // summary), and every other shown candidate under `rejected` with // its `cause`: a failed check (`failure_probability_resize_failed`, // `accuracy_target_missed`, `no_accuracy_guarantee`, `raw_data_not_retained`, @@ -619,7 +620,7 @@ fn plan_selection_json( json!({ "target_metric": accepted.target_metric, "accepted_metric": accepted.accepted_metric, - "accepted_without_guarantee": accepted.accepted_without_guarantee, + "accepted_regardless_of_guarantee": accepted.accepted_regardless_of_guarantee, "sketches": accepted.sketches, "summary": accepted.to_string(), }) diff --git a/crates/logical-asap-optimization/src/lib.rs b/crates/logical-asap-optimization/src/lib.rs index 38468c54..9c5e441e 100644 --- a/crates/logical-asap-optimization/src/lib.rs +++ b/crates/logical-asap-optimization/src/lib.rs @@ -54,7 +54,6 @@ pub mod realization; #[cfg(any(test, feature = "test-support"))] pub mod test_support; -pub use accuracy::PropagationStats; pub use deployment_capabilities::{ DeploymentCapabilities, DeploymentCapabilitiesError, SupportedSummary, }; diff --git a/crates/plan-selection/src/accuracy.rs b/crates/plan-selection/src/accuracy.rs index d9b76b05..40c7c4a5 100644 --- a/crates/plan-selection/src/accuracy.rs +++ b/crates/plan-selection/src/accuracy.rs @@ -7,7 +7,8 @@ //! with the model here, whether a guarantee meets its query's target, how //! guarantees compose once the data workload and the deployment's evidence //! ([`AccuracyEvidenceProvider`]) bind the statistics Stage 1 left unknown, -//! and whether an output with no guarantee at all is accepted. +//! and whether the deployment accepts an output directly, whatever its +//! guarantee says. use asap_logical_asap_optimization::accuracy::{ self as analytical, OperandDomain, PropagationStats, @@ -50,13 +51,14 @@ pub trait AccuracyModel { fn satisfies(&self, guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool; /// Whether the deployment accepts `output`, one query's output in a - /// candidate, for `target` although the output has no guarantee: Stage 1 - /// had no analytical rule for it and the deployment's evidence supplies - /// none ([`AccuracyEvidenceProvider::output_guarantee`]). Stage 3 asks only - /// then; a guarantee from either source is checked with - /// [`Self::satisfies`] instead. The default accepts none, and Stage 3 - /// rejects such a candidate (`RejectionCause::NoAccuracyGuarantee`). - fn accepts_output_without_guarantee( + /// candidate, for `target` directly, whatever its guarantee says: + /// missing (Stage 1 had no analytical rule), met or missed. Stage 3 asks + /// this first for every query output; when it returns `true`, no + /// guarantee or evidence check runs for that output. The default accepts + /// none, so Stage 3 checks the Stage 1 guarantee, then the evidence + /// ([`AccuracyEvidenceProvider::output_guarantee`]) with + /// [`Self::satisfies`]. + fn accepts_output_regardless_of_guarantee( &self, _output: &QueryRoot, _target: &AccuracyTarget, @@ -67,7 +69,8 @@ pub trait AccuracyModel { /// The planner's built-in analytical rules as an [`AccuracyModel`]: the /// trait's default composition, the conservative target check Stage 1's -/// `accuracy::satisfies` makes, and no output accepted without a guarantee. +/// `accuracy::satisfies` makes, and no output accepted regardless of its +/// guarantee. /// The default in [`PlanningModels::builtin`](crate::PlanningModels::builtin). #[derive(Debug, Default, Clone, Copy)] pub struct BuiltinAccuracyModel; diff --git a/crates/plan-selection/src/lib.rs b/crates/plan-selection/src/lib.rs index 5425b251..5936e78a 100644 --- a/crates/plan-selection/src/lib.rs +++ b/crates/plan-selection/src/lib.rs @@ -7,9 +7,9 @@ //! from the data workload and the deployment's operand-domain contracts), //! or when the deployment's empirical evidence shows this //! candidate's output does (`AccuracyEvidenceProvider::output_guarantee`, -//! read only here). An output with no guarantee from either source is -//! accepted only when the deployment's accuracy model accepts it without one -//! (`AccuracyModel::accepts_output_without_guarantee`). For a query with a target, Count-Min is admitted only +//! read only here). Before either, the deployment's accuracy model may accept +//! the output directly, whatever its guarantee says +//! (`AccuracyModel::accepts_output_regardless_of_guarantee`). For a query with a target, Count-Min is admitted only //! over weights proven non-negative; a query without one constrains nothing. //! A miss rejects the candidate as invalid, with a reason; only this stage //! discards valid candidates. @@ -590,8 +590,8 @@ pub enum RejectionCause { AccuracyTargetMissed, /// A query's output has no accuracy guarantee (Stage 1 has no /// analytical rule for it and the deployment's evidence supplies none), - /// and the deployment's accuracy model does not accept it without one - /// ([`AccuracyModel::accepts_output_without_guarantee`]). + /// and the deployment's accuracy model did not accept it directly + /// ([`AccuracyModel::accepts_output_regardless_of_guarantee`]). NoAccuracyGuarantee, /// It scans raw data at query time in a deployment that does not /// retain raw data. @@ -646,14 +646,14 @@ pub struct AcceptedAccuracy { /// The target's metric; `None` when the user did not set one. pub target_metric: Option, /// The metric the accepted guarantee is stated in; `None` for an exact - /// output, which meets every metric, and for one accepted without a - /// guarantee. + /// output, which meets every metric, and for one the deployment's model + /// accepted regardless of its guarantee (no guarantee was checked). pub accepted_metric: Option, - /// Whether the output has no guarantee and the deployment's accuracy - /// model accepted it anyway - /// ([`AccuracyModel::accepts_output_without_guarantee`]); `false` - /// otherwise. - pub accepted_without_guarantee: bool, + /// Whether the deployment's accuracy model accepted the output directly, + /// whatever its guarantee says + /// ([`AccuracyModel::accepts_output_regardless_of_guarantee`]); `false` + /// when a guarantee met the target. + pub accepted_regardless_of_guarantee: bool, /// The sketch algorithms the query's output reads, in plan order /// (e.g. `["Kll"]`); empty for an exact plan. pub sketches: Vec, @@ -662,7 +662,7 @@ pub struct AcceptedAccuracy { impl std::fmt::Display for AcceptedAccuracy { /// E.g. `metric not set; accepted rank (Kll)`, or /// `metric relative; accepted relative (DDSketch)`, or - /// `metric rank; accepted without a guarantee (Kll)`. + /// `metric rank; accepted by the deployment's accuracy model regardless of guarantee (Kll)`. fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let name = |metric: AccuracyMetric| { serde_json::to_value(metric) @@ -675,7 +675,10 @@ impl std::fmt::Display for AcceptedAccuracy { Some(metric) => write!(f, "metric {}; ", name(metric))?, } match self.accepted_metric { - _ if self.accepted_without_guarantee => write!(f, "accepted without a guarantee")?, + _ if self.accepted_regardless_of_guarantee => write!( + f, + "accepted by the deployment's accuracy model regardless of guarantee" + )?, None => write!(f, "accepted exact")?, Some(metric) => write!(f, "accepted {}", name(metric))?, } @@ -1417,12 +1420,13 @@ pub fn resize_for_failure_probability( /// its ε, δ and scope ([`AccuracyModel::satisfies`]); one that sets a metric /// accepts only guarantees in it (or convertible to it). /// -/// An output with no guarantee from either source (Stage 1 had no rule, -/// and the evidence has none) is accepted only when the deployment's -/// accuracy model accepts it without one -/// ([`AccuracyModel::accepts_output_without_guarantee`]); otherwise the -/// candidate is rejected as [`RejectionCause::NoAccuracyGuarantee`]. Any -/// other miss is [`RejectionCause::AccuracyTargetMissed`]. +/// The deployment's accuracy model is asked first +/// ([`AccuracyModel::accepts_output_regardless_of_guarantee`]): an output it +/// accepts is accepted whatever its guarantee says, and none of the checks +/// above runs for it. Otherwise, an output with no guarantee from either +/// source (Stage 1 had no rule, and the evidence has none) is rejected as +/// [`RejectionCause::NoAccuracyGuarantee`], and any other miss as +/// [`RejectionCause::AccuracyTargetMissed`]. fn accuracy_check( candidate: &PhysicalCandidate, targets: &[AccuracyTarget], @@ -1433,6 +1437,9 @@ fn accuracy_check( for (query, (root, target)) in candidate.roots.iter().zip(targets).enumerate() { let query = query + 1; let mut sketches: Vec = Vec::new(); + // The first Count-Min whose update weights are not proven + // non-negative, which its bound assumes, as a rejection reason. + let mut unproven_weights: Option = None; for node in root_operators(root) .iter() .flat_map(|node| OperatorNode::reachable(node)) @@ -1462,16 +1469,32 @@ fn accuracy_check( } _ => String::new(), }; - return Err(( - RejectionCause::AccuracyTargetMissed, + unproven_weights.get_or_insert_with(|| { format!( "q{query}: {} needs non-negative update weights, and these are not \ proven non-negative{unconfirmed}", family_name(family) - ), - )); + ) + }); } } + // The deployment's model may accept the output directly, whatever + // its guarantee says; no guarantee or evidence check runs then. + if models + .accuracy + .accepts_output_regardless_of_guarantee(root, target) + { + accepted.push(AcceptedAccuracy { + target_metric: target.accuracy_metric, + accepted_metric: None, + accepted_regardless_of_guarantee: true, + sketches, + }); + continue; + } + if let Some(reason) = unproven_weights { + return Err((RejectionCause::AccuracyTargetMissed, reason)); + } // A guarantee per value is stated over the whole result's `G` // values for a whole-result target. let series = series_read(root, data); @@ -1496,31 +1519,17 @@ fn accuracy_check( accepted.push(AcceptedAccuracy { target_metric: target.accuracy_metric, accepted_metric: (!met.is_exact()).then_some(met.spec.accuracy_metric), - accepted_without_guarantee: false, + accepted_regardless_of_guarantee: false, sketches, }); continue; } - // No guarantee from either source: the deployment's model decides. if guarantee.is_none() && evidence.is_none() { - if models - .accuracy - .accepts_output_without_guarantee(root, target) - { - accepted.push(AcceptedAccuracy { - target_metric: target.accuracy_metric, - accepted_metric: None, - accepted_without_guarantee: true, - sketches, - }); - continue; - } return Err(( RejectionCause::NoAccuracyGuarantee, format!( - "q{query}: the output has no guarantee, the deployment has no evidence \ - for it, and its accuracy model does not accept it for target {target:?} \ - without one" + "q{query}: the output has no guarantee, and the deployment has no evidence \ + for it, so it cannot meet target {target:?}" ), )); } @@ -1535,24 +1544,23 @@ fn accuracy_check( { format!( "q{query}: the output's guarantee holds for each value, and the number \ - of values one evaluation returns is not bound (no declared series count \ - for a source a PromQL vector reads; no bound on a SQL result's groups \ - yet), \ - so it does not meet whole-result target {target:?}, and the deployment \ - has no evidence that it meets it" + of values one evaluation returns is not bound (no declared series \ + count for a source a PromQL vector reads; no bound on a SQL result's \ + groups yet), so it does not meet whole-result target {target:?}, and \ + the deployment has no evidence that it meets it" ) } Some(g) => format!( - "q{query}: the output guarantees a {:?} bound {:?}, failure probability {:?}, \ - which misses target {target:?}, and the deployment has no evidence that it \ - meets it", + "q{query}: the output guarantees a {:?} bound {:?}, failure probability \ + {:?}, which misses target {target:?}, and the deployment has no evidence \ + that it meets it", g.spec.accuracy_metric, g.spec.error.evaluate(), g.spec.failure_probability.evaluate(), ), None => format!( - "q{query}: the output has no guarantee, and the deployment has no evidence \ - that it meets target {target:?}" + "q{query}: the output has no guarantee, and the deployment has no \ + evidence that it meets target {target:?}" ), }, )); @@ -4250,6 +4258,91 @@ mod tests { } } + /// Deployment evidence of a large absolute error (1.0) for every output. + struct EveryOutputMeasuredLoose; + + impl AccuracyEvidenceProvider for EveryOutputMeasuredLoose { + fn output_guarantee(&self, _output: &QueryRoot) -> Option { + Some(AccuracyGuarantee::new( + AccuracyMetric::ABSOLUTE, + BoundExpr::Constant { value: 1.0 }, + ProbabilityExpr::Constant { value: 1e-6 }, + vec![], + )) + } + } + + /// A model that accepts every output directly, whatever its guarantee. + struct AcceptsEveryOutput; + + impl AccuracyModel for AcceptsEveryOutput { + fn satisfies(&self, guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool { + BuiltinAccuracyModel.satisfies(guarantee, target) + } + + fn accepts_output_regardless_of_guarantee( + &self, + _output: &QueryRoot, + _target: &AccuracyTarget, + ) -> bool { + true + } + } + + /// The model's hook has priority over every guarantee. The summary + /// candidates of a difference of quantile estimates have no theoretical + /// guarantee: with evidence that misses the target the built-in model + /// rejects them as `AccuracyTargetMissed`, and with no evidence as + /// `NoAccuracyGuarantee`; a model that accepts every output accepts them + /// in both cases, regardless of guarantee. + #[test] + fn the_accuracy_model_accepts_an_output_whatever_its_guarantee() { + let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); + let candidates = all_candidates( + "quantile_over_time(0.99, m[1m]) - quantile_over_time(0.5, m[1m])", + &target, + ); + let summaries: Vec<_> = candidates + .iter() + .filter(|c| has_summary_estimate(c)) + .map(|c| c.id.as_str()) + .collect(); + assert!(!summaries.is_empty()); + let select = |accuracy: &dyn AccuracyModel, evidence: &dyn AccuracyEvidenceProvider| { + select_plan( + &candidates, + &[entry_with( + target.clone().accuracy_metric(AccuracyMetric::ABSOLUTE), + repeating(10_000), + )], + &data(), + PlanningModels::builtin() + .with_accuracy(accuracy) + .with_evidence(evidence), + ) + .unwrap() + }; + for (evidence, builtin_cause) in [ + ( + &EveryOutputMeasuredLoose as &dyn AccuracyEvidenceProvider, + RejectionCause::AccuracyTargetMissed, + ), + (&NoAccuracyEvidence, RejectionCause::NoAccuracyGuarantee), + ] { + let builtin = select(&BuiltinAccuracyModel, evidence); + let accepting = select(&AcceptsEveryOutput, evidence); + for id in &summaries { + let rejection = builtin.rejected.iter().find(|r| r.id == *id).unwrap(); + assert_eq!(rejection.cause, builtin_cause, "{}", rejection.reason); + assert!(accepting.costs.contains_key(*id), "{id} is valid"); + assert!( + accepting.accepted[*id][0].accepted_regardless_of_guarantee, + "{id}" + ); + } + } + } + /// `candidates` with every state node (a summary build or an exact /// aggregate) kept at ingestion time, so selection charges its kept /// bytes every second (a `MatAtQuery` state is held only for one diff --git a/crates/planner/tests/deployment_accuracy_model.rs b/crates/planner/tests/deployment_accuracy_model.rs index 2a68e89d..c357e1fe 100644 --- a/crates/planner/tests/deployment_accuracy_model.rs +++ b/crates/planner/tests/deployment_accuracy_model.rs @@ -166,16 +166,16 @@ async fn the_deployment_accuracy_model_decides_only_in_plan_selection() { assert!(!reads_sketch(&exact_only.plans[0].root)); } -/// A deployment whose own measurements cover every output that has no -/// analytical guarantee: it accepts each such output for any target. -struct AcceptsOutputsWithoutGuarantee; +/// A deployment that has measured every output itself and accepts each one +/// for any target, whatever its guarantee says. +struct AcceptsEveryOutput; -impl AccuracyModel for AcceptsOutputsWithoutGuarantee { +impl AccuracyModel for AcceptsEveryOutput { fn satisfies(&self, guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool { BuiltinAccuracyModel.satisfies(guarantee, target) } - fn accepts_output_without_guarantee( + fn accepts_output_regardless_of_guarantee( &self, _output: &QueryRoot, _target: &AccuracyTarget, @@ -184,28 +184,45 @@ impl AccuracyModel for AcceptsOutputsWithoutGuarantee { } } -/// The candidates for which Stage 3 accepted some query's output that has -/// no guarantee. -fn accepted_without_guarantee(output: &PlanOutput) -> BTreeSet { +/// The candidates for which Stage 3 accepted some query's output because +/// the deployment's model accepted it regardless of its guarantee. +fn accepted_regardless_of_guarantee(output: &PlanOutput) -> BTreeSet { let selection = output.selection.as_ref().expect("plan selection ran"); selection .accepted .iter() - .filter(|(_, queries)| queries.iter().any(|q| q.accepted_without_guarantee)) + .filter(|(_, queries)| queries.iter().any(|q| q.accepted_regardless_of_guarantee)) .map(|(id, _)| id.clone()) .collect() } +/// The ids of the candidates `output`'s selection rejected with `cause`. +fn rejected_with(output: &PlanOutput, cause: RejectionCause) -> BTreeSet { + output + .selection + .as_ref() + .expect("plan selection ran") + .rejected + .iter() + .filter(|r| r.cause == cause) + .map(|r| r.id.clone()) + .collect() +} + +/// A deployment model that accepts an output directly also accepts one with +/// no guarantee: the built-in model rejects the summary candidates of a +/// difference of two quantile estimates as having none, and the accepting +/// model makes them valid, and Stage 3 selects one. #[tokio::test] -async fn the_deployment_accuracy_model_decides_outputs_without_a_guarantee() { +async fn the_deployment_accuracy_model_accepts_outputs_without_a_guarantee() { // The user's workload: a difference of two quantile estimates, 1% rank // error. Stage 1 has no analytical rule for a difference of rank // estimates, so it keeps the summary candidates with no guarantee. let workload = rank_target_workload("quantile(0.99, rate(x[5m])) - quantile(0.5, rate(x[5m]))"); let promql = || FrontendInput::Promql { histograms: None }; - // The built-in model accepts no output without a guarantee: Stage 3 - // rejects every summary candidate as having none and selects exact. + // The built-in model (default hook: accepts nothing directly) rejects + // every summary candidate as having no guarantee and selects exact. let builtin = e2e_plan(UserInput::new( &workload, promql(), @@ -213,18 +230,14 @@ async fn the_deployment_accuracy_model_decides_outputs_without_a_guarantee() { )) .await .expect("plans with the built-in models"); - let builtin_selection = builtin.selection.as_ref().unwrap(); - let unguaranteed: BTreeSet = builtin_selection - .rejected - .iter() - .filter(|r| r.cause == RejectionCause::NoAccuracyGuarantee) - .map(|r| r.id.clone()) - .collect(); + // The candidates rejected for having no guarantee: P6 … P47. + let unguaranteed = rejected_with(&builtin, RejectionCause::NoAccuracyGuarantee); assert!(!unguaranteed.is_empty()); + assert!(accepted_regardless_of_guarantee(&builtin).is_empty()); assert!(!reads_sketch(&builtin.plans[0].root)); - // The deployment's model accepts those outputs through the hook. - let accepting_model = AcceptsOutputsWithoutGuarantee; + // The deployment's model accepts every output directly. + let accepting_model = AcceptsEveryOutput; let accepting = e2e_plan(UserInput::new( &workload, promql(), @@ -242,16 +255,56 @@ async fn the_deployment_accuracy_model_decides_outputs_without_a_guarantee() { ) ); // Every candidate the built-in model rejected for having no guarantee - // is now valid and priced, accepted without a guarantee. - assert_eq!(accepted_without_guarantee(&accepting), unguaranteed); + // is now valid and priced, accepted regardless of its guarantee, and no + // candidate fails the accuracy check. + assert!(accepted_regardless_of_guarantee(&accepting).is_superset(&unguaranteed)); for id in &unguaranteed { assert!(accepting_selection.costs.contains_key(id), "{id}"); } - assert!(accepting_selection - .rejected - .iter() - .all(|r| r.cause != RejectionCause::NoAccuracyGuarantee)); + assert!(accepting_selection.rejected.iter().all(|r| !matches!( + r.cause, + RejectionCause::NoAccuracyGuarantee | RejectionCause::AccuracyTargetMissed + ))); // Stage 3 selects one of them, a plan that reads a sketch. assert!(unguaranteed.contains(&accepting_selection.selected)); assert!(reads_sketch(&accepting.plans[0].root)); } + +/// The hook has priority over a guarantee that misses: a candidate whose +/// guarantee misses its target under the built-in models (a relative bound +/// for a rank target) is accepted under the accepting model. +#[tokio::test] +async fn the_deployment_accuracy_model_accepts_outputs_whose_guarantee_misses() { + // The user's workload: one p99 query with a 1% rank-error target. + let workload = rank_target_workload("quantile_over_time(0.99, lat[5m])"); + let promql = || FrontendInput::Promql { histograms: None }; + // Under the built-in models: the candidates whose guarantee misses the + // target (P3, a relative bound 0.01 for a rank target). + let builtin = e2e_plan(UserInput::new( + &workload, + promql(), + PlanningModels::builtin(), + )) + .await + .expect("plans with the built-in models"); + let missed = rejected_with(&builtin, RejectionCause::AccuracyTargetMissed); + assert!(!missed.is_empty()); + // The same workload with the model that accepts every output directly. + let accepting_model = AcceptsEveryOutput; + let accepting = e2e_plan(UserInput::new( + &workload, + promql(), + PlanningModels::builtin().with_accuracy(&accepting_model), + )) + .await + .expect("plans with the accepting model"); + // Each of them is now valid, priced and accepted regardless of guarantee. + let accepting_selection = accepting.selection.as_ref().unwrap(); + for id in &missed { + assert!(accepting_selection.costs.contains_key(id), "{id}"); + assert!( + accepting_selection.accepted[id][0].accepted_regardless_of_guarantee, + "{id}" + ); + } +} diff --git a/tools/dag-viewer/examples/planner-layering-example1.json b/tools/dag-viewer/examples/planner-layering-example1.json index 5f67a4b3..709134cf 100644 --- a/tools/dag-viewer/examples/planner-layering-example1.json +++ b/tools/dag-viewer/examples/planner-layering-example1.json @@ -103554,14 +103554,14 @@ "P1": [ { "accepted_metric": null, - "accepted_without_guarantee": false, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, - "accepted_without_guarantee": false, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103570,14 +103570,14 @@ "P2": [ { "accepted_metric": null, - "accepted_without_guarantee": false, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, - "accepted_without_guarantee": false, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103586,14 +103586,14 @@ "P3": [ { "accepted_metric": null, - "accepted_without_guarantee": false, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, - "accepted_without_guarantee": false, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103602,14 +103602,14 @@ "P4": [ { "accepted_metric": null, - "accepted_without_guarantee": false, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, - "accepted_without_guarantee": false, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103618,14 +103618,14 @@ "P4-m1": [ { "accepted_metric": null, - "accepted_without_guarantee": false, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, - "accepted_without_guarantee": false, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103634,14 +103634,14 @@ "P5": [ { "accepted_metric": null, - "accepted_without_guarantee": false, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, - "accepted_without_guarantee": false, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103650,14 +103650,14 @@ "P5-m1": [ { "accepted_metric": null, - "accepted_without_guarantee": false, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, - "accepted_without_guarantee": false, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103666,14 +103666,14 @@ "P6": [ { "accepted_metric": null, - "accepted_without_guarantee": false, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, - "accepted_without_guarantee": false, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103682,14 +103682,14 @@ "P6-m1": [ { "accepted_metric": null, - "accepted_without_guarantee": false, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, - "accepted_without_guarantee": false, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103698,14 +103698,14 @@ "P7": [ { "accepted_metric": null, - "accepted_without_guarantee": false, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, - "accepted_without_guarantee": false, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103714,14 +103714,14 @@ "P7-m1": [ { "accepted_metric": null, - "accepted_without_guarantee": false, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, - "accepted_without_guarantee": false, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103730,14 +103730,14 @@ "P8": [ { "accepted_metric": null, - "accepted_without_guarantee": false, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, - "accepted_without_guarantee": false, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103746,14 +103746,14 @@ "P8-m1": [ { "accepted_metric": null, - "accepted_without_guarantee": false, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, - "accepted_without_guarantee": false, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103762,14 +103762,14 @@ "P8-m2": [ { "accepted_metric": null, - "accepted_without_guarantee": false, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, - "accepted_without_guarantee": false, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103778,14 +103778,14 @@ "P8-m3": [ { "accepted_metric": null, - "accepted_without_guarantee": false, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, - "accepted_without_guarantee": false, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103794,14 +103794,14 @@ "P9": [ { "accepted_metric": null, - "accepted_without_guarantee": false, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, - "accepted_without_guarantee": false, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103810,14 +103810,14 @@ "P9-m1": [ { "accepted_metric": null, - "accepted_without_guarantee": false, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, - "accepted_without_guarantee": false, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103826,14 +103826,14 @@ "P9-m2": [ { "accepted_metric": null, - "accepted_without_guarantee": false, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, - "accepted_without_guarantee": false, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103842,14 +103842,14 @@ "P9-m3": [ { "accepted_metric": null, - "accepted_without_guarantee": false, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, - "accepted_without_guarantee": false, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -105778,192 +105778,192 @@ { "cause": "no_accuracy_guarantee", "id": "P10", - "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { "cause": "no_accuracy_guarantee", "id": "P11", - "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { "cause": "no_accuracy_guarantee", "id": "P12", - "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { "cause": "no_accuracy_guarantee", "id": "P13", - "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { "cause": "no_accuracy_guarantee", "id": "P13-m1", - "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { "cause": "no_accuracy_guarantee", "id": "P14", - "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { "cause": "no_accuracy_guarantee", "id": "P14-m1", - "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { "cause": "no_accuracy_guarantee", "id": "P15", - "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { "cause": "no_accuracy_guarantee", "id": "P15-m1", - "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { "cause": "no_accuracy_guarantee", "id": "P16", - "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { "cause": "no_accuracy_guarantee", "id": "P16-m1", - "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { "cause": "no_accuracy_guarantee", "id": "P17", - "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { "cause": "no_accuracy_guarantee", "id": "P17-m1", - "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { "cause": "no_accuracy_guarantee", "id": "P17-m2", - "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { "cause": "no_accuracy_guarantee", "id": "P17-m3", - "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { "cause": "no_accuracy_guarantee", "id": "P18", - "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { "cause": "no_accuracy_guarantee", "id": "P18-m1", - "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { "cause": "no_accuracy_guarantee", "id": "P18-m2", - "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { "cause": "no_accuracy_guarantee", "id": "P18-m3", - "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { "cause": "no_accuracy_guarantee", "id": "P19", - "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { "cause": "no_accuracy_guarantee", "id": "P20", - "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { "cause": "no_accuracy_guarantee", "id": "P21", - "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { "cause": "no_accuracy_guarantee", "id": "P22", - "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { "cause": "no_accuracy_guarantee", "id": "P22-m1", - "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { "cause": "no_accuracy_guarantee", "id": "P23", - "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { "cause": "no_accuracy_guarantee", "id": "P23-m1", - "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { "cause": "no_accuracy_guarantee", "id": "P24", - "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { "cause": "no_accuracy_guarantee", "id": "P24-m1", - "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { "cause": "no_accuracy_guarantee", "id": "P25", - "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { "cause": "no_accuracy_guarantee", "id": "P25-m1", - "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { "cause": "no_accuracy_guarantee", "id": "P26", - "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { "cause": "no_accuracy_guarantee", "id": "P26-m1", - "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { "cause": "no_accuracy_guarantee", "id": "P26-m2", - "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { "cause": "no_accuracy_guarantee", "id": "P26-m3", - "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { "cause": "no_accuracy_guarantee", "id": "P27", - "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { "cause": "no_accuracy_guarantee", "id": "P27-m1", - "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { "cause": "no_accuracy_guarantee", "id": "P27-m2", - "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { "cause": "no_accuracy_guarantee", "id": "P27-m3", - "reason": "q2: the output has no guarantee, the deployment has no evidence for it, and its accuracy model does not accept it for target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult } without one" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { "cause": "tied_lost_on_enumeration_order", From 775b04b43683e51740a6e1a182e2ba9f6a6f4baa Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sun, 18 Oct 2026 16:39:54 +0000 Subject: [PATCH 5/6] fix(plan-selection)!: reject unproven Count-Min weights before the accuracy model's hook The executor cannot run Count-Min (CmsWithHeap, Hydra Count-Min) over update weights not proven non-negative, and Stage 3's weight check is the only place that rejects such a plan. accepts_output_regardless_of_guarantee skipped it. The check now runs before the hook and rejects with the new failed-check cause RejectionCause::CountMinWeightsNotProvenNonNegative (count_min_weights_not_proven_non_negative); the hook still overrides every guarantee. A test also pins that the hook overrides guarantees that meet the target. BREAKING CHANGE: a plan rejected for unproven Count-Min weights now has cause count_min_weights_not_proven_non_negative instead of accuracy_target_missed. --- crates/devtools/src/bin/stage_pipeline.rs | 3 +- crates/plan-selection/src/accuracy.rs | 8 +- crates/plan-selection/src/lib.rs | 90 ++++++++++++++----- crates/plan-selection/src/pass/mod.rs | 4 +- .../tests/deployment_accuracy_model.rs | 12 +++ tools/dag-viewer/stages.js | 3 +- 6 files changed, 93 insertions(+), 27 deletions(-) diff --git a/crates/devtools/src/bin/stage_pipeline.rs b/crates/devtools/src/bin/stage_pipeline.rs index 2e20af98..20b20f88 100644 --- a/crates/devtools/src/bin/stage_pipeline.rs +++ b/crates/devtools/src/bin/stage_pipeline.rs @@ -37,7 +37,8 @@ // the sketches read and a one-line // summary), and every other shown candidate under `rejected` with // its `cause`: a failed check (`failure_probability_resize_failed`, -// `accuracy_target_missed`, `no_accuracy_guarantee`, `raw_data_not_retained`, +// `accuracy_target_missed`, `count_min_weights_not_proven_non_negative`, +// `no_accuracy_guarantee`, `raw_data_not_retained`, // `latency_bound_exceeded`, `over_cluster_capacity`) or a loss under // the cost model (`costlier`, `slower`, `more_memory`, // `not_lowest_latency_on_pareto_front`, `dominated`, diff --git a/crates/plan-selection/src/accuracy.rs b/crates/plan-selection/src/accuracy.rs index 40c7c4a5..9255a477 100644 --- a/crates/plan-selection/src/accuracy.rs +++ b/crates/plan-selection/src/accuracy.rs @@ -53,8 +53,12 @@ pub trait AccuracyModel { /// Whether the deployment accepts `output`, one query's output in a /// candidate, for `target` directly, whatever its guarantee says: /// missing (Stage 1 had no analytical rule), met or missed. Stage 3 asks - /// this first for every query output; when it returns `true`, no - /// guarantee or evidence check runs for that output. The default accepts + /// this for every query output before any guarantee check; when it + /// returns `true`, no guarantee or evidence check runs for that output. + /// It does not override the executability check before it: a plan that + /// reads Count-Min over update weights not proven non-negative is + /// rejected (`RejectionCause::CountMinWeightsNotProvenNonNegative`) + /// whatever this returns. The default accepts /// none, so Stage 3 checks the Stage 1 guarantee, then the evidence /// ([`AccuracyEvidenceProvider::output_guarantee`]) with /// [`Self::satisfies`]. diff --git a/crates/plan-selection/src/lib.rs b/crates/plan-selection/src/lib.rs index 5936e78a..48b62f52 100644 --- a/crates/plan-selection/src/lib.rs +++ b/crates/plan-selection/src/lib.rs @@ -9,8 +9,10 @@ //! candidate's output does (`AccuracyEvidenceProvider::output_guarantee`, //! read only here). Before either, the deployment's accuracy model may accept //! the output directly, whatever its guarantee says -//! (`AccuracyModel::accepts_output_regardless_of_guarantee`). For a query with a target, Count-Min is admitted only -//! over weights proven non-negative; a query without one constrains nothing. +//! (`AccuracyModel::accepts_output_regardless_of_guarantee`). Before that, for +//! a query with a target, Count-Min is admitted only over weights proven +//! non-negative, which the executor needs to run it; a query without one +//! constrains nothing. //! A miss rejects the candidate as invalid, with a reason; only this stage //! discards valid candidates. //! A candidate whose evaluation of some query takes longer than that @@ -574,7 +576,7 @@ pub struct Rejection { pub reason: String, } -/// Why Stage 3 did not select a candidate. The first six mean the +/// Why Stage 3 did not select a candidate. The first seven mean the /// candidate failed a check and is invalid ([`Self::failed_a_check`]); the /// rest mean it was valid and lost under the cost model. JSON: the /// snake_case variant name, e.g. `"costlier"`. @@ -585,9 +587,13 @@ pub enum RejectionCause { /// not rebuild into a plan. FailureProbabilityResizeFailed, /// A query's accuracy target is not met: its output's guarantee misses - /// it (or does not hold, as for a Count-Min sketch whose update weights - /// are not proven non-negative) and no deployment evidence meets it. + /// it and no deployment evidence meets it. AccuracyTargetMissed, + /// A query's plan reads a Count-Min sketch whose update weights are not + /// proven non-negative (e.g. samples of a metric not declared a + /// counter). The executor cannot run it, so the deployment's accuracy + /// model cannot accept it either. + CountMinWeightsNotProvenNonNegative, /// A query's output has no accuracy guarantee (Stage 1 has no /// analytical rule for it and the deployment's evidence supplies none), /// and the deployment's accuracy model did not accept it directly @@ -627,6 +633,7 @@ impl RejectionCause { self, Self::FailureProbabilityResizeFailed | Self::AccuracyTargetMissed + | Self::CountMinWeightsNotProvenNonNegative | Self::NoAccuracyGuarantee | Self::RawDataNotRetained | Self::LatencyBoundExceeded @@ -1412,7 +1419,7 @@ pub fn resize_for_failure_probability( /// or ratio of certified estimates can be unbounded. The root guarantee's /// symbolic statistics are bound first ([`bound_root_guarantee`]); one still /// unknown leaves the target unmet. Count-Min is admitted only over update -/// weights proven non-negative, which its bound assumes: an open +/// weights proven non-negative, which its bound and the executor assume: an open /// [`WeightDomain::SamplesOfMetric`] requirement is met when `data` declares /// the metric a counter ([`confirmed_weights`]), and unmet otherwise. /// @@ -1420,10 +1427,12 @@ pub fn resize_for_failure_probability( /// its ε, δ and scope ([`AccuracyModel::satisfies`]); one that sets a metric /// accepts only guarantees in it (or convertible to it). /// -/// The deployment's accuracy model is asked first +/// The Count-Min weight check runs first and rejects as +/// [`RejectionCause::CountMinWeightsNotProvenNonNegative`]: the executor +/// cannot run such a plan. Then the deployment's accuracy model is asked /// ([`AccuracyModel::accepts_output_regardless_of_guarantee`]): an output it -/// accepts is accepted whatever its guarantee says, and none of the checks -/// above runs for it. Otherwise, an output with no guarantee from either +/// accepts is accepted whatever its guarantee says, and no guarantee or +/// evidence check runs for it. Otherwise, an output with no guarantee from either /// source (Stage 1 had no rule, and the evidence has none) is rejected as /// [`RejectionCause::NoAccuracyGuarantee`], and any other miss as /// [`RejectionCause::AccuracyTargetMissed`]. @@ -1437,9 +1446,6 @@ fn accuracy_check( for (query, (root, target)) in candidate.roots.iter().zip(targets).enumerate() { let query = query + 1; let mut sketches: Vec = Vec::new(); - // The first Count-Min whose update weights are not proven - // non-negative, which its bound assumes, as a rejection reason. - let mut unproven_weights: Option = None; for node in root_operators(root) .iter() .flat_map(|node| OperatorNode::reachable(node)) @@ -1469,13 +1475,17 @@ fn accuracy_check( } _ => String::new(), }; - unproven_weights.get_or_insert_with(|| { + // The executor cannot run Count-Min over weights not proven + // non-negative, so this rejects the plan whatever the + // deployment's accuracy model accepts. + return Err(( + RejectionCause::CountMinWeightsNotProvenNonNegative, format!( "q{query}: {} needs non-negative update weights, and these are not \ proven non-negative{unconfirmed}", family_name(family) - ) - }); + ), + )); } } // The deployment's model may accept the output directly, whatever @@ -1492,9 +1502,6 @@ fn accuracy_check( }); continue; } - if let Some(reason) = unproven_weights { - return Err((RejectionCause::AccuracyTargetMissed, reason)); - } // A guarantee per value is stated over the whole result's `G` // values for a whole-result target. let series = series_read(root, data); @@ -3381,7 +3388,10 @@ mod tests { ) .unwrap(); let p2 = selection.rejected.iter().find(|r| r.id == "P2").unwrap(); - assert_eq!(p2.cause, RejectionCause::AccuracyTargetMissed); + assert_eq!( + p2.cause, + RejectionCause::CountMinWeightsNotProvenNonNegative + ); assert!(p2.reason.contains("non-negative"), "{}", p2.reason); } @@ -3416,11 +3426,11 @@ mod tests { let p2 = selection.rejected.iter().find(|r| r.id == "P2").unwrap(); // Admitted weights leave the top-k output without a guarantee, // which the built-in model does not accept; unproven weights - // miss the target outright. + // reject the plan before any accuracy check. let expected_cause = if admitted { RejectionCause::NoAccuracyGuarantee } else { - RejectionCause::AccuracyTargetMissed + RejectionCause::CountMinWeightsNotProvenNonNegative }; assert_eq!(p2.cause, expected_cause); assert_eq!( @@ -4289,6 +4299,44 @@ mod tests { } } + /// The hook overrides guarantees, not the executor's requirement on + /// Count-Min update weights: over samples of a metric declared a gauge, + /// the CmsWithHeap candidate (P2) is rejected as + /// `CountMinWeightsNotProvenNonNegative` under a model that accepts + /// every output, as under the built-in one. + #[test] + fn unproven_count_min_weights_reject_even_an_accepting_model() { + let target = AccuracyTarget::max_error(0.01) + .failure_probability(0.001) + .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1); + let gauge = with_metric_types(data(), &[("m", MetricType::Gauge)]); + for accuracy in [ + &AcceptsEveryOutput as &dyn AccuracyModel, + &BuiltinAccuracyModel, + ] { + let selection = select_plan( + &candidates(), + &[entry_with(target.clone(), repeating(10_000))], + &gauge, + PlanningModels::builtin().with_accuracy(accuracy), + ) + .unwrap(); + let p2 = selection.rejected.iter().find(|r| r.id == "P2").unwrap(); + assert_eq!( + p2.cause, + RejectionCause::CountMinWeightsNotProvenNonNegative, + "{}", + p2.reason + ); + assert!( + p2.reason.contains("not declared a counter"), + "{}", + p2.reason + ); + assert!(p2.cause.failed_a_check()); + } + } + /// The model's hook has priority over every guarantee. The summary /// candidates of a difference of quantile estimates have no theoretical /// guarantee: with evidence that misses the target the built-in model diff --git a/crates/plan-selection/src/pass/mod.rs b/crates/plan-selection/src/pass/mod.rs index 5b1790e4..59d28368 100644 --- a/crates/plan-selection/src/pass/mod.rs +++ b/crates/plan-selection/src/pass/mod.rs @@ -59,8 +59,8 @@ static DEFAULT_DEPLOYMENT: std::sync::LazyLock = #[non_exhaustive] pub struct PlanningModels<'a> { /// Decides whether a candidate's accuracy guarantee meets its query's - /// target, and whether an output with no guarantee is accepted; - /// [`BuiltinAccuracyModel`] by default. + /// target, and whether the deployment accepts an output regardless of + /// its guarantee; [`BuiltinAccuracyModel`] by default. pub accuracy: &'a dyn AccuracyModel, /// Deployment facts the guarantees need (operand domains, observed /// guarantees); [`NoAccuracyEvidence`] by default. diff --git a/crates/planner/tests/deployment_accuracy_model.rs b/crates/planner/tests/deployment_accuracy_model.rs index c357e1fe..04c1df43 100644 --- a/crates/planner/tests/deployment_accuracy_model.rs +++ b/crates/planner/tests/deployment_accuracy_model.rs @@ -307,4 +307,16 @@ async fn the_deployment_accuracy_model_accepts_outputs_whose_guarantee_misses() "{id}" ); } + // The hook also overrides guarantees that meet the target: every + // accepted output, including the exact P1 and the KLL P2, is accepted + // regardless of its guarantee. + for id in ["P1", "P2"] { + assert!(accepting_selection.accepted.contains_key(id), "{id}"); + } + for (id, queries) in &accepting_selection.accepted { + assert!( + queries.iter().all(|q| q.accepted_regardless_of_guarantee), + "{id}" + ); + } } diff --git a/tools/dag-viewer/stages.js b/tools/dag-viewer/stages.js index e6cad9cd..e42f70a5 100644 --- a/tools/dag-viewer/stages.js +++ b/tools/dag-viewer/stages.js @@ -168,7 +168,8 @@ function uniqueIds(list, where, errors) { // of a candidate that failed a check (`RejectionCause::failed_a_check`): // it is not priced. const FAILED_CHECK_CAUSES = [ - 'failure_probability_resize_failed', 'accuracy_target_missed', 'no_accuracy_guarantee', + 'failure_probability_resize_failed', 'accuracy_target_missed', + 'count_min_weights_not_proven_non_negative', 'no_accuracy_guarantee', 'raw_data_not_retained', 'latency_bound_exceeded', 'over_cluster_capacity', ]; // Every `stage3_plan_selection.rejected[].cause` Stage 3 writes: the failed From c0bb1562fe9faba6378dca31dd5de65715ef01a8 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sun, 18 Oct 2026 17:17:58 +0000 Subject: [PATCH 6/6] refactor(plan-selection)!: drop AccuracyModel::propagate; name Hydra Count-Min in the weights rejection Stage 3 composed through the model only for a ratio of relative estimates with enforced operand domains (domain_bound, ExactDivision); every other guarantee comes from Stage 1's built-in rules. The trait now has satisfies and accepts_output_regardless_of_guarantee, and domain_bound calls accuracy::composition::propagate directly. A Hydra (SharedMultiSubpopulation) summary is now named by its Hydra kind, so the unproven-weights reason reads "HydraCms needs non-negative update weights" rather than "Cms". Document which Count-Min forms the weight check covers and why: the executor refuses CmsWithHeap and HydraCms at plan build, and plain Count-Min fails at the first negative weight. BREAKING CHANGE: AccuracyModel::propagate is removed; a deployment model cannot change how guarantees compose. --- crates/plan-selection/src/accuracy.rs | 52 +++++------- crates/plan-selection/src/lib.rs | 84 ++++++++++++++++--- .../tests/deployment_accuracy_model.rs | 3 +- 3 files changed, 94 insertions(+), 45 deletions(-) diff --git a/crates/plan-selection/src/accuracy.rs b/crates/plan-selection/src/accuracy.rs index 9255a477..33a2e77e 100644 --- a/crates/plan-selection/src/accuracy.rs +++ b/crates/plan-selection/src/accuracy.rs @@ -4,42 +4,26 @@ //! Stage 1 composes each candidate's guarantee with the planner's built-in //! analytical rules (`asap_logical_asap_optimization::accuracy`) and keeps a //! candidate it has no rule for with `guarantee = None`. This stage decides, -//! with the model here, whether a guarantee meets its query's target, how -//! guarantees compose once the data workload and the deployment's evidence -//! ([`AccuracyEvidenceProvider`]) bind the statistics Stage 1 left unknown, +//! with the model here, whether a guarantee meets its query's target (after +//! the data workload and the deployment's evidence, +//! [`AccuracyEvidenceProvider`], bind the statistics Stage 1 left unknown), //! and whether the deployment accepts an output directly, whatever its //! guarantee says. -use asap_logical_asap_optimization::accuracy::{ - self as analytical, OperandDomain, PropagationStats, -}; -use asap_types::ir::properties::{ - AccuracyError, AccuracyGuarantee, AccuracyTarget, CompositionOperator, -}; +use asap_logical_asap_optimization::accuracy::{self as analytical, OperandDomain}; +use asap_types::ir::properties::{AccuracyGuarantee, AccuracyTarget}; use asap_types::ir::{OperatorNode, QueryRoot}; -/// The deployment-extensible accuracy algebra plan selection uses. This -/// crate ships [`BuiltinAccuracyModel`]; a deployment with a proof for a -/// composition the default rejects (a registered cross-metric conversion, -/// say) implements this trait and passes it in +/// The deployment's accuracy decisions in plan selection: whether a +/// guarantee meets a query's target, and whether the deployment accepts an +/// output directly, whatever its guarantee. This crate ships +/// [`BuiltinAccuracyModel`]; a deployment with its own target check (a +/// registered cross-metric conversion, say) or its own measurements +/// implements this trait and passes it in /// [`PlanningModels::accuracy`](crate::PlanningModels::accuracy). +/// Guarantees are composed with the built-in rules only +/// (`asap_logical_asap_optimization::accuracy::composition::propagate`). pub trait AccuracyModel { - /// Compose `inputs`' guarantees (in the parent's child order) with the - /// parent's own `local` guarantee under `op`, given the statistics - /// plan selection bound (`stats`). `Err` is the fail-closed answer: no - /// registered rule, or a missing input guarantee. The default is the - /// built-in rules (`asap_logical_asap_optimization::accuracy::composition::propagate`); - /// a deployment overrides it only to add a rule. - fn propagate( - &self, - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - local: Option<&AccuracyGuarantee>, - stats: &PropagationStats, - ) -> Result { - analytical::composition::propagate(op, inputs, local, stats) - } - /// Whether `guarantee` meets `target`: a bound within its ε, and within /// its δ when it states one, over the values the target's scope covers /// (a per-value δ does not meet a whole-result target). When the target @@ -56,9 +40,11 @@ pub trait AccuracyModel { /// this for every query output before any guarantee check; when it /// returns `true`, no guarantee or evidence check runs for that output. /// It does not override the executability check before it: a plan that - /// reads Count-Min over update weights not proven non-negative is - /// rejected (`RejectionCause::CountMinWeightsNotProvenNonNegative`) - /// whatever this returns. The default accepts + /// reads Count-Min (plain, CmsWithHeap or Hydra Count-Min) over update + /// weights not proven non-negative is rejected + /// (`RejectionCause::CountMinWeightsNotProvenNonNegative`) whatever this + /// returns, because the executor refuses CmsWithHeap and HydraCms when it + /// builds the plan and plain Count-Min fails at the first negative weight. The default accepts /// none, so Stage 3 checks the Stage 1 guarantee, then the evidence /// ([`AccuracyEvidenceProvider::output_guarantee`]) with /// [`Self::satisfies`]. @@ -72,7 +58,7 @@ pub trait AccuracyModel { } /// The planner's built-in analytical rules as an [`AccuracyModel`]: the -/// trait's default composition, the conservative target check Stage 1's +/// conservative target check Stage 1's /// `accuracy::satisfies` makes, and no output accepted regardless of its /// guarantee. /// The default in [`PlanningModels::builtin`](crate::PlanningModels::builtin). diff --git a/crates/plan-selection/src/lib.rs b/crates/plan-selection/src/lib.rs index 48b62f52..19d98bd4 100644 --- a/crates/plan-selection/src/lib.rs +++ b/crates/plan-selection/src/lib.rs @@ -1475,9 +1475,12 @@ fn accuracy_check( } _ => String::new(), }; - // The executor cannot run Count-Min over weights not proven - // non-negative, so this rejects the plan whatever the - // deployment's accuracy model accepts. + // Count-Min (plain, CmsWithHeap or Hydra Count-Min) over + // weights not proven non-negative does not run: the executor + // refuses CmsWithHeap and HydraCms when it builds the plan, + // and plain Count-Min fails at the first negative weight. So + // this rejects the plan whatever the deployment's accuracy + // model accepts. return Err(( RejectionCause::CountMinWeightsNotProvenNonNegative, format!( @@ -1895,13 +1898,16 @@ fn domain_bound( division_operand_domains: Some([dx, dy]), ..PropagationStats::default() }; - return models - .accuracy - .propagate(&CompositionOperator::ExactDivision, &[gx, gy], None, &stats) - .ok()? - .spec - .error - .evaluate(); + return asap_logical_asap_optimization::accuracy::composition::propagate( + &CompositionOperator::ExactDivision, + &[gx, gy], + None, + &stats, + ) + .ok()? + .spec + .error + .evaluate(); } let valid = |d: &OperandDomain| d.lower.is_finite() && d.upper.is_finite() && d.lower <= d.upper; @@ -1940,6 +1946,10 @@ fn domain_bound( fn family_name(family: &FieldDataType) -> String { match family { + // A shared (Hydra) structure is named by its Hydra kind, e.g. `HydraCms`. + FieldDataType::Sketch(_, GroupingStrategy::SharedMultiSubpopulation { kind, .. }) => { + format!("{kind:?}") + } FieldDataType::Sketch(kind, _) => format!("{:?}", kind.algorithm()), FieldDataType::ExactAggregate(kind, _) => format!("exact {kind:?} accumulator"), other => format!("{other:?}"), @@ -4337,6 +4347,60 @@ mod tests { } } + /// `node` with every CmsWithHeap summary build replaced by a Hydra + /// Count-Min over the same update. + fn with_hydra_count_min(node: &Rc) -> Rc { + use asap_types::ir::schema::{HydraKind, HydraParams, SketchParams}; + let mut rebuilt = node.with_new_children(with_hydra_count_min).unwrap(); + if let Operator::ASAP(ASAPOp::SummaryAgg { family, .. }) = &mut rebuilt.operator { + if let FieldDataType::Sketch(kind, _) = family { + if let SketchParams::CmsWithHeap { width, depth, .. } = kind.params() { + let (width, depth) = (*width, *depth); + *family = FieldDataType::Sketch( + SketchKind::new(SketchAlgorithm::Cms, SketchParams::Cms { width, depth }), + GroupingStrategy::SharedMultiSubpopulation { + kind: HydraKind::HydraCms, + params: HydraParams::HydraCms { + width, + depth, + shared_rows: depth, + shared_columns: width, + }, + }, + ); + } + } + } + Rc::new(rebuilt) + } + + /// A Hydra Count-Min over unproven weights is rejected by its own name: + /// the reason says `HydraCms`, not `Cms`. + #[test] + fn unproven_hydra_count_min_weights_name_hydra_cms() { + let target = AccuracyTarget::max_error(0.01) + .failure_probability(0.001) + .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1); + let gauge = with_metric_types(data(), &[("m", MetricType::Gauge)]); + let mut p2 = candidates().into_iter().find(|c| c.id == "P2").unwrap(); + p2.roots = p2 + .roots + .iter() + .map(|root| match root { + QueryRoot::Operator(node) => QueryRoot::Operator(with_hydra_count_min(node)), + other => other.clone(), + }) + .collect(); + let (cause, reason) = + accuracy_check(&p2, &[target], &gauge, &PlanningModels::builtin()).unwrap_err(); + assert_eq!(cause, RejectionCause::CountMinWeightsNotProvenNonNegative); + assert!( + reason.contains("HydraCms needs non-negative update weights"), + "{reason}" + ); + assert!(reason.contains("not declared a counter"), "{reason}"); + } + /// The model's hook has priority over every guarantee. The summary /// candidates of a difference of quantile estimates have no theoretical /// guarantee: with evidence that misses the target the built-in model diff --git a/crates/planner/tests/deployment_accuracy_model.rs b/crates/planner/tests/deployment_accuracy_model.rs index 04c1df43..0a432945 100644 --- a/crates/planner/tests/deployment_accuracy_model.rs +++ b/crates/planner/tests/deployment_accuracy_model.rs @@ -20,8 +20,7 @@ use asap_types::workload::{ }; /// A deployment that trusts no approximate estimate: only an exact -/// guarantee meets a target. Composition is the trait's default (the -/// built-in rules). +/// guarantee meets a target. struct ExactOnlyAccuracyModel; impl AccuracyModel for ExactOnlyAccuracyModel {