diff --git a/crates/devtools/src/bin/stage_pipeline.rs b/crates/devtools/src/bin/stage_pipeline.rs index e88b8157..20b20f88 100644 --- a/crates/devtools/src/bin/stage_pipeline.rs +++ b/crates/devtools/src/bin/stage_pipeline.rs @@ -31,10 +31,14 @@ // - stage3_plan_selection: the selected candidate, chosen among all candidates; // the shown candidates' costs and accepted accuracy (per query: the // target's metric or null when not set, the accepted guarantee's -// metric or null when exact, the sketches read and a one-line +// metric or null when exact or accepted regardless of guarantee, +// whether the deployment's accuracy model accepted it directly, +// whatever its guarantee says (`accepted_regardless_of_guarantee`), +// the sketches read and a one-line // summary), and every other shown candidate under `rejected` with // its `cause`: a failed check (`failure_probability_resize_failed`, -// `accuracy_target_missed`, `raw_data_not_retained`, +// `accuracy_target_missed`, `count_min_weights_not_proven_non_negative`, +// `no_accuracy_guarantee`, `raw_data_not_retained`, // `latency_bound_exceeded`, `over_cluster_capacity`) or a loss under // the cost model (`costlier`, `slower`, `more_memory`, // `not_lowest_latency_on_pareto_front`, `dominated`, @@ -617,6 +621,7 @@ fn plan_selection_json( json!({ "target_metric": accepted.target_metric, "accepted_metric": accepted.accepted_metric, + "accepted_regardless_of_guarantee": accepted.accepted_regardless_of_guarantee, "sketches": accepted.sketches, "summary": accepted.to_string(), }) diff --git a/crates/frontend-promql/tests/univmon_candidates.rs b/crates/frontend-promql/tests/univmon_candidates.rs index a84cc3ad..56a22bdd 100644 --- a/crates/frontend-promql/tests/univmon_candidates.rs +++ b/crates/frontend-promql/tests/univmon_candidates.rs @@ -1,6 +1,7 @@ use std::rc::Rc; -use asap_logical_asap_optimization::accuracy::{AccuracyModel, DefaultAccuracyModel}; +use asap_logical_asap_optimization::accuracy::estimators::local_guarantee; +use asap_logical_asap_optimization::accuracy::satisfies; mod support; use asap_types::ir::cse::share_common_sub_dags; use asap_types::ir::properties::AccuracyMetric; @@ -88,9 +89,7 @@ fn four_evaluations_size_their_value_frequency_states_and_keep_honest_guarantees Some(true) ); assert_eq!(guarantee.spec.failure_probability.evaluate(), None); - assert!(DefaultAccuracyModel - .local_guarantee(family, query) - .is_none()); + assert!(local_guarantee(family, query).is_none()); } 1 => { assert!(matches!( @@ -102,7 +101,7 @@ fn four_evaluations_size_their_value_frequency_states_and_keep_honest_guarantees 2 => { let guarantee = guarantee.expect("an L2 guarantee"); assert_eq!(guarantee.spec.accuracy_metric, AccuracyMetric::Relative); - assert!(DefaultAccuracyModel.satisfies( + assert!(satisfies( guarantee, &AccuracyTarget::max_error(0.02).accuracy_metric(AccuracyMetric::Relative) )); @@ -111,9 +110,7 @@ fn four_evaluations_size_their_value_frequency_states_and_keep_honest_guarantees // Entropy has no certified bound, so composition certifies nothing. _ => { assert!(guarantee.is_none()); - assert!(DefaultAccuracyModel - .local_guarantee(family, query) - .is_none()); + assert!(local_guarantee(family, query).is_none()); } } post_asap_dag(root); diff --git a/crates/integration-tests/tests/promql_to_post_asap.rs b/crates/integration-tests/tests/promql_to_post_asap.rs index 3830f8a1..0a8f74c3 100644 --- a/crates/integration-tests/tests/promql_to_post_asap.rs +++ b/crates/integration-tests/tests/promql_to_post_asap.rs @@ -11,8 +11,9 @@ use std::rc::Rc; use asap_integration_tests::fixtures::lower_promql; use asap_integration_tests::planning::logical_candidates; use asap_integration_tests::post_asap::{at_ingestion, at_ingestion_post_asap_dag, post_asap_dag}; -use asap_logical_asap_optimization::accuracy::{AccuracyEvidenceProvider, OperandDomain}; +use asap_logical_asap_optimization::accuracy::OperandDomain; use asap_physical_asap_optimization::PhysicalCandidate; +use asap_plan_selection::AccuracyEvidenceProvider; use asap_plan_selection::PlanningModels; use asap_plan_selection::Selection; use asap_types::ir::operator::operator_properties::Reduction; diff --git a/crates/logical-asap-optimization/src/accuracy/composition.rs b/crates/logical-asap-optimization/src/accuracy/composition.rs index dcac1f39..d5787166 100644 --- a/crates/logical-asap-optimization/src/accuracy/composition.rs +++ b/crates/logical-asap-optimization/src/accuracy/composition.rs @@ -78,471 +78,468 @@ pub fn parse_source_norm_statistic(statistic: &str) -> Option<(&'static str, &st }) } -impl DefaultAccuracyModel { - fn additive( - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - local: &AccuracyGuarantee, - rule: &str, - ) -> AccuracyGuarantee { - let mut terms: Vec = inputs.iter().map(|g| g.spec.error.clone()).collect(); - terms.push(local.spec.error.clone()); - let mut deltas: Vec = inputs - .iter() - .map(|g| g.spec.failure_probability.clone()) - .collect(); - deltas.push(local.spec.failure_probability.clone()); - AccuracyGuarantee::new( - local.spec.accuracy_metric, - BoundExpr::Sum { terms }, - ProbabilityExpr::UnionBound { terms: deltas }, - composed_provenance(op, inputs, local, rule), - ) - } +fn additive( + op: &CompositionOperator, + inputs: &[AccuracyGuarantee], + local: &AccuracyGuarantee, + rule: &str, +) -> AccuracyGuarantee { + let mut terms: Vec = inputs.iter().map(|g| g.spec.error.clone()).collect(); + terms.push(local.spec.error.clone()); + let mut deltas: Vec = inputs + .iter() + .map(|g| g.spec.failure_probability.clone()) + .collect(); + deltas.push(local.spec.failure_probability.clone()); + AccuracyGuarantee::new( + local.spec.accuracy_metric, + BoundExpr::Sum { terms }, + ProbabilityExpr::UnionBound { terms: deltas }, + composed_provenance(op, inputs, local, rule), + ) +} - /// `(1 + ε_total) = Π (1 + ε_i)` ⇒ for two factors - /// `ε_in + ε_out + ε_in·ε_out`; written out as the sum of all - /// cross-products so the expression DAG is exact for any input count. - fn multiplicative( - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - local: &AccuracyGuarantee, - ) -> AccuracyGuarantee { - let factors: Vec<&BoundExpr> = inputs - .iter() - .map(|g| &g.spec.error) - .chain(std::iter::once(&local.spec.error)) - .collect(); - // Every non-empty subset's product: Π(1+ε_i) − 1 = Σ_{S≠∅} Π_{i∈S} ε_i. - let mut terms = Vec::new(); - for mask in 1..(1u32 << factors.len()) { - let subset: Vec = factors - .iter() - .enumerate() - .filter(|(i, _)| mask & (1 << i) != 0) - .map(|(_, b)| (*b).clone()) - .collect(); - terms.push(if subset.len() == 1 { - subset.into_iter().next().expect("one element") - } else { - BoundExpr::Product { factors: subset } - }); - } - let mut deltas: Vec = inputs +/// `(1 + ε_total) = Π (1 + ε_i)` ⇒ for two factors +/// `ε_in + ε_out + ε_in·ε_out`; written out as the sum of all +/// cross-products so the expression DAG is exact for any input count. +fn multiplicative( + op: &CompositionOperator, + inputs: &[AccuracyGuarantee], + local: &AccuracyGuarantee, +) -> AccuracyGuarantee { + let factors: Vec<&BoundExpr> = inputs + .iter() + .map(|g| &g.spec.error) + .chain(std::iter::once(&local.spec.error)) + .collect(); + // Every non-empty subset's product: Π(1+ε_i) − 1 = Σ_{S≠∅} Π_{i∈S} ε_i. + let mut terms = Vec::new(); + for mask in 1..(1u32 << factors.len()) { + let subset: Vec = factors .iter() - .map(|g| g.spec.failure_probability.clone()) + .enumerate() + .filter(|(i, _)| mask & (1 << i) != 0) + .map(|(_, b)| (*b).clone()) .collect(); - deltas.push(local.spec.failure_probability.clone()); - AccuracyGuarantee::new( - AccuracyMetric::Relative, - BoundExpr::Sum { terms }, - ProbabilityExpr::UnionBound { terms: deltas }, - composed_provenance(op, inputs, local, "relative_cross_term_union_bound"), - ) - } - - fn lipschitz( - op: &CompositionOperator, - constant: f64, - inputs: &[AccuracyGuarantee], - local: Option<&AccuracyGuarantee>, - ) -> AccuracyGuarantee { - let input = &inputs[0]; - let scaled = BoundExpr::Scaled { - factor: constant, - inner: Box::new(input.spec.error.clone()), - }; - let (bound, delta) = match local { - Some(local) => ( - BoundExpr::Sum { - terms: vec![scaled, local.spec.error.clone()], - }, - ProbabilityExpr::UnionBound { - terms: vec![ - input.spec.failure_probability.clone(), - local.spec.failure_probability.clone(), - ], - }, - ), - None => (scaled, input.spec.failure_probability.clone()), - }; - let exact_local = AccuracyGuarantee::exact("deterministic Lipschitz transformation"); - AccuracyGuarantee::new( - AccuracyMetric::ABSOLUTE, - bound, - delta, - composed_provenance( - op, - inputs, - local.unwrap_or(&exact_local), - "lipschitz_union_bound", - ), - ) + terms.push(if subset.len() == 1 { + subset.into_iter().next().expect("one element") + } else { + BoundExpr::Product { factors: subset } + }); } + let mut deltas: Vec = inputs + .iter() + .map(|g| g.spec.failure_probability.clone()) + .collect(); + deltas.push(local.spec.failure_probability.clone()); + AccuracyGuarantee::new( + AccuracyMetric::Relative, + BoundExpr::Sum { terms }, + ProbabilityExpr::UnionBound { terms: deltas }, + composed_provenance(op, inputs, local, "relative_cross_term_union_bound"), + ) +} - /// Exact `sum` over approximate inputs: `B ≤ Σ B_i`, `δ ≤ Σ δ_i`. The - /// planner composes one *per-value* child guarantee over an unknown - /// number of input rows, so both the bound and the union bound scale by - /// `stats.input_row_count` — an [`BoundExpr::Unknown`] leaf when it is - /// not supplied. Each input's normalized bound is first converted to - /// absolute units via the statistic its metric is normalized by (also - /// unknown unless supplied); a `Rank` input has no such conversion. - fn exact_sum( - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - stats: &PropagationStats, - ) -> Result { - if all_relative(inputs) { - return Self::relative_fold(op, inputs, stats, true, "exact_sum_relative"); - } - let mut terms = Vec::with_capacity(inputs.len()); - let mut deltas = Vec::with_capacity(inputs.len()); - let mut provenance = Vec::new(); - for (i, input) in inputs.iter().enumerate() { - let absolute = - absolute_bound(input).ok_or_else(|| AccuracyError::UnsupportedComposition { - operator: op.clone(), - input_metrics: inputs.iter().map(|g| g.spec.accuracy_metric).collect(), - local_metric: None, - reason: format!( - "input {i} carries a {:?} guarantee, which has no registered \ - conversion to an absolute value error", - input.spec.accuracy_metric - ), - })?; - if let BoundExpr::Product { factors } = &absolute { - for f in factors { - if let BoundExpr::Unknown { statistic } = f { - provenance.push(GuaranteeSource::UnavailableStatistic { - statistic: statistic.clone(), - }); - } - } - } - terms.push(absolute); - deltas.push(input.spec.failure_probability.clone()); - } - let count = row_count(stats, &mut provenance); - let exact_local = AccuracyGuarantee::exact("ExactAggregate(Sum)"); - provenance.extend(composed_provenance( - op, - inputs, - &exact_local, - "exact_sum_union_bound", - )); - Ok(AccuracyGuarantee::new( - AccuracyMetric::ABSOLUTE, - BoundExpr::Product { - factors: vec![count.clone(), BoundExpr::Sum { terms }], +fn lipschitz( + op: &CompositionOperator, + constant: f64, + inputs: &[AccuracyGuarantee], + local: Option<&AccuracyGuarantee>, +) -> AccuracyGuarantee { + let input = &inputs[0]; + let scaled = BoundExpr::Scaled { + factor: constant, + inner: Box::new(input.spec.error.clone()), + }; + let (bound, delta) = match local { + Some(local) => ( + BoundExpr::Sum { + terms: vec![scaled, local.spec.error.clone()], }, - ProbabilityExpr::Scaled { - count, - inner: Box::new(ProbabilityExpr::UnionBound { terms: deltas }), + ProbabilityExpr::UnionBound { + terms: vec![ + input.spec.failure_probability.clone(), + local.spec.failure_probability.clone(), + ], }, - provenance, - )) - } - - /// Exact arithmetic mean over values with absolute-error guarantees. - /// Averaging cannot amplify the largest absolute input error. The event - /// that every row respects its bound is still protected conservatively - /// by a union bound over the input row count. - fn exact_average( - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - stats: &PropagationStats, - ) -> Result { - if all_relative(inputs) { - return Self::relative_fold(op, inputs, stats, true, "exact_average_relative"); - } - if inputs - .iter() - .any(|input| input.spec.accuracy_metric != AccuracyMetric::ABSOLUTE) - { - return Err(AccuracyError::UnsupportedComposition { - operator: op.clone(), - input_metrics: inputs.iter().map(|g| g.spec.accuracy_metric).collect(), - local_metric: None, - reason: "exact average requires absolute-error input guarantees".into(), - }); - } - let mut provenance = Vec::new(); - let count = row_count(stats, &mut provenance); - let exact_local = AccuracyGuarantee::exact("ExactAggregate(Average)"); - provenance.extend(composed_provenance( + ), + None => (scaled, input.spec.failure_probability.clone()), + }; + let exact_local = AccuracyGuarantee::exact("deterministic Lipschitz transformation"); + AccuracyGuarantee::new( + AccuracyMetric::ABSOLUTE, + bound, + delta, + composed_provenance( op, inputs, - &exact_local, - "exact_average_union_bound", - )); - Ok(AccuracyGuarantee::new( - AccuracyMetric::ABSOLUTE, - BoundExpr::Max { - terms: inputs.iter().map(|g| g.spec.error.clone()).collect(), - }, - ProbabilityExpr::Scaled { - count, - inner: Box::new(ProbabilityExpr::UnionBound { - terms: inputs - .iter() - .map(|g| g.spec.failure_probability.clone()) - .collect(), - }), - }, - provenance, - )) - } + local.unwrap_or(&exact_local), + "lipschitz_union_bound", + ), + ) +} - /// Exact `max`/`min` over approximate inputs of one shared metric: the - /// returned value's error is at most the largest input bound (order - /// statistics are monotone under a uniform perturbation), with - /// probability by the union bound over every input row. This bounds the - /// returned *value*; it does not identify the true winning key. - fn exact_extremum( - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - stats: &PropagationStats, - ) -> Result { - if all_relative(inputs) { - return Self::relative_fold(op, inputs, stats, false, "exact_extremum_relative"); - } - let metric = inputs[0].spec.accuracy_metric; - if inputs - .iter() - .any(|g| g.spec.accuracy_metric != AccuracyMetric::ABSOLUTE) - { - return Err(AccuracyError::UnsupportedComposition { - operator: op.clone(), - input_metrics: inputs.iter().map(|g| g.spec.accuracy_metric).collect(), - local_metric: None, - reason: "exact max/min requires absolute-error input guarantees".into(), - }); - } - let mut provenance = Vec::new(); - let count = row_count(stats, &mut provenance); - let exact_local = AccuracyGuarantee::exact("ExactAggregate(Max)"); - provenance.extend(composed_provenance( - op, - inputs, - &exact_local, - "exact_extremum_union_bound", - )); - Ok(AccuracyGuarantee::new( - metric, - BoundExpr::Max { - terms: inputs.iter().map(|g| g.spec.error.clone()).collect(), - }, - ProbabilityExpr::Scaled { - count, - inner: Box::new(ProbabilityExpr::UnionBound { - terms: inputs - .iter() - .map(|g| g.spec.failure_probability.clone()) - .collect(), - }), - }, - provenance, - )) +/// Exact `sum` over approximate inputs: `B ≤ Σ B_i`, `δ ≤ Σ δ_i`. The +/// planner composes one *per-value* child guarantee over an unknown +/// number of input rows, so both the bound and the union bound scale by +/// `stats.input_row_count` — an [`BoundExpr::Unknown`] leaf when it is +/// not supplied. Each input's normalized bound is first converted to +/// absolute units via the statistic its metric is normalized by (also +/// unknown unless supplied); a `Rank` input has no such conversion. +fn exact_sum( + op: &CompositionOperator, + inputs: &[AccuracyGuarantee], + stats: &PropagationStats, +) -> Result { + if all_relative(inputs) { + return relative_fold(op, inputs, stats, true, "exact_sum_relative"); } - - /// An exact sum, mean, max or min of values each within relative `α_i` - /// is within relative `max α_i`. A max or min holds whatever the signs; - /// a sum or mean only over non-negative values (no cancellation), so - /// with `needs_sign` and the sign unknown the bound is the symbolic - /// [`VALUES_NON_NEGATIVE`]. The failure probability is the union bound - /// over every input row. - fn relative_fold( - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - stats: &PropagationStats, - needs_sign: bool, - rule: &str, - ) -> Result { - if needs_sign && stats.values_non_negative == Some(false) { - return Err(AccuracyError::UnsupportedComposition { + let mut terms = Vec::with_capacity(inputs.len()); + let mut deltas = Vec::with_capacity(inputs.len()); + let mut provenance = Vec::new(); + for (i, input) in inputs.iter().enumerate() { + let absolute = + absolute_bound(input).ok_or_else(|| AccuracyError::UnsupportedComposition { operator: op.clone(), input_metrics: inputs.iter().map(|g| g.spec.accuracy_metric).collect(), local_metric: None, - reason: "relative error of a sum over signed values is unbounded".into(), - }); - } - let mut provenance = Vec::new(); - let count = row_count(stats, &mut provenance); - let bound = if needs_sign && stats.values_non_negative.is_none() { - match &stats.sign_statistic { - // The sign of one metric's samples: the bound, gated on it. - Some(statistic) => { + reason: format!( + "input {i} carries a {:?} guarantee, which has no registered \ + conversion to an absolute value error", + input.spec.accuracy_metric + ), + })?; + if let BoundExpr::Product { factors } = &absolute { + for f in factors { + if let BoundExpr::Unknown { statistic } = f { provenance.push(GuaranteeSource::UnavailableStatistic { statistic: statistic.clone(), }); - BoundExpr::Product { - factors: vec![ - BoundExpr::Max { - terms: inputs.iter().map(|g| g.spec.error.clone()).collect(), - }, - BoundExpr::Unknown { - statistic: statistic.clone(), - }, - ], - } - } - None => { - provenance.push(GuaranteeSource::UnavailableStatistic { - statistic: VALUES_NON_NEGATIVE.into(), - }); - BoundExpr::Unknown { - statistic: VALUES_NON_NEGATIVE.into(), - } } } - } else { - BoundExpr::Max { - terms: inputs.iter().map(|g| g.spec.error.clone()).collect(), - } - }; - provenance.extend(composed_provenance( - op, - inputs, - &AccuracyGuarantee::exact("exact aggregation"), - rule, - )); - Ok(AccuracyGuarantee::new( - AccuracyMetric::Relative, - bound, - ProbabilityExpr::Scaled { - count, - inner: Box::new(ProbabilityExpr::UnionBound { - terms: inputs - .iter() - .map(|g| g.spec.failure_probability.clone()) - .collect(), - }), - }, - provenance, - )) + } + terms.push(absolute); + deltas.push(input.spec.failure_probability.clone()); } + let count = row_count(stats, &mut provenance); + let exact_local = AccuracyGuarantee::exact("ExactAggregate(Sum)"); + provenance.extend(composed_provenance( + op, + inputs, + &exact_local, + "exact_sum_union_bound", + )); + Ok(AccuracyGuarantee::new( + AccuracyMetric::ABSOLUTE, + BoundExpr::Product { + factors: vec![count.clone(), BoundExpr::Sum { terms }], + }, + ProbabilityExpr::Scaled { + count, + inner: Box::new(ProbabilityExpr::UnionBound { terms: deltas }), + }, + provenance, + )) +} - /// Exact division of two relative-value estimates. If the numerator is - /// within `a` and the denominator within `b`, their ratio is within - /// `(a + b) / (1 - b)`. DDSketch supplies those deterministic bounds. - fn exact_division( - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - stats: &PropagationStats, - ) -> Result { - let unsupported = |reason: String| AccuracyError::UnsupportedComposition { +/// Exact arithmetic mean over values with absolute-error guarantees. +/// Averaging cannot amplify the largest absolute input error. The event +/// that every row respects its bound is still protected conservatively +/// by a union bound over the input row count. +fn exact_average( + op: &CompositionOperator, + inputs: &[AccuracyGuarantee], + stats: &PropagationStats, +) -> Result { + if all_relative(inputs) { + return relative_fold(op, inputs, stats, true, "exact_average_relative"); + } + if inputs + .iter() + .any(|input| input.spec.accuracy_metric != AccuracyMetric::ABSOLUTE) + { + return Err(AccuracyError::UnsupportedComposition { operator: op.clone(), input_metrics: inputs.iter().map(|g| g.spec.accuracy_metric).collect(), local_metric: None, - reason, - }; - if inputs.len() != 2 - || inputs.iter().any(|input| { - input.spec.accuracy_metric != AccuracyMetric::Relative && !input.is_exact() - }) - { - return Err(unsupported( - "division needs two relative-value or exact guarantees".into(), - )); - } - let Some(numerator) = inputs[0].spec.error.evaluate() else { - return Err(unsupported( - "numerator relative bound is unavailable".into(), - )); - }; - let Some(denominator) = inputs[1].spec.error.evaluate() else { - return Err(unsupported( - "denominator relative bound is unavailable".into(), - )); - }; - if !(numerator.is_finite() - && denominator.is_finite() - && numerator >= 0.0 - && (0.0..1.0).contains(&denominator)) - { - return Err(unsupported( - "division needs finite non-negative bounds and a denominator bound below one" - .into(), - )); - } - let Some(domains) = &stats.division_operand_domains else { - return Err(unsupported( - "division needs finite operand domains and a nonzero denominator proof".into(), - )); - }; - for domain in domains { - if !domain.lower.is_finite() - || !domain.upper.is_finite() - || domain.lower > domain.upper - || domain.contract.trim().is_empty() - { - return Err(unsupported("invalid division operand domain".into())); + reason: "exact average requires absolute-error input guarantees".into(), + }); + } + let mut provenance = Vec::new(); + let count = row_count(stats, &mut provenance); + let exact_local = AccuracyGuarantee::exact("ExactAggregate(Average)"); + provenance.extend(composed_provenance( + op, + inputs, + &exact_local, + "exact_average_union_bound", + )); + Ok(AccuracyGuarantee::new( + AccuracyMetric::ABSOLUTE, + BoundExpr::Max { + terms: inputs.iter().map(|g| g.spec.error.clone()).collect(), + }, + ProbabilityExpr::Scaled { + count, + inner: Box::new(ProbabilityExpr::UnionBound { + terms: inputs + .iter() + .map(|g| g.spec.failure_probability.clone()) + .collect(), + }), + }, + provenance, + )) +} + +/// Exact `max`/`min` over approximate inputs of one shared metric: the +/// returned value's error is at most the largest input bound (order +/// statistics are monotone under a uniform perturbation), with +/// probability by the union bound over every input row. This bounds the +/// returned *value*; it does not identify the true winning key. +fn exact_extremum( + op: &CompositionOperator, + inputs: &[AccuracyGuarantee], + stats: &PropagationStats, +) -> Result { + if all_relative(inputs) { + return relative_fold(op, inputs, stats, false, "exact_extremum_relative"); + } + let metric = inputs[0].spec.accuracy_metric; + if inputs + .iter() + .any(|g| g.spec.accuracy_metric != AccuracyMetric::ABSOLUTE) + { + return Err(AccuracyError::UnsupportedComposition { + operator: op.clone(), + input_metrics: inputs.iter().map(|g| g.spec.accuracy_metric).collect(), + local_metric: None, + reason: "exact max/min requires absolute-error input guarantees".into(), + }); + } + let mut provenance = Vec::new(); + let count = row_count(stats, &mut provenance); + let exact_local = AccuracyGuarantee::exact("ExactAggregate(Max)"); + provenance.extend(composed_provenance( + op, + inputs, + &exact_local, + "exact_extremum_union_bound", + )); + Ok(AccuracyGuarantee::new( + metric, + BoundExpr::Max { + terms: inputs.iter().map(|g| g.spec.error.clone()).collect(), + }, + ProbabilityExpr::Scaled { + count, + inner: Box::new(ProbabilityExpr::UnionBound { + terms: inputs + .iter() + .map(|g| g.spec.failure_probability.clone()) + .collect(), + }), + }, + provenance, + )) +} + +/// An exact sum, mean, max or min of values each within relative `α_i` +/// is within relative `max α_i`. A max or min holds whatever the signs; +/// a sum or mean only over non-negative values (no cancellation), so +/// with `needs_sign` and the sign unknown the bound is the symbolic +/// [`VALUES_NON_NEGATIVE`]. The failure probability is the union bound +/// over every input row. +fn relative_fold( + op: &CompositionOperator, + inputs: &[AccuracyGuarantee], + stats: &PropagationStats, + needs_sign: bool, + rule: &str, +) -> Result { + if needs_sign && stats.values_non_negative == Some(false) { + return Err(AccuracyError::UnsupportedComposition { + operator: op.clone(), + input_metrics: inputs.iter().map(|g| g.spec.accuracy_metric).collect(), + local_metric: None, + reason: "relative error of a sum over signed values is unbounded".into(), + }); + } + let mut provenance = Vec::new(); + let count = row_count(stats, &mut provenance); + let bound = if needs_sign && stats.values_non_negative.is_none() { + match &stats.sign_statistic { + // The sign of one metric's samples: the bound, gated on it. + Some(statistic) => { + provenance.push(GuaranteeSource::UnavailableStatistic { + statistic: statistic.clone(), + }); + BoundExpr::Product { + factors: vec![ + BoundExpr::Max { + terms: inputs.iter().map(|g| g.spec.error.clone()).collect(), + }, + BoundExpr::Unknown { + statistic: statistic.clone(), + }, + ], + } + } + None => { + provenance.push(GuaranteeSource::UnavailableStatistic { + statistic: VALUES_NON_NEGATIVE.into(), + }); + BoundExpr::Unknown { + statistic: VALUES_NON_NEGATIVE.into(), + } } } - if domains[1].lower <= 0.0 && domains[1].upper >= 0.0 { - return Err(unsupported("denominator domain includes zero".into())); + } else { + BoundExpr::Max { + terms: inputs.iter().map(|g| g.spec.error.clone()).collect(), } - // Keep the true and perturbed quotients finite and out of the - // subnormal range, where Float64 division loses relative accuracy. - // A nonzero numerator interval touching zero cannot prove this. - let num = &domains[0]; - if num.lower <= 0.0 && num.upper >= 0.0 && (num.lower != 0.0 || num.upper != 0.0) { - return Err(unsupported( - "numerator domain cannot exclude underflow near zero".into(), - )); + }; + provenance.extend(composed_provenance( + op, + inputs, + &AccuracyGuarantee::exact("exact aggregation"), + rule, + )); + Ok(AccuracyGuarantee::new( + AccuracyMetric::Relative, + bound, + ProbabilityExpr::Scaled { + count, + inner: Box::new(ProbabilityExpr::UnionBound { + terms: inputs + .iter() + .map(|g| g.spec.failure_probability.clone()) + .collect(), + }), + }, + provenance, + )) +} + +/// Exact division of two relative-value estimates. If the numerator is +/// within `a` and the denominator within `b`, their ratio is within +/// `(a + b) / (1 - b)`. DDSketch supplies those deterministic bounds. +fn exact_division( + op: &CompositionOperator, + inputs: &[AccuracyGuarantee], + stats: &PropagationStats, +) -> Result { + let unsupported = |reason: String| AccuracyError::UnsupportedComposition { + operator: op.clone(), + input_metrics: inputs.iter().map(|g| g.spec.accuracy_metric).collect(), + local_metric: None, + reason, + }; + if inputs.len() != 2 + || inputs.iter().any(|input| { + input.spec.accuracy_metric != AccuracyMetric::Relative && !input.is_exact() + }) + { + return Err(unsupported( + "division needs two relative-value or exact guarantees".into(), + )); + } + let Some(numerator) = inputs[0].spec.error.evaluate() else { + return Err(unsupported( + "numerator relative bound is unavailable".into(), + )); + }; + let Some(denominator) = inputs[1].spec.error.evaluate() else { + return Err(unsupported( + "denominator relative bound is unavailable".into(), + )); + }; + if !(numerator.is_finite() + && denominator.is_finite() + && numerator >= 0.0 + && (0.0..1.0).contains(&denominator)) + { + return Err(unsupported( + "division needs finite non-negative bounds and a denominator bound below one".into(), + )); + } + let Some(domains) = &stats.division_operand_domains else { + return Err(unsupported( + "division needs finite operand domains and a nonzero denominator proof".into(), + )); + }; + for domain in domains { + if !domain.lower.is_finite() + || !domain.upper.is_finite() + || domain.lower > domain.upper + || domain.contract.trim().is_empty() + { + return Err(unsupported("invalid division operand domain".into())); } - for n in [num.lower, num.upper] { - for d in [domains[1].lower, domains[1].upper] { - for nf in [1.0 - numerator, 1.0 + numerator] { - for df in [1.0 - denominator, 1.0 + denominator] { - let quotient = (n * nf) / (d * df); - if !(n * nf).is_finite() - || !(d * df).is_finite() - || !quotient.is_finite() - || (n != 0.0 && quotient.abs() < f64::MIN_POSITIVE) - { - return Err(unsupported( - "division may overflow or underflow Float64".into(), - )); - } + } + if domains[1].lower <= 0.0 && domains[1].upper >= 0.0 { + return Err(unsupported("denominator domain includes zero".into())); + } + // Keep the true and perturbed quotients finite and out of the + // subnormal range, where Float64 division loses relative accuracy. + // A nonzero numerator interval touching zero cannot prove this. + let num = &domains[0]; + if num.lower <= 0.0 && num.upper >= 0.0 && (num.lower != 0.0 || num.upper != 0.0) { + return Err(unsupported( + "numerator domain cannot exclude underflow near zero".into(), + )); + } + for n in [num.lower, num.upper] { + for d in [domains[1].lower, domains[1].upper] { + for nf in [1.0 - numerator, 1.0 + numerator] { + for df in [1.0 - denominator, 1.0 + denominator] { + let quotient = (n * nf) / (d * df); + if !(n * nf).is_finite() + || !(d * df).is_finite() + || !quotient.is_finite() + || (n != 0.0 && quotient.abs() < f64::MIN_POSITIVE) + { + return Err(unsupported( + "division may overflow or underflow Float64".into(), + )); } } } } - Ok(AccuracyGuarantee::new( - AccuracyMetric::Relative, - BoundExpr::Constant { - value: (numerator + denominator) / (1.0 - denominator), - }, - ProbabilityExpr::UnionBound { - terms: inputs - .iter() - .map(|input| input.spec.failure_probability.clone()) - .collect(), - }, - inputs + } + Ok(AccuracyGuarantee::new( + AccuracyMetric::Relative, + BoundExpr::Constant { + value: (numerator + denominator) / (1.0 - denominator), + }, + ProbabilityExpr::UnionBound { + terms: inputs .iter() - .enumerate() - .map(|(input_index, guarantee)| GuaranteeSource::ChildGuarantee { - input_index, - guarantee: Box::new(guarantee.clone()), - }) - .chain(domains.iter().enumerate().map(|(input_index, domain)| { - GuaranteeSource::InputValueDomain { - input_index, - lower: domain.lower, - upper: domain.upper, - max_samples: domain.max_samples, - contract: domain.contract.clone(), - } - })) - .chain(std::iter::once(GuaranteeSource::CompositionStep { - operator: op.clone(), - rule: "relative_division".into(), - })) + .map(|input| input.spec.failure_probability.clone()) .collect(), - )) - } + }, + inputs + .iter() + .enumerate() + .map(|(input_index, guarantee)| GuaranteeSource::ChildGuarantee { + input_index, + guarantee: Box::new(guarantee.clone()), + }) + .chain(domains.iter().enumerate().map(|(input_index, domain)| { + GuaranteeSource::InputValueDomain { + input_index, + lower: domain.lower, + upper: domain.upper, + max_samples: domain.max_samples, + contract: domain.contract.clone(), + } + })) + .chain(std::iter::once(GuaranteeSource::CompositionStep { + operator: op.clone(), + rule: "relative_division".into(), + })) + .collect(), + )) } /// Whether `guarantee` is a sketch's distinct-count estimate. @@ -697,7 +694,13 @@ fn composed_provenance( provenance } -pub(super) fn propagate( +/// The built-in composition rule: `inputs`' guarantees (in the parent's +/// child order) composed with the parent's own `local` guarantee under `op`, +/// given the statistics in `stats` (all unknown in Stage 1; plan selection +/// binds some). `Err` is the fail-closed answer: no registered rule, or a +/// missing input guarantee. It means "no rule", not "infeasible": Pass 1 +/// keeps the candidate with `guarantee = None`. +pub fn propagate( op: &CompositionOperator, inputs: &[AccuracyGuarantee], local: Option<&AccuracyGuarantee>, @@ -806,12 +809,7 @@ pub(super) fn propagate( )); } match local.spec.accuracy_metric { - AccuracyMetric::ABSOLUTE => Ok(DefaultAccuracyModel::additive( - op, - inputs, - local, - "additive_union_bound", - )), + AccuracyMetric::ABSOLUTE => Ok(additive(op, inputs, local, "additive_union_bound")), AccuracyMetric::Relative => { if stats.values_non_negative == Some(false) { return Err(unsupported( @@ -840,7 +838,7 @@ pub(super) fn propagate( provenance, )); } - Ok(DefaultAccuracyModel::multiplicative(op, inputs, local)) + Ok(multiplicative(op, inputs, local)) } AccuracyMetric::Rank | AccuracyMetric::ABSOLUTE_STREAM_L1 @@ -867,18 +865,12 @@ pub(super) fn propagate( "Lipschitz rule needs an absolute-error local guarantee".into(), )); } - Ok(DefaultAccuracyModel::lipschitz( - op, *constant, inputs, local, - )) - } - CompositionOperator::ExactSum => DefaultAccuracyModel::exact_sum(op, inputs, stats), - CompositionOperator::ExactAverage => DefaultAccuracyModel::exact_average(op, inputs, stats), - CompositionOperator::ExactExtremum => { - DefaultAccuracyModel::exact_extremum(op, inputs, stats) - } - CompositionOperator::ExactDivision => { - DefaultAccuracyModel::exact_division(op, inputs, stats) + Ok(lipschitz(op, *constant, inputs, local)) } + CompositionOperator::ExactSum => exact_sum(op, inputs, stats), + CompositionOperator::ExactAverage => exact_average(op, inputs, stats), + CompositionOperator::ExactExtremum => exact_extremum(op, inputs, stats), + CompositionOperator::ExactDivision => exact_division(op, inputs, stats), CompositionOperator::CounterRate | CompositionOperator::InstantCounterRate | CompositionOperator::CounterIncrease => Err(unsupported( @@ -1025,29 +1017,22 @@ mod tests { fn checked_division_propagates_value_bounds_and_rejects_rank_bounds() { let op = CompositionOperator::CheckedRelativeDivision; let inputs = [rel(0.01), rel(0.01)]; - let g = DefaultAccuracyModel - .propagate(&op, &inputs, None, &Default::default()) - .unwrap(); + let g = propagate(&op, &inputs, None, &Default::default()).unwrap(); assert!((g.spec.error.evaluate().unwrap() - 0.02 / 0.99).abs() < 1e-14); let mut rank = inputs[0].clone(); rank.spec.accuracy_metric = AccuracyMetric::Rank; - assert!(DefaultAccuracyModel - .propagate(&op, &[rank.clone(), rank], None, &Default::default()) - .is_err()); - assert!(DefaultAccuracyModel - .propagate(&op, &[rel(0.01), rel(1.0)], None, &Default::default()) - .is_err()); + assert!(propagate(&op, &[rank.clone(), rank], None, &Default::default()).is_err()); + assert!(propagate(&op, &[rel(0.01), rel(1.0)], None, &Default::default()).is_err()); } #[test] fn relative_division_requires_operand_domains() { - assert!(DefaultAccuracyModel - .propagate( - &CompositionOperator::ExactDivision, - &[rel(0.01), rel(0.01)], - None, - &PropagationStats::default() - ) - .is_err()); + assert!(propagate( + &CompositionOperator::ExactDivision, + &[rel(0.01), rel(0.01)], + None, + &PropagationStats::default() + ) + .is_err()); } #[test] fn relative_division_preserves_asymmetric_bound_and_domain_provenance() { @@ -1056,14 +1041,13 @@ mod tests { division_operand_domains: Some([domain(-20., -2.), denominator]), ..Default::default() }; - let got = DefaultAccuracyModel - .propagate( - &CompositionOperator::ExactDivision, - &[rel(0.02), rel(0.03)], - None, - &stats, - ) - .unwrap(); + let got = propagate( + &CompositionOperator::ExactDivision, + &[rel(0.02), rel(0.03)], + None, + &stats, + ) + .unwrap(); assert!((got.spec.error.evaluate().unwrap() - 0.05 / 0.97).abs() < 1e-14); assert_eq!(got.spec.failure_probability.evaluate(), Some(0.)); assert_eq!( @@ -1089,55 +1073,51 @@ mod tests { division_operand_domains: Some(domains), ..Default::default() }; - assert!(DefaultAccuracyModel - .propagate( - &CompositionOperator::ExactDivision, - &[rel(0.01), rel(0.01)], - None, - &stats - ) - .is_err()); + assert!(propagate( + &CompositionOperator::ExactDivision, + &[rel(0.01), rel(0.01)], + None, + &stats + ) + .is_err()); } let stats = PropagationStats { division_operand_domains: Some([domain(1., 2.), domain(1., 2.)]), ..Default::default() }; for bound in [1., f64::NAN, f64::INFINITY, -0.1] { - assert!(DefaultAccuracyModel - .propagate( - &CompositionOperator::ExactDivision, - &[rel(0.01), rel(bound)], - None, - &stats - ) - .is_err()); + assert!(propagate( + &CompositionOperator::ExactDivision, + &[rel(0.01), rel(bound)], + None, + &stats + ) + .is_err()); } } #[test] fn exact_child_contributes_zero_error() { let local = abs(0.05, 0.01); - let out = DefaultAccuracyModel - .propagate( - &CompositionOperator::ApproximateAggregate, - &[AccuracyGuarantee::exact("sum")], - Some(&local), - &PropagationStats::default(), - ) - .unwrap(); + let out = propagate( + &CompositionOperator::ApproximateAggregate, + &[AccuracyGuarantee::exact("sum")], + Some(&local), + &PropagationStats::default(), + ) + .unwrap(); assert_eq!(out.spec.error.evaluate(), Some(0.05)); assert_eq!(out.spec.failure_probability.evaluate(), Some(0.01)); assert_eq!(out.spec.accuracy_metric, AccuracyMetric::ABSOLUTE); } #[test] fn additive_bounds_and_delta_union_bound_compose() { - let out = DefaultAccuracyModel - .propagate( - &CompositionOperator::ApproximateAggregate, - &[abs(0.02, 0.01)], - Some(&abs(0.03, 0.02)), - &PropagationStats::default(), - ) - .unwrap(); + let out = propagate( + &CompositionOperator::ApproximateAggregate, + &[abs(0.02, 0.01)], + Some(&abs(0.03, 0.02)), + &PropagationStats::default(), + ) + .unwrap(); assert!((out.spec.error.evaluate().unwrap() - 0.05).abs() < 1e-12); // Union bound, not 1 − (1−0.01)(1−0.02) = 0.0298. assert!((out.spec.failure_probability.evaluate().unwrap() - 0.03).abs() < 1e-12); @@ -1152,30 +1132,28 @@ mod tests { values_non_negative: Some(true), ..Default::default() }; - let out = DefaultAccuracyModel - .propagate( - &CompositionOperator::ApproximateAggregate, - &[rel(0.1)], - Some(&rel(0.2)), - &stats, - ) - .unwrap(); + let out = propagate( + &CompositionOperator::ApproximateAggregate, + &[rel(0.1)], + Some(&rel(0.2)), + &stats, + ) + .unwrap(); // 0.1 + 0.2 + 0.1·0.2 = 0.32, not 0.3. assert!((out.spec.error.evaluate().unwrap() - 0.32).abs() < 1e-12); assert_eq!(out.spec.accuracy_metric, AccuracyMetric::Relative); } #[test] fn relative_error_without_sign_knowledge_remains_symbolic() { - let unknown = DefaultAccuracyModel - .propagate( - &CompositionOperator::ApproximateAggregate, - &[rel(0.1)], - Some(&rel(0.2)), - &PropagationStats::default(), - ) - .unwrap(); + let unknown = propagate( + &CompositionOperator::ApproximateAggregate, + &[rel(0.1)], + Some(&rel(0.2)), + &PropagationStats::default(), + ) + .unwrap(); assert!(unknown.has_unknown()); - let signed = DefaultAccuracyModel.propagate( + let signed = propagate( &CompositionOperator::ApproximateAggregate, &[rel(0.1)], Some(&rel(0.2)), @@ -1189,14 +1167,13 @@ mod tests { #[test] fn incompatible_metrics_are_rejected_not_treated_as_exact() { // HLL cardinality error under a CMS frequency guarantee. - let err = DefaultAccuracyModel - .propagate( - &CompositionOperator::ApproximateAggregate, - &[with_metric(AccuracyMetric::Relative, 0.01)], - Some(&with_metric(AccuracyMetric::ABSOLUTE_STREAM_L1, 0.01)), - &PropagationStats::default(), - ) - .unwrap_err(); + let err = propagate( + &CompositionOperator::ApproximateAggregate, + &[with_metric(AccuracyMetric::Relative, 0.01)], + Some(&with_metric(AccuracyMetric::ABSOLUTE_STREAM_L1, 0.01)), + &PropagationStats::default(), + ) + .unwrap_err(); assert!(matches!( err, AccuracyError::UnsupportedComposition { @@ -1206,51 +1183,47 @@ mod tests { } if input_metrics == vec![AccuracyMetric::Relative] )); // Quantile rank error under value-additive logic. - let err = DefaultAccuracyModel - .propagate( - &CompositionOperator::ApproximateAggregate, - &[with_metric(AccuracyMetric::Rank, 0.01)], - Some(&abs(0.01, 0.0)), - &PropagationStats::default(), - ) - .unwrap_err(); + let err = propagate( + &CompositionOperator::ApproximateAggregate, + &[with_metric(AccuracyMetric::Rank, 0.01)], + Some(&abs(0.01, 0.0)), + &PropagationStats::default(), + ) + .unwrap_err(); assert!(matches!(err, AccuracyError::UnsupportedComposition { .. })); } #[test] fn same_metric_rank_over_rank_has_no_registered_rule() { - let err = DefaultAccuracyModel - .propagate( - &CompositionOperator::ApproximateAggregate, - &[with_metric(AccuracyMetric::Rank, 0.01)], - Some(&with_metric(AccuracyMetric::Rank, 0.01)), - &PropagationStats::default(), - ) - .unwrap_err(); + let err = propagate( + &CompositionOperator::ApproximateAggregate, + &[with_metric(AccuracyMetric::Rank, 0.01)], + Some(&with_metric(AccuracyMetric::Rank, 0.01)), + &PropagationStats::default(), + ) + .unwrap_err(); assert!(matches!(err, AccuracyError::UnsupportedComposition { .. })); } #[test] fn lipschitz_scales_the_input_bound() { - let out = DefaultAccuracyModel - .propagate( - &CompositionOperator::Lipschitz { constant: 3.0 }, - &[abs(0.1, 0.01)], - Some(&abs(0.05, 0.02)), - &PropagationStats::default(), - ) - .unwrap(); + let out = propagate( + &CompositionOperator::Lipschitz { constant: 3.0 }, + &[abs(0.1, 0.01)], + Some(&abs(0.05, 0.02)), + &PropagationStats::default(), + ) + .unwrap(); assert!((out.spec.error.evaluate().unwrap() - 0.35).abs() < 1e-12); assert!((out.spec.failure_probability.evaluate().unwrap() - 0.03).abs() < 1e-12); } #[test] fn exact_sum_over_approximate_sums_bounds_and_keeps_unknown_row_count_unknown() { - let out = DefaultAccuracyModel - .propagate( - &CompositionOperator::ExactSum, - &[abs(0.1, 0.01)], - None, - &PropagationStats::default(), - ) - .unwrap(); + let out = propagate( + &CompositionOperator::ExactSum, + &[abs(0.1, 0.01)], + None, + &PropagationStats::default(), + ) + .unwrap(); assert_eq!(out.spec.accuracy_metric, AccuracyMetric::ABSOLUTE); assert_eq!( out.spec.error.evaluate(), @@ -1261,20 +1234,19 @@ mod tests { s, GuaranteeSource::UnavailableStatistic { statistic } if statistic == "input_row_count" ))); - assert!(!DefaultAccuracyModel.satisfies(&out, &AccuracyTarget::max_error(1.0))); + assert!(!satisfies(&out, &AccuracyTarget::max_error(1.0))); let known = PropagationStats { input_row_count: Some(4), ..Default::default() }; - let out = DefaultAccuracyModel - .propagate( - &CompositionOperator::ExactSum, - &[abs(0.1, 0.01)], - None, - &known, - ) - .unwrap(); + let out = propagate( + &CompositionOperator::ExactSum, + &[abs(0.1, 0.01)], + None, + &known, + ) + .unwrap(); assert!((out.spec.error.evaluate().unwrap() - 0.4).abs() < 1e-12); assert!((out.spec.failure_probability.evaluate().unwrap() - 0.04).abs() < 1e-12); } @@ -1284,14 +1256,13 @@ mod tests { input_row_count: Some(2), ..Default::default() }; - let out = DefaultAccuracyModel - .propagate( - &CompositionOperator::ExactExtremum, - &[abs(0.1, 0.01), abs(0.3, 0.01)], - None, - &known, - ) - .unwrap(); + let out = propagate( + &CompositionOperator::ExactExtremum, + &[abs(0.1, 0.01), abs(0.3, 0.01)], + None, + &known, + ) + .unwrap(); assert!((out.spec.error.evaluate().unwrap() - 0.3).abs() < 1e-12); assert!((out.spec.failure_probability.evaluate().unwrap() - 0.04).abs() < 1e-12); } @@ -1299,17 +1270,16 @@ mod tests { /// `max α_i` of the true extremum, whatever the signs. #[test] fn exact_extremum_keeps_a_relative_value_bound() { - let out = DefaultAccuracyModel - .propagate( - &CompositionOperator::ExactExtremum, - &[rel(0.01), rel(0.02)], - None, - &PropagationStats { - input_row_count: Some(3), - ..Default::default() - }, - ) - .unwrap(); + let out = propagate( + &CompositionOperator::ExactExtremum, + &[rel(0.01), rel(0.02)], + None, + &PropagationStats { + input_row_count: Some(3), + ..Default::default() + }, + ) + .unwrap(); assert_eq!(out.spec.accuracy_metric, AccuracyMetric::Relative); assert_eq!(out.spec.error.evaluate(), Some(0.02)); assert_eq!(out.spec.failure_probability.evaluate(), Some(0.0)); @@ -1324,7 +1294,7 @@ mod tests { CompositionOperator::ExactAverage, ] { let with = |values_non_negative| { - DefaultAccuracyModel.propagate( + propagate( &op, &[rel(0.01)], None, @@ -1362,18 +1332,17 @@ mod tests { CompositionOperator::ExactSum, CompositionOperator::ExactAverage, ] { - let out = DefaultAccuracyModel - .propagate( - &op, - &[rel(0.01)], - None, - &PropagationStats { - sign_statistic: Some(statistic.clone()), - input_row_count: Some(3), - ..Default::default() - }, - ) - .unwrap(); + let out = propagate( + &op, + &[rel(0.01)], + None, + &PropagationStats { + sign_statistic: Some(statistic.clone()), + input_row_count: Some(3), + ..Default::default() + }, + ) + .unwrap(); assert_eq!(out.spec.error.evaluate(), None, "{op:?}"); let proven = out.bind(&|s| (s == statistic).then_some(1.0)); assert_eq!(proven.spec.error.evaluate(), Some(0.01), "{op:?}"); @@ -1384,51 +1353,48 @@ mod tests { #[test] fn exact_average_has_its_own_absolute_error_rule() { - let out = DefaultAccuracyModel - .propagate( - &CompositionOperator::ExactAverage, - &[abs(0.25, 0.01)], - None, - &PropagationStats { - input_row_count: Some(4), - ..PropagationStats::default() - }, - ) - .unwrap(); + let out = propagate( + &CompositionOperator::ExactAverage, + &[abs(0.25, 0.01)], + None, + &PropagationStats { + input_row_count: Some(4), + ..PropagationStats::default() + }, + ) + .unwrap(); assert_eq!(out.spec.accuracy_metric, AccuracyMetric::ABSOLUTE); assert_eq!(out.spec.error.evaluate(), Some(0.25)); assert_eq!(out.spec.failure_probability.evaluate(), Some(0.04)); } #[test] fn topk_selection_requires_a_separated_margin_certificate() { - let unknown = DefaultAccuracyModel - .propagate( - &CompositionOperator::TopKSelection, - &[abs(0.1, 0.01)], - None, - &PropagationStats::default(), - ) - .unwrap(); + let unknown = propagate( + &CompositionOperator::TopKSelection, + &[abs(0.1, 0.01)], + None, + &PropagationStats::default(), + ) + .unwrap(); assert!(unknown.has_unknown()); - let certified = DefaultAccuracyModel - .propagate( - &CompositionOperator::TopKSelection, - &[abs(0.1, 0.01)], - None, - &PropagationStats { - topk_selected_lower_bound: Some(101.0), - topk_excluded_upper_bound: Some(100.0), - topk_interval_failure_probability: Some(0.005), - ..Default::default() - }, - ) - .unwrap(); + let certified = propagate( + &CompositionOperator::TopKSelection, + &[abs(0.1, 0.01)], + None, + &PropagationStats { + topk_selected_lower_bound: Some(101.0), + topk_excluded_upper_bound: Some(100.0), + topk_interval_failure_probability: Some(0.005), + ..Default::default() + }, + ) + .unwrap(); assert_eq!(certified.spec.accuracy_metric, AccuracyMetric::ABSOLUTE); assert_eq!(certified.spec.error.evaluate(), Some(0.1)); assert_eq!(certified.spec.failure_probability.evaluate(), Some(0.005)); - let overlapping = DefaultAccuracyModel.propagate( + let overlapping = propagate( &CompositionOperator::TopKSelection, &[abs(0.1, 0.01)], None, @@ -1441,22 +1407,21 @@ mod tests { ); assert!(overlapping.is_err()); - let partial = DefaultAccuracyModel - .propagate( - &CompositionOperator::TopKSelection, - &[abs(0.1, 0.01)], - None, - &PropagationStats { - topk_selected_lower_bound: Some(101.0), - topk_interval_failure_probability: Some(0.005), - ..Default::default() - }, - ) - .unwrap(); + let partial = propagate( + &CompositionOperator::TopKSelection, + &[abs(0.1, 0.01)], + None, + &PropagationStats { + topk_selected_lower_bound: Some(101.0), + topk_interval_failure_probability: Some(0.005), + ..Default::default() + }, + ) + .unwrap(); assert!(partial.has_unknown()); assert_eq!(partial.spec.failure_probability.evaluate(), Some(0.005)); - let invalid_partial = DefaultAccuracyModel.propagate( + let invalid_partial = propagate( &CompositionOperator::TopKSelection, &[abs(0.1, 0.01)], None, diff --git a/crates/logical-asap-optimization/src/accuracy/estimators/cms.rs b/crates/logical-asap-optimization/src/accuracy/estimators/cms.rs index cbf1b5a6..e7a85f36 100644 --- a/crates/logical-asap-optimization/src/accuracy/estimators/cms.rs +++ b/crates/logical-asap-optimization/src/accuracy/estimators/cms.rs @@ -45,17 +45,16 @@ mod tests { use asap_types::ir::schema::{GroupingStrategy, SketchKind}; let c = asap_types::ir::operator::agg_intent::default_cardinality(); let params = default_size_params(SketchAlgorithm::Cms, &c, 0.01, 0.001); - let g = DefaultAccuracyModel - .local_guarantee( - &FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::Cms, params), - GroupingStrategy::default(), - ), - &SketchStatistic::Cardinality, - ) - .unwrap(); + let g = local_guarantee( + &FieldDataType::Sketch( + SketchKind::new(SketchAlgorithm::Cms, params), + GroupingStrategy::default(), + ), + &SketchStatistic::Cardinality, + ) + .unwrap(); assert_eq!(g.spec.accuracy_metric, AccuracyMetric::ABSOLUTE_STREAM_L1); - assert!(DefaultAccuracyModel.satisfies( + assert!(satisfies( &g, &AccuracyTarget::max_error(0.01) .failure_probability(0.001) @@ -72,15 +71,14 @@ mod tests { depth: 5, heap_size: 10, }; - let topk_frequency = DefaultAccuracyModel - .local_guarantee( - &FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::CmsWithHeap, cms_heap), - GroupingStrategy::default(), - ), - &SketchStatistic::TopK { k: 10 }, - ) - .expect("heap sketch still provides per-key frequency intervals"); + let topk_frequency = local_guarantee( + &FieldDataType::Sketch( + SketchKind::new(SketchAlgorithm::CmsWithHeap, cms_heap), + GroupingStrategy::default(), + ), + &SketchStatistic::TopK { k: 10 }, + ) + .expect("heap sketch still provides per-key frequency intervals"); assert_eq!( topk_frequency.spec.accuracy_metric, AccuracyMetric::ABSOLUTE_STREAM_L1 diff --git a/crates/logical-asap-optimization/src/accuracy/estimators/count_sketch.rs b/crates/logical-asap-optimization/src/accuracy/estimators/count_sketch.rs index e5bf5c44..8e6ed51b 100644 --- a/crates/logical-asap-optimization/src/accuracy/estimators/count_sketch.rs +++ b/crates/logical-asap-optimization/src/accuracy/estimators/count_sketch.rs @@ -75,23 +75,22 @@ mod tests { use asap_types::ir::schema::{GroupingStrategy, SketchKind}; let intent = default_cardinality(); let count_sketch = default_size_params(SketchAlgorithm::CountSketch, &intent, 0.01, 0.01); - let guarantee = DefaultAccuracyModel - .local_guarantee( - &FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::CountSketch, count_sketch), - GroupingStrategy::default(), - ), - &SketchStatistic::PointCount { - key: asap_types::ir::scalar::expr_ir::ColumnRef::SampleValue, - value: None, - }, - ) - .expect("CountSketch has a parameter-derived L2 guarantee"); + let guarantee = local_guarantee( + &FieldDataType::Sketch( + SketchKind::new(SketchAlgorithm::CountSketch, count_sketch), + GroupingStrategy::default(), + ), + &SketchStatistic::PointCount { + key: asap_types::ir::scalar::expr_ir::ColumnRef::SampleValue, + value: None, + }, + ) + .expect("CountSketch has a parameter-derived L2 guarantee"); assert_eq!( guarantee.spec.accuracy_metric, AccuracyMetric::ABSOLUTE_STREAM_L2 ); - assert!(DefaultAccuracyModel.satisfies( + assert!(satisfies( &guarantee, &AccuracyTarget::max_error(0.01) .failure_probability(0.01) diff --git a/crates/logical-asap-optimization/src/accuracy/estimators/hll.rs b/crates/logical-asap-optimization/src/accuracy/estimators/hll.rs index 083e43fc..38bf9c77 100644 --- a/crates/logical-asap-optimization/src/accuracy/estimators/hll.rs +++ b/crates/logical-asap-optimization/src/accuracy/estimators/hll.rs @@ -481,22 +481,21 @@ mod tests { use asap_types::ir::schema::{GroupingStrategy, SketchKind}; let c = default_cardinality(); let params = default_size_params(SketchAlgorithm::Hll, &c, 0.01, 0.01); - let g = DefaultAccuracyModel - .local_guarantee( - &FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::Hll, params), - GroupingStrategy::default(), - ), - &SketchStatistic::Cardinality, - ) - .unwrap(); + let g = local_guarantee( + &FieldDataType::Sketch( + SketchKind::new(SketchAlgorithm::Hll, params), + GroupingStrategy::default(), + ), + &SketchStatistic::Cardinality, + ) + .unwrap(); assert_eq!(g.spec.accuracy_metric, AccuracyMetric::Relative); assert_eq!(g.spec.failure_probability.evaluate(), None); - assert!(DefaultAccuracyModel.satisfies( + assert!(satisfies( &g, &AccuracyTarget::max_error(0.01).accuracy_metric(AccuracyMetric::Relative) )); - assert!(!DefaultAccuracyModel.satisfies( + assert!(!satisfies( &g, &AccuracyTarget::max_error(0.01) .failure_probability(0.01) diff --git a/crates/logical-asap-optimization/src/accuracy/estimators/kll.rs b/crates/logical-asap-optimization/src/accuracy/estimators/kll.rs index b9b19e2b..a39cecc0 100644 --- a/crates/logical-asap-optimization/src/accuracy/estimators/kll.rs +++ b/crates/logical-asap-optimization/src/accuracy/estimators/kll.rs @@ -191,22 +191,21 @@ mod tests { use asap_types::ir::schema::{GroupingStrategy, SketchKind}; let q = default_quantile(0.99); let params = default_size_params(SketchAlgorithm::Kll, &q, 0.01, 0.01); - let g = DefaultAccuracyModel - .local_guarantee( - &FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::Kll, params), - GroupingStrategy::default(), - ), - &SketchStatistic::Quantile { q: 0.99 }, - ) - .unwrap(); + let g = local_guarantee( + &FieldDataType::Sketch( + SketchKind::new(SketchAlgorithm::Kll, params), + GroupingStrategy::default(), + ), + &SketchStatistic::Quantile { q: 0.99 }, + ) + .unwrap(); assert_eq!(g.spec.accuracy_metric, AccuracyMetric::Rank); - assert!(DefaultAccuracyModel.satisfies( + assert!(satisfies( &g, &AccuracyTarget::max_error(0.01).accuracy_metric(AccuracyMetric::Rank) )); assert_eq!(g.spec.failure_probability.evaluate(), Some(0.01)); - assert!(DefaultAccuracyModel.satisfies( + assert!(satisfies( &g, &AccuracyTarget::max_error(0.01) .failure_probability(0.01) @@ -240,7 +239,7 @@ mod tests { &SketchStatistic::Quantile { q: 0.99 }, ) .unwrap(); - assert!(DefaultAccuracyModel.satisfies( + assert!(satisfies( &g, &AccuracyTarget::max_error(0.01) .failure_probability(0.001) diff --git a/crates/logical-asap-optimization/src/accuracy/estimators/mod.rs b/crates/logical-asap-optimization/src/accuracy/estimators/mod.rs index 052180d1..7d88fb7a 100644 --- a/crates/logical-asap-optimization/src/accuracy/estimators/mod.rs +++ b/crates/logical-asap-optimization/src/accuracy/estimators/mod.rs @@ -54,7 +54,15 @@ fn bounded_guarantee( ) } -pub(super) fn local_guarantee( +/// The built-in guarantee of reading `query` out of a summary of family +/// `family` built over an **exact** input, derived from the family's +/// committed parameters by inverting the same sizing formulas +/// [`crate::realization::default_size_params`] uses. Exact for a plain value +/// or an exact aggregate. `None` when the family has no error model +/// (`Sample`, `Wavelet`, `StatModel`, and sketch evaluations without a +/// certified bound): Pass 1 then leaves the guarantee `None` and keeps the +/// candidate for plan selection to decide. +pub fn local_guarantee( family: &FieldDataType, query: &SketchStatistic, ) -> Option { diff --git a/crates/logical-asap-optimization/src/accuracy/estimators/univmon.rs b/crates/logical-asap-optimization/src/accuracy/estimators/univmon.rs index 82592d91..78e4027a 100644 --- a/crates/logical-asap-optimization/src/accuracy/estimators/univmon.rs +++ b/crates/logical-asap-optimization/src/accuracy/estimators/univmon.rs @@ -378,7 +378,7 @@ mod tests { } fn meets(guarantee: &AccuracyGuarantee, epsilon: f64, delta: f64) -> bool { - DefaultAccuracyModel.satisfies( + satisfies( guarantee, &crate::accuracy::native_target( &AccuracyTarget::max_error(epsilon).failure_probability(delta), diff --git a/crates/logical-asap-optimization/src/accuracy/evidence.rs b/crates/logical-asap-optimization/src/accuracy/evidence.rs index fce69d5d..211e8db3 100644 --- a/crates/logical-asap-optimization/src/accuracy/evidence.rs +++ b/crates/logical-asap-optimization/src/accuracy/evidence.rs @@ -1,6 +1,5 @@ -//! Scoped source contracts and evidence required by accuracy rules. +//! Scoped source contracts and statistics the accuracy rules consult. use super::*; -use asap_types::ir::QueryRoot; /// An enforced domain for the true values of one operand of an exact /// arithmetic operation, in every evaluation: they are finite and lie in @@ -59,31 +58,3 @@ pub struct PropagationStats { /// certificate. pub topk_interval_failure_probability: Option, } - -/// Supplies typed planning-time evidence required by propagation rules. -pub trait AccuracyEvidenceProvider { - /// Enforced domain of `operand`'s true values ([`OperandDomain`]), scoped - /// to this complete expression, including its source, filters, grouping - /// and window: a quantile ratio's operands, or the operands of a - /// product or quotient of estimates. `None` means unknown, including - /// emptiness. - fn operand_domain(&self, _operand: &OperatorNode) -> Option { - None - } - - /// The deployment's empirical accuracy for one candidate's whole query - /// output, as a guarantee. Plan selection (#509 Stage 3) accepts a - /// candidate whose output has no theoretical guarantee that meets the - /// target (e.g. a difference of two quantile estimates) only when this - /// shows the output meets it. Stage 1 composition never reads it: root - /// guarantees there come from the built-in theoretical model. `None` - /// means no evidence. - fn output_guarantee(&self, _output: &QueryRoot) -> Option { - None - } -} - -#[derive(Debug, Default, Clone, Copy)] -pub struct NoAccuracyEvidence; - -impl AccuracyEvidenceProvider for NoAccuracyEvidence {} diff --git a/crates/logical-asap-optimization/src/accuracy/mod.rs b/crates/logical-asap-optimization/src/accuracy/mod.rs index 3866a4c0..93b9629d 100644 --- a/crates/logical-asap-optimization/src/accuracy/mod.rs +++ b/crates/logical-asap-optimization/src/accuracy/mod.rs @@ -1,9 +1,10 @@ -//! Planner accuracy interfaces and model dispatch. +//! The planner's built-in analytical accuracy rules: estimators, composition, budget split and target check. //! //! Estimator models derive local guarantees, composition propagates them, //! the budget split sizes each approximate input of a composition for an -//! equal share of its target, and evidence supplies scoped contracts. -//! Unknown evidence may retain a candidate but does not authorize selection. +//! equal share of its target, and `evidence` holds the scoped contracts and +//! statistics the rules consult. The deployment's evidence provider and +//! accuracy model live in plan selection (`asap_plan_selection::accuracy`). //! See `docs/design_docs/concepts/accuracy-models.md` for the design. pub mod budget; @@ -12,7 +13,7 @@ pub mod estimators; pub mod evidence; pub use budget::per_input_target; -pub use evidence::{AccuracyEvidenceProvider, NoAccuracyEvidence, OperandDomain, PropagationStats}; +pub use evidence::{OperandDomain, PropagationStats}; use asap_types::ir::properties::{ AccuracyError, AccuracyGuarantee, AccuracyMetric, BoundExpr, CompositionOperator, @@ -20,101 +21,36 @@ use asap_types::ir::properties::{ }; use asap_types::ir::properties::{AccuracyScope, AccuracyTarget}; use asap_types::ir::schema::{FieldDataType, SketchAlgorithm, SketchParams, SketchStatistic}; -use asap_types::ir::OperatorNode; - -/// The deployment-extensible accuracy algebra. This crate ships -/// [`DefaultAccuracyModel`]; a deployment with a proof for a composition the -/// default rejects (a registered cross-metric conversion, say) implements -/// this trait and passes it to plan selection in -/// `asap_plan_selection::PlanningModels::accuracy`. -pub trait AccuracyModel { - /// The guarantee of reading `query` out of a summary of family `family` - /// built over an **exact** input — derived from the family's committed - /// parameters by inverting the same sizing formulas - /// [`crate::realization::default_size_params`] uses. `None` when this - /// model has no error model for the family (the default has none for - /// `Sample`/`Wavelet`/`StatModel`). - fn local_guarantee( - &self, - family: &FieldDataType, - query: &SketchStatistic, - ) -> Option; - - /// Compose `inputs`' guarantees (in the parent's child order) with the - /// parent's own `local` guarantee under `op`. `Err` is the fail-closed - /// answer: no registered rule, or a missing input guarantee. - fn propagate( - &self, - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - local: Option<&AccuracyGuarantee>, - stats: &PropagationStats, - ) -> Result; - - /// Whether `guarantee` meets `target`: a bound within its ε, and within - /// its δ when it states one, over the values the target's scope covers - /// (a per-value δ does not meet a whole-result target). When the target - /// sets a metric, the bound must be in that metric (or one with a sound - /// direct comparison): a bound in another metric does not satisfy, - /// however small. When it sets none, a bound in any metric does, each - /// compared in its own metric. Unknown required dimensions fail; - /// selection separately excludes missing accuracy evidence. - fn satisfies(&self, guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool; -} - -/// The built-in estimator and composition models, with conservative target checks. -#[derive(Debug, Default, Clone, Copy)] -pub struct DefaultAccuracyModel; /// Small relative tolerance for comparing an evaluated bound against a /// target, so a parameter sized by `⌈·⌉` to *exactly* meet ε is not rejected /// by floating-point noise. const SATISFACTION_TOLERANCE: f64 = 1e-9; -impl DefaultAccuracyModel { - /// Derive the guarantee for the committed estimator parameters and evaluation. - pub fn sketch_guarantee( - algorithm: &SketchAlgorithm, - params: &SketchParams, - query: &SketchStatistic, - ) -> Option { - estimators::sketch_guarantee(algorithm, params, query) - } -} - -impl AccuracyModel for DefaultAccuracyModel { - fn local_guarantee( - &self, - family: &FieldDataType, - query: &SketchStatistic, - ) -> Option { - estimators::local_guarantee(family, query) - } - fn propagate( - &self, - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - local: Option<&AccuracyGuarantee>, - stats: &PropagationStats, - ) -> Result { - composition::propagate(op, inputs, local, stats) - } - fn satisfies(&self, guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool { - let within = |value: Option, limit: f64| { - value.is_some_and(|v| v <= limit * (1.0 + SATISFACTION_TOLERANCE) + f64::EPSILON) - }; - if target.is_exact() || guarantee.is_exact() { - return guarantee.is_exact(); - } - target - .accuracy_metric - .is_none_or(|metric| metric_meets(guarantee.spec.accuracy_metric, metric)) - && scope_meets(guarantee, target) - && within(guarantee.spec.error.evaluate(), target.max_error) - && target - .failure_probability - .is_none_or(|delta| within(guarantee.spec.failure_probability.evaluate(), delta)) +/// The built-in conservative check of whether `guarantee` meets `target`: +/// a bound within its ε, and within its δ when it states one, over the +/// values the target's scope covers (a per-value δ does not meet a +/// whole-result target). When the target sets a metric, the bound must be +/// in that metric (or one with a sound direct comparison): a bound in +/// another metric does not satisfy, however small. When it sets none, a +/// bound in any metric does, each compared in its own metric. Unknown +/// required dimensions fail. Plan selection's default accuracy model +/// (`asap_plan_selection::BuiltinAccuracyModel`) delegates to it. +pub fn satisfies(guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool { + let within = |value: Option, limit: f64| { + value.is_some_and(|v| v <= limit * (1.0 + SATISFACTION_TOLERANCE) + f64::EPSILON) + }; + if target.is_exact() || guarantee.is_exact() { + return guarantee.is_exact(); } + target + .accuracy_metric + .is_none_or(|metric| metric_meets(guarantee.spec.accuracy_metric, metric)) + && scope_meets(guarantee, target) + && within(guarantee.spec.error.evaluate(), target.max_error) + && target + .failure_probability + .is_none_or(|delta| within(guarantee.spec.failure_probability.evaluate(), delta)) } /// Whether an error bound stated in `guarantee` bounds the error `target` @@ -175,10 +111,12 @@ mod tests { unknown.spec.error = BoundExpr::Unknown { statistic: "x".into(), }; - assert!(!DefaultAccuracyModel.satisfies(&unknown, &AccuracyTarget::max_error(1.0))); - assert!(!DefaultAccuracyModel.satisfies(&abs(0.0, 0.01), &AccuracyTarget::exact())); - assert!(DefaultAccuracyModel - .satisfies(&AccuracyGuarantee::exact("x"), &AccuracyTarget::exact())); + assert!(!satisfies(&unknown, &AccuracyTarget::max_error(1.0))); + assert!(!satisfies(&abs(0.0, 0.01), &AccuracyTarget::exact())); + assert!(satisfies( + &AccuracyGuarantee::exact("x"), + &AccuracyTarget::exact() + )); } fn guarantee(metric: AccuracyMetric, bound: f64, delta: f64) -> AccuracyGuarantee { @@ -198,7 +136,6 @@ mod tests { /// meets every metric. #[test] fn satisfies_requires_the_targets_metric() { - let model = DefaultAccuracyModel; let target = |metric| { AccuracyTarget::max_error(0.5) .failure_probability(0.5) @@ -224,12 +161,12 @@ mod tests { (AccuracyMetric::Rank, AccuracyMetric::Rank, true), ] { assert_eq!( - model.satisfies(&guarantee(bound, 0.01, 0.01), &target(wanted)), + satisfies(&guarantee(bound, 0.01, 0.01), &target(wanted)), meets, "{bound:?} against {wanted:?}" ); } - assert!(model.satisfies( + assert!(satisfies( &AccuracyGuarantee::exact("x"), &target(AccuracyMetric::Rank) )); @@ -243,8 +180,8 @@ mod tests { g.spec.failure_probability = ProbabilityExpr::Unknown { statistic: "x".into(), }; - assert!(DefaultAccuracyModel.satisfies(&g, &AccuracyTarget::max_error(0.01))); - assert!(!DefaultAccuracyModel.satisfies( + assert!(satisfies(&g, &AccuracyTarget::max_error(0.01))); + assert!(!satisfies( &g, &AccuracyTarget::max_error(0.01).failure_probability(0.5) )); @@ -257,14 +194,13 @@ mod tests { /// directly. #[test] fn per_value_guarantees_meet_whole_result_targets_over_g_values() { - let model = DefaultAccuracyModel; let g = guarantee(AccuracyMetric::ABSOLUTE, 0.01, 0.001); assert_eq!(g.spec.accuracy_scope, AccuracyScope::PerValue); let whole = AccuracyTarget::max_error(0.01).failure_probability(0.01); assert_eq!(whole.accuracy_scope, AccuracyScope::WholeResult); - assert!(!model.satisfies(&g, &whole)); + assert!(!satisfies(&g, &whole)); let per_value = whole.clone().accuracy_scope(AccuracyScope::PerValue); - assert!(model.satisfies(&g, &per_value)); + assert!(satisfies(&g, &per_value)); let over = |values| g.over_whole_result(values); assert_eq!( over(Some(10)).spec.accuracy_scope, @@ -274,10 +210,10 @@ mod tests { over(Some(10)).spec.failure_probability.evaluate(), Some(0.01) ); - assert!(model.satisfies(&over(Some(10)), &whole)); - assert!(!model.satisfies(&over(Some(11)), &whole)); - assert!(!model.satisfies(&over(None), &whole)); - assert!(model.satisfies(&over(Some(10)), &per_value)); + assert!(satisfies(&over(Some(10)), &whole)); + assert!(!satisfies(&over(Some(11)), &whole)); + assert!(!satisfies(&over(None), &whole)); + assert!(satisfies(&over(Some(10)), &per_value)); } /// The scope bounds only the failure probability: a target without a δ @@ -285,9 +221,8 @@ mod tests { /// target whatever the number of values. #[test] fn scope_matters_only_with_a_failure_probability() { - let model = DefaultAccuracyModel; let g = guarantee(AccuracyMetric::ABSOLUTE, 0.01, 0.001); - assert!(model.satisfies(&g, &AccuracyTarget::max_error(0.01))); + assert!(satisfies(&g, &AccuracyTarget::max_error(0.01))); let never_fails = AccuracyGuarantee::new( AccuracyMetric::Relative, BoundExpr::Constant { value: 0.01 }, @@ -297,8 +232,8 @@ mod tests { let whole = AccuracyTarget::max_error(0.01) .failure_probability(0.01) .accuracy_metric(AccuracyMetric::Relative); - assert!(model.satisfies(&never_fails, &whole)); - assert!(model.satisfies(&never_fails.over_whole_result(None), &whole)); + assert!(satisfies(&never_fails, &whole)); + assert!(satisfies(&never_fails.over_whole_result(None), &whole)); } /// A target whose metric the user did not set accepts a guarantee in @@ -307,7 +242,6 @@ mod tests { /// or failure probability still misses it. #[test] fn a_target_without_a_metric_accepts_any_metric() { - let model = DefaultAccuracyModel; let target = AccuracyTarget::max_error(0.01) .failure_probability(0.01) .accuracy_scope(AccuracyScope::PerValue); @@ -320,15 +254,15 @@ mod tests { AccuracyMetric::Rank, ] { assert!( - model.satisfies(&guarantee(metric, 0.01, 0.01), &target), + satisfies(&guarantee(metric, 0.01, 0.01), &target), "{metric:?}" ); assert!( - !model.satisfies(&guarantee(metric, 0.02, 0.01), &target), + !satisfies(&guarantee(metric, 0.02, 0.01), &target), "{metric:?}" ); assert!( - !model.satisfies(&guarantee(metric, 0.01, 0.02), &target), + !satisfies(&guarantee(metric, 0.01, 0.02), &target), "{metric:?}" ); } diff --git a/crates/logical-asap-optimization/src/lib.rs b/crates/logical-asap-optimization/src/lib.rs index e8cece06..9c5e441e 100644 --- a/crates/logical-asap-optimization/src/lib.rs +++ b/crates/logical-asap-optimization/src/lib.rs @@ -30,16 +30,17 @@ //! - [`realization`] — the summary families that can serve an aggregate //! intent, the intent's accuracy target and `(ε, δ)` budget, and default //! sketch sizing, shared by Pass 1 and Pass 2. -//! - [`accuracy`] — the analytical accuracy model: per-family estimators, -//! composition and budget allocation, and the -//! [`AccuracyModel`](accuracy::AccuracyModel) extension point (issue #172) -//! that derives a machine-readable +//! - [`accuracy`] — the built-in analytical accuracy rules: per-family +//! estimators, composition and budget allocation. They derive a +//! machine-readable //! [`AccuracyGuarantee`](asap_types::ir::properties::AccuracyGuarantee) -//! for every finalized value and propagates it through +//! for every finalized value and propagate it through //! approximate-over-approximate compositions under conservative rules -//! (no independence assumptions, unknown statistics stay unknown). Plan -//! selection checks each candidate's guarantee against its query's -//! `AccuracyTarget` before pricing it. +//! (no independence assumptions, unknown statistics stay unknown). A +//! composition with no rule leaves the guarantee `None` and keeps the +//! candidate. Stage 1 has no pluggable accuracy model: plan selection +//! (Stage 3) checks each candidate's guarantee against its query's +//! `AccuracyTarget` with the deployment's `AccuracyModel` before pricing it. //! - [`deployment_capabilities`] — the deployment's supported summaries. //! - [`maintained_population`] — builds a maintained-population sub-DAG for //! an instant top-k/quantile/sum query; Pass 1 does not offer it yet (#681). @@ -53,10 +54,6 @@ pub mod realization; #[cfg(any(test, feature = "test-support"))] pub mod test_support; -pub use accuracy::{ - AccuracyEvidenceProvider, AccuracyModel, DefaultAccuracyModel, NoAccuracyEvidence, - PropagationStats, -}; pub use deployment_capabilities::{ DeploymentCapabilities, DeploymentCapabilitiesError, SupportedSummary, }; diff --git a/crates/logical-asap-optimization/src/per_computation_candidates.rs b/crates/logical-asap-optimization/src/per_computation_candidates.rs index cc1d0236..75f09cb6 100644 --- a/crates/logical-asap-optimization/src/per_computation_candidates.rs +++ b/crates/logical-asap-optimization/src/per_computation_candidates.rs @@ -113,13 +113,15 @@ use asap_types::ir::{ use asap_types::physical::Materialization; use thiserror::Error; +use crate::accuracy::composition::propagate; use crate::accuracy::composition::samples_sign_statistic; use crate::accuracy::composition::{ additive_as_absolute, StreamNorm, DIVISION_OPERAND_DOMAINS, GROUP_ROW_COUNT, PRODUCT_OPERAND_DOMAINS, QUOTIENT_OPERAND_DOMAINS, }; +use crate::accuracy::estimators::local_guarantee; use crate::accuracy::estimators::{hll, univmon}; -use crate::accuracy::{per_input_target, AccuracyModel, DefaultAccuracyModel, PropagationStats}; +use crate::accuracy::{per_input_target, PropagationStats}; use crate::deployment_capabilities::DeploymentCapabilities; use crate::function_rules::approximate_input_composition; use crate::realization::{accuracy_budget, default_size_params, sizing_target, summary_candidates}; @@ -1361,7 +1363,7 @@ fn with_guarantees( } /// The theoretical guarantee of a composed node, given its children's, -/// composed with the built-in accuracy model ([`DefaultAccuracyModel`]); +/// composed with the built-in analytical rules ([`propagate`]); /// Stage 1 reads no data statistics or deployment evidence. Exact over exact /// inputs; a summary estimate has its sized family's bound when /// [`estimate_has_bound`], composed with its input's when that is @@ -1428,9 +1430,7 @@ fn composed_guarantee(node: &OperatorNode) -> Option { ExactKind::IRate => CompositionOperator::InstantCounterRate, ExactKind::Increase => CompositionOperator::CounterIncrease, }; - DefaultAccuracyModel - .propagate(&op, &[input], None, &stats) - .ok() + propagate(&op, &[input], None, &stats).ok() } _ => None, }, @@ -1450,7 +1450,7 @@ fn composed_guarantee(node: &OperatorNode) -> Option { let local = match estimate_bound_over_samples(family, update, query) { Some(gated) => gated, None => source_distinct_guarantee(family, query, input) - .or_else(|| DefaultAccuracyModel.local_guarantee(family, query))?, + .or_else(|| local_guarantee(family, query))?, }; if exact_inputs(summary_input) { Some(local) @@ -1498,14 +1498,13 @@ fn composed_guarantee(node: &OperatorNode) -> Option { .. }) if filters.iter().all(Option::is_none) && !groups_by_estimate(reduction, child) => { match measures.as_slice() { - [intent] => DefaultAccuracyModel - .propagate( - &approximate_input_composition(intent)?, - &[operand_guarantee(child)?], - None, - &fold_stats(reduction, child)?, - ) - .ok(), + [intent] => propagate( + &approximate_input_composition(intent)?, + &[operand_guarantee(child)?], + None, + &fold_stats(reduction, child)?, + ) + .ok(), _ => None, } } @@ -1765,9 +1764,7 @@ fn compose( inputs: Vec, local: Option<&AccuracyGuarantee>, ) -> Option { - DefaultAccuracyModel - .propagate(op, &inputs, local, &PropagationStats::default()) - .ok() + propagate(op, &inputs, local, &PropagationStats::default()).ok() } /// A projection that computes over its child's sample value: the guarantee @@ -2215,17 +2212,16 @@ fn lipschitz_guarantee(args: Vec) -> Option Sketc let metric = statistic(intent) .ok() .and_then(|query| { - DefaultAccuracyModel.local_guarantee( + local_guarantee( &FieldDataType::Sketch(whole.clone(), GroupingStrategy::default()), &query, ) @@ -4528,9 +4524,7 @@ mod tests { }; let gated = estimate_bound_over_samples(&family, &update, &query).expect("gated"); assert_eq!(gated.spec.error.evaluate(), None); - let local = DefaultAccuracyModel - .local_guarantee(&family, &query) - .unwrap(); + let local = local_guarantee(&family, &query).unwrap(); let statistic = samples_sign_statistic("m"); // Every other statistic (a stream norm) bound alike. let others = |s: &str| Some(if s == statistic { 1.0 } else { 1_000.0 }); diff --git a/crates/plan-selection/src/accuracy.rs b/crates/plan-selection/src/accuracy.rs new file mode 100644 index 00000000..33a2e77e --- /dev/null +++ b/crates/plan-selection/src/accuracy.rs @@ -0,0 +1,105 @@ +//! The deployment's accuracy model, read only by plan selection (#509 +//! Stage 3). +//! +//! Stage 1 composes each candidate's guarantee with the planner's built-in +//! analytical rules (`asap_logical_asap_optimization::accuracy`) and keeps a +//! candidate it has no rule for with `guarantee = None`. This stage decides, +//! with the model here, whether a guarantee meets its query's target (after +//! the data workload and the deployment's evidence, +//! [`AccuracyEvidenceProvider`], bind the statistics Stage 1 left unknown), +//! and whether the deployment accepts an output directly, whatever its +//! guarantee says. + +use asap_logical_asap_optimization::accuracy::{self as analytical, OperandDomain}; +use asap_types::ir::properties::{AccuracyGuarantee, AccuracyTarget}; +use asap_types::ir::{OperatorNode, QueryRoot}; + +/// The deployment's accuracy decisions in plan selection: whether a +/// guarantee meets a query's target, and whether the deployment accepts an +/// output directly, whatever its guarantee. This crate ships +/// [`BuiltinAccuracyModel`]; a deployment with its own target check (a +/// registered cross-metric conversion, say) or its own measurements +/// implements this trait and passes it in +/// [`PlanningModels::accuracy`](crate::PlanningModels::accuracy). +/// Guarantees are composed with the built-in rules only +/// (`asap_logical_asap_optimization::accuracy::composition::propagate`). +pub trait AccuracyModel { + /// Whether `guarantee` meets `target`: a bound within its ε, and within + /// its δ when it states one, over the values the target's scope covers + /// (a per-value δ does not meet a whole-result target). When the target + /// sets a metric, the bound must be in that metric (or one with a sound + /// direct comparison): a bound in another metric does not satisfy, + /// however small. When it sets none, a bound in any metric does, each + /// compared in its own metric. Unknown required dimensions fail; + /// selection separately excludes missing accuracy evidence. + fn satisfies(&self, guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool; + + /// Whether the deployment accepts `output`, one query's output in a + /// candidate, for `target` directly, whatever its guarantee says: + /// missing (Stage 1 had no analytical rule), met or missed. Stage 3 asks + /// this for every query output before any guarantee check; when it + /// returns `true`, no guarantee or evidence check runs for that output. + /// It does not override the executability check before it: a plan that + /// reads Count-Min (plain, CmsWithHeap or Hydra Count-Min) over update + /// weights not proven non-negative is rejected + /// (`RejectionCause::CountMinWeightsNotProvenNonNegative`) whatever this + /// returns, because the executor refuses CmsWithHeap and HydraCms when it + /// builds the plan and plain Count-Min fails at the first negative weight. The default accepts + /// none, so Stage 3 checks the Stage 1 guarantee, then the evidence + /// ([`AccuracyEvidenceProvider::output_guarantee`]) with + /// [`Self::satisfies`]. + fn accepts_output_regardless_of_guarantee( + &self, + _output: &QueryRoot, + _target: &AccuracyTarget, + ) -> bool { + false + } +} + +/// The planner's built-in analytical rules as an [`AccuracyModel`]: the +/// conservative target check Stage 1's +/// `accuracy::satisfies` makes, and no output accepted regardless of its +/// guarantee. +/// The default in [`PlanningModels::builtin`](crate::PlanningModels::builtin). +#[derive(Debug, Default, Clone, Copy)] +pub struct BuiltinAccuracyModel; + +impl AccuracyModel for BuiltinAccuracyModel { + fn satisfies(&self, guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool { + analytical::satisfies(guarantee, target) + } +} + +/// The deployment's planning-time accuracy evidence, read only by plan +/// selection: enforced operand domains the propagation rules need, and +/// empirical guarantees for whole query outputs. Passed in +/// [`PlanningModels::evidence`](crate::PlanningModels::evidence). The trait +/// moved here from `asap_logical_asap_optimization::accuracy`. +pub trait AccuracyEvidenceProvider { + /// Enforced domain of `operand`'s true values ([`OperandDomain`]), scoped + /// to this complete expression, including its source, filters, grouping + /// and window: a quantile ratio's operands, or the operands of a + /// product or quotient of estimates. `None` means unknown, including + /// emptiness. + fn operand_domain(&self, _operand: &OperatorNode) -> Option { + None + } + + /// The deployment's empirical accuracy for one candidate's whole query + /// output, as a guarantee. Plan selection (#509 Stage 3) accepts a + /// candidate whose output has no theoretical guarantee that meets the + /// target (e.g. a difference of two quantile estimates) when this shows + /// the output meets it. Stage 1 never reads it: root guarantees there + /// come from the built-in analytical rules. `None` means no evidence. + fn output_guarantee(&self, _output: &QueryRoot) -> Option { + None + } +} + +/// No deployment evidence: every operand domain and output guarantee is +/// unknown. The default in [`PlanningModels::builtin`](crate::PlanningModels::builtin). +#[derive(Debug, Default, Clone, Copy)] +pub struct NoAccuracyEvidence; + +impl AccuracyEvidenceProvider for NoAccuracyEvidence {} diff --git a/crates/plan-selection/src/lib.rs b/crates/plan-selection/src/lib.rs index c9d4051e..19d98bd4 100644 --- a/crates/plan-selection/src/lib.rs +++ b/crates/plan-selection/src/lib.rs @@ -3,12 +3,16 @@ //! Each CandidatePhysicalASAPDAG from Stage 2 is checked against every //! query's accuracy target: the query's output meets it when its root //! guarantee does (theoretical: Stage 1 composes it with the built-in -//! accuracy model, and this stage binds the statistics it left symbolic +//! analytical rules, and this stage binds the statistics it left symbolic //! from the data workload and the deployment's operand-domain contracts), //! or when the deployment's empirical evidence shows this //! candidate's output does (`AccuracyEvidenceProvider::output_guarantee`, -//! read only here). For a query with a target, Count-Min is admitted only -//! over weights proven non-negative; a query without one constrains nothing. +//! read only here). Before either, the deployment's accuracy model may accept +//! the output directly, whatever its guarantee says +//! (`AccuracyModel::accepts_output_regardless_of_guarantee`). Before that, for +//! a query with a target, Count-Min is admitted only over weights proven +//! non-negative, which the executor needs to run it; a query without one +//! constrains nothing. //! A miss rejects the candidate as invalid, with a reason; only this stage //! discards valid candidates. //! A candidate whose evaluation of some query takes longer than that @@ -207,6 +211,10 @@ use asap_logical_asap_optimization::per_computation_candidates::{ }; use asap_physical_asap_optimization::PhysicalCandidate; +pub mod accuracy; +pub use accuracy::{ + AccuracyEvidenceProvider, AccuracyModel, BuiltinAccuracyModel, NoAccuracyEvidence, +}; pub mod cost; /// Costed reuse of compatible physical pane producers. pub mod pane_sharing; @@ -568,7 +576,7 @@ pub struct Rejection { pub reason: String, } -/// Why Stage 3 did not select a candidate. The first five mean the +/// Why Stage 3 did not select a candidate. The first seven mean the /// candidate failed a check and is invalid ([`Self::failed_a_check`]); the /// rest mean it was valid and lost under the cost model. JSON: the /// snake_case variant name, e.g. `"costlier"`. @@ -579,9 +587,18 @@ pub enum RejectionCause { /// not rebuild into a plan. FailureProbabilityResizeFailed, /// A query's accuracy target is not met: its output's guarantee misses - /// it (or does not hold, as for a Count-Min sketch whose update weights - /// are not proven non-negative) and no deployment evidence meets it. + /// it and no deployment evidence meets it. AccuracyTargetMissed, + /// A query's plan reads a Count-Min sketch whose update weights are not + /// proven non-negative (e.g. samples of a metric not declared a + /// counter). The executor cannot run it, so the deployment's accuracy + /// model cannot accept it either. + CountMinWeightsNotProvenNonNegative, + /// A query's output has no accuracy guarantee (Stage 1 has no + /// analytical rule for it and the deployment's evidence supplies none), + /// and the deployment's accuracy model did not accept it directly + /// ([`AccuracyModel::accepts_output_regardless_of_guarantee`]). + NoAccuracyGuarantee, /// It scans raw data at query time in a deployment that does not /// retain raw data. RawDataNotRetained, @@ -616,6 +633,8 @@ impl RejectionCause { self, Self::FailureProbabilityResizeFailed | Self::AccuracyTargetMissed + | Self::CountMinWeightsNotProvenNonNegative + | Self::NoAccuracyGuarantee | Self::RawDataNotRetained | Self::LatencyBoundExceeded | Self::OverClusterCapacity @@ -634,8 +653,14 @@ pub struct AcceptedAccuracy { /// The target's metric; `None` when the user did not set one. pub target_metric: Option, /// The metric the accepted guarantee is stated in; `None` for an exact - /// output, which meets every metric. + /// output, which meets every metric, and for one the deployment's model + /// accepted regardless of its guarantee (no guarantee was checked). pub accepted_metric: Option, + /// Whether the deployment's accuracy model accepted the output directly, + /// whatever its guarantee says + /// ([`AccuracyModel::accepts_output_regardless_of_guarantee`]); `false` + /// when a guarantee met the target. + pub accepted_regardless_of_guarantee: bool, /// The sketch algorithms the query's output reads, in plan order /// (e.g. `["Kll"]`); empty for an exact plan. pub sketches: Vec, @@ -643,7 +668,8 @@ pub struct AcceptedAccuracy { impl std::fmt::Display for AcceptedAccuracy { /// E.g. `metric not set; accepted rank (Kll)`, or - /// `metric relative; accepted relative (DDSketch)`. + /// `metric relative; accepted relative (DDSketch)`, or + /// `metric rank; accepted by the deployment's accuracy model regardless of guarantee (Kll)`. fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let name = |metric: AccuracyMetric| { serde_json::to_value(metric) @@ -656,6 +682,10 @@ impl std::fmt::Display for AcceptedAccuracy { Some(metric) => write!(f, "metric {}; ", name(metric))?, } match self.accepted_metric { + _ if self.accepted_regardless_of_guarantee => write!( + f, + "accepted by the deployment's accuracy model regardless of guarantee" + )?, None => write!(f, "accepted exact")?, Some(metric) => write!(f, "accepted {}", name(metric))?, } @@ -763,10 +793,10 @@ pub fn select_plan( }; let accepted_accuracy = match accuracy_check(&candidate, targets, data, &models) { Ok(accepted_accuracy) => accepted_accuracy, - Err(reason) => { + Err((cause, reason)) => { rejected.push(Rejection { id: candidate.id.clone(), - cause: RejectionCause::AccuracyTargetMissed, + cause, reason, }); continue; @@ -1384,24 +1414,34 @@ pub fn resize_for_failure_probability( /// first query whose output is not shown to meet its target, as a reason. An output meets its target when its theoretical root guarantee /// (composed in Stage 1 with the built-in model) satisfies it, or when the /// deployment's empirical evidence shows that this candidate's output does -/// ([`asap_logical_asap_optimization::accuracy::AccuracyEvidenceProvider::output_guarantee`]). A +/// ([`AccuracyEvidenceProvider::output_guarantee`]). A /// certified estimate inside the plan is not enough on its own: a difference /// or ratio of certified estimates can be unbounded. The root guarantee's /// symbolic statistics are bound first ([`bound_root_guarantee`]); one still /// unknown leaves the target unmet. Count-Min is admitted only over update -/// weights proven non-negative, which its bound assumes: an open +/// weights proven non-negative, which its bound and the executor assume: an open /// [`WeightDomain::SamplesOfMetric`] requirement is met when `data` declares /// the metric a counter ([`confirmed_weights`]), and unmet otherwise. /// /// A target that sets no metric accepts a guarantee in any metric that meets -/// its ε, δ and scope ([`asap_logical_asap_optimization::accuracy::AccuracyModel::satisfies`]); one that sets a metric +/// its ε, δ and scope ([`AccuracyModel::satisfies`]); one that sets a metric /// accepts only guarantees in it (or convertible to it). +/// +/// The Count-Min weight check runs first and rejects as +/// [`RejectionCause::CountMinWeightsNotProvenNonNegative`]: the executor +/// cannot run such a plan. Then the deployment's accuracy model is asked +/// ([`AccuracyModel::accepts_output_regardless_of_guarantee`]): an output it +/// accepts is accepted whatever its guarantee says, and no guarantee or +/// evidence check runs for it. Otherwise, an output with no guarantee from either +/// source (Stage 1 had no rule, and the evidence has none) is rejected as +/// [`RejectionCause::NoAccuracyGuarantee`], and any other miss as +/// [`RejectionCause::AccuracyTargetMissed`]. fn accuracy_check( candidate: &PhysicalCandidate, targets: &[AccuracyTarget], data: &DataWorkload, models: &PlanningModels<'_>, -) -> Result, String> { +) -> Result, (RejectionCause, String)> { let mut accepted = Vec::with_capacity(targets.len()); for (query, (root, target)) in candidate.roots.iter().zip(targets).enumerate() { let query = query + 1; @@ -1435,13 +1475,36 @@ fn accuracy_check( } _ => String::new(), }; - return Err(format!( - "q{query}: {} needs non-negative update weights, and these are not proven \ - non-negative{unconfirmed}", - family_name(family) + // Count-Min (plain, CmsWithHeap or Hydra Count-Min) over + // weights not proven non-negative does not run: the executor + // refuses CmsWithHeap and HydraCms when it builds the plan, + // and plain Count-Min fails at the first negative weight. So + // this rejects the plan whatever the deployment's accuracy + // model accepts. + return Err(( + RejectionCause::CountMinWeightsNotProvenNonNegative, + format!( + "q{query}: {} needs non-negative update weights, and these are not \ + proven non-negative{unconfirmed}", + family_name(family) + ), )); } } + // The deployment's model may accept the output directly, whatever + // its guarantee says; no guarantee or evidence check runs then. + if models + .accuracy + .accepts_output_regardless_of_guarantee(root, target) + { + accepted.push(AcceptedAccuracy { + target_metric: target.accuracy_metric, + accepted_metric: None, + accepted_regardless_of_guarantee: true, + sketches, + }); + continue; + } // A guarantee per value is stated over the whole result's `G` // values for a whole-result target. let series = series_read(root, data); @@ -1452,50 +1515,65 @@ fn accuracy_check( }; let guarantee = bound_root_guarantee(root, series, data, models).map(over_scope); let meets = |g: &AccuracyGuarantee| models.accuracy.satisfies(g, target); - let met = guarantee.clone().filter(meets).or_else(|| { - models - .evidence - .output_guarantee(root) - .map(over_scope) - .filter(meets) - }); + // The deployment's evidence is read only when the theoretical + // guarantee does not meet the target. + let evidence = || models.evidence.output_guarantee(root).map(over_scope); + let (met, evidence) = match guarantee.clone().filter(meets) { + Some(met) => (Some(met), None), + None => { + let evidence = evidence(); + (evidence.clone().filter(meets), evidence) + } + }; if let Some(met) = met { accepted.push(AcceptedAccuracy { target_metric: target.accuracy_metric, accepted_metric: (!met.is_exact()).then_some(met.spec.accuracy_metric), + accepted_regardless_of_guarantee: false, sketches, }); continue; } - return Err(match guarantee { - Some(g) - if values.is_none() - && target.accuracy_scope == AccuracyScope::WholeResult - && target.failure_probability.is_some() - && g.spec.failure_probability.evaluate().is_none() => - { + if guarantee.is_none() && evidence.is_none() { + return Err(( + RejectionCause::NoAccuracyGuarantee, format!( - "q{query}: the output's guarantee holds for each value, and the number \ - of values one evaluation returns is not bound (no declared series count \ - for a source a PromQL vector reads; no bound on a SQL result's groups \ - yet), \ - so it does not meet whole-result target {target:?}, and the deployment \ - has no evidence that it meets it" - ) - } - Some(g) => format!( - "q{query}: the output guarantees a {:?} bound {:?}, failure probability {:?}, \ - which misses target {target:?}, and the deployment has no evidence that it \ - meets it", - g.spec.accuracy_metric, - g.spec.error.evaluate(), - g.spec.failure_probability.evaluate(), - ), - None => format!( - "q{query}: the output has no guarantee, and the deployment has no evidence \ - that it meets target {target:?}" - ), - }); + "q{query}: the output has no guarantee, and the deployment has no evidence \ + for it, so it cannot meet target {target:?}" + ), + )); + } + return Err(( + RejectionCause::AccuracyTargetMissed, + match guarantee { + Some(g) + if values.is_none() + && target.accuracy_scope == AccuracyScope::WholeResult + && target.failure_probability.is_some() + && g.spec.failure_probability.evaluate().is_none() => + { + format!( + "q{query}: the output's guarantee holds for each value, and the number \ + of values one evaluation returns is not bound (no declared series \ + count for a source a PromQL vector reads; no bound on a SQL result's \ + groups yet), so it does not meet whole-result target {target:?}, and \ + the deployment has no evidence that it meets it" + ) + } + Some(g) => format!( + "q{query}: the output guarantees a {:?} bound {:?}, failure probability \ + {:?}, which misses target {target:?}, and the deployment has no evidence \ + that it meets it", + g.spec.accuracy_metric, + g.spec.error.evaluate(), + g.spec.failure_probability.evaluate(), + ), + None => format!( + "q{query}: the output has no guarantee, and the deployment has no \ + evidence that it meets target {target:?}" + ), + }, + )); } Ok(accepted) } @@ -1820,13 +1898,16 @@ fn domain_bound( division_operand_domains: Some([dx, dy]), ..PropagationStats::default() }; - return models - .accuracy - .propagate(&CompositionOperator::ExactDivision, &[gx, gy], None, &stats) - .ok()? - .spec - .error - .evaluate(); + return asap_logical_asap_optimization::accuracy::composition::propagate( + &CompositionOperator::ExactDivision, + &[gx, gy], + None, + &stats, + ) + .ok()? + .spec + .error + .evaluate(); } let valid = |d: &OperandDomain| d.lower.is_finite() && d.upper.is_finite() && d.lower <= d.upper; @@ -1865,6 +1946,10 @@ fn domain_bound( fn family_name(family: &FieldDataType) -> String { match family { + // A shared (Hydra) structure is named by its Hydra kind, e.g. `HydraCms`. + FieldDataType::Sketch(_, GroupingStrategy::SharedMultiSubpopulation { kind, .. }) => { + format!("{kind:?}") + } FieldDataType::Sketch(kind, _) => format!("{:?}", kind.algorithm()), FieldDataType::ExactAggregate(kind, _) => format!("exact {kind:?} accumulator"), other => format!("{other:?}"), @@ -3104,7 +3189,7 @@ mod tests { settings, ) } - use asap_logical_asap_optimization::accuracy::AccuracyEvidenceProvider; + use crate::AccuracyEvidenceProvider; use asap_logical_asap_optimization::deployment_capabilities::DeploymentCapabilities; use asap_logical_asap_optimization::test_support::{entry, entry_with, lower_promql}; use asap_physical_asap_optimization::optimize_physical_asap; @@ -3313,7 +3398,10 @@ mod tests { ) .unwrap(); let p2 = selection.rejected.iter().find(|r| r.id == "P2").unwrap(); - assert_eq!(p2.cause, RejectionCause::AccuracyTargetMissed); + assert_eq!( + p2.cause, + RejectionCause::CountMinWeightsNotProvenNonNegative + ); assert!(p2.reason.contains("non-negative"), "{}", p2.reason); } @@ -3346,7 +3434,15 @@ mod tests { ) .unwrap(); let p2 = selection.rejected.iter().find(|r| r.id == "P2").unwrap(); - assert_eq!(p2.cause, RejectionCause::AccuracyTargetMissed); + // Admitted weights leave the top-k output without a guarantee, + // which the built-in model does not accept; unproven weights + // reject the plan before any accuracy check. + let expected_cause = if admitted { + RejectionCause::NoAccuracyGuarantee + } else { + RejectionCause::CountMinWeightsNotProvenNonNegative + }; + assert_eq!(p2.cause, expected_cause); assert_eq!( !p2.reason.contains("non-negative"), admitted, @@ -4158,7 +4254,7 @@ mod tests { .unwrap(); for id in &summaries { let rejection = builtin.rejected.iter().find(|r| r.id == *id).unwrap(); - assert_eq!(rejection.cause, RejectionCause::AccuracyTargetMissed); + assert_eq!(rejection.cause, RejectionCause::NoAccuracyGuarantee); assert!( rejection.reason.contains("no guarantee"), "{}", @@ -4182,6 +4278,183 @@ mod tests { } } + /// Deployment evidence of a large absolute error (1.0) for every output. + struct EveryOutputMeasuredLoose; + + impl AccuracyEvidenceProvider for EveryOutputMeasuredLoose { + fn output_guarantee(&self, _output: &QueryRoot) -> Option { + Some(AccuracyGuarantee::new( + AccuracyMetric::ABSOLUTE, + BoundExpr::Constant { value: 1.0 }, + ProbabilityExpr::Constant { value: 1e-6 }, + vec![], + )) + } + } + + /// A model that accepts every output directly, whatever its guarantee. + struct AcceptsEveryOutput; + + impl AccuracyModel for AcceptsEveryOutput { + fn satisfies(&self, guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool { + BuiltinAccuracyModel.satisfies(guarantee, target) + } + + fn accepts_output_regardless_of_guarantee( + &self, + _output: &QueryRoot, + _target: &AccuracyTarget, + ) -> bool { + true + } + } + + /// The hook overrides guarantees, not the executor's requirement on + /// Count-Min update weights: over samples of a metric declared a gauge, + /// the CmsWithHeap candidate (P2) is rejected as + /// `CountMinWeightsNotProvenNonNegative` under a model that accepts + /// every output, as under the built-in one. + #[test] + fn unproven_count_min_weights_reject_even_an_accepting_model() { + let target = AccuracyTarget::max_error(0.01) + .failure_probability(0.001) + .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1); + let gauge = with_metric_types(data(), &[("m", MetricType::Gauge)]); + for accuracy in [ + &AcceptsEveryOutput as &dyn AccuracyModel, + &BuiltinAccuracyModel, + ] { + let selection = select_plan( + &candidates(), + &[entry_with(target.clone(), repeating(10_000))], + &gauge, + PlanningModels::builtin().with_accuracy(accuracy), + ) + .unwrap(); + let p2 = selection.rejected.iter().find(|r| r.id == "P2").unwrap(); + assert_eq!( + p2.cause, + RejectionCause::CountMinWeightsNotProvenNonNegative, + "{}", + p2.reason + ); + assert!( + p2.reason.contains("not declared a counter"), + "{}", + p2.reason + ); + assert!(p2.cause.failed_a_check()); + } + } + + /// `node` with every CmsWithHeap summary build replaced by a Hydra + /// Count-Min over the same update. + fn with_hydra_count_min(node: &Rc) -> Rc { + use asap_types::ir::schema::{HydraKind, HydraParams, SketchParams}; + let mut rebuilt = node.with_new_children(with_hydra_count_min).unwrap(); + if let Operator::ASAP(ASAPOp::SummaryAgg { family, .. }) = &mut rebuilt.operator { + if let FieldDataType::Sketch(kind, _) = family { + if let SketchParams::CmsWithHeap { width, depth, .. } = kind.params() { + let (width, depth) = (*width, *depth); + *family = FieldDataType::Sketch( + SketchKind::new(SketchAlgorithm::Cms, SketchParams::Cms { width, depth }), + GroupingStrategy::SharedMultiSubpopulation { + kind: HydraKind::HydraCms, + params: HydraParams::HydraCms { + width, + depth, + shared_rows: depth, + shared_columns: width, + }, + }, + ); + } + } + } + Rc::new(rebuilt) + } + + /// A Hydra Count-Min over unproven weights is rejected by its own name: + /// the reason says `HydraCms`, not `Cms`. + #[test] + fn unproven_hydra_count_min_weights_name_hydra_cms() { + let target = AccuracyTarget::max_error(0.01) + .failure_probability(0.001) + .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1); + let gauge = with_metric_types(data(), &[("m", MetricType::Gauge)]); + let mut p2 = candidates().into_iter().find(|c| c.id == "P2").unwrap(); + p2.roots = p2 + .roots + .iter() + .map(|root| match root { + QueryRoot::Operator(node) => QueryRoot::Operator(with_hydra_count_min(node)), + other => other.clone(), + }) + .collect(); + let (cause, reason) = + accuracy_check(&p2, &[target], &gauge, &PlanningModels::builtin()).unwrap_err(); + assert_eq!(cause, RejectionCause::CountMinWeightsNotProvenNonNegative); + assert!( + reason.contains("HydraCms needs non-negative update weights"), + "{reason}" + ); + assert!(reason.contains("not declared a counter"), "{reason}"); + } + + /// The model's hook has priority over every guarantee. The summary + /// candidates of a difference of quantile estimates have no theoretical + /// guarantee: with evidence that misses the target the built-in model + /// rejects them as `AccuracyTargetMissed`, and with no evidence as + /// `NoAccuracyGuarantee`; a model that accepts every output accepts them + /// in both cases, regardless of guarantee. + #[test] + fn the_accuracy_model_accepts_an_output_whatever_its_guarantee() { + let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); + let candidates = all_candidates( + "quantile_over_time(0.99, m[1m]) - quantile_over_time(0.5, m[1m])", + &target, + ); + let summaries: Vec<_> = candidates + .iter() + .filter(|c| has_summary_estimate(c)) + .map(|c| c.id.as_str()) + .collect(); + assert!(!summaries.is_empty()); + let select = |accuracy: &dyn AccuracyModel, evidence: &dyn AccuracyEvidenceProvider| { + select_plan( + &candidates, + &[entry_with( + target.clone().accuracy_metric(AccuracyMetric::ABSOLUTE), + repeating(10_000), + )], + &data(), + PlanningModels::builtin() + .with_accuracy(accuracy) + .with_evidence(evidence), + ) + .unwrap() + }; + for (evidence, builtin_cause) in [ + ( + &EveryOutputMeasuredLoose as &dyn AccuracyEvidenceProvider, + RejectionCause::AccuracyTargetMissed, + ), + (&NoAccuracyEvidence, RejectionCause::NoAccuracyGuarantee), + ] { + let builtin = select(&BuiltinAccuracyModel, evidence); + let accepting = select(&AcceptsEveryOutput, evidence); + for id in &summaries { + let rejection = builtin.rejected.iter().find(|r| r.id == *id).unwrap(); + assert_eq!(rejection.cause, builtin_cause, "{}", rejection.reason); + assert!(accepting.costs.contains_key(*id), "{id} is valid"); + assert!( + accepting.accepted[*id][0].accepted_regardless_of_guarantee, + "{id}" + ); + } + } + } + /// `candidates` with every state node (a summary build or an exact /// aggregate) kept at ingestion time, so selection charges its kept /// bytes every second (a `MatAtQuery` state is held only for one diff --git a/crates/plan-selection/src/pass/mod.rs b/crates/plan-selection/src/pass/mod.rs index 97d6983d..59d28368 100644 --- a/crates/plan-selection/src/pass/mod.rs +++ b/crates/plan-selection/src/pass/mod.rs @@ -26,10 +26,10 @@ use asap_types::ir::{OperatorNode, QueryRoot}; use asap_types::workload::parsed_workload::ParsedWorkload; use asap_types::workload::WorkloadError; -use crate::{PlanSelectionSettings, Selection, SelectionError}; -use asap_logical_asap_optimization::accuracy::{ - AccuracyEvidenceProvider, AccuracyModel, DefaultAccuracyModel, NoAccuracyEvidence, +use crate::accuracy::{ + AccuracyEvidenceProvider, AccuracyModel, BuiltinAccuracyModel, NoAccuracyEvidence, }; +use crate::{PlanSelectionSettings, Selection, SelectionError}; use asap_logical_asap_optimization::deployment_capabilities::DeploymentCapabilities; use asap_logical_asap_optimization::per_computation_candidates::PerComputationCandidateError; @@ -38,7 +38,7 @@ pub use stage1_to_stage3::{ PrunedCandidate, Stage1ToStage3Pass, Stage1ToStage3Run, }; -static DEFAULT_ACCURACY_MODEL: DefaultAccuracyModel = DefaultAccuracyModel; +static BUILTIN_ACCURACY_MODEL: BuiltinAccuracyModel = BuiltinAccuracyModel; static NO_ACCURACY_EVIDENCE: NoAccuracyEvidence = NoAccuracyEvidence; static DEFAULT_PLAN_SELECTION: std::sync::LazyLock = std::sync::LazyLock::new(PlanSelectionSettings::default); @@ -59,7 +59,8 @@ static DEFAULT_DEPLOYMENT: std::sync::LazyLock = #[non_exhaustive] pub struct PlanningModels<'a> { /// Decides whether a candidate's accuracy guarantee meets its query's - /// target; built-in [`DefaultAccuracyModel`] by default. + /// target, and whether the deployment accepts an output regardless of + /// its guarantee; [`BuiltinAccuracyModel`] by default. pub accuracy: &'a dyn AccuracyModel, /// Deployment facts the guarantees need (operand domains, observed /// guarantees); [`NoAccuracyEvidence`] by default. @@ -93,7 +94,7 @@ impl<'a> PlanningModels<'a> { /// The built-in models. pub fn builtin() -> PlanningModels<'static> { PlanningModels { - accuracy: &DEFAULT_ACCURACY_MODEL, + accuracy: &BUILTIN_ACCURACY_MODEL, evidence: &NO_ACCURACY_EVIDENCE, plan_selection: &DEFAULT_PLAN_SELECTION, deployment_capabilities: &DEFAULT_DEPLOYMENT, diff --git a/crates/planner/tests/deployment_accuracy_model.rs b/crates/planner/tests/deployment_accuracy_model.rs new file mode 100644 index 00000000..0a432945 --- /dev/null +++ b/crates/planner/tests/deployment_accuracy_model.rs @@ -0,0 +1,321 @@ +//! The deployment's accuracy model plugs into plan selection (#509 Stage 3) +//! only. Stage 1 composes guarantees with the built-in analytical rules, so +//! a different model leaves the candidates unchanged and changes only which +//! of them Stage 3 accepts. + +use std::collections::BTreeSet; +use std::rc::Rc; + +use asap_frontend_promql::{lower_promql_query_workload, promql_metric_names}; +use asap_plan_selection::{ + AccuracyModel, BuiltinAccuracyModel, PlanOutput, PlanningModels, RejectionCause, +}; +use asap_planner::{e2e_plan, FrontendInput, UserInput}; +use asap_types::ir::properties::{AccuracyGuarantee, AccuracyMetric, AccuracyTarget}; +use asap_types::ir::schema::FieldDataType; +use asap_types::ir::{ASAPOp, OperatorNode, QueryRoot}; +use asap_types::workload::{ + DataArrival, DataWorkload, DurationMs, PlanningWorkload, Predictability, Query, QueryLanguage, + QueryRequirements, QueryWorkload, RepeatedDemand, RepeatingEntry, RepetitionInterval, +}; + +/// A deployment that trusts no approximate estimate: only an exact +/// guarantee meets a target. +struct ExactOnlyAccuracyModel; + +impl AccuracyModel for ExactOnlyAccuracyModel { + fn satisfies(&self, guarantee: &AccuracyGuarantee, _target: &AccuracyTarget) -> bool { + guarantee.is_exact() + } +} + +/// `query` with a 1% rank error, repeated every ten minutes over metrics +/// sampled every 15 s. +fn rank_target_workload(query: &str) -> PlanningWorkload { + PlanningWorkload { + query_workload: QueryWorkload { + language: QueryLanguage::PromQL, + query_batch: None, + repeating_queries: Some(vec![RepeatingEntry { + query: Query(query.into()), + demand: RepeatedDemand::FixedInterval(RepetitionInterval(600_000)), + requirements: QueryRequirements { + accuracy: AccuracyTarget::max_error(0.01).accuracy_metric(AccuracyMetric::Rank), + max_latency_ms: None, + }, + predictability: Predictability::Unknown, + }]), + }, + data_workload: Some(DataWorkload { + arrival: DataArrival::Ingesting { + rows_per_second: 1.0, + }, + ..DataWorkload::sampled_every(promql_metric_names(query).unwrap(), DurationMs(15_000)) + }), + } +} + +/// Whether `root` reads a sketch state. +fn reads_sketch(root: &Rc) -> bool { + OperatorNode::reachable(root).into_iter().any(|node| { + matches!( + node.asap(), + Some(ASAPOp::SummaryAgg { + family: FieldDataType::Sketch(..), + .. + }) + ) + }) +} + +/// Every Stage 2 candidate for `workload` under `models`, rendered in full: +/// its id and its query roots with every node's guarantee and summary +/// parameters. Two runs that differ only in Stage 3's models must agree. +fn stage2_candidates(workload: &PlanningWorkload, models: PlanningModels<'_>) -> Vec { + let roots = lower_promql_query_workload(workload).expect("lowers"); + let run = asap_plan_selection::run_stage1_to_stage3( + roots.into_iter().enumerate().collect(), + &workload.query_workload.entries().collect::>(), + workload.data_workload.as_ref().unwrap(), + models, + ) + .expect("Stage 1 accepts the workload"); + run.stage2_physical_asap_optimization + .iter() + .map(|candidate| format!("{}: {:?}", candidate.id, candidate.roots)) + .collect() +} + +/// Every candidate Stage 3 was given: the valid ones it priced and the +/// ones it rejected. +fn stage3_candidate_ids(output: &PlanOutput) -> BTreeSet { + let selection = output.selection.as_ref().expect("plan selection ran"); + selection + .costs + .keys() + .cloned() + .chain(selection.rejected.iter().map(|r| r.id.clone())) + .collect() +} + +/// The candidates for which Stage 3 accepted an approximate output for some +/// query. +fn approximate_accepted(output: &PlanOutput) -> BTreeSet { + let selection = output.selection.as_ref().expect("plan selection ran"); + selection + .accepted + .iter() + .filter(|(_, queries)| queries.iter().any(|q| q.accepted_metric.is_some())) + .map(|(id, _)| id.clone()) + .collect() +} + +#[tokio::test] +async fn the_deployment_accuracy_model_decides_only_in_plan_selection() { + // The user's workload: one p99 query with a 1% rank-error target. + let workload = rank_target_workload("quantile_over_time(0.99, lat[5m])"); + + // The built-in models: Stage 3 checks guarantees with BuiltinAccuracyModel. + let builtin = e2e_plan(UserInput::new( + &workload, + FrontendInput::Promql { histograms: None }, + PlanningModels::builtin(), + )) + .await + .expect("plans with the built-in models"); + // The candidates whose approximate output the built-in model accepts: + // the sketch candidates, sized in Stage 1 for the 1% target. + let builtin_approximate = approximate_accepted(&builtin); + assert!(!builtin_approximate.is_empty()); + + // The same workload with a deployment model that accepts only exact + // outputs, plugged into Stage 3 through `PlanningModels::with_accuracy`. + let exact_only_model = ExactOnlyAccuracyModel; + let exact_only = e2e_plan(UserInput::new( + &workload, + FrontendInput::Promql { histograms: None }, + PlanningModels::builtin().with_accuracy(&exact_only_model), + )) + .await + .expect("plans with the exact-only model"); + + // Stages 1 and 2 do not read the model: Stage 3 got the same + // candidates, with the same guarantees and summary parameters. + assert_eq!( + stage3_candidate_ids(&builtin), + stage3_candidate_ids(&exact_only) + ); + assert_eq!( + stage2_candidates(&workload, PlanningModels::builtin()), + stage2_candidates( + &workload, + PlanningModels::builtin().with_accuracy(&exact_only_model) + ) + ); + // Stage 3 now rejects each of those candidates for missing its target, + // accepts no approximate output and selects an exact plan. + let exact_only_selection = exact_only.selection.as_ref().unwrap(); + for id in &builtin_approximate { + assert!(exact_only_selection + .rejected + .iter() + .any(|r| &r.id == id && r.cause == RejectionCause::AccuracyTargetMissed)); + } + assert!(approximate_accepted(&exact_only).is_empty()); + assert!(!reads_sketch(&exact_only.plans[0].root)); +} + +/// A deployment that has measured every output itself and accepts each one +/// for any target, whatever its guarantee says. +struct AcceptsEveryOutput; + +impl AccuracyModel for AcceptsEveryOutput { + fn satisfies(&self, guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool { + BuiltinAccuracyModel.satisfies(guarantee, target) + } + + fn accepts_output_regardless_of_guarantee( + &self, + _output: &QueryRoot, + _target: &AccuracyTarget, + ) -> bool { + true + } +} + +/// The candidates for which Stage 3 accepted some query's output because +/// the deployment's model accepted it regardless of its guarantee. +fn accepted_regardless_of_guarantee(output: &PlanOutput) -> BTreeSet { + let selection = output.selection.as_ref().expect("plan selection ran"); + selection + .accepted + .iter() + .filter(|(_, queries)| queries.iter().any(|q| q.accepted_regardless_of_guarantee)) + .map(|(id, _)| id.clone()) + .collect() +} + +/// The ids of the candidates `output`'s selection rejected with `cause`. +fn rejected_with(output: &PlanOutput, cause: RejectionCause) -> BTreeSet { + output + .selection + .as_ref() + .expect("plan selection ran") + .rejected + .iter() + .filter(|r| r.cause == cause) + .map(|r| r.id.clone()) + .collect() +} + +/// A deployment model that accepts an output directly also accepts one with +/// no guarantee: the built-in model rejects the summary candidates of a +/// difference of two quantile estimates as having none, and the accepting +/// model makes them valid, and Stage 3 selects one. +#[tokio::test] +async fn the_deployment_accuracy_model_accepts_outputs_without_a_guarantee() { + // The user's workload: a difference of two quantile estimates, 1% rank + // error. Stage 1 has no analytical rule for a difference of rank + // estimates, so it keeps the summary candidates with no guarantee. + let workload = rank_target_workload("quantile(0.99, rate(x[5m])) - quantile(0.5, rate(x[5m]))"); + let promql = || FrontendInput::Promql { histograms: None }; + + // The built-in model (default hook: accepts nothing directly) rejects + // every summary candidate as having no guarantee and selects exact. + let builtin = e2e_plan(UserInput::new( + &workload, + promql(), + PlanningModels::builtin(), + )) + .await + .expect("plans with the built-in models"); + // The candidates rejected for having no guarantee: P6 … P47. + let unguaranteed = rejected_with(&builtin, RejectionCause::NoAccuracyGuarantee); + assert!(!unguaranteed.is_empty()); + assert!(accepted_regardless_of_guarantee(&builtin).is_empty()); + assert!(!reads_sketch(&builtin.plans[0].root)); + + // The deployment's model accepts every output directly. + let accepting_model = AcceptsEveryOutput; + let accepting = e2e_plan(UserInput::new( + &workload, + promql(), + PlanningModels::builtin().with_accuracy(&accepting_model), + )) + .await + .expect("plans with the accepting model"); + let accepting_selection = accepting.selection.as_ref().unwrap(); + // Stages 1 and 2 produced the same candidates under both models. + assert_eq!( + stage2_candidates(&workload, PlanningModels::builtin()), + stage2_candidates( + &workload, + PlanningModels::builtin().with_accuracy(&accepting_model) + ) + ); + // Every candidate the built-in model rejected for having no guarantee + // is now valid and priced, accepted regardless of its guarantee, and no + // candidate fails the accuracy check. + assert!(accepted_regardless_of_guarantee(&accepting).is_superset(&unguaranteed)); + for id in &unguaranteed { + assert!(accepting_selection.costs.contains_key(id), "{id}"); + } + assert!(accepting_selection.rejected.iter().all(|r| !matches!( + r.cause, + RejectionCause::NoAccuracyGuarantee | RejectionCause::AccuracyTargetMissed + ))); + // Stage 3 selects one of them, a plan that reads a sketch. + assert!(unguaranteed.contains(&accepting_selection.selected)); + assert!(reads_sketch(&accepting.plans[0].root)); +} + +/// The hook has priority over a guarantee that misses: a candidate whose +/// guarantee misses its target under the built-in models (a relative bound +/// for a rank target) is accepted under the accepting model. +#[tokio::test] +async fn the_deployment_accuracy_model_accepts_outputs_whose_guarantee_misses() { + // The user's workload: one p99 query with a 1% rank-error target. + let workload = rank_target_workload("quantile_over_time(0.99, lat[5m])"); + let promql = || FrontendInput::Promql { histograms: None }; + // Under the built-in models: the candidates whose guarantee misses the + // target (P3, a relative bound 0.01 for a rank target). + let builtin = e2e_plan(UserInput::new( + &workload, + promql(), + PlanningModels::builtin(), + )) + .await + .expect("plans with the built-in models"); + let missed = rejected_with(&builtin, RejectionCause::AccuracyTargetMissed); + assert!(!missed.is_empty()); + // The same workload with the model that accepts every output directly. + let accepting_model = AcceptsEveryOutput; + let accepting = e2e_plan(UserInput::new( + &workload, + promql(), + PlanningModels::builtin().with_accuracy(&accepting_model), + )) + .await + .expect("plans with the accepting model"); + // Each of them is now valid, priced and accepted regardless of guarantee. + let accepting_selection = accepting.selection.as_ref().unwrap(); + for id in &missed { + assert!(accepting_selection.costs.contains_key(id), "{id}"); + assert!( + accepting_selection.accepted[id][0].accepted_regardless_of_guarantee, + "{id}" + ); + } + // The hook also overrides guarantees that meet the target: every + // accepted output, including the exact P1 and the KLL P2, is accepted + // regardless of its guarantee. + for id in ["P1", "P2"] { + assert!(accepting_selection.accepted.contains_key(id), "{id}"); + } + for (id, queries) in &accepting_selection.accepted { + assert!( + queries.iter().all(|q| q.accepted_regardless_of_guarantee), + "{id}" + ); + } +} diff --git a/crates/planner/tests/summary_sharing.rs b/crates/planner/tests/summary_sharing.rs index f7dbb486..c4c2317c 100644 --- a/crates/planner/tests/summary_sharing.rs +++ b/crates/planner/tests/summary_sharing.rs @@ -447,7 +447,7 @@ fn combined_scalar_references_keep_summary_candidates() { .expect("not selected"); assert_eq!( rejection.cause, - RejectionCause::AccuracyTargetMissed, + RejectionCause::NoAccuracyGuarantee, "{rejection:?}" ); assert!( diff --git a/crates/types/src/ir/properties/guarantee.rs b/crates/types/src/ir/properties/guarantee.rs index 2e88a761..45c38a1d 100644 --- a/crates/types/src/ir/properties/guarantee.rs +++ b/crates/types/src/ir/properties/guarantee.rs @@ -7,10 +7,11 @@ //! could silently consume an approximate child. This module is the //! *vocabulary* that fixes that — the typed metric, the symbolic bound and //! failure-probability expressions, the provenance trail, and the typed -//! rejection reasons. The *algebra* that composes these (the `AccuracyModel` -//! trait, its default conservative rules, and budget allocation) lives one -//! layer up in `asap_logical_asap_optimization::accuracy`: this crate defines the -//! shapes, the planning crate decides. +//! rejection reasons. The *algebra* that composes these (the built-in +//! conservative rules and budget allocation) lives one layer up in +//! `asap_logical_asap_optimization::accuracy`, and the deployment's +//! pluggable `AccuracyModel` trait in `asap_plan_selection::accuracy`: this +//! crate defines the shapes, the planning crates decide. //! //! ## What a guarantee says //! @@ -268,7 +269,7 @@ impl ProbabilityExpr { } /// How a parent operator consumes its inputs' values — the shape an -/// `AccuracyModel::propagate` rule is registered against. `#[non_exhaustive]` +/// `propagate` rule is registered against. `#[non_exhaustive]` /// for the same reason [`AccuracyMetric`] is. #[non_exhaustive] #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] diff --git a/tools/dag-viewer/examples/planner-layering-example1.json b/tools/dag-viewer/examples/planner-layering-example1.json index db38b862..709134cf 100644 --- a/tools/dag-viewer/examples/planner-layering-example1.json +++ b/tools/dag-viewer/examples/planner-layering-example1.json @@ -103554,12 +103554,14 @@ "P1": [ { "accepted_metric": null, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103568,12 +103570,14 @@ "P2": [ { "accepted_metric": null, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103582,12 +103586,14 @@ "P3": [ { "accepted_metric": null, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103596,12 +103602,14 @@ "P4": [ { "accepted_metric": null, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103610,12 +103618,14 @@ "P4-m1": [ { "accepted_metric": null, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103624,12 +103634,14 @@ "P5": [ { "accepted_metric": null, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103638,12 +103650,14 @@ "P5-m1": [ { "accepted_metric": null, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103652,12 +103666,14 @@ "P6": [ { "accepted_metric": null, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103666,12 +103682,14 @@ "P6-m1": [ { "accepted_metric": null, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103680,12 +103698,14 @@ "P7": [ { "accepted_metric": null, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103694,12 +103714,14 @@ "P7-m1": [ { "accepted_metric": null, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103708,12 +103730,14 @@ "P8": [ { "accepted_metric": null, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103722,12 +103746,14 @@ "P8-m1": [ { "accepted_metric": null, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103736,12 +103762,14 @@ "P8-m2": [ { "accepted_metric": null, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103750,12 +103778,14 @@ "P8-m3": [ { "accepted_metric": null, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103764,12 +103794,14 @@ "P9": [ { "accepted_metric": null, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103778,12 +103810,14 @@ "P9-m1": [ { "accepted_metric": null, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103792,12 +103826,14 @@ "P9-m2": [ { "accepted_metric": null, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103806,12 +103842,14 @@ "P9-m3": [ { "accepted_metric": null, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null }, { "accepted_metric": null, + "accepted_regardless_of_guarantee": false, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -105738,194 +105776,194 @@ "front": [], "rejected": [ { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P10", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P11", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P12", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P13", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P13-m1", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P14", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P14-m1", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P15", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P15-m1", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P16", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P16-m1", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P17", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P17-m1", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P17-m2", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P17-m3", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P18", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P18-m1", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P18-m2", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P18-m3", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P19", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P20", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P21", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P22", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P22-m1", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P23", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P23-m1", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P24", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P24-m1", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P25", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P25-m1", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P26", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P26-m1", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P26-m2", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P26-m3", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P27", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P27-m1", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P27-m2", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { - "cause": "accuracy_target_missed", + "cause": "no_accuracy_guarantee", "id": "P27-m3", - "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" + "reason": "q2: the output has no guarantee, and the deployment has no evidence for it, so it cannot meet target AccuracyTarget { accuracy_metric: None, max_error: 0.01, failure_probability: Some(0.001), accuracy_scope: WholeResult }" }, { "cause": "tied_lost_on_enumeration_order", diff --git a/tools/dag-viewer/stages.js b/tools/dag-viewer/stages.js index d55e2305..e42f70a5 100644 --- a/tools/dag-viewer/stages.js +++ b/tools/dag-viewer/stages.js @@ -168,8 +168,9 @@ function uniqueIds(list, where, errors) { // of a candidate that failed a check (`RejectionCause::failed_a_check`): // it is not priced. const FAILED_CHECK_CAUSES = [ - 'failure_probability_resize_failed', 'accuracy_target_missed', 'raw_data_not_retained', - 'latency_bound_exceeded', 'over_cluster_capacity', + 'failure_probability_resize_failed', 'accuracy_target_missed', + 'count_min_weights_not_proven_non_negative', 'no_accuracy_guarantee', + 'raw_data_not_retained', 'latency_bound_exceeded', 'over_cluster_capacity', ]; // Every `stage3_plan_selection.rejected[].cause` Stage 3 writes: the failed // checks, then the losses under the cost model (priced, not selected). diff --git a/tools/dag-viewer/test_viewer.py b/tools/dag-viewer/test_viewer.py index a87a00f9..424ae5d0 100644 --- a/tools/dag-viewer/test_viewer.py +++ b/tools/dag-viewer/test_viewer.py @@ -176,7 +176,8 @@ def test_status_follows_the_rejection_cause_not_the_cost(self): model and "rejected" when it failed a check, whether or not the document carries its cost.""" selection = self.doc["stage3_plan_selection"] - failed = next(r for r in selection["rejected"] if r["cause"] == "accuracy_target_missed") + failed_a_check = set(json.loads(self.js.eval("JSON.stringify(FAILED_CHECK_CAUSES)"))) + failed = next(r for r in selection["rejected"] if r["cause"] in failed_a_check) lost = next(r for r in selection["rejected"] if r["cause"] == "costlier") selection["costs"][failed["id"]] = selection["costs"][lost["id"]] del selection["costs"][lost["id"]]