diff --git a/crates/asap-aware-mapping/src/accuracy/estimators/cardinality.rs b/crates/asap-aware-mapping/src/accuracy/estimators/cardinality.rs index 35207ac51..b49a40209 100644 --- a/crates/asap-aware-mapping/src/accuracy/estimators/cardinality.rs +++ b/crates/asap-aware-mapping/src/accuracy/estimators/cardinality.rs @@ -1,30 +1,215 @@ -//! KMV and Theta cardinality bounds using variance and Chebyshev at 99%. +//! KMV's provable distinct-count bound and Theta's Chebyshev calibration. +//! +//! # KMV (#694) +//! +//! `asap_sketchlib`'s KMV keeps the `k` smallest distinct 64-bit hashes of +//! its items and estimates (`src/sketches/kmv.rs`): +//! - the number of hashes it keeps, exactly, while it keeps fewer than `k` +//! (fewer than `k` distinct items); +//! - `(k-1)/U`, where `U = (h+1)/2^64` is the `k`-th smallest hash `h` +//! mapped into (0, 1], once it keeps `k`. +//! +//! A merge keeps the `k` smallest distinct hashes of both states, so a +//! merged state is the one a single build over the union keeps. +//! +//! Model: the hashes of the `N` distinct items are independent, uniform on +//! (0, 1] and distinct (the 64-bit range's `2^-64` granularity and hash +//! collisions are not modelled). For `N < k` the estimate is `N`. For +//! `N ≥ k`, let `X(u) ~ Binomial(N, u)` count the hashes below `u`: +//! - overestimate: `(k-1)/U > (1+ε)N` iff `U < u = (k-1)/((1+ε)N)`, iff +//! `X(u) ≥ k`. With `μ = E[X(u)] = (k-1)/(1+ε)`, `k > (1+ε)μ`, so by +//! Chernoff `P ≤ exp(-ε²μ/(2+ε)) = exp(-ε²(k-1)/((1+ε)(2+ε)))`; +//! - underestimate: `(k-1)/U < (1-ε)N` iff `U > u' = (k-1)/((1-ε)N)`, iff +//! `X(u') ≤ k-1 = (1-ε)μ'` with `μ' = (k-1)/(1-ε)`, so +//! `P ≤ exp(-ε²μ'/2) = exp(-ε²(k-1)/(2(1-ε)))` (and `0` for `ε ≥ 1`). +//! +//! Neither bound depends on `N`, so no distinct bound is needed: the +//! failure probability ([`kmv_failure_probability`]) is their sum, per +//! output row (each group's estimate). Sizing ([`kmv_k`]) takes the +//! smallest `k` with that sum at most `δ`, about +//! `1 + (1+ε)(2+ε)·ln(2/δ)/ε²` (below `3·ln(2/δ)/ε² + 1` for `ε ≤ 1`). use super::*; +/// The smallest `k` `asap_sketchlib`'s KMV estimate `(k-1)/U` is defined for. +const KMV_MIN_K: u32 = 2; +/// The largest `k` sizing returns. +const KMV_MAX_K: u32 = 1 << 26; + pub(super) fn guarantee( algorithm: &SketchAlgorithm, params: &SketchParams, query: &SketchStatistic, ) -> Option { - let (SketchParams::Kmv { k } | SketchParams::Theta { k }) = params else { - return None; + match params { + SketchParams::Kmv { k, relative_error } => { + // Not sized for a target: its relative standard error. + let eps = relative_error + .unwrap_or_else(|| 1.0 / f64::from(k.saturating_sub(2).max(1)).sqrt()); + Some(super::bounded_guarantee( + algorithm, + params, + query, + ErrorMetric::Relative, + eps, + ProbabilityExpr::Constant { + value: kmv_failure_probability(*k, eps), + }, + "kmv_exact_below_k_chernoff_v1", + )) + } + SketchParams::Theta { k } => Some(super::bounded_guarantee( + algorithm, + params, + query, + ErrorMetric::Relative, + 10.0 / f64::from(k.saturating_sub(2).max(1)).sqrt(), + ProbabilityExpr::Constant { value: 0.01 }, + "theta_variance_chebyshev_99_v1", + )), + _ => None, + } +} + +/// The probability that `asap_sketchlib`'s KMV with `k` misses relative +/// error `eps` on any number of distinct items (the module's bound): `1` +/// when it certifies nothing. +pub(crate) fn kmv_failure_probability(k: u32, eps: f64) -> f64 { + if k < KMV_MIN_K || !(eps.is_finite() && eps > 0.0) { + return 1.0; + } + let ranks = f64::from(k - 1); + let over = (-eps * eps * ranks / ((1.0 + eps) * (2.0 + eps))).exp(); + let under = if eps < 1.0 { + (-eps * eps * ranks / (2.0 * (1.0 - eps))).exp() + } else { + 0.0 + }; + // Rounded up, so rounding never certifies a smaller probability. + ((over + under) * (1.0 + 1e-12)).min(1.0) +} + +/// The smallest `k` whose KMV failure probability at relative error `eps` +/// is at most `delta` ([`kmv_failure_probability`]), up to `2^26`; a +/// degenerate `eps` or `delta` saturates to `2^26`. +pub(crate) fn kmv_k(eps: f64, delta: f64) -> u32 { + if !(eps.is_finite() && eps > 0.0 && delta.is_finite() && delta > 0.0) { + return KMV_MAX_K; + } + // The overestimate term alone at most `δ` bounds `k` below; both terms + // at most `δ/2` bound it above. + let k_at = |ln_inverse: f64| { + saturating_ceil( + 1.0 + (1.0 + eps) * (2.0 + eps) * ln_inverse / (eps * eps), + KMV_MIN_K, + KMV_MAX_K, + ) }; - Some(super::bounded_guarantee( - algorithm, - params, - query, - ErrorMetric::Relative, - 10.0 / f64::from(k.saturating_sub(2).max(1)).sqrt(), - ProbabilityExpr::Constant { value: 0.01 }, - match params { - SketchParams::Kmv { .. } => "kmv_unbiased_variance_chebyshev_99_v1", - _ => "theta_variance_chebyshev_99_v1", - }, - )) + let (mut lo, mut hi) = (k_at((1.0 / delta).ln()), k_at((2.0 / delta).ln())); + if kmv_failure_probability(hi, eps) > delta { + return KMV_MAX_K; + } + while lo < hi { + let mid = lo + (hi - lo) / 2; + if kmv_failure_probability(mid, eps) <= delta { + hi = mid; + } else { + lo = mid + 1; + } + } + hi } -/// 99%-confidence KMV/Theta relative bound via Chebyshev, using +/// 99%-confidence Theta relative bound via Chebyshev, using /// `RSE <= 1/sqrt(k-2)` and a ten-standard-deviation interval. -pub(crate) fn kmv_k_99(eps: f64) -> u32 { +pub(crate) fn theta_k_99(eps: f64) -> u32 { saturating_ceil(100.0 / (eps * eps) + 2.0, 16, 1 << 26) } + +#[cfg(test)] +mod tests { + use super::*; + use asap_types::pre_asap::agg_intent::default_cardinality; + + fn error_and_delta(params: &SketchParams) -> (f64, f64) { + let g = guarantee(&SketchAlgorithm::Kmv, params, &SketchStatistic::Cardinality).unwrap(); + ( + g.spec.error.evaluate().unwrap(), + g.spec.failure_probability.evaluate().unwrap(), + ) + } + + /// KMV sized for (ε, δ) certifies both (#700): its failure probability + /// is the requested δ's, not a fixed 0.01. + #[test] + fn kmv_sized_for_a_failure_probability_meets_it() { + for (eps, delta) in [(0.01, 0.001), (0.05, 0.01), (0.02, 1e-6)] { + let params = + super::super::size_params(SketchAlgorithm::Kmv, &default_cardinality(), eps, delta); + let (error, failure_probability) = error_and_delta(¶ms); + assert!(error <= eps, "{params:?}: {error}"); + assert!( + failure_probability <= delta, + "{params:?}: {failure_probability}" + ); + } + } + + /// KMV's sizing grows `k` for a smaller δ at the same ε, near the + /// module's closed form, and a KMV sized for one δ is enlarged for a + /// smaller one at the same relative error. + #[test] + fn kmv_k_follows_the_failure_probability() { + // Ten times smaller than the former Chebyshev calibration's 40,002. + assert_eq!(kmv_k(0.05, 0.01), 4323); + assert_eq!(kmv_k(0.02, 0.01), 26656); + assert!(kmv_k(0.05, 1e-6) > kmv_k(0.05, 0.01)); + assert!(kmv_failure_probability(kmv_k(0.05, 0.01) - 1, 0.05) > 0.01); + assert_eq!(kmv_k(0.0, 0.01), KMV_MAX_K); + let sized = + super::super::size_params(SketchAlgorithm::Kmv, &default_cardinality(), 0.05, 0.01); + let enlarged = super::super::with_failure_probability(&sized, 1e-4); + let (error, delta) = error_and_delta(&enlarged); + assert_eq!(error, 0.05); + assert!(delta <= 1e-4); + assert_ne!(enlarged, sized); + // Not sized for a relative error: nothing to enlarge for. + let unsized_kmv = SketchParams::Kmv { + k: 1024, + relative_error: None, + }; + assert_eq!( + super::super::with_failure_probability(&unsized_kmv, 1e-4), + unsized_kmv + ); + assert_eq!(error_and_delta(&unsized_kmv).1, 1.0); + } + + /// The bound dominates the exact failure probability of the order + /// statistic for every number of distinct items checked: exact below + /// `k`, and both binomial tails from `k` on. + #[test] + fn kmv_bound_dominates_the_exact_order_statistic_tails() { + use super::super::binomial_upper_tail; + for k in [16u32, 64, 256] { + for eps in [0.1, 0.3, 0.6] { + let bound = kmv_failure_probability(k, eps); + for n in (k..8 * k).step_by(7).chain([k, 20 * k]) { + let nf = f64::from(n); + let ranks = f64::from(k - 1); + let over = binomial_upper_tail(n, ranks / ((1.0 + eps) * nf), k); + let under_p = ranks / ((1.0 - eps) * nf); + let under = if under_p >= 1.0 { + 0.0 + } else { + 1.0 - binomial_upper_tail(n, under_p, k) + }; + assert!( + over + under <= bound + 1e-9, + "k={k} eps={eps} n={n}: {}>{bound}", + over + under + ); + } + } + } + } +} diff --git a/crates/asap-aware-mapping/src/accuracy/estimators/hll.rs b/crates/asap-aware-mapping/src/accuracy/estimators/hll.rs index 1e3d1d711..3aeff8d71 100644 --- a/crates/asap-aware-mapping/src/accuracy/estimators/hll.rs +++ b/crates/asap-aware-mapping/src/accuracy/estimators/hll.rs @@ -11,7 +11,7 @@ pub(super) fn generic_guarantee( params: &SketchParams, query: &SketchStatistic, ) -> Option { - let SketchParams::Hll { precision } = params else { + let SketchParams::Hll { precision, .. } = params else { return None; }; Some(super::bounded_guarantee( @@ -31,43 +31,70 @@ fn relative_standard_error(precision: u8) -> f64 { 1.04 / 2f64.powi(i32::from(precision)).sqrt() } -/// Names classic HLL's failure probability at `precision` as a function of -/// `max_distinct[source]`, the deployment's bound on distinct items per -/// evaluation of `source`, which plan selection binds. +/// Names classic HLL's failure probability at `precision` and a relative +/// error as a function of `max_distinct[source]`, the deployment's bound on +/// distinct items per evaluation of `source`, which plan selection binds. const CLASSIC_FAILURE_PROBABILITY: &str = "classic_hll_failure_probability"; const MAX_DISTINCT: &str = "max_distinct"; -/// The statistic classic HLL's failure probability at `precision` over -/// `source` is left as ([`classic_guarantee`]). -pub(crate) fn classic_failure_statistic(precision: u8, source: &str) -> String { - format!("{CLASSIC_FAILURE_PROBABILITY}[p={precision}]({MAX_DISTINCT}[{source}])") +/// The relative error a classic HLL's guarantee states: the one its +/// precision was sized for, else its relative standard error. +pub(crate) fn classic_relative_error(precision: u8, relative_error: Option) -> f64 { + relative_error.unwrap_or_else(|| relative_standard_error(precision)) } -/// The precision and source a [`classic_failure_statistic`] names. -pub(crate) fn parse_classic_failure_statistic(statistic: &str) -> Option<(u8, &str)> { +/// The statistic classic HLL's failure probability at `precision` and +/// `relative_error` over `source` is left as ([`classic_guarantee`]). +/// `relative_error` is written in Rust's shortest round-trip form, so it +/// parses back to the same value. +pub(crate) fn classic_failure_statistic( + precision: u8, + relative_error: f64, + source: &str, +) -> String { + format!( + "{CLASSIC_FAILURE_PROBABILITY}[p={precision},eps={relative_error}]({MAX_DISTINCT}[{source}])" + ) +} + +/// The precision, relative error and source a [`classic_failure_statistic`] +/// names. +pub(crate) fn parse_classic_failure_statistic(statistic: &str) -> Option<(u8, f64, &str)> { let rest = statistic .strip_prefix(CLASSIC_FAILURE_PROBABILITY)? .strip_prefix("[p=")?; - let (precision, rest) = rest.split_once(']')?; + let (params, rest) = rest.split_once(']')?; + let (precision, relative_error) = params.split_once(",eps=")?; let source = rest .strip_prefix('(')? .strip_prefix(MAX_DISTINCT)? .strip_prefix('[')? .strip_suffix("])")?; - Some((precision.parse().ok()?, source)) + Some(( + precision.parse().ok()?, + relative_error.parse().ok()?, + source, + )) } -/// Classic HLL's guarantee for a distinct count over `source`'s own rows: -/// relative error at most its relative standard error, with a failure -/// probability ([`ClassicHllConfidence`]) left symbolic in the source's -/// distinct bound ([`classic_failure_statistic`]). `None` for a precision -/// outside the contract's. -pub(crate) fn classic_guarantee(precision: u8, source: &str) -> Option { +/// Classic HLL's guarantee for a distinct count over `source`'s own rows +/// (one instance per group, over the source's scan at most behind filters +/// and windows): relative error at most the one its precision was sized for +/// ([`classic_relative_error`]), with a failure probability +/// ([`ClassicHllConfidence`]) at that error left symbolic in the source's +/// distinct bound ([`classic_failure_statistic`]). The failure probability +/// is per output row: each group's estimate fails with at most it. `None` +/// for a precision outside the contract's. +pub(crate) fn classic_guarantee( + precision: u8, + relative_error: Option, + source: &str, +) -> Option { if !(4..=18).contains(&precision) { return None; } - let statistic = classic_failure_statistic(precision, source); - let relative_error = relative_standard_error(precision); + let relative_error = classic_relative_error(precision, relative_error); + let statistic = classic_failure_statistic(precision, relative_error, source); Some(AccuracyGuarantee::new( ErrorMetric::Relative, BoundExpr::Constant { @@ -91,17 +118,25 @@ pub(crate) fn classic_guarantee(precision: u8, source: &str) -> Option f64 { +/// the contract's range for this precision and relative error. +pub(crate) fn classic_failure_probability( + precision: u8, + relative_error: f64, + max_distinct: u64, +) -> f64 { u32::try_from(max_distinct) .ok() - .and_then(|n| ClassicHllConfidence::new(n, relative_standard_error(precision))) + .and_then(|n| ClassicHllConfidence::new(n, relative_error)) .and_then(|model| model.failure_probability(precision)) .unwrap_or(1.0) } const CLASSIC_CONTRACT: &str = "classic_hll_linear_counting_collision_bound_v1"; +/// The largest distinct bound the contract supports at any precision: +/// `m/2` at precision 18. Evaluating a bound is O(bound). +const MAX_CONTRACT_DISTINCT: u32 = 1 << 17; + /// A finite-population contract for `m * ln(m / zero_registers)` with the /// classic HLL small-range switch. Hashing is assumed independent and uniform. /// The deployment must establish the population bound; observations alone do @@ -115,7 +150,7 @@ pub struct ClassicHllConfidence { impl ClassicHllConfidence { pub fn new(max_distinct: u32, relative_error: f64) -> Option { (max_distinct > 0 - && max_distinct <= 4096 + && max_distinct <= MAX_CONTRACT_DISTINCT && relative_error.is_finite() && (1e-6..1.0).contains(&relative_error)) .then_some(Self { @@ -193,64 +228,120 @@ impl ClassicHllConfidence { } /// The smallest precision from `precision` up whose classic failure -/// probability over a source of at most `max_distinct` distinct items is at -/// most `delta` (#669b); `precision` when none up to 18 is. -pub(crate) fn precision_for(precision: u8, max_distinct: u64, delta: f64) -> u8 { +/// probability at `relative_error` over a source of at most `max_distinct` +/// distinct items is at most `delta`; `precision` when none up to 18 is. +/// Plan selection's enlargement (#669b) applies this rule to the source's +/// bound, Pass 1's sizing ([`hll_precision`]) to a bound of `⌈1/ε⌉`. +pub(crate) fn precision_for( + precision: u8, + relative_error: f64, + max_distinct: u64, + delta: f64, +) -> u8 { (precision..=18) - .find(|&p| classic_failure_probability(p, max_distinct) <= delta) + .find(|&p| classic_failure_probability(p, relative_error, max_distinct) <= delta) .unwrap_or(precision) } -/// HLL RSE-magnitude inversion: sized from ε alone. Its confidence comes -/// from the source's distinct bound ([`classic_guarantee`]). -pub(crate) fn hll_precision(eps: f64) -> u8 { - saturating_ceil((1.04 / eps).powi(2).log2(), 4, 18) as u8 +/// Classic HLL's precision for relative error `eps` with failure +/// probability `delta`, without data (#694). Below `1/ε` distinct items a +/// single bucket collision already exceeds the error, and the chance of +/// one is about `N²/(2m)`; a certificate for any bound of at least `1/ε` +/// therefore needs `m ≥ 1/(2ε²δ)`. This is the smallest such precision at +/// which [`ClassicHllConfidence`] certifies `delta` at `eps` for every +/// bound up to `⌈1/ε⌉` (18 when none up to 18 does). Plan selection then +/// binds the source's own bound, and enlarges the precision by the same +/// rule ([`precision_for`]) when the bound or a smaller failure +/// probability needs it. +pub(crate) fn hll_precision(eps: f64, delta: f64) -> u8 { + if !(eps.is_finite() && eps > 0.0 && delta.is_finite() && delta > 0.0) { + return 18; + } + let start = saturating_ceil((1.0 / (2.0 * eps * eps * delta.min(1.0))).log2(), 4, 18) as u8; + let reference = (1.0 / eps).ceil().min(f64::from(MAX_CONTRACT_DISTINCT)) as u64; + (start..=18) + .find(|&p| classic_failure_probability(p, eps, reference) <= delta) + .unwrap_or(18) } #[cfg(test)] mod tests { use super::*; - /// The failure-probability statistic names its precision and source, any source name. + /// The failure-probability statistic names its precision, relative + /// error and source, any source name. #[test] fn classic_failure_statistics_round_trip() { for source in ["flows_src_ip", "a[b]", "x])(", ""] { - let statistic = classic_failure_statistic(12, source); - assert_eq!( - parse_classic_failure_statistic(&statistic), - Some((12, source)) - ); + for eps in [0.02, 0.1 + 0.2, 1.04 / 64.0] { + let statistic = classic_failure_statistic(12, eps, source); + assert_eq!( + parse_classic_failure_statistic(&statistic), + Some((12, eps, source)) + ); + } } for other in [ "max_distinct[m]", - "classic_hll_failure_probability[p=x](max_distinct[m])", + "classic_hll_failure_probability[p=x,eps=0.02](max_distinct[m])", + "classic_hll_failure_probability[p=12](max_distinct[m])", ] { assert_eq!(parse_classic_failure_statistic(other), None); } } - /// The symbolic guarantee binds to the model's probability; an + /// The symbolic guarantee states the relative error the precision was + /// sized for and binds to the model's probability at that error; an /// out-of-range bound certifies nothing. #[test] fn classic_guarantee_binds_the_distinct_bound() { - let guarantee = classic_guarantee(12, "m").unwrap(); + let guarantee = classic_guarantee(17, Some(0.02), "m").unwrap(); + assert_eq!(guarantee.spec.error.evaluate(), Some(0.02)); assert_eq!(guarantee.spec.failure_probability.evaluate(), None); - let rse = relative_standard_error(12); - let expected = ClassicHllConfidence::new(9, rse) + let expected = ClassicHllConfidence::new(1_000, 0.02) .unwrap() - .failure_probability(12) + .failure_probability(17) .unwrap(); let bound = guarantee.bind(&|statistic| { - let (precision, source) = parse_classic_failure_statistic(statistic)?; - (source == "m").then(|| classic_failure_probability(precision, 9)) + let (precision, eps, source) = parse_classic_failure_statistic(statistic)?; + (source == "m").then(|| classic_failure_probability(precision, eps, 1_000)) }); assert_eq!(bound.spec.failure_probability.evaluate(), Some(expected)); assert!(expected <= 0.01); - assert!(classic_failure_probability(12, 10) > 0.01); - for too_large in [4097, u64::MAX] { - assert_eq!(classic_failure_probability(12, too_large), 1.0); + for too_large in [u64::from(MAX_CONTRACT_DISTINCT) + 1, u64::MAX] { + assert_eq!(classic_failure_probability(18, 0.05, too_large), 1.0); + } + // Not sized for a target: its relative standard error. + let unsized_guarantee = classic_guarantee(12, None, "m").unwrap(); + assert_eq!( + unsized_guarantee.spec.error.evaluate(), + Some(relative_standard_error(12)) + ); + assert!(classic_guarantee(3, Some(0.02), "m").is_none()); + } + + /// The largest distinct bound certified at δ = 0.01 for a target ε, by + /// precision (#694's table): past the former 4096 computation cap at + /// precision 18. + #[test] + fn largest_certified_bounds_by_precision() { + for (eps, precision, largest) in [ + (0.05, 15, 1176), + (0.05, 18, 9636), + (0.02, 17, 1842), + (0.02, 18, 3791), + (0.01, 18, 72), + ] { + assert!( + classic_failure_probability(precision, eps, largest) <= 0.01, + "eps={eps} p={precision} n={largest}" + ); + assert!( + classic_failure_probability(precision, eps, largest + 1) > 0.01, + "eps={eps} p={precision} n={}", + largest + 1 + ); } - assert!(classic_guarantee(3, "m").is_none()); } /// A supported estimator contract supplies a probability, unlike generic HLL RSE. @@ -274,7 +365,12 @@ mod tests { assert!(model.guarantee(3).is_none()); assert!(model.guarantee(19).is_none()); assert!(model.guarantee(7).is_none()); - for (n, e) in [(0, 0.05), (4097, 0.05), (128, 0.0), (128, f64::NAN)] { + for (n, e) in [ + (0, 0.05), + (MAX_CONTRACT_DISTINCT + 1, 0.05), + (128, 0.0), + (128, f64::NAN), + ] { assert!(ClassicHllConfidence::new(n, e).is_none()); } } @@ -357,6 +453,28 @@ mod tests { assert_eq!(left.estimate(), expected); assert!((expected as f64 - 128.0).abs() / 128.0 <= 0.05); } + /// Pass 1 sizes classic HLL from (ε, δ) without data (#694): the + /// smallest precision with `m ≥ 1/(2ε²δ)` that the tail formula + /// certifies at the target ε, which its guarantee states as its error + /// bound. + #[test] + fn precision_is_sized_from_epsilon_and_delta() { + use crate::realization::default_size_params; + use asap_types::pre_asap::agg_intent::default_cardinality; + for (eps, delta, precision) in [(0.05, 0.01, 15), (0.02, 0.01, 17)] { + let params = + default_size_params(SketchAlgorithm::Hll, &default_cardinality(), eps, delta); + let SketchParams::Hll { + precision: sized, + relative_error, + } = params + else { + panic!("{params:?}") + }; + assert_eq!((sized, relative_error), (precision, Some(eps))); + } + } + #[test] fn generic_rse_sizing_does_not_certify_confidence() { use crate::realization::default_size_params; diff --git a/crates/asap-aware-mapping/src/accuracy/estimators/mod.rs b/crates/asap-aware-mapping/src/accuracy/estimators/mod.rs index 67aca9271..6eb50753e 100644 --- a/crates/asap-aware-mapping/src/accuracy/estimators/mod.rs +++ b/crates/asap-aware-mapping/src/accuracy/estimators/mod.rs @@ -74,11 +74,13 @@ pub(super) fn local_guarantee( /// on enlarged, never reduced, so that each value it answers fails with /// probability at most `delta` and its error bound is unchanged or smaller /// (#669b): Count-Min's depth, Count Sketch's depth (the binomial tail of -/// its median), KLL's failure probability (its δ scale) and a distinct -/// count UnivMon's heap. Other parameters are returned as they are: -/// DDSketch never fails; classic HLL's failure probability comes from its -/// source's distinct bound ([`hll::precision_for`]); KMV and Theta are -/// calibrated at a fixed 99%; an L2 or entropy UnivMon is not enlarged. +/// its median), KLL's failure probability (its δ scale), a distinct count +/// UnivMon's heap, and KMV's `k` at the relative error it was sized for +/// (#694). Other parameters are returned as they are: DDSketch never fails; +/// classic HLL's failure probability comes from its source's distinct bound +/// ([`hll::precision_for`]); Theta is calibrated at a fixed 99%; a KMV not +/// sized for a relative error and an L2 or entropy UnivMon are not +/// enlarged. pub(crate) fn with_failure_probability(params: &SketchParams, delta: f64) -> SketchParams { if !(delta > 0.0 && delta < 1.0) { return params.clone(); @@ -111,6 +113,13 @@ pub(crate) fn with_failure_probability(params: &SketchParams, delta: f64) -> Ske heap_size: *heap_size, }, SketchParams::Kll { .. } => kll::with_failure_probability(params, delta), + SketchParams::Kmv { + k, + relative_error: Some(eps), + } => SketchParams::Kmv { + k: (*k).max(cardinality::kmv_k(*eps, delta)), + relative_error: Some(*eps), + }, SketchParams::UnivMon { .. } => univmon::with_failure_probability(params, delta), _ => params.clone(), } @@ -130,7 +139,8 @@ pub(crate) fn size_params( depth: cms::cms_depth(delta), }, SketchAlgorithm::Hll => SketchParams::Hll { - precision: hll::hll_precision(eps), + precision: hll::hll_precision(eps, delta), + relative_error: Some(eps), }, SketchAlgorithm::CmsWithHeap => { let k = match intent { @@ -148,10 +158,11 @@ pub(crate) fn size_params( // them; sized here so that wiring is local. SketchAlgorithm::DDSketch => ddsketch::size_params(eps), SketchAlgorithm::Theta => SketchParams::Theta { - k: cardinality::kmv_k_99(eps), + k: cardinality::theta_k_99(eps), }, SketchAlgorithm::Kmv => SketchParams::Kmv { - k: cardinality::kmv_k_99(eps), + k: cardinality::kmv_k(eps, delta), + relative_error: Some(eps), }, SketchAlgorithm::CountSketch => { let width = count_sketch::count_sketch_width(eps); @@ -308,13 +319,32 @@ pub(crate) fn shared_size_params(params: &[&SketchParams]) -> Option P::Hll { + ( + P::Hll { + precision, + relative_error, + }, + P::Hll { + precision: p, + relative_error: e, + }, + ) => P::Hll { precision: precision.max(*p), + relative_error: min_recorded(relative_error, *e), }, (P::DDSketch { alpha }, P::DDSketch { alpha: a }) => P::DDSketch { alpha: alpha.min(*a), }, - (P::Kmv { k }, P::Kmv { k: k2 }) => P::Kmv { k: k.max(*k2) }, + ( + P::Kmv { k, relative_error }, + P::Kmv { + k: k2, + relative_error: e, + }, + ) => P::Kmv { + k: k.max(*k2), + relative_error: min_recorded(relative_error, *e), + }, (P::Theta { k }, P::Theta { k: k2 }) => P::Theta { k: k.max(*k2) }, _ => return None, }) @@ -421,7 +451,13 @@ mod shared_size_tests { Some(SketchParams::DDSketch { alpha: 0.001 }) ); assert_eq!( - shared_size_params(&[&kll(200, None), &SketchParams::Hll { precision: 12 }]), + shared_size_params(&[ + &kll(200, None), + &SketchParams::Hll { + precision: 12, + relative_error: None + } + ]), None ); assert_eq!(shared_size_params(&[]), None); @@ -551,10 +587,10 @@ mod tests { /// source's distinct bound meets the target, up to 18. #[test] fn classic_hll_precision_grows_for_its_failure_probability() { - let p = hll::precision_for(4, 64, 0.01); + let p = hll::precision_for(4, 0.05, 64, 0.01); assert!(p > 4); - assert!(hll::classic_failure_probability(p, 64) <= 0.01); - assert!(hll::classic_failure_probability(p - 1, 64) > 0.01); - assert_eq!(hll::precision_for(12, 4096, 1e-30), 12); + assert!(hll::classic_failure_probability(p, 0.05, 64) <= 0.01); + assert!(hll::classic_failure_probability(p - 1, 0.05, 64) > 0.01); + assert_eq!(hll::precision_for(12, 0.05, 4096, 1e-30), 12); } } diff --git a/crates/asap-aware-mapping/src/cost_model.rs b/crates/asap-aware-mapping/src/cost_model.rs index 8d8191191..944aa40d2 100644 --- a/crates/asap-aware-mapping/src/cost_model.rs +++ b/crates/asap-aware-mapping/src/cost_model.rs @@ -1349,7 +1349,13 @@ mod tests { ExactParams::Sum, )); let sketch = default_cse_shared_maintenance_cost(&FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::Hll, SketchParams::Hll { precision: 12 }), + SketchKind::new( + SketchAlgorithm::Hll, + SketchParams::Hll { + precision: 12, + relative_error: None, + }, + ), GroupingStrategy::default(), )); assert!( diff --git a/crates/asap-aware-mapping/src/empirical_cost.rs b/crates/asap-aware-mapping/src/empirical_cost.rs index 7f044a4bc..9596a6363 100644 --- a/crates/asap-aware-mapping/src/empirical_cost.rs +++ b/crates/asap-aware-mapping/src/empirical_cost.rs @@ -366,9 +366,9 @@ fn valid_params(algorithm: &SketchAlgorithm, params: &SketchParams) -> bool { ) } (SketchAlgorithm::Kll, SketchParams::Kll { k, .. }) - | (SketchAlgorithm::Kmv, SketchParams::Kmv { k }) + | (SketchAlgorithm::Kmv, SketchParams::Kmv { k, .. }) | (SketchAlgorithm::Theta, SketchParams::Theta { k }) => *k > 0, - (SketchAlgorithm::Hll, SketchParams::Hll { precision }) => (4..=18).contains(precision), + (SketchAlgorithm::Hll, SketchParams::Hll { precision, .. }) => (4..=18).contains(precision), (SketchAlgorithm::DDSketch, SketchParams::DDSketch { alpha }) => { alpha.is_finite() && *alpha > 0.0 && *alpha < 1.0 } @@ -675,7 +675,10 @@ mod tests { .value = f64::NAN; assert!(bad.validate().is_err()); let mut bad = artifact.clone(); - bad.records[0].params = SketchParams::Hll { precision: 14 }; + bad.records[0].params = SketchParams::Hll { + precision: 14, + relative_error: None, + }; assert!(bad.validate().is_err()); let mut bad = artifact.clone(); bad.records[0].provenance.repetitions = 0; diff --git a/crates/asap-aware-mapping/src/per_computation_candidates.rs b/crates/asap-aware-mapping/src/per_computation_candidates.rs index 28880a6e3..b10dc2507 100644 --- a/crates/asap-aware-mapping/src/per_computation_candidates.rs +++ b/crates/asap-aware-mapping/src/per_computation_candidates.rs @@ -1340,7 +1340,10 @@ fn source_distinct_guarantee( } source_scan(input)?; match kind.params() { - SketchParams::Hll { precision } => hll::classic_guarantee(*precision, &source(input)?), + SketchParams::Hll { + precision, + relative_error, + } => hll::classic_guarantee(*precision, *relative_error, &source(input)?), params @ SketchParams::UnivMon { .. } => { univmon::distinct_guarantee(params, &source(input)?) } @@ -4044,4 +4047,53 @@ mod tests { .evaluate(); assert_eq!(unconfirmed, None); } + + /// Classic HLL's failure probability is left symbolic in a source's + /// distinct bound only for one instance per group over that source's + /// own rows (#693's scope condition): not over a join of two sources, + /// not over computed rows, and not for one Hydra structure shared by + /// every group, whose distinct items the source's bound does not bound. + #[test] + fn classic_hll_confidence_needs_one_instance_over_a_sources_own_rows() { + use asap_types::post_asap::{HydraKind, HydraParams}; + let hll = |grouping| { + FieldDataType::Sketch( + SketchKind::new( + SketchAlgorithm::Hll, + SketchParams::Hll { + precision: 15, + relative_error: Some(0.05), + }, + ), + grouping, + ) + }; + let input = |query| crate::test_support::lower_promql(query, RequestedAccuracy::exact()); + let guarantee = |family: &FieldDataType, query| { + source_distinct_guarantee(family, &SketchStatistic::Cardinality, &input(query)) + }; + let own = hll(GroupingStrategy::PerSubpopulationInstance); + let symbolic = guarantee(&own, "a").expect("a source's own rows"); + assert_eq!( + symbolic.spec.failure_probability, + ProbabilityExpr::Unknown { + statistic: crate::accuracy::estimators::hll::classic_failure_statistic( + 15, 0.05, "a" + ) + } + ); + for (query, over) in [("a * on() b", "a join"), ("a * 2", "computed rows")] { + assert!(guarantee(&own, query).is_none(), "{over}: {query}"); + } + let hydra = hll(GroupingStrategy::SharedMultiSubpopulation { + kind: HydraKind::HydraCms, + params: HydraParams::HydraCms { + width: 8, + depth: 2, + shared_rows: 2, + shared_columns: 8, + }, + }); + assert!(guarantee(&hydra, "a").is_none()); + } } diff --git a/crates/asap-aware-mapping/src/replacement.rs b/crates/asap-aware-mapping/src/replacement.rs index f73ed16bc..45e6ac41a 100644 --- a/crates/asap-aware-mapping/src/replacement.rs +++ b/crates/asap-aware-mapping/src/replacement.rs @@ -7394,13 +7394,18 @@ mod tests { ); } + /// HLL is sized from (ε, δ) (#694): ε = 0.008125 at δ = 0.01 needs + /// `m ≥ 1/(2ε²δ)`, past precision 18, the largest. #[test] - fn default_cardinality_sizes_hll_to_its_rse_magnitude() { + fn default_cardinality_sizes_hll_from_epsilon_and_delta() { assert_eq!( preferred(&default_cardinality()), Realization::Sketch(SketchKind::new( SketchAlgorithm::Hll, - SketchParams::Hll { precision: 14 }, + SketchParams::Hll { + precision: 18, + relative_error: Some(0.008125) + }, )) ); } diff --git a/crates/asap-aware-mapping/src/stage3_plan_selection.rs b/crates/asap-aware-mapping/src/stage3_plan_selection.rs index 730ec52c5..0f1991c77 100644 --- a/crates/asap-aware-mapping/src/stage3_plan_selection.rs +++ b/crates/asap-aware-mapping/src/stage3_plan_selection.rs @@ -480,11 +480,22 @@ pub fn sized_roots( let resize = |state: &OperatorNode, params: &SketchParams| { let delta = *required.get(&std::ptr::from_ref(state))?; Some(match params { - SketchParams::Hll { precision } => { + SketchParams::Hll { + precision, + relative_error, + } => { let source = summary_source(state)?; let bound = data.max_distinct_per_evaluation.value_at(ANY_TIME)?; + // The guarantee keeps the relative error Pass 1 sized for. + let relative_error = hll::classic_relative_error(*precision, *relative_error); SketchParams::Hll { - precision: hll::precision_for(*precision, *bound.get(&source)?, delta), + precision: hll::precision_for( + *precision, + relative_error, + *bound.get(&source)?, + delta, + ), + relative_error: Some(relative_error), } } params => with_failure_probability(params, delta), @@ -724,10 +735,13 @@ fn workload_statistic(statistic: &str, data: &DataWorkload) -> Option { .value_at(ANY_TIME) .map(|rows| *rows as f64); } - if let Some((precision, source)) = hll::parse_classic_failure_statistic(statistic) { + if let Some((precision, relative_error, source)) = + hll::parse_classic_failure_statistic(statistic) + { let bound = data.max_distinct_per_evaluation.value_at(ANY_TIME)?; return Some(hll::classic_failure_probability( precision, + relative_error, *bound.get(source)?, )); } @@ -1595,8 +1609,8 @@ fn summary_shape(family: &FieldDataType) -> (f64, u64) { ), SketchParams::Kll { k, .. } => (1.0, 8 * kll_items(*k)), SketchParams::DDSketch { alpha } => (1.0, ddsketch_bins(*alpha).saturating_mul(8)), - SketchParams::Hll { precision } => (1.0, 1u64 << (*precision).min(63)), - SketchParams::Kmv { k } | SketchParams::Theta { k } => (1.0, 8 * u64::from(*k)), + SketchParams::Hll { precision, .. } => (1.0, 1u64 << (*precision).min(63)), + SketchParams::Kmv { k, .. } | SketchParams::Theta { k } => (1.0, 8 * u64::from(*k)), SketchParams::UnivMon { heap_size, sketch_rows, @@ -1703,6 +1717,27 @@ mod tests { } } + /// Classic HLL's failure probability over a source binds only that + /// source's distinct bound: a bound keyed to another source leaves it + /// unknown. + #[test] + fn classic_hll_failure_probability_binds_only_its_own_sources_bound() { + let statistic = hll::classic_failure_statistic(15, 0.05, "a"); + let bounded = |source: &str| DataWorkload { + max_distinct_per_evaluation: Evidence { + value: Some(std::collections::BTreeMap::from([(source.into(), 100)])), + ..Default::default() + }, + ..data() + }; + assert_eq!(workload_statistic(&statistic, &bounded("b")), None); + assert_eq!( + workload_statistic(&statistic, &bounded("a")), + Some(hll::classic_failure_probability(15, 0.05, 100)) + ); + assert!(hll::classic_failure_probability(15, 0.05, 100) <= 0.01); + } + /// Exact (P1), whole-expression CMS+heap (P2) and CountSketch+heap (P3) realizations of /// one approximate top-k query, in Pass 1 catalog order. fn candidates() -> Vec { @@ -2601,11 +2636,23 @@ mod tests { .1 }; assert_eq!( - bytes(SketchAlgorithm::Hll, SketchParams::Hll { precision: 14 }), + bytes( + SketchAlgorithm::Hll, + SketchParams::Hll { + precision: 14, + relative_error: None + } + ), 1 << 14 ); assert_eq!( - bytes(SketchAlgorithm::Kmv, SketchParams::Kmv { k: 1_000 }), + bytes( + SketchAlgorithm::Kmv, + SketchParams::Kmv { + k: 1_000, + relative_error: None + } + ), 8 * 1_000 ); assert_eq!( diff --git a/crates/asap-physical-operators/src/capability.rs b/crates/asap-physical-operators/src/capability.rs index 6e0441ec2..17d3ad7e5 100644 --- a/crates/asap-physical-operators/src/capability.rs +++ b/crates/asap-physical-operators/src/capability.rs @@ -59,8 +59,8 @@ pub fn validate_summary_kernel( { false } - (A::Hll, P::Hll { precision }) if (4..=18).contains(precision) => false, - (A::Kmv, P::Kmv { k }) if *k >= crate::summary_kernels::kmv::KMV_MIN_K => false, + (A::Hll, P::Hll { precision, .. }) if (4..=18).contains(precision) => false, + (A::Kmv, P::Kmv { k, .. }) if *k >= crate::summary_kernels::kmv::KMV_MIN_K => false, (A::Cms, P::Cms { width, depth }) if valid_matrix(*width, *depth) => true, // The median needs an odd row count. (A::CountSketch, P::CountSketch { width, depth }) diff --git a/crates/asap-physical-operators/src/summary_kernels/factory.rs b/crates/asap-physical-operators/src/summary_kernels/factory.rs index caa1e6b17..2f6422c6f 100644 --- a/crates/asap-physical-operators/src/summary_kernels/factory.rs +++ b/crates/asap-physical-operators/src/summary_kernels/factory.rs @@ -626,13 +626,13 @@ pub fn create_planner_accumulator( .map_err(|e| e.to_string())?, ) } - (SketchAlgorithm::Hll, SketchParams::Hll { precision }) => Box::new(HllUpdater { + (SketchAlgorithm::Hll, SketchParams::Hll { precision, .. }) => Box::new(HllUpdater { acc: HllSketchAccumulator::new( asap_sketchlib::HllVariant::Regular, u32::from(*precision), ), }), - (SketchAlgorithm::Kmv, SketchParams::Kmv { k }) => { + (SketchAlgorithm::Kmv, SketchParams::Kmv { k, .. }) => { Box::new(crate::summary_kernels::kmv::KmvAccumulator::new(*k)) } ( diff --git a/crates/asap-physical-operators/src/summary_kernels/kmv.rs b/crates/asap-physical-operators/src/summary_kernels/kmv.rs index 543794ed0..e6ae02e5d 100644 --- a/crates/asap-physical-operators/src/summary_kernels/kmv.rs +++ b/crates/asap-physical-operators/src/summary_kernels/kmv.rs @@ -141,8 +141,8 @@ mod tests { ); } - // Above k the estimate is within KMV's relative standard error - // 1/sqrt(k - 2) times ten (the 99% Chebyshev bound Stage 1 sizes for). + // Above k the estimate is within ten times KMV's relative standard + // error 1/sqrt(k - 2). #[test] fn estimates_many_distinct_items_within_its_error() { let k = 1024; @@ -154,6 +154,21 @@ mod tests { } } + // From k distinct items on, the estimate is `(k - 1) / U`, with `U` the + // k-th smallest hash mapped into (0, 1]: the estimator the planner's + // KMV bound is proved for (`accuracy::estimators::cardinality`, #694). + #[test] + fn estimates_k_minus_one_over_the_kth_smallest_hash() { + let k = 256; + for exact in [k, 5_000] { + let kmv = build(k, 0..exact); + let kth = kmv.inner.k_vals.iter().copied().max().unwrap(); + let u = (kth as f64 + 1.0) / 18_446_744_073_709_551_616.0; + assert_eq!(cardinality(&kmv), f64::from(k - 1) / u); + } + assert_eq!(cardinality(&build(k, 0..k - 1)), f64::from(k - 1)); + } + // Merging two overlapping states keeps the same k smallest distinct // hashes as building one over the union, so both estimate alike. #[test] diff --git a/crates/asap-physical-operators/src/values.rs b/crates/asap-physical-operators/src/values.rs index 830a624a8..0c5c25ad9 100644 --- a/crates/asap-physical-operators/src/values.rs +++ b/crates/asap-physical-operators/src/values.rs @@ -320,11 +320,11 @@ fn validate_state(family: &SummaryFamilyType, state: &dyn AggregateCore) -> Resu .as_any() .downcast_ref::() .is_some_and(|s| s.inner.alpha == *alpha), - SketchParams::Hll { precision } => state + SketchParams::Hll { precision, .. } => state .as_any() .downcast_ref::() .is_some_and(|s| s.inner.precision == u32::from(*precision)), - SketchParams::Kmv { k } => state + SketchParams::Kmv { k, .. } => state .as_any() .downcast_ref::() .is_some_and(|s| s.inner.k == *k as usize), diff --git a/crates/integration-tests/tests/planner_layering_example2.rs b/crates/integration-tests/tests/planner_layering_example2.rs index 6c356740a..36944c9a3 100644 --- a/crates/integration-tests/tests/planner_layering_example2.rs +++ b/crates/integration-tests/tests/planner_layering_example2.rs @@ -581,9 +581,9 @@ fn stage3_plan_selection_judges_univmon_with_an_accuracy_model() { } } -/// The physical candidates that answer Q1 with HLL and Q2 and Q3 exactly: -/// their validity depends only on Q1's HLL estimate. -fn hll_only_candidates(run: &Run) -> Vec<&str> { +/// The physical candidates that answer Q1 with `algorithm` and Q2 and Q3 +/// exactly: their validity depends only on Q1's estimate. +fn q1_only_candidates<'a>(run: &'a Run, algorithm: &str) -> Vec<&'a str> { let ids: Vec<_> = run .logical .iter() @@ -591,28 +591,38 @@ fn hll_only_candidates(run: &Run) -> Vec<&str> { !shares_a_summary(c) && (0..3) .map(|q| query_option(&c.dag, &c.query_roots, q)) - .eq(["Hll", "exact", "exact"]) + .eq([algorithm, "exact", "exact"]) }) .flat_map(|c| run.physical_of(c).map(|p| p.id.as_str())) .collect(); - assert!(!ids.is_empty(), "a candidate answers Q1 with HLL alone"); + assert!( + !ids.is_empty(), + "a candidate answers Q1 with {algorithm} alone" + ); ids } +fn hll_only_candidates(run: &Run) -> Vec<&str> { + q1_only_candidates(run, "Hll") +} + /// With the deployment's bound on distinct items per evaluation, plan /// selection binds classic HLL's failure probability and accepts Q1's HLL /// estimate; without a bound it stays unknown and the target unmet. Pass 1 -/// sizes HLL from ε = 0.02 alone (precision 12), which certifies δ = 0.01 -/// for at most 9 distinct items. +/// sizes HLL from ε = 0.02 and δ = 0.01 (#694): `m ≥ 1/(2ε²δ)` gives +/// precision 17, which certifies δ = 0.01 at ε = 0.02 for bounds up to +/// 1,842 distinct items. #[test] fn stage3_plan_selection_accepts_hll_given_a_distinct_bound() { - let bounded = pipeline_with_distinct_bound(Some(9)); - for id in hll_only_candidates(&bounded) { - assert!( - bounded.cost(id).is_some(), - "{id}: {:?}", - bounded.invalid().get(id) - ); + for bound in [9, 1_842] { + let bounded = pipeline_with_distinct_bound(Some(bound)); + for id in hll_only_candidates(&bounded) { + assert!( + bounded.cost(id).is_some(), + "bound {bound}: {id}: {:?}", + bounded.invalid().get(id) + ); + } } let unbounded = pipeline_with_distinct_bound(None); let invalid = unbounded.invalid(); @@ -626,20 +636,19 @@ fn stage3_plan_selection_accepts_hll_given_a_distinct_bound() { } } -/// A bound too large for the precision Pass 1 chose (12 certifies δ = 0.01 -/// for at most 9 distinct items) is met by a larger precision: plan -/// selection enlarges HLL until its failure probability over the bound -/// meets the target, up to precision 18 (#669b): a bound of 10 is met. The -/// model certifies a bound of 100 at no precision (a larger precision also -/// shrinks the relative error it certifies), and one outside its range (at -/// most 4096) certifies nothing. +/// A bound too large for the precision Pass 1 chose (17 certifies at most +/// 1,842 distinct items) is met by a larger precision: plan selection +/// enlarges HLL until its failure probability over the bound at ε = 0.02 +/// meets the target, up to precision 18 (#669b), which certifies up to +/// 3,791. Past that, up to Example 2's 10M source IPs, classic HLL's +/// linear-counting contract certifies nothing. #[test] fn stage3_plan_selection_enlarges_hll_for_a_larger_distinct_bound() { - let run = pipeline_with_distinct_bound(Some(10)); + let run = pipeline_with_distinct_bound(Some(3_000)); for id in hll_only_candidates(&run) { assert!(run.cost(id).is_some(), "{id}: {:?}", run.invalid().get(id)); } - for bound in [100, 10_000_000] { + for bound in [10_000, 10_000_000] { let run = pipeline_with_distinct_bound(Some(bound)); let invalid = run.invalid(); for id in hll_only_candidates(&run) { @@ -655,6 +664,79 @@ fn stage3_plan_selection_enlarges_hll_for_a_larger_distinct_bound() { } } +/// KMV certifies Q1 over the 10M source IPs without a distinct bound +/// (#694): its bound holds for any number of distinct items, and Pass 1 +/// sizes `k` for the requested δ (#700), so a stricter δ = 10⁻⁴ is met +/// too, by a larger `k`. +#[test] +fn stage3_plan_selection_accepts_kmv_for_any_failure_probability_without_a_bound() { + let mut ks = Vec::new(); + for delta in [0.01, 1e-4] { + let mut workload = standin_workload(); + let q1 = &mut workload + .query_workload + .repeating_queries + .as_mut() + .expect("repeating queries")[0]; + q1.requirements.accuracy = AccuracyRequirement::Explicit(RequestedAccuracy { + scope: AccuracyScope::PerValue, + ..RequestedAccuracy::epsilon_delta(0.02, delta) + }); + let run = run_stages(&workload, lower_promql(&workload)); + for id in q1_only_candidates(&run, "Kmv") { + assert!( + run.cost(id).is_some(), + "δ = {delta}: {id}: {:?}", + run.invalid().get(id) + ); + let k: BTreeSet<_> = sketch_builds(run.physical(id).dag()) + .into_iter() + .filter_map(|(_, _, params)| match params { + SketchParams::Kmv { k, .. } => Some(k), + _ => None, + }) + .collect(); + ks.push((delta, k)); + } + } + assert!( + ks.iter().any(|(d, k)| *d == 0.01 && k.contains(&26_656)), + "{ks:?}" + ); + assert!( + ks.iter() + .any(|(d, k)| *d == 1e-4 && k.iter().all(|k| *k > 26_656)), + "{ks:?}" + ); +} + +/// The physical candidates among `ids` that build their sketch as a +/// tumbling-pane `PaneChain`. +fn tumbling<'a>(run: &Run, ids: Vec<&'a str>) -> Vec<&'a str> { + ids.into_iter() + .filter(|id| { + let dag = run.physical(id).dag(); + sketch_builds(dag).iter().any(|(build, _, _)| { + matches!(window_form(dag, *build), WindowForm::Tumbling { .. }) + }) + }) + .collect() +} + +/// Plan selection's enlargement reaches a sketch carried by a tumbling-pane +/// `PaneChain` too (#713): Q1's HLL over a bound of 3,000 is valid only at +/// precision 18, above Pass 1's 17, so a valid tumbling-pane HLL candidate +/// had its chain's precision enlarged. +#[test] +fn stage3_plan_selection_enlarges_tumbling_pane_hll() { + let run = pipeline_with_distinct_bound(Some(3_000)); + let hll = tumbling(&run, hll_only_candidates(&run)); + assert!(!hll.is_empty(), "a tumbling-pane HLL candidate answers Q1"); + for id in hll { + assert!(run.cost(id).is_some(), "{id}: {:?}", run.invalid().get(id)); + } +} + /// Stage 1 does not read the distinct bound: its candidates are the same /// with and without one, and Q1's HLL estimate leaves its failure /// probability symbolic in the source's bound. @@ -682,8 +764,9 @@ fn stage1_logical_asap_optimization_leaves_the_hll_failure_probability_symbolic( assert_eq!( guarantee.spec.failure_probability, ProbabilityExpr::Unknown { - statistic: "classic_hll_failure_probability[p=12](max_distinct[flows_src_ip])" - .into() + statistic: + "classic_hll_failure_probability[p=17,eps=0.02](max_distinct[flows_src_ip])" + .into() }, "{}", c.id diff --git a/crates/integration-tests/tests/sql_to_post_asap.rs b/crates/integration-tests/tests/sql_to_post_asap.rs index d29df4d91..a97c1ad48 100644 --- a/crates/integration-tests/tests/sql_to_post_asap.rs +++ b/crates/integration-tests/tests/sql_to_post_asap.rs @@ -735,9 +735,10 @@ async fn sql_quantile_binds_kll_sketch_over_named_column() { /// — unlike `Quantile`/`Count`/`TopK`, its preferred candidate is HLL, not /// KLL/CMS (`realization::summary_candidates`), so this exercises a /// distinct branch of the sketch-vs-exact decision than the quantile test -/// above. +/// above. HLL is sized from ε and the default δ = 0.01 (#694): `m ≥ +/// 1/(2ε²δ)` reaches precision 18, the largest. #[tokio::test] -async fn sql_count_distinct_with_epsilon_binds_hll_rse_over_named_column() { +async fn sql_count_distinct_with_epsilon_binds_hll_over_named_column() { let pre_asap = lower( "SELECT COUNT(DISTINCT service) FROM metrics", RequestedAccuracy::epsilon(0.01), @@ -772,7 +773,13 @@ async fn sql_count_distinct_with_epsilon_binds_hll_rse_over_named_column() { assert_eq!( family, &FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::Hll, SketchParams::Hll { precision: 14 }), + SketchKind::new( + SketchAlgorithm::Hll, + SketchParams::Hll { + precision: 18, + relative_error: Some(0.01) + } + ), GroupingStrategy::default() ) ); diff --git a/crates/types/src/post_asap/family_capability.rs b/crates/types/src/post_asap/family_capability.rs index 25f087ee2..08a3a3176 100644 --- a/crates/types/src/post_asap/family_capability.rs +++ b/crates/types/src/post_asap/family_capability.rs @@ -333,7 +333,13 @@ mod tests { heap_size: 10, }, ); - let hll = sketch(SketchAlgorithm::Hll, SketchParams::Hll { precision: 12 }); + let hll = sketch( + SketchAlgorithm::Hll, + SketchParams::Hll { + precision: 12, + relative_error: None, + }, + ); for (family, yes, no) in [ ( &kll, diff --git a/crates/types/src/post_asap/sketch.rs b/crates/types/src/post_asap/sketch.rs index 9d9da9d12..b8b1eb7fc 100644 --- a/crates/types/src/post_asap/sketch.rs +++ b/crates/types/src/post_asap/sketch.rs @@ -117,6 +117,12 @@ pub enum SketchParams { }, Hll { precision: u8, + /// The relative error `precision` was sized for (#694): the error + /// bound of its guarantee, at which its failure probability is + /// evaluated. `None`: not sized for a target; its error bound is + /// the relative standard error `1.04/√m`. Ignored by equality. + #[serde(default, skip_serializing_if = "Option::is_none")] + relative_error: Option, }, DDSketch { alpha: f64, @@ -128,6 +134,12 @@ pub enum SketchParams { }, Kmv { k: u32, + /// The relative error `k` was sized for (#694): the error bound of + /// its guarantee, at which its failure probability is evaluated. + /// `None`: not sized for a target; its error bound is the relative + /// standard error `1/√(k-2)`. Ignored by equality. + #[serde(default, skip_serializing_if = "Option::is_none")] + relative_error: Option, }, Theta { k: u32, @@ -208,9 +220,11 @@ impl PartialEq for SketchParams { heap_size: b2, }, ) => (a0, a1, a2) == (b0, b1, b2), - (P::Hll { precision: a }, P::Hll { precision: b }) => a == b, + (P::Hll { precision: a, .. }, P::Hll { precision: b, .. }) => a == b, (P::DDSketch { alpha: a }, P::DDSketch { alpha: b }) => a == b, - (P::Kmv { k: a }, P::Kmv { k: b }) | (P::Theta { k: a }, P::Theta { k: b }) => a == b, + (P::Kmv { k: a, .. }, P::Kmv { k: b, .. }) | (P::Theta { k: a }, P::Theta { k: b }) => { + a == b + } // Different variants. Listed by `self`'s variant, not `_`, so a // new variant fails to compile here until it is compared above. (P::UnivMon { .. }, _) @@ -821,7 +835,13 @@ mod tests { assert_eq!(kll(331, Some(0.001)), kll(331, None)); assert_eq!(kll(331, Some(0.001)), kll(331, Some(0.0001))); assert_ne!(kll(331, None), kll(269, None)); - assert_ne!(kll(331, None), SketchParams::Kmv { k: 331 }); + assert_ne!( + kll(331, None), + SketchParams::Kmv { + k: 331, + relative_error: None + } + ); let family = |params| { crate::post_asap::FieldDataType::Sketch( SketchKind::new(SketchAlgorithm::Kll, params), @@ -834,7 +854,13 @@ mod tests { #[test] #[should_panic(expected = "SketchKind parameter mismatch")] fn sketch_kind_rejects_params_from_another_algorithm() { - SketchKind::new(SketchAlgorithm::Kll, SketchParams::Hll { precision: 14 }); + SketchKind::new( + SketchAlgorithm::Kll, + SketchParams::Hll { + precision: 14, + relative_error: None, + }, + ); } #[test] diff --git a/crates/types/src/workload.rs b/crates/types/src/workload.rs index 06137ac12..98f14caae 100644 --- a/crates/types/src/workload.rs +++ b/crates/types/src/workload.rs @@ -552,10 +552,14 @@ pub struct DataWorkload { /// Upper bound per source (PromQL metric or SQL table name) on the /// distinct items any one distinct-count summary over the source's own /// rows counts in one complete evaluation, including every merged pane. - /// A filter or a shorter window only lowers it. The deployment must + /// One bound serves every query over the source, so it must hold for + /// the longest evaluation window any of them uses; a filter or a + /// shorter window only lowers it. Each group's own items count + /// separately: the bound is per output row. The deployment must /// enforce it; an observed distinct count is not a bound. Plan selection - /// binds it to give classic HLL its failure probability; a source - /// without a bound leaves that probability unknown. + /// binds it to give classic HLL (and a distinct-count UnivMon) its + /// failure probability per output row; a source without a bound leaves + /// that probability unknown. KMV's guarantee needs no bound. #[serde(default)] pub max_distinct_per_evaluation: Evidence>, /// Declared type of each metric, by metric name (Prometheus `# TYPE` diff --git a/crates/types/tests/summary_merge_structure.rs b/crates/types/tests/summary_merge_structure.rs index 7d709f6c4..7d6ce2ced 100644 --- a/crates/types/tests/summary_merge_structure.rs +++ b/crates/types/tests/summary_merge_structure.rs @@ -116,7 +116,13 @@ fn kll() -> FieldDataType { ) } fn hll() -> FieldDataType { - sketch(SketchAlgorithm::Hll, SketchParams::Hll { precision: 12 }) + sketch( + SketchAlgorithm::Hll, + SketchParams::Hll { + precision: 12, + relative_error: None, + }, + ) } fn cms_with_heap() -> FieldDataType { sketch( @@ -214,7 +220,13 @@ fn family_capability_table() { // KMV keeps the k smallest hashes under one shared seed: the merge // is the k smallest of the union and ignores repeated hashes. ( - sketch(SketchAlgorithm::Kmv, SketchParams::Kmv { k: 1024 }), + sketch( + SketchAlgorithm::Kmv, + SketchParams::Kmv { + k: 1024, + relative_error: None, + }, + ), Some((OverlapAllowed, SameAsUnionBuild)), false, ),