From 0a56ced8e3ae721353cd4311906e635ae05fd1a8 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Tue, 13 Oct 2026 22:24:30 +1100 Subject: [PATCH 1/6] feat(stack-encrypt): size a match query's fetch with target_limit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR-0009 decision 5: a caller who wants `want` real matches from a probabilistic query fetches `want + Poisson_quantile(λ = p·N, confidence)` rows, where `p` is the mean over stored filters of `f^b`, `f` a filter's fill and `b` the bits the query sets. - `Probabilistic::target_limit(&self, want, &Stats) -> Limit`. - `op::Stats::declared(rows)` uses a new `MatchConfig::expected_tokens()` prior, a default method returning `None`, so existing configurations compile unchanged; it is not part of the index's identity. - `op::Stats::measured(rows, histogram)` takes `(set bits, rows)` buckets from one sampled aggregate over the column, and takes the mean of the power, never the power of the mean fill. An empty histogram falls back to the prior. - `op::Limit` is `Rows(n)` or `Unbounded`, the latter when nothing is known of the stored filters or the confidence is not in (0, 1). `fetch_or(page)` reads either as a fetch size. - The quantile sums the Poisson mass exactly in log space from `λ − 10√λ`, so it costs about `12√λ` terms; past `λ = 1e7` it is the conservative `λ + 8√λ`. Unit tests reproduce the ADR's worked table and its mixed-length column, and pin the quantile to reference values computed with `lgamma`. Closes #1184 Claude-Session: https://claude.ai/code/session_01VYKAWbd361xDLmqBdLoUcB --- packages/stack-encrypt/CHANGELOG.md | 8 + ...and-a-query-term-is-not-the-stored-term.md | 35 +- .../stack-encrypt/src/{op.rs => op/mod.rs} | 16 + packages/stack-encrypt/src/op/stats.rs | 381 ++++++++++++++++++ packages/stack-encrypt/src/sem/mod.rs | 26 ++ packages/stack-encrypt/tests/op.rs | 102 ++++- ...nexact_operator_needs_probabilistic.stderr | 2 +- .../tests/ui/op_target_does_not_answer.stderr | 2 +- 8 files changed, 556 insertions(+), 16 deletions(-) rename packages/stack-encrypt/src/{op.rs => op/mod.rs} (95%) create mode 100644 packages/stack-encrypt/src/op/stats.rs diff --git a/packages/stack-encrypt/CHANGELOG.md b/packages/stack-encrypt/CHANGELOG.md index db149485f..c4055d9d0 100644 --- a/packages/stack-encrypt/CHANGELOG.md +++ b/packages/stack-encrypt/CHANGELOG.md @@ -45,6 +45,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 copies of the text. An exact term has no `verify`, and none of this crate's indexes can answer `op::Matches` but with a probabilistic term (`op::TermFor`). +- **`target_limit` says how many rows a match query should fetch.** + `query.target_limit(want, &stats)` is `want` plus the false positives + expected at the confidence `stats` names (99% by default), as an + `op::Limit`. `op::Stats::declared(rows)` uses the new + `MatchConfig::expected_tokens()` prior; `op::Stats::measured(rows, + histogram)` uses a histogram of the stored filters' set-bit counts. With + neither, the limit is `Limit::Unbounded`, and the caller pages with a + cursor. - **Errors now expose stable codes, help text, and structured details.** All error types implement `miette::Diagnostic` with codes such as diff --git a/packages/stack-encrypt/docs/adr/0009-operators-select-the-index-and-a-query-term-is-not-the-stored-term.md b/packages/stack-encrypt/docs/adr/0009-operators-select-the-index-and-a-query-term-is-not-the-stored-term.md index 04cfc2e32..0b9457132 100644 --- a/packages/stack-encrypt/docs/adr/0009-operators-select-the-index-and-a-query-term-is-not-the-stored-term.md +++ b/packages/stack-encrypt/docs/adr/0009-operators-select-the-index-and-a-query-term-is-not-the-stored-term.md @@ -363,9 +363,10 @@ index declaration. The library takes either as numbers in `Stats` and does not care which it was given. A column has the prior from the day it is declared. -**Declared.** A match index declares `expected_tokens`, the typical token -count of the values it will hold, beside `m` and `k`: 20 for a name column, -200 for a notes column. From it `Stats::declared(rows)` takes +**Declared.** A match configuration declares `expected_tokens()`, the typical +token count of the values it will hold, beside the `m` and `k` of its +options: 20 for a name column, 200 for a notes column. It is not part of the +index's identity and changes no term. From it `Stats::declared(rows)` takes `f = 1 − e^(−k·expected_tokens/m)` and `p = f^b`. This is a prior, and `rows` is a planning estimate too. It is enough to warn before a query runs (a three-letter needle against a notes column) and to size a first fetch; it @@ -388,8 +389,16 @@ query has. The sample bounds the cost on a large table; `TABLESAMPLE` is a page sample, which is fine for a distribution and unsuited to an exact count. It is also a sample of pages, so on a table of a few thousand rows a 1% sample can select none; below that size the clause is dropped and the whole -table is read, which costs nothing worth saving. `rows` comes from `SELECT reltuples FROM pg_class WHERE oid = -'users'::regclass`, a catalog read. +table is read, which costs nothing worth saving. `rows` comes from +`SELECT reltuples FROM pg_class WHERE oid = 'users'::regclass`, a catalog +read. An empty histogram falls back to the declared prior. + +`target_limit` returns a `Limit`: `Rows(n)`, or `Unbounded` when the +statistics say nothing about the stored filters (no prior and no +histogram) or the confidence is not below 1. `fetch_or(page)` reads either +as a fetch size, for a caller that pages anyway. Past ten million expected +false positives the quantile is the conservative `λ + 8√λ`, since the query +is hopeless at that point. Where the numbers live and who keeps them in step was asked on the review, so it is written down. They are read from the server by the application, at @@ -458,7 +467,7 @@ shot, with a bound, and the completeness check from decision 5: ```rust let term = cipher.query(op::Matches, "ali").on(&users::name).await?; -let fetch = term.target_limit(10, &stats).fetch(); +let fetch = term.target_limit(10, &stats).fetch_or(50); let rows: Vec = sqlx::query_as( "SELECT id, name FROM users WHERE name @@ $1 ORDER BY id LIMIT $2", @@ -719,7 +728,7 @@ the operator is `@@`, column on the left. The query selects the ciphertext ```rust let term = cipher.query(op::Matches, "ali").on(&users::name).await?; -let fetch = term.target_limit(10, &stats).fetch(); +let fetch = term.target_limit(10, &stats).fetch_or(50); let rows: Vec = sqlx::query_as( "SELECT id, name FROM users WHERE name @@ $1::eql_v4.query_text_match ORDER BY id LIMIT $2", @@ -741,12 +750,12 @@ The `Stats` it used were read once, at startup, by the application: ```rust let rows: i64 = sqlx::query_scalar("SELECT reltuples::bigint FROM pg_class WHERE oid = 'users'::regclass") .fetch_one(&pool).await?; -let histogram: Vec<(i16, i64)> = sqlx::query_as( +let histogram: Vec<(i32, i64)> = sqlx::query_as( "SELECT cardinality(eql_v4.match_term(name)) AS bits, count(*) FROM users TABLESAMPLE SYSTEM (1) GROUP BY 1", ) .fetch_all(&pool).await?; -let stats = Stats::measured(rows as u64, histogram.into_iter().map(|(b, n)| (b as u16, n as u64))); +let stats = Stats::measured(rows as u64, histogram.into_iter().map(|(b, n)| (b as u32, n as u64))); ``` or, before anyone has measured, `Stats::declared(1_000_000)`, which uses the @@ -766,7 +775,7 @@ let rows: Vec = sqlx::query_as( ) .bind(sqlx::types::Json(&name)) .bind(sqlx::types::Json(&age)) -.bind(name.target_limit(10, &stats).fetch() as i64) +.bind(name.target_limit(10, &stats).fetch_or(50) as i64) .fetch_all(&pool).await?; let hits: Vec = cipher.open(rows).using(&users_plan).await? .into_iter().filter(|u| name.verify(&u.name)).take(10).collect(); @@ -908,7 +917,7 @@ it binds as a parameter in any library built on `database/sql` or pgx: ```go q, _ := users.Fields.Name.Matches(ctx, cipher, "ali") -limit := q.TargetLimit(10, stats).Fetch() +limit := q.TargetLimit(10, stats).FetchOr(50) rows, err := db.QueryContext(ctx, `SELECT id, name FROM users WHERE name @@ $1::eql_v4.query_text_match ORDER BY id LIMIT $2`, @@ -1014,7 +1023,7 @@ TypeScript programs use: ```ts // pg -const limit = name.targetLimit(10, stats).fetch() +const limit = name.targetLimit(10, stats).fetchOr(50) const { rows } = await pool.query( 'SELECT id, name FROM users WHERE name @@ $1::eql_v4.query_text_match ORDER BY id LIMIT $2', [name, limit], @@ -1058,7 +1067,7 @@ const q = await ops.matches(users.name, 'ali') // { sql, verify, target const rows = await db.select().from(users) .where(q.sql) // name @@ '…'::eql_v4.query_text_match .orderBy(users.id) - .limit(q.targetLimit(10, stats).fetch()) + .limit(q.targetLimit(10, stats).fetchOr(50)) const hits = (await cipher.decrypt(users, rows)).filter((u) => q.verify(u.name)).slice(0, 10) ``` diff --git a/packages/stack-encrypt/src/op.rs b/packages/stack-encrypt/src/op/mod.rs similarity index 95% rename from packages/stack-encrypt/src/op.rs rename to packages/stack-encrypt/src/op/mod.rs index faccd5fc0..1cd92b2ec 100644 --- a/packages/stack-encrypt/src/op.rs +++ b/packages/stack-encrypt/src/op/mod.rs @@ -57,6 +57,9 @@ use std::marker::PhantomData; use stack_kms::MaybeSend; +pub(crate) mod stats; +pub use stats::{Limit, Stats, DEFAULT_CONFIDENCE}; + use crate::sem::{DefaultMatch, EqualityTerm, MatchConfig, MatchQuery, MatchTerms, QueryTokens}; use crate::target::{Equality, Index, Match}; use crate::PlanError; @@ -223,6 +226,19 @@ pub trait Probabilistic: QueryTerm { /// the index was declared over, so every outcome other than a match is /// `false`, never an error. fn verify(&self, value: &Self::Plaintext) -> bool; + + /// How many candidate rows to fetch so that, at the confidence `stats` + /// names, `want` of them [`verify`](Self::verify), when the table holds + /// that many matches. + /// + /// A probabilistic bound, not a guarantee: at 99% confidence one query + /// in a hundred draws more false positives than it allows for. The + /// candidate count says which case a fetch is in. If the server + /// returned fewer rows than the limit, the predicate is exhausted, and + /// fewer than `want` verified rows means fewer than `want` matches. If + /// it returned exactly the limit, the result is a page: carry on from + /// its last row with a cursor. + fn target_limit(&self, want: u64, stats: &Stats) -> Limit; } /// The query terms that may answer the operator `Op`: what diff --git a/packages/stack-encrypt/src/op/stats.rs b/packages/stack-encrypt/src/op/stats.rs new file mode 100644 index 000000000..edefda45d --- /dev/null +++ b/packages/stack-encrypt/src/op/stats.rs @@ -0,0 +1,381 @@ +//! How many rows to fetch for a probabilistic query: [`Stats`] in, +//! [`Limit`] out (ADR-0009, decision 5). +//! +//! A Bloom match returns every real match and some rows that matched by +//! chance. A non-matching row whose stored filter has fill `f` (the +//! fraction of its bits set) passes a query that sets `b` bits with +//! probability about `f^b`. Over a table the pass rate `p` is the *mean* of +//! each row's `f^b`, and the false positives a query scanning `N` rows +//! draws are `Poisson(λ = p·N)`. To expect `want` real matches at a given +//! confidence, fetch +//! +//! ```text +//! want + Poisson_quantile(λ, confidence) +//! ``` +//! +//! rows. The bound is probabilistic: at 99% confidence one query in a +//! hundred draws more false positives than the quantile. + +/// What a probabilistic query's [`target_limit`] knows of the table: how +/// many rows the predicate scans, how full their stored filters are, and +/// the confidence the fetch should meet. +/// +/// The numbers come from the server, read by the application at startup or +/// on a timer and kept as plain numbers. None of them is secret: the server +/// holds the filters and the catalog holds the row count. A stale `Stats` +/// moves the confidence, never the correctness, because +/// [`verify`](super::Probabilistic::verify) still decides every row. +/// +/// [`target_limit`]: super::Probabilistic::target_limit +#[derive(Clone, Debug, PartialEq)] +pub struct Stats { + rows: u64, + fill: Fill, + confidence: f64, +} + +/// Where a [`Stats`] takes the stored filters' fill from. +#[derive(Clone, Debug, PartialEq)] +enum Fill { + /// The index's declared `expected_tokens` prior. + Declared, + /// A histogram of set-bit counts: `(bits, rows)` pairs. + Measured(Vec<(u32, u64)>), +} + +/// The confidence a [`Stats`] targets unless told otherwise. +pub const DEFAULT_CONFIDENCE: f64 = 0.99; + +impl Stats { + /// Statistics from the index's declared prior + /// ([`MatchConfig::expected_tokens`](crate::sem::MatchConfig::expected_tokens)) + /// and an estimate of the rows the query scans, such as + /// `pg_class.reltuples`. + /// + /// The prior knows nothing about the spread of value lengths, so it + /// underestimates a column of mixed lengths; measure the column + /// ([`measured`](Self::measured)) once it holds data. An index that + /// declares no prior gives [`Limit::Unbounded`]. + pub fn declared(rows: u64) -> Self { + Self { + rows, + fill: Fill::Declared, + confidence: DEFAULT_CONFIDENCE, + } + } + + /// Statistics from a measured histogram of the stored filters' set-bit + /// counts, as `(bits, rows)` pairs, and an estimate of the rows the + /// query scans. + /// + /// One aggregate over the column gives the histogram, with at most `m` + /// buckets; a page sample bounds its cost on a large table: + /// + /// ```sql + /// SELECT cardinality(eql_v4.match_term(name)) AS bits, count(*) + /// FROM users TABLESAMPLE SYSTEM (1) + /// GROUP BY 1 + /// ``` + /// + /// A page sample can select no page of a small table, so drop the + /// `TABLESAMPLE` there. An empty histogram falls back to the declared + /// prior. + pub fn measured(rows: u64, histogram: impl IntoIterator) -> Self { + Self { + rows, + fill: Fill::Measured(histogram.into_iter().collect()), + confidence: DEFAULT_CONFIDENCE, + } + } + + /// Target `confidence` instead of 99%: the probability that the fetch + /// holds `want` real matches when the table has that many. Outside the + /// open interval `(0, 1)` there is no finite fetch to promise, and + /// [`target_limit`](super::Probabilistic::target_limit) answers + /// [`Limit::Unbounded`]. + pub fn with_confidence(mut self, confidence: f64) -> Self { + self.confidence = confidence; + self + } + + /// The rows the query scans. + pub fn rows(&self) -> u64 { + self.rows + } + + /// The confidence the fetch targets. + pub fn confidence(&self) -> f64 { + self.confidence + } + + /// The probability that one non-matching row passes a query that sets + /// `bits` of an `m`-bit filter, or `None` when nothing is known of the + /// stored filters' fill. `declared` is the fill of one stored filter + /// under the index's prior, if it declares one. + pub(crate) fn pass_rate(&self, bits: u32, m: u32, declared: Option) -> Option { + let measured = match &self.fill { + Fill::Measured(histogram) => mean_power(histogram, bits, m), + Fill::Declared => None, + }; + measured.or_else(|| declared.map(|fill| fill.powi(bits as i32))) + } + + /// The fetch for `want` real matches at pass rate `p`. + pub(crate) fn limit(&self, want: u64, pass_rate: Option) -> Limit { + let Some(p) = pass_rate else { + return Limit::Unbounded; + }; + let promisable = self.confidence > 0.0 && self.confidence < 1.0; + if !promisable || !p.is_finite() { + return Limit::Unbounded; + } + let lambda = p.clamp(0.0, 1.0) * self.rows as f64; + Limit::Rows(want.saturating_add(poisson_quantile(lambda, self.confidence))) + } +} + +/// The mean of `(bits_i / m)^b` over a histogram of set-bit counts, or +/// `None` for an empty one. A count above `m` (a histogram from another +/// configuration) is a full filter. +fn mean_power(histogram: &[(u32, u64)], b: u32, m: u32) -> Option { + let rows: u64 = histogram.iter().map(|&(_, rows)| rows).sum(); + if rows == 0 || m == 0 { + return None; + } + let total: f64 = histogram + .iter() + .map(|&(bits, rows)| { + let fill = (f64::from(bits) / f64::from(m)).min(1.0); + rows as f64 * fill.powi(b as i32) + }) + .sum(); + Some(total / rows as f64) +} + +/// The fill of one stored filter of `tokens` tokens, each setting `k` of +/// `m` bits: `1 − e^(−k·tokens/m)`. +pub(crate) fn expected_fill(tokens: u32, k: usize, m: u32) -> f64 { + 1.0 - (-(k as f64) * f64::from(tokens) / f64::from(m)).exp() +} + +/// How many rows a probabilistic query should fetch. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Limit { + /// Fetch this many rows: `want` and the false positives expected at the + /// stated confidence. + Rows(u64), + /// No finite fetch can be promised: the statistics say nothing about + /// the stored filters, or the confidence is not below 1. Page through + /// the candidates with a cursor until `want` rows verify. + Unbounded, +} + +impl Limit { + /// The rows to fetch, if a finite fetch can be promised. + pub fn rows(self) -> Option { + match self { + Self::Rows(rows) => Some(rows), + Self::Unbounded => None, + } + } + + /// The rows to fetch, or `page` when no finite fetch can be promised, + /// for a caller that pages either way. + pub fn fetch_or(self, page: u64) -> u64 { + self.rows().unwrap_or(page) + } +} + +/// The smallest `q` with `P(X ≤ q) ≥ confidence` for `X ~ Poisson(λ)`. +/// +/// Exact up to floating point: the probabilities are summed in log space +/// from where the mass begins, `λ − 10√λ` for a large `λ` (the mass below +/// it is under `1e-23`), so the work is about `12√λ` terms. Past `1e7` +/// expected false positives the query is hopeless anyway and the answer is +/// the conservative `λ + 8√λ`, which covers any confidence below +/// `1 − 1e-15`. +pub(crate) fn poisson_quantile(lambda: f64, confidence: f64) -> u64 { + if lambda <= 0.0 || lambda.is_nan() { + return 0; + } + if lambda > 1e7 { + return (lambda + 8.0 * lambda.sqrt()).ceil() as u64; + } + // Saturates to 0 for a small `λ`, where the mass starts at 0. + let start = (lambda - 10.0 * lambda.sqrt()).floor().max(0.0) as u64; + let ln_lambda = lambda.ln(); + let mut log_term = -lambda + start as f64 * ln_lambda - ln_factorial(start); + let mut cdf = 0.0; + let mut k = start; + loop { + cdf += log_term.exp(); + // Past the mode, a term this small adds nothing a float can hold: + // the remaining mass is below any confidence that is not 1. + if cdf >= confidence || (k as f64 > lambda && log_term < -50.0) { + return k; + } + k += 1; + log_term += ln_lambda - (k as f64).ln(); + } +} + +/// `ln(n!)`: summed exactly below 20, Stirling's series above, where it is +/// accurate to about `1e-12`. +fn ln_factorial(n: u64) -> f64 { + if n < 20 { + return (2..=n).map(|i| (i as f64).ln()).sum(); + } + let x = n as f64 + 1.0; + (x - 0.5) * x.ln() - x + 0.5 * (2.0 * std::f64::consts::PI).ln() + 1.0 / (12.0 * x) + - 1.0 / (360.0 * x.powi(3)) + + 1.0 / (1260.0 * x.powi(5)) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Reference quantiles, computed exactly with `math.lgamma` in log space. + #[test] + fn the_poisson_quantile_matches_reference_values() { + for (lambda, confidence, expected) in [ + (0.034, 0.99, 1), + (0.439, 0.99, 3), + (1.0, 0.5, 1), + (1.0, 0.99, 4), + (10.0, 0.99, 18), + (57.8, 0.99, 76), + (100.0, 0.99, 124), + (245.0, 0.99, 282), + (1000.0, 0.999, 1099), + (3800.0, 0.99, 3944), + (7600.0, 0.99, 7804), + (1e6, 0.99, 1_002_327), + ] { + assert_eq!( + poisson_quantile(lambda, confidence), + expected, + "λ = {lambda}, confidence = {confidence}" + ); + } + assert_eq!(poisson_quantile(0.0, 0.99), 0); + assert_eq!(poisson_quantile(-1.0, 0.99), 0); + } + + #[test] + fn past_ten_million_the_quantile_is_conservative() { + let lambda = 4e7; + let q = poisson_quantile(lambda, 0.999_999); + assert!(q as f64 >= lambda + 4.9 * lambda.sqrt(), "{q}"); + assert!(q as f64 <= lambda + 8.0 * lambda.sqrt() + 1.0, "{q}"); + } + + #[test] + fn a_confidence_at_one_still_ends() { + // A float CDF may never reach 1; the tail cut-off ends the sum. + let q = poisson_quantile(5.0, 1.0); + assert!(q > 20, "{q}"); + } + + #[test] + fn ln_factorial_is_continuous_where_the_series_takes_over() { + let exact: f64 = (2..=20u64).map(|i| (i as f64).ln()).sum(); + assert!((ln_factorial(20) - exact).abs() < 1e-10); + assert_eq!(ln_factorial(0), 0.0); + assert_eq!(ln_factorial(1), 0.0); + assert!((ln_factorial(5) - 120f64.ln()).abs() < 1e-12); + } + + /// ADR-0009 decision 5's worked table: EQL v4 defaults (`k = 6`, + /// `m = 2048`), a million rows, 99%. + #[test] + fn the_declared_prior_reproduces_the_worked_table() { + let stats = Stats::declared(1_000_000); + for (tokens, bits, want_extra) in [(20, 6, 1), (200, 6, 7805), (200, 18, 3)] { + let fill = expected_fill(tokens, 6, 2048); + let p = stats.pass_rate(bits, 2048, Some(fill)); + assert_eq!( + stats.limit(10, p), + Limit::Rows(10 + want_extra), + "{tokens} stored tokens, {bits} query bits" + ); + } + } + + /// The mixed-length column of decision 5: half the rows hold 20 tokens, + /// half 200. The mean of `f^b` is what the bound needs; the mean fill + /// raised to `b` would promise a fetch about fifteen times too small. + #[test] + fn a_measured_histogram_uses_the_mean_of_the_power_not_the_power_of_the_mean() { + let short = (expected_fill(20, 6, 2048) * 2048.0).round() as u32; + let long = (expected_fill(200, 6, 2048) * 2048.0).round() as u32; + let stats = Stats::measured(1_000_000, [(short, 500_000), (long, 500_000)]); + let p = stats.pass_rate(6, 2048, None).unwrap(); + let lambda = p * 1e6; + assert!((3700.0..3900.0).contains(&lambda), "λ = {lambda}"); + + let mean_fill = (f64::from(short) + f64::from(long)) / 2.0 / 2048.0; + assert!(mean_fill.powi(6) * 1e6 < 300.0); + let Limit::Rows(rows) = stats.limit(10, Some(p)) else { + panic!("a measured column has a finite fetch"); + }; + assert!(rows > 3800, "{rows}"); + } + + #[test] + fn nothing_known_is_unbounded() { + let stats = Stats::declared(1_000_000); + assert_eq!( + stats.limit(10, stats.pass_rate(6, 2048, None)), + Limit::Unbounded + ); + + let empty = Stats::measured(1_000_000, []); + assert_eq!(empty.pass_rate(6, 2048, None), None); + assert_eq!( + empty.pass_rate(6, 2048, Some(0.5)), + Some(0.5f64.powi(6)), + "an empty histogram falls back to the prior" + ); + } + + #[test] + fn a_confidence_outside_the_open_interval_is_unbounded() { + for confidence in [0.0, 1.0, -0.5, 1.5, f64::NAN] { + let stats = Stats::declared(1000).with_confidence(confidence); + assert_eq!( + stats.limit(10, Some(0.01)), + Limit::Unbounded, + "{confidence}" + ); + } + let stats = Stats::declared(1000).with_confidence(0.5); + assert_eq!(stats.confidence(), 0.5); + assert_eq!(stats.rows(), 1000); + assert_eq!(stats.limit(10, Some(0.01)), Limit::Rows(10 + 10)); + } + + #[test] + fn a_count_above_the_filter_size_is_a_full_filter() { + let stats = Stats::measured(100, [(5000, 1)]); + assert_eq!(stats.pass_rate(6, 2048, None), Some(1.0)); + // Every row passes: λ = 100, whose 99% quantile is 124. + assert_eq!(stats.limit(1, Some(1.0)), Limit::Rows(1 + 124)); + } + + #[test] + fn a_limit_reads_back_as_rows_or_a_page() { + assert_eq!(Limit::Rows(12).rows(), Some(12)); + assert_eq!(Limit::Rows(12).fetch_or(50), 12); + assert_eq!(Limit::Unbounded.rows(), None); + assert_eq!(Limit::Unbounded.fetch_or(50), 50); + } + + #[test] + fn want_does_not_overflow() { + assert_eq!( + Stats::declared(10).limit(u64::MAX, Some(0.5)), + Limit::Rows(u64::MAX) + ); + } +} diff --git a/packages/stack-encrypt/src/sem/mod.rs b/packages/stack-encrypt/src/sem/mod.rs index 635f38629..8b1b65220 100644 --- a/packages/stack-encrypt/src/sem/mod.rs +++ b/packages/stack-encrypt/src/sem/mod.rs @@ -471,6 +471,16 @@ impl MatchOptions { /// construction. Define your own by implementing this on a marker type. pub trait MatchConfig: Send + Sync + 'static { fn options() -> MatchOptions; + + /// The typical number of tokens in a value this index stores: 20 for a + /// name column, 200 for a notes column. A prior, used to size a query's + /// fetch ([`Stats::declared`](crate::op::Stats::declared)) before the + /// column's filters have been measured. It is not part of the index's + /// identity and changes no term. `None`, the default, declares no + /// prior. + fn expected_tokens() -> Option { + None + } } /// The default [`MatchConfig`]: [`MatchOptions::default`]. @@ -709,6 +719,22 @@ impl crate::op::Probabilistic for MatchQuery { let have: HashSet<&str> = tokens.iter().map(String::as_str).collect(); !self.tokens.0.is_empty() && self.tokens.0.iter().all(|t| have.contains(t.as_str())) } + + /// `want` plus the false positives expected among the rows `stats` + /// scans: the query sets `b` bits ([`positions`](Self::positions)), and + /// a stored filter of fill `f` holds them all with probability `f^b`. + /// The fill comes from the measured histogram in `stats`, or else from + /// the configuration's + /// [`expected_tokens`](MatchConfig::expected_tokens) prior; with + /// neither, the answer is + /// [`Limit::Unbounded`](crate::op::Limit::Unbounded). + fn target_limit(&self, want: u64, stats: &crate::op::Stats) -> crate::op::Limit { + let options = O::options(); + let declared = O::expected_tokens() + .map(|tokens| crate::op::stats::expected_fill(tokens, options.k, options.m)); + let bits = u32::try_from(self.positions().len()).unwrap_or(u32::MAX); + stats.limit(want, stats.pass_rate(bits, options.m, declared)) + } } /// Positions only: the tokens are plaintext. diff --git a/packages/stack-encrypt/tests/op.rs b/packages/stack-encrypt/tests/op.rs index 55eb8f8a6..f5f78e085 100644 --- a/packages/stack-encrypt/tests/op.rs +++ b/packages/stack-encrypt/tests/op.rs @@ -6,7 +6,7 @@ mod common; use common::{loads_counting_cipher, stack_cipher}; use stack_encrypt::kms::FakeDataKeySource; -use stack_encrypt::op::{self, MatchCase, Probabilistic}; +use stack_encrypt::op::{self, Limit, MatchCase, Probabilistic, Stats}; use stack_encrypt::plan::{Field, FieldSchema, Fields, PlanError}; use stack_encrypt::sem::{EqualityTerm, MatchConfig, MatchOptions, MatchQuery, MatchTerms}; use stack_encrypt::target::{Encrypted, IndexSpec}; @@ -721,3 +721,103 @@ async fn verify_is_a_token_test_not_a_substring_test() { "a word is a token, not a prefix" ); } + +/// EQL v4's match parameters, with a notes column's prior. +struct Notes; +impl MatchConfig for Notes { + fn options() -> MatchOptions { + MatchOptions { + k: 6, + m: 2048, + ..Default::default() + } + } + fn expected_tokens() -> Option { + Some(200) + } +} +impl MatchCase for Notes { + type Case = op::CaseInsensitive; +} + +#[tokio::test] +async fn target_limit_sizes_the_fetch_from_the_query_and_the_prior() { + let cipher = stack_cipher().await; + let plan = Plan::context("notes/body") + .with::(Match::::new()) + .build() + .unwrap(); + let query = |text: &'static str| { + cipher + .query(op::Matches, text) + .via(Match::::new()) + .on(&plan) + }; + let stats = Stats::declared(1_000_000); + + let short = query("ali").await.unwrap(); + let long = query("alice nguyen").await.unwrap(); + let (Limit::Rows(short_rows), Limit::Rows(long_rows)) = ( + short.target_limit(10, &stats), + long.target_limit(10, &stats), + ) else { + panic!("a declared prior gives a finite fetch"); + }; + assert!( + short_rows > 1_000, + "three letters against long text match thousands of rows by chance: {short_rows}" + ); + assert!(long_rows < 20, "a long query sets enough bits: {long_rows}"); + assert!(long_rows >= 10, "never fewer than `want`"); + + // A tighter confidence asks for more. + let strict = short.target_limit(10, &stats.clone().with_confidence(0.9999)); + assert!(strict.rows().unwrap() > short_rows); +} + +#[tokio::test] +async fn target_limit_without_a_prior_needs_a_measured_column() { + let cipher = stack_cipher().await; + let plan = people_plan(); + let email_plan = plan.field("email").unwrap(); + let query = cipher + .query(op::Matches, "bob") + .on(&email_plan) + .await + .unwrap(); + + // The default configuration declares no prior. + assert_eq!( + query.target_limit(10, &Stats::declared(1_000)), + Limit::Unbounded + ); + + // Measure the column: the set-bit count of each stored filter. + let mut histogram = std::collections::BTreeMap::::new(); + for email in [ + "bob@example.com", + "alice@example.org", + "carol.smith@example.net", + ] { + let mut row = cipher + .encrypt(&Person { + email: email.into(), + name: "x".repeat(4), + }) + .using(&plan) + .await + .unwrap(); + let stored: Encrypted<(EqualityTerm, MatchTerms)> = row.take("email").unwrap(); + *histogram + .entry(stored.terms.1.positions().len() as u32) + .or_default() += 1; + } + let stats = Stats::measured(3, histogram); + let Limit::Rows(rows) = query.target_limit(1, &stats) else { + panic!("a measured column gives a finite fetch"); + }; + assert!( + (1..=4).contains(&rows), + "1 wanted, at most every row: {rows}" + ); +} diff --git a/packages/stack-encrypt/tests/ui/op_inexact_operator_needs_probabilistic.stderr b/packages/stack-encrypt/tests/ui/op_inexact_operator_needs_probabilistic.stderr index ec190c313..ceaa770be 100644 --- a/packages/stack-encrypt/tests/ui/op_inexact_operator_needs_probabilistic.stderr +++ b/packages/stack-encrypt/tests/ui/op_inexact_operator_needs_probabilistic.stderr @@ -13,7 +13,7 @@ help: the trait `Probabilistic` is implemented for `MatchQuery` | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ = note: required for `EqualityTerm` to implement `TermFor` note: required by a bound in `stack_encrypt::op::Supports::Query` - --> src/op.rs + --> src/op/mod.rs | | type Query: TermFor; | ^^^^^^^^^^^ required by this bound in `Supports::Query` diff --git a/packages/stack-encrypt/tests/ui/op_target_does_not_answer.stderr b/packages/stack-encrypt/tests/ui/op_target_does_not_answer.stderr index 8fe0edf23..fa0080642 100644 --- a/packages/stack-encrypt/tests/ui/op_target_does_not_answer.stderr +++ b/packages/stack-encrypt/tests/ui/op_target_does_not_answer.stderr @@ -7,7 +7,7 @@ error[E0277]: `EqualityTerm` does not answer `stack_encrypt::op::Matches` over ` = note: a target answers the operators its type implements `Answers` for; `.via(index)` names an index the target declares help: the trait `Answers` is not implemented for `EqualityTerm` but trait `Answers` is implemented for it - --> src/op.rs + --> src/op/mod.rs | | impl Answers for EqualityTerm { | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From 13e250a89377a726543f5e1c1ab4c39c72a6c816 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Tue, 13 Oct 2026 23:06:25 +1100 Subject: [PATCH 2/6] fix(stack-encrypt): address the review of target_limit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - The Poisson quantile is a bounded, normalised sum over `[λ − 10√λ, λ + 12√λ + 50]`. Each term is a log relative to the window's first, so a large `λ` no longer loses the starting term to cancellation, and the sums run from the top, where the terms are smallest, so a confidence near 1 is compared against a tail rounding did not swallow. It matches an exact reference to within one up to 1 − 1e-12 (was off by thousands at λ ≈ 4.75e6), and no mutation of it can loop forever. `ln_factorial` is gone. - A `Limit` never exceeds `op::MAX_ROWS` (`i64::MAX`), the largest SQL `LIMIT`; `reltuples` of −1 read as a `u64` no longer overflows a bind. The ADR's statements use `greatest(reltuples, 0)`. - Histogram counts are summed as floats, so huge counts cannot overflow. - The independence assumption is stated in ADR-0009 decision 5 and on `target_limit`: a row sharing tokens with the needle passes more often, so the confidence is nominal on natural text; `verify` still decides. - Docs: the stats SQL filters NULL values; `target_limit` says a fetch that returned the limit is a page only when fewer than `want` verified; `expected_tokens` counts distinct tokens; decision 4's code shows `target_limit` on `Probabilistic`. - Tests: exact references at high confidence and at λ = 1e7 (which kills the threshold mutant), the `MAX_ROWS` cap, float-summed counts, and the measured-column test bounds the fetch against the histogram's own λ. Claude-Session: https://claude.ai/code/session_01VYKAWbd361xDLmqBdLoUcB --- packages/stack-encrypt/CHANGELOG.md | 5 +- ...and-a-query-term-is-not-the-stored-term.md | 27 +++- packages/stack-encrypt/src/op/mod.rs | 12 +- packages/stack-encrypt/src/op/stats.rs | 140 ++++++++++++------ packages/stack-encrypt/src/sem/mod.rs | 4 +- packages/stack-encrypt/tests/op.rs | 18 ++- 6 files changed, 138 insertions(+), 68 deletions(-) diff --git a/packages/stack-encrypt/CHANGELOG.md b/packages/stack-encrypt/CHANGELOG.md index c4055d9d0..eafc00d40 100644 --- a/packages/stack-encrypt/CHANGELOG.md +++ b/packages/stack-encrypt/CHANGELOG.md @@ -52,7 +52,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `MatchConfig::expected_tokens()` prior; `op::Stats::measured(rows, histogram)` uses a histogram of the stored filters' set-bit counts. With neither, the limit is `Limit::Unbounded`, and the caller pages with a - cursor. + cursor. A limit never exceeds `op::MAX_ROWS` (`i64::MAX`), the largest + SQL `LIMIT`. The bound assumes a non-matching row shares no token with + the needle, so on natural text its confidence is nominal; `verify` still + decides every row. - **Errors now expose stable codes, help text, and structured details.** All error types implement `miette::Diagnostic` with codes such as diff --git a/packages/stack-encrypt/docs/adr/0009-operators-select-the-index-and-a-query-term-is-not-the-stored-term.md b/packages/stack-encrypt/docs/adr/0009-operators-select-the-index-and-a-query-term-is-not-the-stored-term.md index 0b9457132..33f58cb72 100644 --- a/packages/stack-encrypt/docs/adr/0009-operators-select-the-index-and-a-query-term-is-not-the-stored-term.md +++ b/packages/stack-encrypt/docs/adr/0009-operators-select-the-index-and-a-query-term-is-not-the-stored-term.md @@ -235,8 +235,7 @@ pub trait QueryTerm {} pub trait Probabilistic: QueryTerm { type Plaintext; fn verify(&self, value: &Self::Plaintext) -> bool; - // `target_limit(&self, want, &Stats) -> Limit` joins it with decision 5 - // (#1184). + fn target_limit(&self, want: u64, stats: &Stats) -> Limit; } pub trait Ranked: QueryTerm { @@ -337,6 +336,15 @@ positives); the mean fill, 0.25, raised to 6 predicts 245. The number the bound needs is therefore `p = mean(f_i^b)`, and the statistics the client holds must let it compute that for the `b` of each query. +The model assumes a non-matching row shares no token with the needle. The +same token sets the same bits in every filter, so a row holding part of the +needle has fewer bits left to chance: "police" already holds two of +"alice"'s three 3-grams, and passes at `f^6` rather than `f^18`. On natural +text the stated confidence is therefore nominal, and a measured histogram +cannot correct it, because it measures fill, not overlap. Correctness does +not depend on it: `verify` decides every row, and the candidate count says +whether a fetch was complete. + False positives are then `Poisson(λ = p·N)`, and the worst case that still satisfies the caller is exactly `want` true matches preceded by every false positive, so @@ -380,6 +388,7 @@ fills. One aggregate gives it, at most `m` buckets: ```sql SELECT cardinality(eql_v4.match_term(name)) AS bits, count(*) FROM users TABLESAMPLE SYSTEM (1) +WHERE name IS NOT NULL GROUP BY 1 ``` @@ -390,8 +399,10 @@ page sample, which is fine for a distribution and unsuited to an exact count. It is also a sample of pages, so on a table of a few thousand rows a 1% sample can select none; below that size the clause is dropped and the whole table is read, which costs nothing worth saving. `rows` comes from -`SELECT reltuples FROM pg_class WHERE oid = 'users'::regclass`, a catalog -read. An empty histogram falls back to the declared prior. +`SELECT greatest(reltuples, 0) FROM pg_class WHERE oid = 'users'::regclass`, +a catalog read; `reltuples` is −1 for a table that has never been vacuumed +or analysed, hence the `greatest`. A `Limit` never exceeds `i64::MAX`, the +largest `LIMIT` PostgreSQL takes. An empty histogram falls back to the declared prior. `target_limit` returns a `Limit`: `Rows(n)`, or `Unbounded` when the statistics say nothing about the stored filters (no prior and no @@ -748,11 +759,11 @@ let hits: Vec = cipher.open(rows).using(&users_plan).await? The `Stats` it used were read once, at startup, by the application: ```rust -let rows: i64 = sqlx::query_scalar("SELECT reltuples::bigint FROM pg_class WHERE oid = 'users'::regclass") +let rows: i64 = sqlx::query_scalar("SELECT greatest(reltuples, 0)::bigint FROM pg_class WHERE oid = 'users'::regclass") .fetch_one(&pool).await?; let histogram: Vec<(i32, i64)> = sqlx::query_as( "SELECT cardinality(eql_v4.match_term(name)) AS bits, count(*) - FROM users TABLESAMPLE SYSTEM (1) GROUP BY 1", + FROM users TABLESAMPLE SYSTEM (1) WHERE name IS NOT NULL GROUP BY 1", ) .fetch_all(&pool).await?; let stats = Stats::measured(rows as u64, histogram.into_iter().map(|(b, n)| (b as u32, n as u64))); @@ -1081,8 +1092,8 @@ application, or from the declared prior: ```ts const stats = Stats.measured( - (await pool.query("SELECT reltuples::bigint AS rows FROM pg_class WHERE oid = 'users'::regclass")).rows[0].rows, - (await pool.query('SELECT cardinality(eql_v4.match_term(name)) AS bits, count(*) FROM users TABLESAMPLE SYSTEM (1) GROUP BY 1')).rows, + (await pool.query("SELECT greatest(reltuples, 0)::bigint AS rows FROM pg_class WHERE oid = 'users'::regclass")).rows[0].rows, + (await pool.query('SELECT cardinality(eql_v4.match_term(name)) AS bits, count(*) FROM users TABLESAMPLE SYSTEM (1) WHERE name IS NOT NULL GROUP BY 1')).rows, ) const prior = Stats.declared(1_000_000) // uses users.name's expectedTokens ``` diff --git a/packages/stack-encrypt/src/op/mod.rs b/packages/stack-encrypt/src/op/mod.rs index 1cd92b2ec..028969c9f 100644 --- a/packages/stack-encrypt/src/op/mod.rs +++ b/packages/stack-encrypt/src/op/mod.rs @@ -58,7 +58,7 @@ use std::marker::PhantomData; use stack_kms::MaybeSend; pub(crate) mod stats; -pub use stats::{Limit, Stats, DEFAULT_CONFIDENCE}; +pub use stats::{Limit, Stats, DEFAULT_CONFIDENCE, MAX_ROWS}; use crate::sem::{DefaultMatch, EqualityTerm, MatchConfig, MatchQuery, MatchTerms, QueryTokens}; use crate::target::{Equality, Index, Match}; @@ -232,12 +232,14 @@ pub trait Probabilistic: QueryTerm { /// that many matches. /// /// A probabilistic bound, not a guarantee: at 99% confidence one query - /// in a hundred draws more false positives than it allows for. The - /// candidate count says which case a fetch is in. If the server + /// in a hundred draws more false positives than it allows for, and the + /// model assumes a non-matching row shares no token with the needle, + /// which natural text does not honour, so the confidence is nominal. + /// The candidate count says which case a fetch is in. If the server /// returned fewer rows than the limit, the predicate is exhausted, and /// fewer than `want` verified rows means fewer than `want` matches. If - /// it returned exactly the limit, the result is a page: carry on from - /// its last row with a cursor. + /// it returned exactly the limit and fewer than `want` verified, the + /// result is a page: carry on from its last row with a cursor. fn target_limit(&self, want: u64, stats: &Stats) -> Limit; } diff --git a/packages/stack-encrypt/src/op/stats.rs b/packages/stack-encrypt/src/op/stats.rs index edefda45d..ad33323d7 100644 --- a/packages/stack-encrypt/src/op/stats.rs +++ b/packages/stack-encrypt/src/op/stats.rs @@ -46,11 +46,16 @@ enum Fill { /// The confidence a [`Stats`] targets unless told otherwise. pub const DEFAULT_CONFIDENCE: f64 = 0.99; +/// The largest fetch a [`Limit`] names: PostgreSQL's `LIMIT` is a signed +/// 64-bit integer. +pub const MAX_ROWS: u64 = i64::MAX as u64; + impl Stats { /// Statistics from the index's declared prior /// ([`MatchConfig::expected_tokens`](crate::sem::MatchConfig::expected_tokens)) /// and an estimate of the rows the query scans, such as - /// `pg_class.reltuples`. + /// `greatest(pg_class.reltuples, 0)` (`reltuples` is −1 for a table + /// never analysed). /// /// The prior knows nothing about the spread of value lengths, so it /// underestimates a column of mixed lengths; measure the column @@ -74,6 +79,7 @@ impl Stats { /// ```sql /// SELECT cardinality(eql_v4.match_term(name)) AS bits, count(*) /// FROM users TABLESAMPLE SYSTEM (1) + /// WHERE name IS NOT NULL /// GROUP BY 1 /// ``` /// @@ -130,7 +136,8 @@ impl Stats { return Limit::Unbounded; } let lambda = p.clamp(0.0, 1.0) * self.rows as f64; - Limit::Rows(want.saturating_add(poisson_quantile(lambda, self.confidence))) + let rows = want.saturating_add(poisson_quantile(lambda, self.confidence)); + Limit::Rows(rows.min(MAX_ROWS)) } } @@ -138,8 +145,9 @@ impl Stats { /// `None` for an empty one. A count above `m` (a histogram from another /// configuration) is a full filter. fn mean_power(histogram: &[(u32, u64)], b: u32, m: u32) -> Option { - let rows: u64 = histogram.iter().map(|&(_, rows)| rows).sum(); - if rows == 0 || m == 0 { + // Summed as floats: counts from a caller need not fit a `u64` total. + let rows: f64 = histogram.iter().map(|&(_, rows)| rows as f64).sum(); + if rows == 0.0 || m == 0 { return None; } let total: f64 = histogram @@ -149,7 +157,7 @@ fn mean_power(histogram: &[(u32, u64)], b: u32, m: u32) -> Option { rows as f64 * fill.powi(b as i32) }) .sum(); - Some(total / rows as f64) + Some(total / rows) } /// The fill of one stored filter of `tokens` tokens, each setting `k` of @@ -158,7 +166,8 @@ pub(crate) fn expected_fill(tokens: u32, k: usize, m: u32) -> f64 { 1.0 - (-(k as f64) * f64::from(tokens) / f64::from(m)).exp() } -/// How many rows a probabilistic query should fetch. +/// How many rows a probabilistic query should fetch. Never more than +/// [`MAX_ROWS`], the largest `LIMIT` PostgreSQL takes. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Limit { /// Fetch this many rows: `want` and the false positives expected at the @@ -188,47 +197,58 @@ impl Limit { /// The smallest `q` with `P(X ≤ q) ≥ confidence` for `X ~ Poisson(λ)`. /// -/// Exact up to floating point: the probabilities are summed in log space -/// from where the mass begins, `λ − 10√λ` for a large `λ` (the mass below -/// it is under `1e-23`), so the work is about `12√λ` terms. Past `1e7` -/// expected false positives the query is hopeless anyway and the answer is -/// the conservative `λ + 8√λ`, which covers any confidence below +/// The probabilities are summed over the window `[λ − 10√λ, λ + 12√λ + 50]`, +/// which holds all but about `1e-23` of the mass, and normalised by the +/// window's total. Each term is kept as a log relative to the window's +/// first, so no term carries the cancellation of `−λ + k·ln λ − ln k!` at a +/// large `λ`, and the sums run from the small end, so a confidence close to +/// 1 compares against a tail summed without losing it to rounding. The +/// answer is exact to within one at any confidence below `1 − 1e-12`. The +/// work is about `22√λ` terms. +/// Past `1e7` expected false positives the query is hopeless anyway and the +/// answer is the conservative `λ + 8√λ`, which covers any confidence below /// `1 − 1e-15`. +/// +/// `λ ≤ 0` and NaN have a window of one term at 0, so the answer is 0. pub(crate) fn poisson_quantile(lambda: f64, confidence: f64) -> u64 { - if lambda <= 0.0 || lambda.is_nan() { - return 0; - } if lambda > 1e7 { return (lambda + 8.0 * lambda.sqrt()).ceil() as u64; } - // Saturates to 0 for a small `λ`, where the mass starts at 0. - let start = (lambda - 10.0 * lambda.sqrt()).floor().max(0.0) as u64; + let spread = lambda.sqrt(); + // A float-to-integer cast saturates, and maps NaN to 0. + let start = (lambda - 10.0 * spread).floor().max(0.0) as u64; + let end = (lambda + 12.0 * spread + 50.0).ceil() as u64; let ln_lambda = lambda.ln(); - let mut log_term = -lambda + start as f64 * ln_lambda - ln_factorial(start); - let mut cdf = 0.0; - let mut k = start; - loop { - cdf += log_term.exp(); - // Past the mode, a term this small adds nothing a float can hold: - // the remaining mass is below any confidence that is not 1. - if cdf >= confidence || (k as f64 > lambda && log_term < -50.0) { - return k; + let mut weights = Vec::with_capacity((end - start + 1) as usize); + let mut log_term = 0.0; + for k in start..=end { + if k > start { + log_term += ln_lambda - (k as f64).ln(); } - k += 1; - log_term += ln_lambda - (k as f64).ln(); + weights.push(log_term); } -} - -/// `ln(n!)`: summed exactly below 20, Stirling's series above, where it is -/// accurate to about `1e-12`. -fn ln_factorial(n: u64) -> f64 { - if n < 20 { - return (2..=n).map(|i| (i as f64).ln()).sum(); + let peak = weights.iter().copied().fold(f64::NEG_INFINITY, f64::max); + for weight in &mut weights { + *weight = (*weight - peak).exp(); + } + // Summed from the top down, where the terms are smallest, so the tail a + // high confidence compares against is not lost to rounding. + let total: f64 = weights.iter().rev().sum(); + let allowed = (1.0 - confidence) * total; + // Walking down from the top, `above` is the mass strictly above `k`. + // The answer is the lowest `k` whose mass above is within what the + // confidence allows. + let mut answer = end; + let mut above = 0.0; + for (offset, weight) in weights.iter().enumerate().rev() { + if above <= allowed { + answer = start + offset as u64; + above += weight; + } else { + break; + } } - let x = n as f64 + 1.0; - (x - 0.5) * x.ln() - x + 0.5 * (2.0 * std::f64::consts::PI).ln() + 1.0 / (12.0 * x) - - 1.0 / (360.0 * x.powi(3)) - + 1.0 / (1260.0 * x.powi(5)) + answer } #[cfg(test)] @@ -272,18 +292,29 @@ mod tests { #[test] fn a_confidence_at_one_still_ends() { - // A float CDF may never reach 1; the tail cut-off ends the sum. + // A float CDF may never reach 1; the window's end ends the sum. let q = poisson_quantile(5.0, 1.0); - assert!(q > 20, "{q}"); + assert!((20..=5 + 27 + 50).contains(&q), "{q}"); + assert_eq!(poisson_quantile(f64::NAN, 0.99), 0); } + /// High confidences over a large `λ`, where summing from `e^(−λ)` + /// loses precision: the window is normalised instead. References are + /// exact (mpmath's regularised incomplete gamma); the sum may land one + /// above, never below. #[test] - fn ln_factorial_is_continuous_where_the_series_takes_over() { - let exact: f64 = (2..=20u64).map(|i| (i as f64).ln()).sum(); - assert!((ln_factorial(20) - exact).abs() < 1e-10); - assert_eq!(ln_factorial(0), 0.0); - assert_eq!(ln_factorial(1), 0.0); - assert!((ln_factorial(5) - 120f64.ln()).abs() < 1e-12); + fn the_quantile_holds_at_high_confidence_and_large_lambda() { + for (lambda, confidence, exact) in [ + (3.4e6, 0.9999, 3_406_860), + (4_750_922.0, 1.0 - 1e-12, 4_766_263), + (1e7, 0.99, 10_007_357), + ] { + let q = poisson_quantile(lambda, confidence); + assert!( + q == exact || q == exact + 1, + "λ = {lambda}, confidence = {confidence}: {q}, exact {exact}" + ); + } } /// ADR-0009 decision 5's worked table: EQL v4 defaults (`k = 6`, @@ -372,10 +403,23 @@ mod tests { } #[test] - fn want_does_not_overflow() { + fn a_limit_fits_a_sql_limit() { assert_eq!( Stats::declared(10).limit(u64::MAX, Some(0.5)), - Limit::Rows(u64::MAX) + Limit::Rows(MAX_ROWS) + ); + // `reltuples` is −1 for a table never analysed; read as a `u64` it is + // the largest count there is. + assert_eq!( + Stats::declared(u64::MAX).limit(10, Some(0.5)), + Limit::Rows(MAX_ROWS) ); } + + #[test] + fn histogram_counts_do_not_overflow() { + let stats = Stats::measured(10, [(1024, u64::MAX), (2048, u64::MAX)]); + let p = stats.pass_rate(1, 2048, None).unwrap(); + assert!((p - 0.75).abs() < 1e-12, "{p}"); + } } diff --git a/packages/stack-encrypt/src/sem/mod.rs b/packages/stack-encrypt/src/sem/mod.rs index 8b1b65220..8068a73e9 100644 --- a/packages/stack-encrypt/src/sem/mod.rs +++ b/packages/stack-encrypt/src/sem/mod.rs @@ -472,8 +472,8 @@ impl MatchOptions { pub trait MatchConfig: Send + Sync + 'static { fn options() -> MatchOptions; - /// The typical number of tokens in a value this index stores: 20 for a - /// name column, 200 for a notes column. A prior, used to size a query's + /// The typical number of distinct tokens in a value this index stores: + /// 20 for a name column, 200 for a notes column. A prior, used to size a query's /// fetch ([`Stats::declared`](crate::op::Stats::declared)) before the /// column's filters have been measured. It is not part of the index's /// identity and changes no term. `None`, the default, declares no diff --git a/packages/stack-encrypt/tests/op.rs b/packages/stack-encrypt/tests/op.rs index f5f78e085..5f5d975f1 100644 --- a/packages/stack-encrypt/tests/op.rs +++ b/packages/stack-encrypt/tests/op.rs @@ -812,12 +812,22 @@ async fn target_limit_without_a_prior_needs_a_measured_column() { .entry(stored.terms.1.positions().len() as u32) .or_default() += 1; } - let stats = Stats::measured(3, histogram); - let Limit::Rows(rows) = query.target_limit(1, &stats) else { + // The pass rate the histogram gives this query, over a large table. + let bits = query.positions().len() as i32; + let rows_seen: u64 = histogram.values().sum(); + let p = histogram + .iter() + .map(|(&set, &n)| n as f64 * (f64::from(set) / 256.0).powi(bits)) + .sum::() + / rows_seen as f64; + let lambda = p * 1_000_000.0; + let stats = Stats::measured(1_000_000, histogram); + let Limit::Rows(rows) = query.target_limit(10, &stats) else { panic!("a measured column gives a finite fetch"); }; + let extra = (rows - 10) as f64; assert!( - (1..=4).contains(&rows), - "1 wanted, at most every row: {rows}" + extra >= lambda && extra <= lambda + 4.0 * lambda.sqrt() + 3.0, + "10 wanted, λ = {lambda}: {rows}" ); } From ff2a4c440db846fc926100bfab16f63096fefe49 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Tue, 13 Oct 2026 23:58:20 +1100 Subject: [PATCH 3/6] fix(stack-encrypt): bound the Poisson quantile's work, and pin its edges MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI's mutants gate found one survivor and three timeouts in `poisson_quantile`: - Shifting each log term by the window's peak never mattered: relative to the window's first term no term exceeds about e^100, so `-` → `+` there was equivalent. The shift is gone. - A mutated threshold or window bound sent the `MAX_ROWS` test's λ of about 9e18 into a window of billions of terms. The window is now `22√λ + 50` wide from its start, capped at 2^20 terms whatever the input, so no mutation can hang a test. - The window's end is the answer when the confidence allows no mass (above 1), now pinned by a test. A focused `cargo mutants` run over `op/stats.rs` leaves no survivor and no timeout. Claude-Session: https://claude.ai/code/session_01VYKAWbd361xDLmqBdLoUcB --- packages/stack-encrypt/src/op/stats.rs | 38 ++++++++++++++++---------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/packages/stack-encrypt/src/op/stats.rs b/packages/stack-encrypt/src/op/stats.rs index ad33323d7..dc2fa036a 100644 --- a/packages/stack-encrypt/src/op/stats.rs +++ b/packages/stack-encrypt/src/op/stats.rs @@ -197,10 +197,10 @@ impl Limit { /// The smallest `q` with `P(X ≤ q) ≥ confidence` for `X ~ Poisson(λ)`. /// -/// The probabilities are summed over the window `[λ − 10√λ, λ + 12√λ + 50]`, -/// which holds all but about `1e-23` of the mass, and normalised by the -/// window's total. Each term is kept as a log relative to the window's -/// first, so no term carries the cancellation of `−λ + k·ln λ − ln k!` at a +/// The probabilities are summed over a window from `λ − 10√λ` (or 0) that is +/// `22√λ + 50` wide, which holds all but about `1e-23` of the mass, and +/// normalised by the window's total. Each term is kept as a log relative to +/// the window's first, so no term carries the cancellation of `−λ + k·ln λ − ln k!` at a /// large `λ`, and the sums run from the small end, so a confidence close to /// 1 compares against a tail summed without losing it to rounding. The /// answer is exact to within one at any confidence below `1 − 1e-12`. The @@ -217,19 +217,21 @@ pub(crate) fn poisson_quantile(lambda: f64, confidence: f64) -> u64 { let spread = lambda.sqrt(); // A float-to-integer cast saturates, and maps NaN to 0. let start = (lambda - 10.0 * spread).floor().max(0.0) as u64; - let end = (lambda + 12.0 * spread + 50.0).ceil() as u64; + // `22√λ + 50` past `start` reaches `λ + 12√λ + 50`. Capped, so no input + // makes the window large: below `1e7` it is at most about 70,000 terms. + let width = ((22.0 * spread + 50.0).ceil() as u64).min(MAX_WINDOW); let ln_lambda = lambda.ln(); - let mut weights = Vec::with_capacity((end - start + 1) as usize); + // Each term is `e^log` with `log` relative to the window's first term. + // From `λ − 10√λ` to the mode that ratio is at most about `e^50`, and + // from 0 to the mode of a `λ` below 100 about `e^λ`, so no term + // overflows and none of the mass is lost to underflow. + let mut weights = Vec::with_capacity(width as usize + 1); let mut log_term = 0.0; - for k in start..=end { + for k in start..=start + width { if k > start { log_term += ln_lambda - (k as f64).ln(); } - weights.push(log_term); - } - let peak = weights.iter().copied().fold(f64::NEG_INFINITY, f64::max); - for weight in &mut weights { - *weight = (*weight - peak).exp(); + weights.push(log_term.exp()); } // Summed from the top down, where the terms are smallest, so the tail a // high confidence compares against is not lost to rounding. @@ -238,7 +240,7 @@ pub(crate) fn poisson_quantile(lambda: f64, confidence: f64) -> u64 { // Walking down from the top, `above` is the mass strictly above `k`. // The answer is the lowest `k` whose mass above is within what the // confidence allows. - let mut answer = end; + let mut answer = start + width; let mut above = 0.0; for (offset, weight) in weights.iter().enumerate().rev() { if above <= allowed { @@ -251,6 +253,10 @@ pub(crate) fn poisson_quantile(lambda: f64, confidence: f64) -> u64 { answer } +/// The most terms [`poisson_quantile`] sums: far more than any `λ` below +/// `1e7` needs, and a bound on the work whatever it is handed. +const MAX_WINDOW: u64 = 1 << 20; + #[cfg(test)] mod tests { use super::*; @@ -294,8 +300,12 @@ mod tests { fn a_confidence_at_one_still_ends() { // A float CDF may never reach 1; the window's end ends the sum. let q = poisson_quantile(5.0, 1.0); - assert!((20..=5 + 27 + 50).contains(&q), "{q}"); + // The window starts at 0 and is `⌈22√5 + 50⌉` wide. + assert!((20..=100).contains(&q), "{q}"); assert_eq!(poisson_quantile(f64::NAN, 0.99), 0); + // A confidence above 1 allows no mass at all: the window's end. + assert_eq!(poisson_quantile(5.0, 1.5), 100); + assert_eq!(poisson_quantile(7600.0, 1.5), 6728 + 1968); } /// High confidences over a large `λ`, where summing from `e^(−λ)` From 6454513eabcbbcd375dfcf74d5fcff7a4a85520b Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Wed, 14 Oct 2026 00:49:30 +1100 Subject: [PATCH 4/6] fix(stack-encrypt): loop the Poisson window by offset, under the cap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI's mutants gate found `start..=start + width` → `start * width` surviving: the loop's range sat outside the cap on the window's width, so the mutation ran for billions of terms instead of failing. The loop now runs over `0..=width`, the capped width itself, and derives each `k` from it. A focused `cargo mutants` run over `op/stats.rs`: 91 mutants, 87 caught, 4 unviable, none missed, no timeouts. Claude-Session: https://claude.ai/code/session_01VYKAWbd361xDLmqBdLoUcB --- packages/stack-encrypt/src/op/stats.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/stack-encrypt/src/op/stats.rs b/packages/stack-encrypt/src/op/stats.rs index dc2fa036a..dd670a5b7 100644 --- a/packages/stack-encrypt/src/op/stats.rs +++ b/packages/stack-encrypt/src/op/stats.rs @@ -227,9 +227,9 @@ pub(crate) fn poisson_quantile(lambda: f64, confidence: f64) -> u64 { // overflows and none of the mass is lost to underflow. let mut weights = Vec::with_capacity(width as usize + 1); let mut log_term = 0.0; - for k in start..=start + width { - if k > start { - log_term += ln_lambda - (k as f64).ln(); + for offset in 0..=width { + if offset > 0 { + log_term += ln_lambda - ((start + offset) as f64).ln(); } weights.push(log_term.exp()); } From 8eb09ee41b3adba7535690177c3b8fe3e16e6aae Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Wed, 14 Oct 2026 11:01:03 +1100 Subject: [PATCH 5/6] docs(stack-encrypt): the query, not the needle, in target_limit's prose Claude-Session: https://claude.ai/code/session_01VYKAWbd361xDLmqBdLoUcB --- packages/stack-encrypt/CHANGELOG.md | 2 +- ...elect-the-index-and-a-query-term-is-not-the-stored-term.md | 4 ++-- packages/stack-encrypt/src/op/mod.rs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/stack-encrypt/CHANGELOG.md b/packages/stack-encrypt/CHANGELOG.md index eafc00d40..6b69e1fdb 100644 --- a/packages/stack-encrypt/CHANGELOG.md +++ b/packages/stack-encrypt/CHANGELOG.md @@ -54,7 +54,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 neither, the limit is `Limit::Unbounded`, and the caller pages with a cursor. A limit never exceeds `op::MAX_ROWS` (`i64::MAX`), the largest SQL `LIMIT`. The bound assumes a non-matching row shares no token with - the needle, so on natural text its confidence is nominal; `verify` still + the query, so on natural text its confidence is nominal; `verify` still decides every row. - **Errors now expose stable codes, help text, and structured details.** diff --git a/packages/stack-encrypt/docs/adr/0009-operators-select-the-index-and-a-query-term-is-not-the-stored-term.md b/packages/stack-encrypt/docs/adr/0009-operators-select-the-index-and-a-query-term-is-not-the-stored-term.md index 33f58cb72..ff2dfea70 100644 --- a/packages/stack-encrypt/docs/adr/0009-operators-select-the-index-and-a-query-term-is-not-the-stored-term.md +++ b/packages/stack-encrypt/docs/adr/0009-operators-select-the-index-and-a-query-term-is-not-the-stored-term.md @@ -336,9 +336,9 @@ positives); the mean fill, 0.25, raised to 6 predicts 245. The number the bound needs is therefore `p = mean(f_i^b)`, and the statistics the client holds must let it compute that for the `b` of each query. -The model assumes a non-matching row shares no token with the needle. The +The model assumes a non-matching row shares no token with the query. The same token sets the same bits in every filter, so a row holding part of the -needle has fewer bits left to chance: "police" already holds two of +query has fewer bits left to chance: "police" already holds two of "alice"'s three 3-grams, and passes at `f^6` rather than `f^18`. On natural text the stated confidence is therefore nominal, and a measured histogram cannot correct it, because it measures fill, not overlap. Correctness does diff --git a/packages/stack-encrypt/src/op/mod.rs b/packages/stack-encrypt/src/op/mod.rs index 028969c9f..34e8dcfc9 100644 --- a/packages/stack-encrypt/src/op/mod.rs +++ b/packages/stack-encrypt/src/op/mod.rs @@ -233,7 +233,7 @@ pub trait Probabilistic: QueryTerm { /// /// A probabilistic bound, not a guarantee: at 99% confidence one query /// in a hundred draws more false positives than it allows for, and the - /// model assumes a non-matching row shares no token with the needle, + /// model assumes a non-matching row shares no token with the query, /// which natural text does not honour, so the confidence is nominal. /// The candidate count says which case a fetch is in. If the server /// returned fewer rows than the limit, the predicate is exhausted, and From 2ec53065ceb18a7133996fca01c957656b8e80c9 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Sat, 17 Oct 2026 12:18:14 +1100 Subject: [PATCH 6/6] fix(stack-encrypt): address the second review of target_limit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A `want` of 0 is `Limit::Rows(0)`. The quantile was added anyway, so a dense column turned a zero-row request into a large fetch. - `Limit::fetch_or(page)` cuts `page` to `MAX_ROWS`, as the type promises, so `Unbounded.fetch_or(u64::MAX)` cannot wrap an `i64` bind. - Past ten million expected false positives the `λ + 8√λ` bound covers a confidence up to `1 − 1e-15` and no more. A higher one is now `Limit::Unbounded`; `poisson_quantile` returns `None` for it. - The row-count recipe no longer clamps `reltuples` to 0. A table never analysed has an unknown count, not an empty one, and 0 gave a confident fetch of exactly `want`. The catalog read now returns no row then, and the docs say to build no `Stats` and page, or run `ANALYZE`. - `rows` counts the non-NULL rows, since the histogram's pass rate is over those: the recipe scales `reltuples` by `pg_stats.null_frac`. Charging NULL rows at the non-NULL pass rate over-fetched in proportion to the column's NULLs. The rustdoc, ADR-0009 decision 5 and its examples, and the CHANGELOG say all of this. --- packages/stack-encrypt/CHANGELOG.md | 7 +- ...and-a-query-term-is-not-the-stored-term.md | 55 ++++++-- packages/stack-encrypt/src/op/stats.rs | 126 ++++++++++++++---- 3 files changed, 145 insertions(+), 43 deletions(-) diff --git a/packages/stack-encrypt/CHANGELOG.md b/packages/stack-encrypt/CHANGELOG.md index 6b69e1fdb..c72a6a08b 100644 --- a/packages/stack-encrypt/CHANGELOG.md +++ b/packages/stack-encrypt/CHANGELOG.md @@ -52,8 +52,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `MatchConfig::expected_tokens()` prior; `op::Stats::measured(rows, histogram)` uses a histogram of the stored filters' set-bit counts. With neither, the limit is `Limit::Unbounded`, and the caller pages with a - cursor. A limit never exceeds `op::MAX_ROWS` (`i64::MAX`), the largest - SQL `LIMIT`. The bound assumes a non-matching row shares no token with + cursor. A `want` of 0 is `Limit::Rows(0)`. A limit, and `fetch_or`'s + page, never exceeds `op::MAX_ROWS` (`i64::MAX`), the largest SQL `LIMIT`. + `rows` counts the non-NULL rows; a table never analysed has an unknown + count, not 0, and gets no `Stats` (the docs give the catalog read). The + bound assumes a non-matching row shares no token with the query, so on natural text its confidence is nominal; `verify` still decides every row. diff --git a/packages/stack-encrypt/docs/adr/0009-operators-select-the-index-and-a-query-term-is-not-the-stored-term.md b/packages/stack-encrypt/docs/adr/0009-operators-select-the-index-and-a-query-term-is-not-the-stored-term.md index ff2dfea70..820eb9aad 100644 --- a/packages/stack-encrypt/docs/adr/0009-operators-select-the-index-and-a-query-term-is-not-the-stored-term.md +++ b/packages/stack-encrypt/docs/adr/0009-operators-select-the-index-and-a-query-term-is-not-the-stored-term.md @@ -398,18 +398,34 @@ query has. The sample bounds the cost on a large table; `TABLESAMPLE` is a page sample, which is fine for a distribution and unsuited to an exact count. It is also a sample of pages, so on a table of a few thousand rows a 1% sample can select none; below that size the clause is dropped and the whole -table is read, which costs nothing worth saving. `rows` comes from -`SELECT greatest(reltuples, 0) FROM pg_class WHERE oid = 'users'::regclass`, -a catalog read; `reltuples` is −1 for a table that has never been vacuumed -or analysed, hence the `greatest`. A `Limit` never exceeds `i64::MAX`, the -largest `LIMIT` PostgreSQL takes. An empty histogram falls back to the declared prior. +table is read, which costs nothing worth saving. `rows` counts the rows the +predicate can match, the non-NULL ones, since the histogram's pass rate is +over those and a NULL never passes. One catalog read gives it: + +```sql +SELECT (c.reltuples * (1 - coalesce(s.null_frac, 0)))::bigint +FROM pg_class c +LEFT JOIN pg_stats s + ON s.schemaname = 'public' AND s.tablename = 'users' AND s.attname = 'name' +WHERE c.oid = 'users'::regclass AND c.reltuples >= 0 +``` + +`reltuples` is −1 for a table that has never been vacuumed or analysed, +such as one just bulk-loaded. That is an unknown count, not an empty +table, so the statement returns no row and the application builds no +`Stats`: it pages with a cursor, as for `Unbounded`, or runs `ANALYZE`. +Clamping −1 to 0 would give λ = 0 and a confident fetch of exactly `want`. +A `Limit` never exceeds `i64::MAX`, the largest `LIMIT` PostgreSQL takes. +An empty histogram falls back to the declared prior. `target_limit` returns a `Limit`: `Rows(n)`, or `Unbounded` when the statistics say nothing about the stored filters (no prior and no -histogram) or the confidence is not below 1. `fetch_or(page)` reads either -as a fetch size, for a caller that pages anyway. Past ten million expected -false positives the quantile is the conservative `λ + 8√λ`, since the query -is hopeless at that point. +histogram) or the confidence is not below 1. A `want` of 0 is `Rows(0)`. +`fetch_or(page)` reads either as a fetch size, for a caller that pages +anyway, and cuts `page` to `i64::MAX`. Past ten million expected false +positives the quantile is the conservative `λ + 8√λ`, since the query is +hopeless at that point; it covers a confidence up to `1 − 1e-15`, and a +higher one there is `Unbounded`. Where the numbers live and who keeps them in step was asked on the review, so it is written down. They are read from the server by the application, at @@ -759,14 +775,23 @@ let hits: Vec = cipher.open(rows).using(&users_plan).await? The `Stats` it used were read once, at startup, by the application: ```rust -let rows: i64 = sqlx::query_scalar("SELECT greatest(reltuples, 0)::bigint FROM pg_class WHERE oid = 'users'::regclass") - .fetch_one(&pool).await?; +// No row while `users` has never been analysed: the count is unknown. +let rows: Option = sqlx::query_scalar( + "SELECT (c.reltuples * (1 - coalesce(s.null_frac, 0)))::bigint + FROM pg_class c + LEFT JOIN pg_stats s + ON s.schemaname = 'public' AND s.tablename = 'users' AND s.attname = 'name' + WHERE c.oid = 'users'::regclass AND c.reltuples >= 0", +) +.fetch_optional(&pool).await?; let histogram: Vec<(i32, i64)> = sqlx::query_as( "SELECT cardinality(eql_v4.match_term(name)) AS bits, count(*) FROM users TABLESAMPLE SYSTEM (1) WHERE name IS NOT NULL GROUP BY 1", ) .fetch_all(&pool).await?; -let stats = Stats::measured(rows as u64, histogram.into_iter().map(|(b, n)| (b as u32, n as u64))); +let stats = rows.map(|rows| { + Stats::measured(rows as u64, histogram.into_iter().map(|(b, n)| (b as u32, n as u64))) +}); ``` or, before anyone has measured, `Stats::declared(1_000_000)`, which uses the @@ -1091,8 +1116,10 @@ one helper that binds one parameter. application, or from the declared prior: ```ts -const stats = Stats.measured( - (await pool.query("SELECT greatest(reltuples, 0)::bigint AS rows FROM pg_class WHERE oid = 'users'::regclass")).rows[0].rows, +// ROWS_SQL is decision 5's rows statement: no row while the count is unknown +const [count] = (await pool.query(ROWS_SQL)).rows +const stats = count && Stats.measured( + count.rows, (await pool.query('SELECT cardinality(eql_v4.match_term(name)) AS bits, count(*) FROM users TABLESAMPLE SYSTEM (1) WHERE name IS NOT NULL GROUP BY 1')).rows, ) const prior = Stats.declared(1_000_000) // uses users.name's expectedTokens diff --git a/packages/stack-encrypt/src/op/stats.rs b/packages/stack-encrypt/src/op/stats.rs index dd670a5b7..e1c7a46e3 100644 --- a/packages/stack-encrypt/src/op/stats.rs +++ b/packages/stack-encrypt/src/op/stats.rs @@ -53,9 +53,25 @@ pub const MAX_ROWS: u64 = i64::MAX as u64; impl Stats { /// Statistics from the index's declared prior /// ([`MatchConfig::expected_tokens`](crate::sem::MatchConfig::expected_tokens)) - /// and an estimate of the rows the query scans, such as - /// `greatest(pg_class.reltuples, 0)` (`reltuples` is −1 for a table - /// never analysed). + /// and an estimate of the rows the query scans. + /// + /// `rows` counts the rows the predicate can match: those whose value is + /// not NULL, since a NULL stores no filter and never passes. One catalog + /// read gives it, and gives no row while the count is unknown: + /// + /// ```sql + /// SELECT (c.reltuples * (1 - coalesce(s.null_frac, 0)))::bigint + /// FROM pg_class c + /// LEFT JOIN pg_stats s + /// ON s.schemaname = 'public' AND s.tablename = 'users' AND s.attname = 'name' + /// WHERE c.oid = 'users'::regclass AND c.reltuples >= 0 + /// ``` + /// + /// `reltuples` is −1 for a table never vacuumed or analysed, such as one + /// just bulk-loaded. That is an unknown count, not an empty table: a + /// count of 0 would promise a fetch of exactly `want`. With no row, + /// build no `Stats`; page with a cursor ([`Limit::Unbounded`]) or run + /// `ANALYZE` first. /// /// The prior knows nothing about the spread of value lengths, so it /// underestimates a column of mixed lengths; measure the column @@ -71,7 +87,7 @@ impl Stats { /// Statistics from a measured histogram of the stored filters' set-bit /// counts, as `(bits, rows)` pairs, and an estimate of the rows the - /// query scans. + /// query scans, read as [`declared`](Self::declared) says. /// /// One aggregate over the column gives the histogram, with at most `m` /// buckets; a page sample bounds its cost on a large table: @@ -98,7 +114,9 @@ impl Stats { /// holds `want` real matches when the table has that many. Outside the /// open interval `(0, 1)` there is no finite fetch to promise, and /// [`target_limit`](super::Probabilistic::target_limit) answers - /// [`Limit::Unbounded`]. + /// [`Limit::Unbounded`]. So it does past ten million expected false + /// positives for a confidence above `1 − 1e-15`, which the conservative + /// bound used there cannot meet. pub fn with_confidence(mut self, confidence: f64) -> Self { self.confidence = confidence; self @@ -126,8 +144,12 @@ impl Stats { measured.or_else(|| declared.map(|fill| fill.powi(bits as i32))) } - /// The fetch for `want` real matches at pass rate `p`. + /// The fetch for `want` real matches at pass rate `p`. Wanting none + /// is a fetch of none, whatever is known of the table. pub(crate) fn limit(&self, want: u64, pass_rate: Option) -> Limit { + if want == 0 { + return Limit::Rows(0); + } let Some(p) = pass_rate else { return Limit::Unbounded; }; @@ -136,8 +158,10 @@ impl Stats { return Limit::Unbounded; } let lambda = p.clamp(0.0, 1.0) * self.rows as f64; - let rows = want.saturating_add(poisson_quantile(lambda, self.confidence)); - Limit::Rows(rows.min(MAX_ROWS)) + match poisson_quantile(lambda, self.confidence) { + Some(extra) => Limit::Rows(want.saturating_add(extra).min(MAX_ROWS)), + None => Limit::Unbounded, + } } } @@ -174,7 +198,8 @@ pub enum Limit { /// stated confidence. Rows(u64), /// No finite fetch can be promised: the statistics say nothing about - /// the stored filters, or the confidence is not below 1. Page through + /// the stored filters, or the confidence is not below 1 (or, past ten + /// million expected false positives, not below `1 − 1e-15`). Page through /// the candidates with a cursor until `want` rows verify. Unbounded, } @@ -189,9 +214,10 @@ impl Limit { } /// The rows to fetch, or `page` when no finite fetch can be promised, - /// for a caller that pages either way. + /// for a caller that pages either way. Never more than [`MAX_ROWS`]: + /// a larger `page` is cut to it. pub fn fetch_or(self, page: u64) -> u64 { - self.rows().unwrap_or(page) + self.rows().unwrap_or(page.min(MAX_ROWS)) } } @@ -206,13 +232,15 @@ impl Limit { /// answer is exact to within one at any confidence below `1 − 1e-12`. The /// work is about `22√λ` terms. /// Past `1e7` expected false positives the query is hopeless anyway and the -/// answer is the conservative `λ + 8√λ`, which covers any confidence below -/// `1 − 1e-15`. +/// answer is the conservative `λ + 8√λ`, which covers any confidence up to +/// [`FALLBACK_CONFIDENCE`]; above it there is no answer (`None`), since an +/// eight-sigma tail is about `6e-16`. /// /// `λ ≤ 0` and NaN have a window of one term at 0, so the answer is 0. -pub(crate) fn poisson_quantile(lambda: f64, confidence: f64) -> u64 { +pub(crate) fn poisson_quantile(lambda: f64, confidence: f64) -> Option { if lambda > 1e7 { - return (lambda + 8.0 * lambda.sqrt()).ceil() as u64; + return (confidence <= FALLBACK_CONFIDENCE) + .then(|| (lambda + 8.0 * lambda.sqrt()).ceil() as u64); } let spread = lambda.sqrt(); // A float-to-integer cast saturates, and maps NaN to 0. @@ -250,9 +278,13 @@ pub(crate) fn poisson_quantile(lambda: f64, confidence: f64) -> u64 { break; } } - answer + Some(answer) } +/// The highest confidence [`poisson_quantile`]'s bound past `1e7` meets: +/// `λ + 8√λ` leaves about `6e-16` of the mass above it. +const FALLBACK_CONFIDENCE: f64 = 1.0 - 1e-15; + /// The most terms [`poisson_quantile`] sums: far more than any `λ` below /// `1e7` needs, and a bound on the work whatever it is handed. const MAX_WINDOW: u64 = 1 << 20; @@ -280,32 +312,55 @@ mod tests { ] { assert_eq!( poisson_quantile(lambda, confidence), - expected, + Some(expected), "λ = {lambda}, confidence = {confidence}" ); } - assert_eq!(poisson_quantile(0.0, 0.99), 0); - assert_eq!(poisson_quantile(-1.0, 0.99), 0); + assert_eq!(poisson_quantile(0.0, 0.99), Some(0)); + assert_eq!(poisson_quantile(-1.0, 0.99), Some(0)); } #[test] fn past_ten_million_the_quantile_is_conservative() { let lambda = 4e7; - let q = poisson_quantile(lambda, 0.999_999); + let q = poisson_quantile(lambda, 0.999_999).unwrap(); assert!(q as f64 >= lambda + 4.9 * lambda.sqrt(), "{q}"); assert!(q as f64 <= lambda + 8.0 * lambda.sqrt() + 1.0, "{q}"); } + /// The bound past ten million covers an eight-sigma tail and no more: + /// a confidence it cannot meet has no finite fetch. + #[test] + fn past_ten_million_a_confidence_beyond_eight_sigma_is_unbounded() { + let at = (4e7 + 8.0 * 4e7f64.sqrt()).ceil() as u64; + assert_eq!(poisson_quantile(4e7, FALLBACK_CONFIDENCE), Some(at)); + let beyond = FALLBACK_CONFIDENCE.next_up(); + assert_eq!(poisson_quantile(4e7, beyond), None); + assert_eq!(poisson_quantile(4e7, 1.0f64.next_down()), None); + assert_eq!( + Stats::declared(u64::MAX) + .with_confidence(beyond) + .limit(10, Some(0.5)), + Limit::Unbounded + ); + assert_eq!( + Stats::declared(u64::MAX) + .with_confidence(FALLBACK_CONFIDENCE) + .limit(10, Some(0.5)), + Limit::Rows(MAX_ROWS) + ); + } + #[test] fn a_confidence_at_one_still_ends() { // A float CDF may never reach 1; the window's end ends the sum. - let q = poisson_quantile(5.0, 1.0); + let q = poisson_quantile(5.0, 1.0).unwrap(); // The window starts at 0 and is `⌈22√5 + 50⌉` wide. assert!((20..=100).contains(&q), "{q}"); - assert_eq!(poisson_quantile(f64::NAN, 0.99), 0); + assert_eq!(poisson_quantile(f64::NAN, 0.99), Some(0)); // A confidence above 1 allows no mass at all: the window's end. - assert_eq!(poisson_quantile(5.0, 1.5), 100); - assert_eq!(poisson_quantile(7600.0, 1.5), 6728 + 1968); + assert_eq!(poisson_quantile(5.0, 1.5), Some(100)); + assert_eq!(poisson_quantile(7600.0, 1.5), Some(6728 + 1968)); } /// High confidences over a large `λ`, where summing from `e^(−λ)` @@ -319,7 +374,7 @@ mod tests { (4_750_922.0, 1.0 - 1e-12, 4_766_263), (1e7, 0.99, 10_007_357), ] { - let q = poisson_quantile(lambda, confidence); + let q = poisson_quantile(lambda, confidence).unwrap(); assert!( q == exact || q == exact + 1, "λ = {lambda}, confidence = {confidence}: {q}, exact {exact}" @@ -410,6 +465,22 @@ mod tests { assert_eq!(Limit::Rows(12).fetch_or(50), 12); assert_eq!(Limit::Unbounded.rows(), None); assert_eq!(Limit::Unbounded.fetch_or(50), 50); + assert_eq!(Limit::Unbounded.fetch_or(MAX_ROWS), MAX_ROWS); + assert_eq!( + Limit::Unbounded.fetch_or(u64::MAX), + MAX_ROWS, + "a page is cut to a SQL LIMIT" + ); + } + + /// Wanting no rows is a fetch of none, even over a table whose every + /// row passes and whose filters nothing is known about. + #[test] + fn wanting_no_rows_fetches_none() { + let stats = Stats::declared(1_000_000); + assert_eq!(stats.limit(0, Some(1.0)), Limit::Rows(0)); + assert_eq!(stats.limit(0, None), Limit::Rows(0)); + assert_eq!(stats.limit(1, Some(1.0)).rows().map(|r| r > 1), Some(true)); } #[test] @@ -418,8 +489,9 @@ mod tests { Stats::declared(10).limit(u64::MAX, Some(0.5)), Limit::Rows(MAX_ROWS) ); - // `reltuples` is −1 for a table never analysed; read as a `u64` it is - // the largest count there is. + // The largest count there is: what `reltuples` −1 becomes if a + // caller casts it to a `u64`, rather than treating it as unknown as + // `Stats::declared` says. Still a valid `LIMIT`. assert_eq!( Stats::declared(u64::MAX).limit(10, Some(0.5)), Limit::Rows(MAX_ROWS)