From 7cdc438dd2ddd463e4d2b716511bb871942a49f4 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Fri, 16 Oct 2026 03:13:19 +1100 Subject: [PATCH 1/8] feat(stack-encrypt): let a binding query a plan field by operator, and verify and size a probabilistic query with no key A binding names a query by field, operator and input, as ADR-0009 has a Rust caller name it by operator type: dynamic::query_term reads the input as the field's declared type, finds the index that answers by the typed path's default table (OpSpec::answered_by), and derives the bytes the typed query chain derives. An operator the field does not answer is refused as a mistyped value is, naming the field. dynamic::verify and dynamic::target_limit run the typed MatchQuery's code under the field's own match options, and the catalog entry of each EQL type lists the operators the engine answers for it with their fidelity. tests/fixtures/query_operators.json records each term, a verified page and fetch limits, checked against the typed chain, for the Go SDK's tests. Part of #1188 --- .../eql-bindings/src/encryption/targets.rs | 5 + packages/stack-encrypt/CHANGELOG.md | 19 + packages/stack-encrypt/src/dynamic/mod.rs | 40 +- packages/stack-encrypt/src/dynamic/query.rs | 1825 +++++++++++++++++ packages/stack-encrypt/src/dynamic/record.rs | 2 +- packages/stack-encrypt/src/dynamic/target.rs | 56 +- packages/stack-encrypt/src/dynamic/term.rs | 4 +- packages/stack-encrypt/src/sem/mod.rs | 53 +- .../stack-encrypt/tests/fixtures/README.md | 52 + .../tests/fixtures/query_operators.json | 283 +++ .../stack-encrypt/tests/query_operators.rs | 354 ++++ .../ui/op_at_path_inherits_fidelity.stderr | 4 +- ...nexact_operator_needs_probabilistic.stderr | 4 +- .../tests/ui/op_json_not_answered.stderr | 10 +- .../tests/ui/op_order_on_text.stderr | 4 +- 15 files changed, 2681 insertions(+), 34 deletions(-) create mode 100644 packages/stack-encrypt/src/dynamic/query.rs create mode 100644 packages/stack-encrypt/tests/fixtures/query_operators.json create mode 100644 packages/stack-encrypt/tests/query_operators.rs diff --git a/packages/eql/crates/eql-bindings/src/encryption/targets.rs b/packages/eql/crates/eql-bindings/src/encryption/targets.rs index 703cf4a15..7654f077f 100644 --- a/packages/eql/crates/eql-bindings/src/encryption/targets.rs +++ b/packages/eql/crates/eql-bindings/src/encryption/targets.rs @@ -52,6 +52,11 @@ //! | `producible` | Whether the engine can produce this type today. [`encrypt`] and [`query`] refuse a type that is not, and a generator should too. | //! | `reason` | Why not, when `producible` is `false`; `null` when it is. | //! +//! The engine adds an eleventh key, `operators`, when it lists the table: +//! the operators it answers for the type, computed from the row by its own +//! rules (`stack_encrypt::dynamic::TargetDescriptor`), so this table holds +//! none of them. +//! //! The table has one row per stored domain in catalog order; query twins are //! not rows (each row names its own). Adding a field is a wire change for //! every reader of `se_targets`; renaming or removing one is a breaking one. diff --git a/packages/stack-encrypt/CHANGELOG.md b/packages/stack-encrypt/CHANGELOG.md index 078ddab80..593a5f8af 100644 --- a/packages/stack-encrypt/CHANGELOG.md +++ b/packages/stack-encrypt/CHANGELOG.md @@ -9,6 +9,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **A binding queries by operator** (`dynamic` feature; ADR-0009, ADR-0007 + amended 2026-10-16). `dynamic::query_term(keyset, &plan, field, op, + input, &resolver)` derives the query term for a field of a data plan, an + operator (`dynamic::OpSpec`, named `eq`, `lt`, `lte`, `gt`, `gte`, + `between` or `matches`) and its input, reading the input as the field's + declared type first. The index that answers is the typed path's default + (`OpSpec::answered_by`), and the term is the bytes the typed query chain + derives for the same query. A field that names an EQL type answers through + the type's query twin. An operator the field does not answer is refused + as a mistyped value is, `dynamic::Error::Source` naming the field with + `Reason::OperatorNotAnswered`; `check_query` refuses the same without a + cipher. `dynamic::verify` checks a page of decrypted values against a + probabilistic query and `dynamic::target_limit` sizes its fetch from the + term's bytes and `Stats` (`Stats::from_value` reads their wire form), by + the code the typed `MatchQuery` runs, with the field's own match options; + neither uses a key. The catalog entry of an EQL type + (`TargetDescriptor::to_value`) lists the operators the engine answers for + it, each with its fidelity (`dynamic::Fidelity`), arity and index + (`TargetDescriptor::operators`). - **A query names an operator, not an index.** `cipher.query(op::Eq, "bob@example.com").on(&plan)` and `cipher.query(op::Matches, "ali").on(&plan)`: the operator comes first, as diff --git a/packages/stack-encrypt/src/dynamic/mod.rs b/packages/stack-encrypt/src/dynamic/mod.rs index e68125f0e..fbaef53dc 100644 --- a/packages/stack-encrypt/src/dynamic/mod.rs +++ b/packages/stack-encrypt/src/dynamic/mod.rs @@ -37,6 +37,10 @@ //! is the build without them. //! * [`TermBytes`] — the term a field lowered from data derives, as its //! frozen bytes. Such a field's plaintext is an [`FfiValue`]. +//! * [`query_term`], [`verify`] and [`target_limit`] — a query named by +//! operator ([`OpSpec`]) on a plan field, and a probabilistic term's two +//! questions, over a page of decrypted values and over [`Stats`](crate::op::Stats) +//! (ADR-0009; ADR-0007, amended 2026-10-16). //! * [`Scope`] — which cipher an opening operation decrypts through. //! //! # What is not here @@ -75,6 +79,7 @@ mod context; mod ffi_value; mod kind; +mod query; pub mod record; mod target; mod term; @@ -94,6 +99,10 @@ pub use ffi_value::{ ValueKind, ValueMap, ValueSeq, ValueSink, }; pub use kind::{admits, read}; +pub use query::{ + check_query, limit_to_value, query_term, target_limit, verify, Fidelity, OpSpec, Operand, + QueryTerm, +}; pub use record::{FieldPlan, Output, Plan}; pub use target::{NoTargets, TargetDescriptor, TargetError, TargetResolver}; pub use term::{term, TermBytes}; @@ -240,8 +249,12 @@ pub enum Error { /// them), a field the plan does not name, a plan field the source does /// not carry or carries twice, or a passthrough or a repeated map key /// under a field the plan seals, or a value of another type than its - /// field declares. Also a query value that cannot be read as its field's - /// type ([`read`]). + /// field declares. Also a query its field does not answer: an operator + /// the field's type does not answer, an input that cannot be read as the + /// field's type ([`read`]) or is not the pair a range takes, a page of + /// values to verify that holds a value of another type, or statistics + /// or a term that are not what [`target_limit`] reads — the same shape of + /// refusal as a mistyped field, naming the field. #[error( "record source does not fit the plan: {reason}{}", in_field_text(field) @@ -444,8 +457,8 @@ macro_rules! reasons { } reasons! { - /// What was wrong with a context, plan, source or stored record: the reason - /// a dynamic input error carries beside the field it names. + /// What was wrong with a context, plan, source, stored record or query: the + /// reason a dynamic input error carries beside the field it names. /// /// One vocabulary for all four, since several reasons apply to more than /// one (a field given twice is a misfit in a source and in a stored @@ -544,6 +557,25 @@ reasons! { /// A stored `"passthrough"` or `"eql"` node is not a passthrough /// carrying a value of the kind it holds. NotPassthrough => "not_passthrough", "a node does not carry a value of the kind it holds"; + /// A plan takes its context from a field of each record, so a query on + /// it has no context: the caller names the context, and sends the plan + /// under it. + QueryContextField => "query_context_field", "the plan takes its context from a field of each record, so a query has none"; + /// An operator name is not one the engine knows ([`OpSpec::parse`]). + UnknownOperator => "unknown_operator", "the operator is not one the engine knows"; + /// The field's type does not answer the operator: no index it declares + /// answers it, or the EQL type it names does not ([`OpSpec::answered_by`]). + OperatorNotAnswered => "operator_not_answered", "the field's type does not answer the operator"; + /// A range's input is not a list of two values. + NotARange => "not_a_range", "a range is not a list of two values"; + /// The operator's answer is exact, so there is nothing to verify and no + /// fetch to size. + ExactOperator => "exact_operator", "the operator's answer is exact, so there is nothing to verify or size"; + /// The term is not one the field's index derives for the operator. + NotAQueryTerm => "not_a_query_term", "the term is not one the field's index derives for the operator"; + /// Statistics are not a row count with an optional histogram and + /// confidence ([`Stats::from_value`](crate::op::Stats::from_value)). + NotStats => "not_stats", "the statistics are not a row count with an optional histogram and confidence"; } } diff --git a/packages/stack-encrypt/src/dynamic/query.rs b/packages/stack-encrypt/src/dynamic/query.rs new file mode 100644 index 000000000..b2100281f --- /dev/null +++ b/packages/stack-encrypt/src/dynamic/query.rs @@ -0,0 +1,1825 @@ +//! Queries named by operator, for a binding. +//! +//! A Rust caller names an operator as a type, `cipher.query(op::Matches, +//! "ali").on(&plan)`, and the compiler finds the index that answers it +//! (ADR-0009). A binding has no type to name: its plan arrives as data and +//! its operator as a string. This module is the runtime form of that query, +//! written once here rather than once per language binding (ADR-0007, +//! amended 2026-10-16): +//! +//! - [`OpSpec`] names an operator, as [`IndexSpec`] names an index, and +//! [`OpSpec::answered_by`] is the table the typed path's `DefaultIndex` +//! impls state: which declared index answers the operator for a field of +//! a kind, if any does. +//! - [`query_term`] derives the query term for a plan field, an operator and +//! its input: the field's own index term for the input, or, for a field +//! that names an EQL type, that type's query value. It refuses an operator +//! the field does not answer as it refuses a value of the wrong type, with +//! [`Error::Source`] naming the field. +//! - [`verify`] and [`target_limit`] are a probabilistic term's two +//! questions, [`Probabilistic::verify`] and +//! [`Probabilistic::target_limit`], over a page of decrypted values and +//! over [`Stats`]. They run the same code the typed `MatchQuery` runs, with +//! the match index's options read from the plan, and need no key. +//! +//! An exact term has neither question: [`verify`] and [`target_limit`] +//! refuse an exact operator ([`Reason::ExactOperator`]) rather than answer +//! `true` for every row, as the typed path does not compile the call. +//! +//! # Which operators a field answers +//! +//! A field with indexes answers what [`OpSpec::answered_by`] says for its +//! declared kind: `eq` through an equality index, `matches` through a match +//! index over text, and the orderings (`lt`, `lte`, `gt`, `gte`, +//! `between`) through an ORE index over an integer, or an OPE index when +//! the field declares no ORE. Those are the typed path's defaults: an +//! ordering over text, bytes or a float is not answered, as `op::Ordered` +//! covers the integers only. +//! +//! A field that names an EQL type answers what its +//! [`TargetDescriptor::operators`] lists: the exact operators of a single +//! value whose index the type carries, for a type the engine produces and +//! that has a query twin. The term is the twin, which carries every term of +//! the type; one query domain per operator is EQL v4's (ADR-0009, decision +//! 8). A probabilistic operator is not answered through a target yet: the +//! descriptor names the type's match index but not its options, and +//! `verify` must cut tokens as the stored filter did. +//! +//! # Wire format +//! +//! An operator is its [`name`](OpSpec::name). A query term is the object +//! [`QueryTerm::to_value`] writes, the statistics the object +//! [`Stats::from_value`] reads, and a limit the value +//! [`limit_to_value`] writes; each is documented where it is written. +//! +//! [`Probabilistic::verify`]: crate::op::Probabilistic::verify +//! [`Probabilistic::target_limit`]: crate::op::Probabilistic::target_limit + +use std::cmp::Ordering; +use std::fmt; + +use stack_kms::DataKeySource; + +use super::record::{FieldPlan, Plan}; +use super::term::{derive, Use}; +use super::ValueKind; +use super::{admits, read, utf8, Error, FfiValue, Reason, TargetDescriptor, TargetResolver}; +use crate::op::{Limit, Stats}; +use crate::sem::{match_limit, MatchOptions, QueryTokens, TermError}; +use crate::target::IndexSpec; +use crate::{KeysetCipher, Pending}; + +/// An operator, named at run time: the runtime form of the +/// [`op`](crate::op) module's operator types, as [`IndexSpec`] is of the +/// index types. +/// +/// Not `#[non_exhaustive]`, for the reason the [module docs](super#stability) +/// give for [`IndexSpec`]: the names are wire format, and a new operator is +/// one every binding has to be taught. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum OpSpec { + /// `"eq"`: [`op::Eq`](struct@crate::op::Eq). + Eq, + /// `"lt"`: [`op::Lt`](crate::op::Lt). + Lt, + /// `"lte"`: [`op::Lte`](crate::op::Lte). + Lte, + /// `"gt"`: [`op::Gt`](crate::op::Gt). + Gt, + /// `"gte"`: [`op::Gte`](crate::op::Gte). + Gte, + /// `"between"`: [`op::Between`](crate::op::Between), whose input is a + /// pair of bounds, low then high, inclusive. + Between, + /// `"matches"`: [`op::Matches`](struct@crate::op::Matches), in the case + /// the field's match index fixes. + Matches, +} + +impl OpSpec { + /// Every operator, in declaration order: the order a catalog lists them + /// in. + pub const ALL: [OpSpec; 7] = [ + OpSpec::Eq, + OpSpec::Lt, + OpSpec::Lte, + OpSpec::Gt, + OpSpec::Gte, + OpSpec::Between, + OpSpec::Matches, + ]; + + /// The operator a name names, or `None` for a name that is not one. + /// Names match exactly: `"eq"`, not `"Eq"` or `"="`. + pub fn parse(name: &str) -> Option { + Self::ALL.into_iter().find(|op| op.name() == name) + } + + /// The operator's wire name. + pub fn name(self) -> &'static str { + match self { + OpSpec::Eq => "eq", + OpSpec::Lt => "lt", + OpSpec::Lte => "lte", + OpSpec::Gt => "gt", + OpSpec::Gte => "gte", + OpSpec::Between => "between", + OpSpec::Matches => "matches", + } + } + + /// How far the server's answer to this operator can be trusted: what + /// the typed path says by the trait the term implements. + pub fn fidelity(self) -> Fidelity { + match self { + OpSpec::Matches => Fidelity::Probabilistic, + _ => Fidelity::Exact, + } + } + + /// Whether the operator takes a pair of values, low then high, rather + /// than one: `between` does. + pub fn takes_pair(self) -> bool { + self == OpSpec::Between + } + + /// The index among `indexes` that answers this operator for a field of + /// `kind`, or `None` when none does: the typed path's `DefaultIndex` + /// table, stated over a kind and the indexes a field declares. + /// + /// ``` + /// use stack_encrypt::dynamic::{OpSpec, ValueKind}; + /// use stack_encrypt::target::IndexSpec; + /// + /// let both = [IndexSpec::Ope, IndexSpec::Ore]; + /// assert_eq!(OpSpec::Lt.answered_by(ValueKind::UInt32, &both), Some(&IndexSpec::Ore)); + /// assert_eq!(OpSpec::Lt.answered_by(ValueKind::String, &both), None); + /// assert_eq!(OpSpec::Eq.answered_by(ValueKind::UInt32, &both), None); + /// ``` + pub fn answered_by(self, kind: ValueKind, indexes: &[IndexSpec]) -> Option<&IndexSpec> { + let declared = |wanted: fn(&IndexSpec) -> bool| indexes.iter().find(|index| wanted(index)); + match self { + OpSpec::Eq => declared(|index| matches!(index, IndexSpec::Equality)), + OpSpec::Matches => declared(|index| matches!(index, IndexSpec::Match(_))), + OpSpec::Lt | OpSpec::Lte | OpSpec::Gt | OpSpec::Gte | OpSpec::Between => { + if !ordered(kind) { + return None; + } + declared(|index| matches!(index, IndexSpec::Ore)) + .or_else(|| declared(|index| matches!(index, IndexSpec::Ope))) + } + } + .filter(|index| admits(kind, index)) + } +} + +impl fmt::Display for OpSpec { + /// Writes [`name`](Self::name). + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.name()) + } +} + +/// The kinds the ordering operators are answered over: the integers, as +/// [`op::Ordered`](crate::op::Ordered) covers them in the typed path. +fn ordered(kind: ValueKind) -> bool { + matches!( + kind, + ValueKind::Int32 | ValueKind::Int64 | ValueKind::UInt32 | ValueKind::UInt64 + ) +} + +/// How far the server's answer to an operator can be trusted (ADR-0009, +/// decision 4), as a catalog reports it. +/// +/// The typed path says this with traits, not a value: a term that +/// implements [`Probabilistic`](crate::op::Probabilistic) can be verified, +/// and one that does not cannot. A binding's generator asks before the +/// program runs, so it needs the answer as data. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Fidelity { + /// `"exact"`: the server's answer is the answer. + Exact, + /// `"probabilistic"`: the answer can hold false positives and no false + /// negatives; [`verify`] decides each row, and [`target_limit`] sizes + /// the fetch. + Probabilistic, +} + +impl Fidelity { + /// The fidelity's wire name. + pub fn name(self) -> &'static str { + match self { + Fidelity::Exact => "exact", + Fidelity::Probabilistic => "probabilistic", + } + } +} + +/// The operators an EQL type answers, as [`TargetDescriptor::operators`] +/// lists them: none for a type the engine does not produce or that has no +/// query twin, and otherwise every exact operator of one value that one of +/// the type's indexes answers for its plaintext kind. See the +/// [module docs](self#which-operators-a-field-answers) for why a +/// probabilistic operator, and `between`, wait. +pub(super) fn target_operators(target: &TargetDescriptor) -> Vec<(OpSpec, IndexSpec)> { + let Some(kind) = target + .plaintext + .filter(|_| target.producible && target.query.is_some()) + else { + return Vec::new(); + }; + let indexes: Vec = target + .indexes + .iter() + .filter_map(|key| IndexSpec::parse(key)) + .collect(); + OpSpec::ALL + .into_iter() + .filter(|op| op.fidelity() == Fidelity::Exact && !op.takes_pair()) + .filter_map(|op| { + op.answered_by(kind, &indexes) + .map(|index| (op, index.clone())) + }) + .collect() +} + +/// A query term a binding asked for by operator: the operand to bind in +/// SQL, and what it is. +/// +/// The bytes are not plaintext: an index term, or an EQL query value, which +/// is what the server compares. The query's own tokens, which +/// [`verify`] needs, are plaintext and are not here; [`verify`] cuts them +/// again from the query's input. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct QueryTerm { + operator: OpSpec, + index: IndexSpec, + domain: Option, + operand: Operand, +} + +/// What a [`QueryTerm`] binds. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Operand { + /// One term: an index term's frozen bytes ([`TermBytes`](super::TermBytes)), or an EQL + /// query value's JSON bytes. + One(Vec), + /// A range's two terms, each the index term of one bound, inclusive. + Range { + /// The low bound's term. + lo: Vec, + /// The high bound's term. + hi: Vec, + }, +} + +impl QueryTerm { + /// The operator the term answers. + pub fn operator(&self) -> OpSpec { + self.operator + } + + /// How far the server's answer to it can be trusted. + pub fn fidelity(&self) -> Fidelity { + self.operator.fidelity() + } + + /// The index that answers the operator for the field. + pub fn index(&self) -> &IndexSpec { + &self.index + } + + /// The PostgreSQL domain the operand is cast to, for a field that names + /// an EQL type; `None` for a field with indexes, whose terms are bytes. + pub fn domain(&self) -> Option<&str> { + self.domain.as_deref() + } + + /// What the term binds. + pub fn operand(&self) -> &Operand { + &self.operand + } + + /// The term in its wire form: an object with every key present, an + /// absent value null. + /// + /// | key | value | + /// |---|---| + /// | `operator` | string: the [operator's name](OpSpec::name), `"matches"` | + /// | `fidelity` | string: `"exact"` or `"probabilistic"` ([`Fidelity::name`]) | + /// | `index` | string: the [key](IndexSpec::key) of the index that answers, `"match"` | + /// | `domain` | string or null: the query value's PostgreSQL domain, for a field that names an EQL type | + /// | `term` | bytes: the term, or for `between` a list of two byte strings, the low bound's term then the high's | + pub fn to_value(&self) -> FfiValue { + let bytes = |b: &[u8]| FfiValue::Bytes(vitaminc_protected::Protected::new(b.to_vec())); + let term = match &self.operand { + Operand::One(term) => bytes(term), + Operand::Range { lo, hi } => FfiValue::Array(vec![bytes(lo), bytes(hi)]), + }; + FfiValue::Object(vec![ + ( + "operator".to_owned(), + FfiValue::String(self.operator.name().into()), + ), + ( + "fidelity".to_owned(), + FfiValue::String(self.fidelity().name().into()), + ), + ( + "index".to_owned(), + FfiValue::String(self.index.key().into()), + ), + ( + "domain".to_owned(), + self.domain + .as_deref() + .map_or(FfiValue::Null, |domain| FfiValue::String(domain.into())), + ), + ("term".to_owned(), term), + ]) + } +} + +/// The field a query is about, and the index that answers its operator. +struct Answer<'p> { + field: &'p FieldPlan, + kind: ValueKind, + index: IndexSpec, + target: Option, +} + +/// The input of a query, read as its field's kind. +enum Input { + One(FfiValue), + Pair(FfiValue, FfiValue), +} + +/// Find `field` in `plan` and the index that answers `operator` for it. +/// +/// # Errors +/// +/// [`Error::Plan`] for a field the plan does not have +/// ([`Reason::NoSuchField`]), [`Error::Target`] for a target the resolver +/// refuses, and [`Error::Source`] +/// ([`Reason::OperatorNotAnswered`]) for an operator the field does not +/// answer. +fn answer<'p>( + plan: &'p Plan, + field: &str, + operator: OpSpec, + resolver: &(impl TargetResolver + ?Sized), +) -> Result, Error> { + let found = plan + .fields() + .iter() + .find(|candidate| candidate.name() == field) + .ok_or_else(|| Error::Plan { + field: Some(field.to_owned()), + reason: Reason::NoSuchField, + })?; + let not_answered = || Error::Source { + field: Some(field.to_owned()), + reason: Reason::OperatorNotAnswered, + }; + let kind = found.field_type().ok_or_else(not_answered)?; + let (index, target) = match found.target() { + Some(name) => { + let target = resolver.resolve(name).map_err(Error::Target)?; + let index = target_operators(&target) + .into_iter() + .find_map(|(op, index)| (op == operator).then_some(index)); + (index, Some(target)) + } + None => (operator.answered_by(kind, &found.indexes()).cloned(), None), + }; + Ok(Answer { + field: found, + kind, + index: index.ok_or_else(not_answered)?, + target, + }) +} + +/// Refuse a plan whose context is a field of each record: a term derives +/// under a context, and such a plan has none until a record names it, so a +/// binding sends the plan under the context its query names. +/// [`verify`] and [`target_limit`] derive nothing and take such a plan. +fn has_context(plan: &Plan, field: &str) -> Result<(), Error> { + match plan.context_field() { + Some(_) => Err(Error::Plan { + field: Some(field.to_owned()), + reason: Reason::QueryContextField, + }), + None => Ok(()), + } +} + +/// Read a query's input as the field's kind ([`read`]): one value, or for +/// `between` a list of two, low then high. +/// +/// # Errors +/// +/// [`Error::Source`] naming the field: [`Reason::FieldType`] for a value +/// that cannot be read as the kind, [`Reason::NotARange`] for a `between` +/// input that is not a list of two values. +fn read_input(answer: &Answer<'_>, operator: OpSpec, input: FfiValue) -> Result { + let name = answer.field.name(); + let one = |value| read(answer.kind, value).map_err(|error| error.in_field(name)); + if !operator.takes_pair() { + return one(input).map(Input::One); + } + match input { + FfiValue::Array(bounds) => match <[FfiValue; 2]>::try_from(bounds) { + Ok([lo, hi]) => Ok(Input::Pair(one(lo)?, one(hi)?)), + Err(_) => Err(not_a_range(name)), + }, + _ => Err(not_a_range(name)), + } +} + +fn not_a_range(field: &str) -> Error { + Error::Source { + field: Some(field.to_owned()), + reason: Reason::NotARange, + } +} + +/// The refusals the typed path makes when a query is awaited, before any +/// term exists ([`Supports::check`](crate::op::Supports::check)): a range +/// whose low bound is above its high bound, and match text the index +/// cannot normalize or that cuts no token. The match index's own +/// derivation refuses the last two again; asking here lets a binding +/// refuse at its boundary. +fn check_input(index: &IndexSpec, input: &Input) -> Result<(), Error> { + let refused = |error: TermError| Error::Cipher(error.into()); + match (index, input) { + (_, Input::Pair(lo, hi)) => { + if integer_order(lo, hi) == Some(Ordering::Greater) { + return Err(refused(TermError::InvertedRange)); + } + } + (IndexSpec::Match(options), Input::One(value)) => { + query_tokens(value, options).map(drop)?; + } + _ => {} + } + Ok(()) +} + +/// The order of two integers of one kind, or `None` for anything else. +fn integer_order(lo: &FfiValue, hi: &FfiValue) -> Option { + match (lo, hi) { + (FfiValue::Int32(lo), FfiValue::Int32(hi)) => Some(lo.cmp(hi)), + (FfiValue::Int64(lo), FfiValue::Int64(hi)) => Some(lo.cmp(hi)), + (FfiValue::UInt32(lo), FfiValue::UInt32(hi)) => Some(lo.cmp(hi)), + (FfiValue::UInt64(lo), FfiValue::UInt64(hi)) => Some(lo.cmp(hi)), + _ => None, + } +} + +/// The tokens a match query of `value` keeps, under the index's options. +/// +/// # Errors +/// +/// [`Error::Cipher`] with [`TermError::Normalize`] for text the index cannot +/// normalize, and [`TermError::EmptyTermText`] for text that cuts no token, +/// whose filter would match every row. [`Error::Internal`] for a value that +/// is not text, which [`read_input`] has ruled out. +fn query_tokens(value: &FfiValue, options: &MatchOptions) -> Result { + let FfiValue::String(text) = value else { + return Err(Error::Internal); + }; + let text = utf8(text).ok_or(Error::Internal)?; + // The query chain's rule, from the same helper. + QueryTokens::of_query(text, options).map_err(|error| Error::Cipher(error.into())) +} + +/// Check a query without a cipher: everything [`query_term`] refuses before +/// it derives anything. A binding calls this at its boundary, so a bad +/// query is refused before a keyset is resolved. +/// +/// # Errors +/// +/// As [`query_term`]. +pub fn check_query( + plan: &Plan, + field: &str, + operator: OpSpec, + input: FfiValue, + resolver: &(impl TargetResolver + ?Sized), +) -> Result<(), Error> { + let answer = answer(plan, field, operator, resolver)?; + has_context(plan, field)?; + let input = read_input(&answer, operator, input)?; + check_input(&answer.index, &input) +} + +/// Derive the query term for one field of a plan, an operator and its +/// input: one value for most operators, a list of two for +/// [`between`](OpSpec::Between). +/// +/// The input is read as the field's declared kind first ([`read`]), so a +/// host's `34` as a float queries a `uint64` field as the `u64` it stored. +/// A field with indexes derives the term of the index that answers the +/// operator ([`OpSpec::answered_by`]) under the field's context, by the +/// derivation its stored terms came from; a range derives one term per +/// bound under that one context. A field that names an EQL type runs the +/// type's query twin through `resolver`. Nothing is requested: the pending +/// settles without I/O, as a typed query's does. +/// +/// # Examples +/// +/// ``` +/// use stack_encrypt::dynamic::{query_term, record, FfiValue, NoTargets, OpSpec, Operand}; +/// use stack_encrypt::kms::FakeDataKeySource; +/// use stack_encrypt::{op, Ore, Plan, StackCipher}; +/// +/// # tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap().block_on(async { +/// let cipher = StackCipher::builder().kms(FakeDataKeySource::new()).init().await?; +/// let keyset = cipher.default_keyset(); +/// let text = |s: &str| FfiValue::String(s.into()); +/// let plan = record::plan(FfiValue::Object(vec![( +/// "age".into(), +/// FfiValue::Object(vec![ +/// ("context".into(), FfiValue::Array(vec![text("users"), text("age")])), +/// ("outputs".into(), FfiValue::Array(vec![text("c"), text("ore")])), +/// ("type".into(), text("uint32")), +/// ]), +/// )]))?; +/// +/// let term = query_term(&keyset, &plan, "age", OpSpec::Lt, FfiValue::UInt32(40), &NoTargets)?.await?; +/// +/// // The typed path's term for the same query. +/// let typed_plan = Plan::context("users/age").with::(Ore).build()?; +/// let typed = cipher.query(op::Lt, &40u32).on(&typed_plan).await?; +/// assert_eq!(term.operand(), &Operand::One(typed.to_bytes())); +/// # Ok::<(), Box>(()) +/// # }).unwrap(); +/// ``` +/// +/// # Errors +/// +/// [`Error::Plan`] for a field the plan does not have +/// ([`Reason::NoSuchField`]), or a plan that takes its context from a field +/// of each record ([`Reason::QueryContextField`]): a query names the +/// context such a plan's records hold, so a binding sends the plan under +/// that context. [`Error::Source`] naming the field for an operator the +/// field does not answer ([`Reason::OperatorNotAnswered`]), an input that +/// cannot be read as the field's kind ([`Reason::FieldType`]) and a +/// `between` input that is not two values ([`Reason::NotARange`]). +/// [`Error::Cipher`] for the query refusals the typed path makes: a range +/// whose low bound is above its high ([`TermError::InvertedRange`]), and +/// match text that cuts no token ([`TermError::EmptyTermText`]) or that +/// the index cannot normalize ([`TermError::Normalize`]). +/// [`Error::Target`] for what the resolver refuses. +pub fn query_term<'a, K>( + cipher: &'a KeysetCipher<'_, K>, + plan: &Plan, + field: &str, + operator: OpSpec, + input: FfiValue, + resolver: &(impl TargetResolver + ?Sized), +) -> Result, Error> +where + K: DataKeySource + Sync + 'static, +{ + let answer = answer(plan, field, operator, resolver)?; + has_context(plan, field)?; + let input = read_input(&answer, operator, input)?; + check_input(&answer.index, &input)?; + let index = answer.index.clone(); + let made = move |domain: Option, operand| QueryTerm { + operator, + index, + domain, + operand, + }; + if let Some(target) = answer.target { + let Input::One(value) = input else { + return Err(Error::Internal); + }; + let domain = target.query_sql_domain; + return Ok(super::record::query(cipher, plan, field, value, resolver)? + .map(move |json| made(domain, Operand::One(json)))); + } + let context = answer.field.context(); + let term = |value: &FfiValue| { + derive(cipher, value, &answer.index, context.clone(), Use::Query) + .map_err(|error| error.in_field(field)) + }; + Ok(match input { + Input::One(value) => { + term(&value)?.map(move |term| made(None, Operand::One(term.into_bytes()))) + } + Input::Pair(lo, hi) => term(&lo)?.zip(term(&hi)?).map(move |(lo, hi)| { + made( + None, + Operand::Range { + lo: lo.into_bytes(), + hi: hi.into_bytes(), + }, + ) + }), + }) +} + +/// The match index that answers a probabilistic operator for the field, +/// or the refusal for an exact one. +fn probabilistic( + plan: &Plan, + field: &str, + operator: OpSpec, + resolver: &(impl TargetResolver + ?Sized), +) -> Result { + let answer = answer(plan, field, operator, resolver)?; + // A match index answers `matches` alone, and `matches` is the one + // probabilistic operator, so the index says which this is. + match answer.index { + IndexSpec::Match(options) => Ok(options), + _ => Err(Error::Source { + field: Some(field.to_owned()), + reason: Reason::ExactOperator, + }), + } +} + +/// Whether each of a page of decrypted values really matches a +/// probabilistic query, in order: [`Probabilistic::verify`] for every +/// value, run by the code the typed `MatchQuery` runs, under the options of +/// the field's match index. +/// +/// The query is named as it was derived, by the plan, the field, the +/// operator and the input, and the input's tokens are cut again here: they +/// are plaintext, so no query term carries them across a binding. The +/// result needs no key. +/// +/// A value that does not hold the query's tokens is `false`, and so is a +/// null (a row whose field is empty holds no token), as +/// [`Probabilistic::verify`] is `false` for every outcome but a match. +/// +/// [`Probabilistic::verify`]: crate::op::Probabilistic::verify +/// +/// # Errors +/// +/// As [`query_term`] for the field, the operator and the input, before any +/// value is read, save that a plan whose context is a field of each record +/// is taken, since verifying derives nothing; [`Error::Source`] with +/// [`Reason::ExactOperator`] for an exact operator, which has nothing to +/// verify; and [`Error::Source`] with [`Reason::FieldType`] for a value +/// that is neither a string nor null. +pub fn verify( + plan: &Plan, + field: &str, + operator: OpSpec, + input: FfiValue, + values: &[FfiValue], + resolver: &(impl TargetResolver + ?Sized), +) -> Result, Error> { + let options = probabilistic(plan, field, operator, resolver)?; + let input = read(ValueKind::String, input).map_err(|error| error.in_field(field))?; + let tokens = query_tokens(&input, &options)?; + values + .iter() + .map(|value| match value { + FfiValue::String(text) => { + Ok(utf8(text).is_some_and(|text| tokens.verify(text, &options))) + } + FfiValue::Null => Ok(false), + _ => Err(Error::Source { + field: Some(field.to_owned()), + reason: Reason::FieldType, + }), + }) + .collect() +} + +/// How many rows to fetch so that, at the confidence `stats` names, `want` +/// of them [`verify`]: [`Probabilistic::target_limit`] for the term +/// [`query_term`] derived, run by the code the typed `MatchQuery` runs. +/// +/// `term` is the term's bytes as [`query_term`] returned them, from which +/// the number of bits the query sets is read. The field's match index +/// declares no `expected_tokens` prior in a data plan, so a [`Stats`] with +/// no measured histogram answers [`Limit::Unbounded`]. The result needs no +/// key. +/// +/// [`Probabilistic::target_limit`]: crate::op::Probabilistic::target_limit +/// +/// # Errors +/// +/// As [`verify`] for the plan, the field and the operator; and +/// [`Error::Source`] with [`Reason::NotAQueryTerm`] for bytes that are not +/// a match query term under the field's options: not whole positions, a +/// position outside the filter, positions out of order or repeated, or +/// none at all. +pub fn target_limit( + plan: &Plan, + field: &str, + operator: OpSpec, + term: &[u8], + want: u64, + stats: &Stats, + resolver: &(impl TargetResolver + ?Sized), +) -> Result { + let options = probabilistic(plan, field, operator, resolver)?; + let bits = query_bits(term, &options).ok_or_else(|| Error::Source { + field: Some(field.to_owned()), + reason: Reason::NotAQueryTerm, + })?; + Ok(match_limit(bits, &options, None, want, stats)) +} + +/// The number of positions a match query term sets, if `term` is one the +/// index derives under `options`: little-endian `u16` positions, each in +/// the filter, strictly ascending, at least one. +fn query_bits(term: &[u8], options: &MatchOptions) -> Option { + if term.is_empty() || !term.len().is_multiple_of(2) { + return None; + } + let positions: Vec = term + .chunks_exact(2) + .map(|pair| u32::from(u16::from_le_bytes([pair[0], pair[1]]))) + .collect(); + let in_filter = positions.iter().all(|&position| position < options.m); + let ascending = positions.windows(2).all(|pair| pair[0] < pair[1]); + (in_filter && ascending).then_some(positions.len()) +} + +impl Stats { + /// Read statistics from their wire form, an object: + /// + /// | key | value | + /// |---|---| + /// | `rows` | integer, required: the rows the query scans | + /// | `histogram` | optional, or null: a list of `[bits, rows]` pairs of integers, a measured histogram of the stored filters' set-bit counts ([`Stats::measured`]); absent, the declared prior ([`Stats::declared`]) | + /// | `confidence` | optional, or null: a float, the confidence to target instead of 99% ([`Stats::with_confidence`]) | + /// + /// Integers are read at any width and must not be negative. + /// + /// # Errors + /// + /// [`Error::Source`] with [`Reason::NotStats`] for anything else: not an + /// object, no `rows`, a key given twice or not one of the three, or a + /// value of another type. + pub fn from_value(value: &FfiValue) -> Result { + let refused = || Error::bad_source(Reason::NotStats); + let FfiValue::Object(entries) = value else { + return Err(refused()); + }; + let mut seen: Vec<&str> = Vec::with_capacity(entries.len()); + let (mut rows, mut histogram, mut confidence) = (None, None, None); + for (key, value) in entries { + if seen.contains(&key.as_str()) { + return Err(refused()); + } + seen.push(key); + match (key.as_str(), value) { + ("histogram" | "confidence", FfiValue::Null) => {} + ("rows", rows_value) => rows = Some(unsigned(rows_value).ok_or_else(refused)?), + ("histogram", FfiValue::Array(buckets)) => { + histogram = Some( + buckets + .iter() + .map(|bucket| match bucket { + FfiValue::Array(pair) => match pair.as_slice() { + [bits, rows] => Some(( + u32::try_from(unsigned(bits)?).ok()?, + unsigned(rows)?, + )), + _ => None, + }, + _ => None, + }) + .collect::>>() + .ok_or_else(refused)?, + ); + } + ("confidence", FfiValue::Float64(c)) => confidence = Some(*c), + ("confidence", FfiValue::Float32(c)) => confidence = Some(f64::from(*c)), + _ => return Err(refused()), + } + } + let rows = rows.ok_or_else(refused)?; + let stats = match histogram { + Some(histogram) => Stats::measured(rows, histogram), + None => Stats::declared(rows), + }; + Ok(match confidence { + Some(confidence) => stats.with_confidence(confidence), + None => stats, + }) + } +} + +/// A non-negative integer of any width. +fn unsigned(value: &FfiValue) -> Option { + match value { + FfiValue::Int32(v) => u64::try_from(*v).ok(), + FfiValue::Int64(v) => u64::try_from(*v).ok(), + FfiValue::UInt32(v) => Some(u64::from(*v)), + FfiValue::UInt64(v) => Some(*v), + _ => None, + } +} + +/// A limit in its wire form: the rows to fetch as an unsigned 64-bit +/// integer, or null for [`Limit::Unbounded`]. +pub fn limit_to_value(limit: Limit) -> FfiValue { + limit.rows().map_or(FfiValue::Null, FfiValue::UInt64) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::dynamic::{record, NoTargets, TargetError}; + use crate::op::{self, Probabilistic as _}; + use crate::sem::MatchTerms; + use crate::{Equality, Label, Match, Ope, Ore, StackCipher}; + use stack_kms::FakeDataKeySource; + use vitaminc_protected::Controlled; + + async fn cipher() -> StackCipher { + StackCipher::builder() + .kms(FakeDataKeySource::new()) + .init() + .await + .expect("build cipher") + } + + fn text(s: &str) -> FfiValue { + FfiValue::String(s.into()) + } + + /// One field of `users`, as a binding spells it. + fn field(name: &str, kind: &str, outputs: Vec) -> (String, FfiValue) { + ( + name.to_owned(), + FfiValue::Object(vec![ + ( + "context".into(), + FfiValue::Array(vec![text("users"), text(name)]), + ), + ("outputs".into(), FfiValue::Array(outputs)), + ("type".into(), text(kind)), + ]), + ) + } + + /// The plan every test queries: an email with equality, a name with + /// match, an age with ORE and OPE, a score with OPE alone, a nickname + /// with ORE over text, and a case-sensitive handle. + fn users() -> Plan { + let case_sensitive = FfiValue::Object(vec![( + "match".into(), + FfiValue::Object(vec![("downcase".into(), FfiValue::Bool(false))]), + )]); + record::plan(FfiValue::Object(vec![ + field("email", "string", vec![text("c"), text("eq")]), + field("name", "string", vec![text("c"), text("match")]), + field("age", "uint32", vec![text("c"), text("ope"), text("ore")]), + field("score", "int64", vec![text("ope")]), + field("nickname", "string", vec![text("c"), text("ore")]), + field("handle", "string", vec![text("c"), case_sensitive]), + ])) + .expect("the plan parses") + } + + async fn query( + keyset: &KeysetCipher<'_, FakeDataKeySource>, + name: &str, + operator: OpSpec, + input: FfiValue, + ) -> Result { + query_term(keyset, &users(), name, operator, input, &NoTargets)? + .await + .map_err(Error::Cipher) + } + + fn reason_of(error: &Error) -> (Option<&str>, Option) { + (error.field(), error.reason()) + } + + fn term_error(error: &Error) -> Option<&TermError> { + match error { + Error::Cipher(crate::Error::Term(term)) => Some(term), + _ => None, + } + } + + #[test] + fn an_operator_reads_back_from_its_name_and_no_other() { + for op in OpSpec::ALL { + assert_eq!(OpSpec::parse(op.name()), Some(op), "{op}"); + assert_eq!(op.to_string(), op.name()); + } + let names: std::collections::BTreeSet<&str> = + OpSpec::ALL.iter().map(|op| op.name()).collect(); + assert_eq!(names.len(), OpSpec::ALL.len(), "names are distinct"); + for not_one in ["Eq", "=", "", "contains", "at_path", "matches "] { + assert_eq!(OpSpec::parse(not_one), None, "{not_one:?}"); + } + } + + #[test] + fn only_matches_is_probabilistic_and_only_between_takes_a_pair() { + for op in OpSpec::ALL { + assert_eq!( + op.fidelity() == Fidelity::Probabilistic, + op == OpSpec::Matches, + "{op}" + ); + assert_eq!(op.takes_pair(), op == OpSpec::Between, "{op}"); + } + assert_eq!(Fidelity::Exact.name(), "exact"); + assert_eq!(Fidelity::Probabilistic.name(), "probabilistic"); + } + + /// The typed path's `DefaultIndex` table, over kinds: equality answers + /// `eq`, match answers `matches` over text, and the orderings go to ORE + /// over an integer, or to OPE when there is no ORE. + #[test] + fn the_answering_index_is_the_typed_paths_default() { + let all = [ + IndexSpec::Equality, + IndexSpec::Match(MatchOptions::default()), + IndexSpec::Ope, + IndexSpec::Ore, + ]; + let answered = + |op: OpSpec, kind, indexes: &[IndexSpec]| op.answered_by(kind, indexes).cloned(); + assert_eq!( + answered(OpSpec::Eq, ValueKind::String, &all), + Some(IndexSpec::Equality) + ); + assert_eq!( + answered(OpSpec::Eq, ValueKind::Bool, &all), + None, + "a boolean has no equality term" + ); + assert_eq!( + answered(OpSpec::Matches, ValueKind::String, &all), + Some(IndexSpec::Match(MatchOptions::default())) + ); + assert_eq!(answered(OpSpec::Matches, ValueKind::UInt64, &all), None); + for op in [ + OpSpec::Lt, + OpSpec::Lte, + OpSpec::Gt, + OpSpec::Gte, + OpSpec::Between, + ] { + for kind in [ + ValueKind::Int32, + ValueKind::Int64, + ValueKind::UInt32, + ValueKind::UInt64, + ] { + assert_eq!( + answered(op, kind, &all), + Some(IndexSpec::Ore), + "{op} {kind}" + ); + assert_eq!( + answered(op, kind, &[IndexSpec::Ope]), + Some(IndexSpec::Ope), + "{op} {kind}" + ); + assert_eq!( + answered(op, kind, &[IndexSpec::Equality]), + None, + "{op} {kind}" + ); + } + for kind in [ + ValueKind::String, + ValueKind::Bytes, + ValueKind::Float64, + ValueKind::Bool, + ] { + assert_eq!( + answered(op, kind, &all), + None, + "{op} is answered over integers only: {kind}" + ); + } + } + for op in OpSpec::ALL { + assert_eq!( + answered(op, ValueKind::UInt32, &[]), + None, + "no index answers nothing" + ); + } + } + + #[tokio::test] + async fn eq_derives_the_typed_paths_equality_term() { + let cipher = cipher().await; + let keyset = cipher.default_keyset(); + let term = query(&keyset, "email", OpSpec::Eq, text("bob@example.com")) + .await + .unwrap(); + let plan = crate::Plan::context("users/email") + .with::(Equality) + .build() + .unwrap(); + let typed = cipher + .query(op::Eq, "bob@example.com") + .on(&plan) + .await + .unwrap(); + assert_eq!(term.operand(), &Operand::One(typed.to_bytes())); + assert_eq!(term.operator(), OpSpec::Eq); + assert_eq!(term.fidelity(), Fidelity::Exact); + assert_eq!(term.index(), &IndexSpec::Equality); + assert_eq!(term.domain(), None, "a field with indexes binds bytes"); + } + + #[tokio::test] + async fn an_ordering_derives_the_typed_paths_ore_term_and_reads_the_input_as_the_fields_kind() { + let cipher = cipher().await; + let keyset = cipher.default_keyset(); + let plan = crate::Plan::context("users/age") + .with::(Ore) + .build() + .unwrap(); + let typed = cipher.query(op::Lt, &40u32).on(&plan).await.unwrap(); + for input in [ + FfiValue::UInt32(40), + FfiValue::Float64(40.0), + FfiValue::Int64(40), + ] { + let term = query(&keyset, "age", OpSpec::Lt, input).await.unwrap(); + assert_eq!(term.operand(), &Operand::One(typed.to_bytes())); + assert_eq!(term.index(), &IndexSpec::Ore, "ORE before OPE"); + } + } + + #[tokio::test] + async fn between_derives_both_bounds_under_one_context_as_the_typed_range() { + let cipher = cipher().await; + let keyset = cipher.default_keyset(); + let plan = crate::Plan::context("users/age") + .with::(Ore) + .build() + .unwrap(); + let typed = cipher + .query(op::Between, &(18u32, 65)) + .on(&plan) + .await + .unwrap(); + let term = query( + &keyset, + "age", + OpSpec::Between, + FfiValue::Array(vec![FfiValue::UInt32(18), FfiValue::UInt32(65)]), + ) + .await + .unwrap(); + assert_eq!( + term.operand(), + &Operand::Range { + lo: typed.lo().to_bytes(), + hi: typed.hi().to_bytes() + } + ); + let one = query( + &keyset, + "age", + OpSpec::Between, + FfiValue::Array(vec![FfiValue::UInt32(7), FfiValue::UInt32(7)]), + ) + .await; + assert!(one.is_ok(), "(v, v) is one value: {one:?}"); + + let plan = crate::Plan::context("users/score") + .with::(Ope) + .build() + .unwrap(); + let typed = cipher + .query(op::Between, &(-5i64, 5)) + .via(Ope) + .on(&plan) + .await + .unwrap(); + let term = query( + &keyset, + "score", + OpSpec::Between, + FfiValue::Array(vec![FfiValue::Int64(-5), FfiValue::Int64(5)]), + ) + .await + .unwrap(); + assert_eq!( + term.index(), + &IndexSpec::Ope, + "OPE when the field declares no ORE" + ); + assert_eq!( + term.operand(), + &Operand::Range { + lo: typed.lo().to_bytes(), + hi: typed.hi().to_bytes() + } + ); + } + + #[tokio::test] + async fn matches_derives_the_typed_paths_filter() { + let cipher = cipher().await; + let keyset = cipher.default_keyset(); + let plan = crate::Plan::context("users/name") + .with::(Match::default()) + .build() + .unwrap(); + let typed = cipher.query(op::Matches, "alice").on(&plan).await.unwrap(); + let term = query(&keyset, "name", OpSpec::Matches, text("alice")) + .await + .unwrap(); + assert_eq!(term.operand(), &Operand::One(typed.to_bytes())); + assert_eq!(term.fidelity(), Fidelity::Probabilistic); + } + + #[tokio::test] + async fn a_query_the_field_does_not_answer_is_refused_naming_the_field() { + let cipher = cipher().await; + let keyset = cipher.default_keyset(); + let cases = [ + ("name", OpSpec::Eq, text("x"), Reason::OperatorNotAnswered), + ( + "email", + OpSpec::Matches, + text("bob"), + Reason::OperatorNotAnswered, + ), + ( + "nickname", + OpSpec::Lt, + text("bob"), + Reason::OperatorNotAnswered, + ), + ( + "age", + OpSpec::Eq, + FfiValue::UInt32(1), + Reason::OperatorNotAnswered, + ), + ("age", OpSpec::Lt, text("40"), Reason::FieldType), + ( + "age", + OpSpec::Lt, + FfiValue::Float64(40.5), + Reason::FieldType, + ), + ( + "age", + OpSpec::Between, + FfiValue::UInt32(40), + Reason::NotARange, + ), + ( + "age", + OpSpec::Between, + FfiValue::Array(vec![FfiValue::UInt32(1)]), + Reason::NotARange, + ), + ( + "age", + OpSpec::Between, + FfiValue::Array(vec![FfiValue::UInt32(1), text("2")]), + Reason::FieldType, + ), + ]; + for (name, op, input, reason) in cases { + let what = format!("{op} on {name}"); + let error = query(&keyset, name, op, input.clone()).await.unwrap_err(); + assert!(matches!(error, Error::Source { .. }), "{what}: {error:?}"); + assert_eq!(reason_of(&error), (Some(name), Some(reason)), "{what}"); + let checked = check_query(&users(), name, op, input, &NoTargets).unwrap_err(); + assert_eq!( + reason_of(&checked), + (Some(name), Some(reason)), + "{what}: checked alike" + ); + } + let error = query(&keyset, "phone", OpSpec::Eq, text("1")) + .await + .unwrap_err(); + assert!(matches!(error, Error::Plan { .. })); + assert_eq!( + reason_of(&error), + (Some("phone"), Some(Reason::NoSuchField)) + ); + } + + #[tokio::test] + async fn the_typed_paths_query_refusals_are_made_before_any_term() { + let cipher = cipher().await; + let keyset = cipher.default_keyset(); + let pair = |lo, hi| FfiValue::Array(vec![FfiValue::UInt32(lo), FfiValue::UInt32(hi)]); + let cases = [ + ("age", OpSpec::Between, pair(65, 18), "inverted"), + ("name", OpSpec::Matches, text("al"), "empty"), + ("name", OpSpec::Matches, text("\u{0378}abc"), "normalize"), + ]; + for (name, op, input, what) in cases { + for error in [ + query(&keyset, name, op, input.clone()).await.unwrap_err(), + check_query(&users(), name, op, input, &NoTargets).unwrap_err(), + ] { + let term = term_error(&error); + let expected = match what { + "inverted" => matches!(term, Some(TermError::InvertedRange)), + "empty" => matches!(term, Some(TermError::EmptyTermText)), + _ => matches!(term, Some(TermError::Normalize(_))), + }; + assert!(expected, "{what}: {error:?}"); + } + } + } + + /// A plan whose context is a field of each record has no context a + /// query could derive under; the binding sends the plan under the + /// context it names. + #[tokio::test] + async fn a_plan_whose_context_is_a_record_field_is_refused() { + let cipher = cipher().await; + let keyset = cipher.default_keyset(); + let one = |name: &str, kind: &str, output: &str| { + ( + name.to_owned(), + FfiValue::Object(vec![ + ("context".into(), FfiValue::Array(vec![text(name)])), + ("outputs".into(), FfiValue::Array(vec![text(output)])), + ("type".into(), text(kind)), + ]), + ) + }; + let plan = record::plan(FfiValue::Object(vec![ + ("context_field".into(), text("tenant")), + one("tenant", "string", "passthrough"), + one("email", "string", "eq"), + ])) + .unwrap(); + let error = query_term(&keyset, &plan, "email", OpSpec::Eq, text("a"), &NoTargets) + .map(drop) + .unwrap_err(); + assert_eq!( + reason_of(&error), + (Some("email"), Some(Reason::QueryContextField)) + ); + let checked = check_query(&plan, "email", OpSpec::Eq, text("a"), &NoTargets).unwrap_err(); + assert_eq!( + reason_of(&checked), + (Some("email"), Some(Reason::QueryContextField)) + ); + // Verifying derives nothing, so it needs no context. + let error = verify(&plan, "email", OpSpec::Eq, text("a"), &[], &NoTargets).unwrap_err(); + assert_eq!( + reason_of(&error), + (Some("email"), Some(Reason::ExactOperator)), + "refused for its operator, not its context" + ); + } + + /// `verify` is the typed `MatchQuery::verify`, value for value, under + /// the options the field declares, and needs no key. + #[tokio::test] + async fn verify_answers_as_the_typed_match_query() { + let cipher = cipher().await; + let plan = crate::Plan::context("users/name") + .with::(Match::default()) + .build() + .unwrap(); + let typed = cipher.query(op::Matches, "alice").on(&plan).await.unwrap(); + let page = ["Alice Smith", "alicia", "Bob", "MALICE", "", "ali ce"]; + let values: Vec = page.iter().map(|v| text(v)).collect(); + let verified = verify( + &users(), + "name", + OpSpec::Matches, + text("alice"), + &values, + &NoTargets, + ) + .unwrap(); + let expected: Vec = page.iter().map(|v| typed.verify(v)).collect(); + assert_eq!(verified, expected); + assert!( + verified.contains(&true) && verified.contains(&false), + "{verified:?}" + ); + + let nulls = verify( + &users(), + "name", + OpSpec::Matches, + text("alice"), + &[FfiValue::Null], + &NoTargets, + ) + .unwrap(); + assert_eq!(nulls, [false], "a null holds no token"); + let empty = verify( + &users(), + "name", + OpSpec::Matches, + text("alice"), + &[], + &NoTargets, + ) + .unwrap(); + assert!(empty.is_empty()); + } + + #[test] + fn verify_cuts_tokens_under_the_fields_own_options() { + let sensitive = |input: &str, value: &str| { + verify( + &users(), + "handle", + OpSpec::Matches, + text(input), + &[text(value)], + &NoTargets, + ) + .unwrap()[0] + }; + assert!(sensitive("Alice", "Alice")); + assert!( + !sensitive("Alice", "alice"), + "the handle's index does not fold case" + ); + let folding = verify( + &users(), + "name", + OpSpec::Matches, + text("Alice"), + &[text("alice")], + &NoTargets, + ) + .unwrap(); + assert_eq!(folding, [true], "the name's index folds case"); + } + + #[test] + fn verify_refuses_what_it_cannot_answer() { + let refused = |name: &str, op: OpSpec, input: FfiValue, values: &[FfiValue]| { + verify(&users(), name, op, input, values, &NoTargets).unwrap_err() + }; + let error = refused("email", OpSpec::Eq, text("a"), &[text("a")]); + assert_eq!( + reason_of(&error), + (Some("email"), Some(Reason::ExactOperator)) + ); + let error = refused("age", OpSpec::Lt, FfiValue::UInt32(1), &[]); + assert_eq!( + reason_of(&error), + (Some("age"), Some(Reason::ExactOperator)) + ); + let error = refused("email", OpSpec::Matches, text("abc"), &[]); + assert_eq!( + reason_of(&error), + (Some("email"), Some(Reason::OperatorNotAnswered)) + ); + let error = refused( + "name", + OpSpec::Matches, + text("alice"), + &[FfiValue::UInt32(1)], + ); + assert_eq!(reason_of(&error), (Some("name"), Some(Reason::FieldType))); + let error = refused("name", OpSpec::Matches, FfiValue::UInt32(1), &[]); + assert_eq!(reason_of(&error), (Some("name"), Some(Reason::FieldType))); + let error = refused("name", OpSpec::Matches, text("al"), &[text("al")]); + assert!( + matches!(term_error(&error), Some(TermError::EmptyTermText)), + "{error:?}" + ); + } + + /// `target_limit` is the typed `MatchQuery::target_limit` for the term + /// `query_term` returned, read from its bytes. + #[tokio::test] + async fn target_limit_answers_as_the_typed_match_query() { + let cipher = cipher().await; + let keyset = cipher.default_keyset(); + let plan = crate::Plan::context("users/name") + .with::(Match::default()) + .build() + .unwrap(); + let typed = cipher.query(op::Matches, "ali").on(&plan).await.unwrap(); + let term = query(&keyset, "name", OpSpec::Matches, text("ali")) + .await + .unwrap(); + let Operand::One(bytes) = term.operand() else { + panic!("one term"); + }; + let measured = Stats::measured(1_000_000, [(40, 500_000), (120, 500_000)]); + for stats in [ + measured.clone(), + measured.clone().with_confidence(0.5), + Stats::declared(1_000_000), + ] { + let limit = target_limit( + &users(), + "name", + OpSpec::Matches, + bytes, + 10, + &stats, + &NoTargets, + ) + .unwrap(); + assert_eq!(limit, typed.target_limit(10, &stats), "{stats:?}"); + } + assert!(matches!( + target_limit(&users(), "name", OpSpec::Matches, bytes, 10, &measured, &NoTargets), + Ok(Limit::Rows(rows)) if rows > 10 + )); + assert_eq!( + target_limit( + &users(), + "name", + OpSpec::Matches, + bytes, + 10, + &Stats::declared(1_000_000), + &NoTargets + ) + .unwrap(), + Limit::Unbounded, + "a data plan's match index declares no prior" + ); + } + + #[test] + fn target_limit_refuses_a_term_its_index_would_not_derive() { + let stats = Stats::measured(100, [(10, 100)]); + let limit = |name: &str, op: OpSpec, term: &[u8]| { + target_limit(&users(), name, op, term, 1, &stats, &NoTargets) + }; + let positions = |p: &[u16]| p.iter().flat_map(|p| p.to_le_bytes()).collect::>(); + assert!(limit("name", OpSpec::Matches, &positions(&[1, 2, 255])).is_ok()); + for (what, bytes) in [ + ("none", positions(&[])), + ("half a position", vec![1, 0, 2]), + ("outside the filter", positions(&[1, 256])), + ("out of order", positions(&[2, 1])), + ("repeated", positions(&[2, 2])), + ] { + let error = limit("name", OpSpec::Matches, &bytes).unwrap_err(); + assert_eq!( + reason_of(&error), + (Some("name"), Some(Reason::NotAQueryTerm)), + "{what}" + ); + } + let error = limit("email", OpSpec::Eq, &[0; 32]).unwrap_err(); + assert_eq!( + reason_of(&error), + (Some("email"), Some(Reason::ExactOperator)) + ); + // A stored filter under the same options reads as a term too: the + // bytes say nothing of which side derived them, and the limit is a + // number, not a query. + let stored = MatchTerms::::from_positions(vec![3, 9]).unwrap(); + assert!(limit("name", OpSpec::Matches, &stored.to_bytes()).is_ok()); + } + + #[test] + fn statistics_read_from_their_wire_form() { + let object = |entries: Vec<(&str, FfiValue)>| { + FfiValue::Object( + entries + .into_iter() + .map(|(k, v)| (k.to_owned(), v)) + .collect(), + ) + }; + let pair = |bits: u32, rows: u64| { + FfiValue::Array(vec![FfiValue::UInt32(bits), FfiValue::UInt64(rows)]) + }; + assert_eq!( + Stats::from_value(&object(vec![("rows", FfiValue::UInt64(9))])).unwrap(), + Stats::declared(9) + ); + assert_eq!( + Stats::from_value(&object(vec![ + ("rows", FfiValue::Int64(9)), + ("histogram", FfiValue::Null), + ("confidence", FfiValue::Null), + ])) + .unwrap(), + Stats::declared(9) + ); + assert_eq!( + Stats::from_value(&object(vec![ + ("histogram", FfiValue::Array(vec![pair(3, 4), pair(5, 6)])), + ("confidence", FfiValue::Float64(0.9)), + ("rows", FfiValue::UInt32(10)), + ])) + .unwrap(), + Stats::measured(10, [(3, 4), (5, 6)]).with_confidence(0.9) + ); + for (what, bad) in [ + ("not an object", FfiValue::UInt64(9)), + ("no rows", object(vec![])), + ("null rows", object(vec![("rows", FfiValue::Null)])), + ("negative rows", object(vec![("rows", FfiValue::Int64(-1))])), + ("float rows", object(vec![("rows", FfiValue::Float64(9.0))])), + ( + "rows twice", + object(vec![ + ("rows", FfiValue::UInt64(1)), + ("rows", FfiValue::UInt64(1)), + ]), + ), + ( + "unknown key", + object(vec![ + ("rows", FfiValue::UInt64(1)), + ("want", FfiValue::Null), + ]), + ), + ( + "bucket not a pair", + object(vec![ + ("rows", FfiValue::UInt64(1)), + ("histogram", FfiValue::Array(vec![FfiValue::UInt32(3)])), + ]), + ), + ( + "bits too wide", + object(vec![ + ("rows", FfiValue::UInt64(1)), + ( + "histogram", + FfiValue::Array(vec![FfiValue::Array(vec![ + FfiValue::UInt64(u64::from(u32::MAX) + 1), + FfiValue::UInt64(1), + ])]), + ), + ]), + ), + ( + "confidence as text", + object(vec![ + ("rows", FfiValue::UInt64(1)), + ("confidence", text("high")), + ]), + ), + ] { + let error = Stats::from_value(&bad).unwrap_err(); + assert_eq!(reason_of(&error), (None, Some(Reason::NotStats)), "{what}"); + } + } + + /// A range is refused the wrong way round for every integer kind a + /// field can declare, and two values of different kinds have no order + /// here: the input was read as one kind first. + #[test] + fn a_range_is_ordered_within_each_integer_kind() { + let cases = [ + (FfiValue::Int32(-1), FfiValue::Int32(1)), + (FfiValue::Int64(-1), FfiValue::Int64(1)), + (FfiValue::UInt32(1), FfiValue::UInt32(2)), + (FfiValue::UInt64(1), FfiValue::UInt64(2)), + ]; + for (lo, hi) in cases { + assert_eq!(integer_order(&lo, &hi), Some(Ordering::Less), "{lo:?}"); + assert_eq!(integer_order(&hi, &lo), Some(Ordering::Greater), "{lo:?}"); + assert_eq!(integer_order(&lo, &lo), Some(Ordering::Equal), "{lo:?}"); + } + assert_eq!( + integer_order(&FfiValue::UInt32(1), &FfiValue::UInt64(2)), + None + ); + assert_eq!(integer_order(&text("a"), &text("b")), None); + } + + #[test] + fn a_count_is_read_at_any_width_and_never_negative() { + assert_eq!(unsigned(&FfiValue::Int32(7)), Some(7)); + assert_eq!(unsigned(&FfiValue::Int64(7)), Some(7)); + assert_eq!(unsigned(&FfiValue::UInt32(7)), Some(7)); + assert_eq!(unsigned(&FfiValue::UInt64(7)), Some(7)); + assert_eq!(unsigned(&FfiValue::Int32(-7)), None); + assert_eq!(unsigned(&FfiValue::Int64(-7)), None); + assert_eq!(unsigned(&FfiValue::Float64(7.0)), None); + let confidence = Stats::from_value(&FfiValue::Object(vec![ + ("rows".into(), FfiValue::Int32(3)), + ("confidence".into(), FfiValue::Float32(0.5)), + ])) + .unwrap(); + assert_eq!(confidence, Stats::declared(3).with_confidence(0.5)); + } + + #[test] + fn a_limit_is_its_rows_or_null() { + assert!(matches!( + limit_to_value(Limit::Rows(11)), + FfiValue::UInt64(11) + )); + assert!(matches!(limit_to_value(Limit::Unbounded), FfiValue::Null)); + } + + #[tokio::test] + async fn a_query_term_writes_every_key_in_order() { + let cipher = cipher().await; + let keyset = cipher.default_keyset(); + let term = query( + &keyset, + "age", + OpSpec::Between, + FfiValue::Array(vec![FfiValue::UInt32(1), FfiValue::UInt32(2)]), + ) + .await + .unwrap(); + let FfiValue::Object(entries) = term.to_value() else { + panic!("an object"); + }; + let keys: Vec<&str> = entries.iter().map(|(k, _)| k.as_str()).collect(); + assert_eq!(keys, ["operator", "fidelity", "index", "domain", "term"]); + assert!(matches!(&entries[0].1, FfiValue::String(s) if s.risky_ref() == b"between")); + assert!(matches!(&entries[1].1, FfiValue::String(s) if s.risky_ref() == b"exact")); + assert!(matches!(&entries[2].1, FfiValue::String(s) if s.risky_ref() == b"ore")); + assert!(matches!(&entries[3].1, FfiValue::Null)); + let Operand::Range { lo, hi } = term.operand() else { + panic!("a range"); + }; + assert!(matches!( + &entries[4].1, + FfiValue::Array(bounds) if matches!( + bounds.as_slice(), + [FfiValue::Bytes(l), FfiValue::Bytes(h)] if l.risky_ref() == lo && h.risky_ref() == hi + ) + )); + } + + fn descriptor( + kind: Option, + indexes: &[&str], + query: bool, + producible: bool, + ) -> TargetDescriptor { + TargetDescriptor::new( + "Some", + "some", + "", + kind, + "public.some", + indexes.iter().map(|i| (*i).to_owned()).collect(), + query.then(|| "SomeQuery".to_owned()), + query.then(|| "eql_v3.query_some".to_owned()), + producible, + (!producible).then(|| "not yet".to_owned()), + ) + } + + /// A type answers the exact operators of one value its indexes answer, + /// and only once the engine produces it and it has a query twin. + #[test] + fn a_target_answers_the_exact_single_value_operators_of_its_indexes() { + let string = Some(ValueKind::String); + let int = Some(ValueKind::Int64); + assert_eq!( + descriptor(string, &["eq"], true, true).operators(), + [OpSpec::Eq] + ); + assert_eq!( + descriptor(int, &["eq", "ore"], true, true).operators(), + [OpSpec::Eq, OpSpec::Lt, OpSpec::Lte, OpSpec::Gt, OpSpec::Gte], + "no between: the twin is one value" + ); + assert!( + descriptor(string, &["match"], true, true) + .operators() + .is_empty(), + "no probabilistic operator until the descriptor carries the index's options" + ); + assert!(descriptor(string, &["eq"], true, false) + .operators() + .is_empty()); + assert!(descriptor(string, &["eq"], false, true) + .operators() + .is_empty()); + assert!(descriptor(None, &["eq"], true, true).operators().is_empty()); + assert!(descriptor(Some(ValueKind::Object), &["json"], true, true) + .operators() + .is_empty()); + } + + #[test] + fn a_targets_operators_are_listed_in_its_wire_form() { + let FfiValue::Object(entries) = + descriptor(Some(ValueKind::Int64), &["ore"], true, true).to_value() + else { + panic!("an object"); + }; + let (key, FfiValue::Array(operators)) = &entries[10] else { + panic!("a list"); + }; + assert_eq!(key, "operators"); + assert_eq!(operators.len(), 4); + let FfiValue::Object(first) = &operators[0] else { + panic!("an object"); + }; + let first: Vec<(&str, &[u8])> = first + .iter() + .map(|(k, v)| match v { + FfiValue::String(s) => (k.as_str(), s.risky_ref()), + _ => panic!("{k} is a string"), + }) + .collect(); + assert_eq!( + first, + [ + ("operator", b"lt".as_slice()), + ("fidelity", b"exact".as_slice()), + ("arity", b"one".as_slice()), + ("index", b"ore".as_slice()) + ] + ); + } + + /// A resolver holding one type that answers `eq`, whose query is a + /// fixed JSON value: what a target field's query term carries. + struct OneType; + + impl TargetResolver for OneType { + fn targets(&self) -> Vec { + vec![descriptor(Some(ValueKind::String), &["eq"], true, true)] + } + fn encrypt<'a, K: 'static>( + &self, + _: &str, + _: &'a KeysetCipher<'_, K>, + _: &Label, + _: FfiValue, + ) -> Result, K>, TargetError> { + unreachable!("a query encrypts nothing") + } + fn decrypt<'a, K: 'static>( + &self, + _: &str, + _: &'a crate::StackCipher, + _: &Label, + _: &[u8], + ) -> Result, TargetError> { + unreachable!("a query decrypts nothing") + } + fn query<'a, K: 'static>( + &self, + _: &str, + keyset: &'a KeysetCipher<'_, K>, + _: &Label, + _: FfiValue, + ) -> Result, K>, TargetError> { + Ok(Pending::ready(keyset, Ok(br#"{"hm":"00"}"#.to_vec()))) + } + } + + #[tokio::test] + async fn a_target_field_answers_with_its_types_query_and_domain() { + let cipher = cipher().await; + let keyset = cipher.default_keyset(); + let plan = record::plan_with( + FfiValue::Object(vec![( + "email".into(), + FfiValue::Object(vec![ + ( + "context".into(), + FfiValue::Array(vec![text("users"), text("email")]), + ), + ("target".into(), text("Some")), + ]), + )]), + &OneType, + ) + .unwrap(); + let term = query_term(&keyset, &plan, "email", OpSpec::Eq, text("a"), &OneType) + .unwrap() + .await + .unwrap(); + assert_eq!(term.operand(), &Operand::One(br#"{"hm":"00"}"#.to_vec())); + assert_eq!(term.domain(), Some("eql_v3.query_some")); + assert_eq!(term.index(), &IndexSpec::Equality); + for op in [OpSpec::Lt, OpSpec::Matches, OpSpec::Between] { + let error = query_term(&keyset, &plan, "email", op, text("a"), &OneType) + .map(drop) + .unwrap_err(); + assert_eq!( + reason_of(&error), + (Some("email"), Some(Reason::OperatorNotAnswered)), + "{op}" + ); + } + let error = verify(&plan, "email", OpSpec::Eq, text("a"), &[], &OneType).unwrap_err(); + assert_eq!( + reason_of(&error), + (Some("email"), Some(Reason::ExactOperator)) + ); + } +} diff --git a/packages/stack-encrypt/src/dynamic/record.rs b/packages/stack-encrypt/src/dynamic/record.rs index 10a6a2ac1..ebfe79e0b 100644 --- a/packages/stack-encrypt/src/dynamic/record.rs +++ b/packages/stack-encrypt/src/dynamic/record.rs @@ -430,7 +430,7 @@ impl FieldPlan { } /// The indexes the field declares, in output order. - fn indexes(&self) -> Vec { + pub(super) fn indexes(&self) -> Vec { self.outputs .iter() .filter_map(|output| match output { diff --git a/packages/stack-encrypt/src/dynamic/target.rs b/packages/stack-encrypt/src/dynamic/target.rs index fe46d87ba..59f43e936 100644 --- a/packages/stack-encrypt/src/dynamic/target.rs +++ b/packages/stack-encrypt/src/dynamic/target.rs @@ -27,7 +27,8 @@ use std::fmt; -use super::{FfiValue, ValueKind}; +use super::query::target_operators; +use super::{FfiValue, OpSpec, ValueKind}; use crate::{KeysetCipher, Label, Pending, StackCipher}; @@ -52,6 +53,21 @@ use crate::{KeysetCipher, Label, Pending, StackCipher}; /// | `query_sql_domain` | string or null: the query twin's PostgreSQL domain | /// | `producible` | bool: whether the engine produces the type today | /// | `reason` | string or null: why not, when it does not | +/// | `operators` | list of objects: the operators the engine answers for the type ([`operators`](Self::operators)), each `{"operator", "fidelity", "arity", "index"}`; empty for a type it does not produce | +/// +/// Each `operators` entry has every key: +/// +/// | key | value | +/// |---|---| +/// | `operator` | string: the operator's name, `eq` ([`OpSpec::name`]) | +/// | `fidelity` | string: `exact` or `probabilistic` ([`Fidelity::name`](super::Fidelity::name)) | +/// | `arity` | string: `one`, or `pair` for an operator whose input is two values, low then high | +/// | `index` | string: the key of the type's index that answers it, `eq` | +/// +/// The first ten keys are the row `eql-bindings` serializes for the type; +/// `operators` is the engine's, computed from the row by the table every +/// plan field is held to ([`OpSpec::answered_by`]), so a generator reads the +/// operators from the engine it embeds rather than from a copy of the rules. #[derive(Clone, Debug, PartialEq, Eq)] #[non_exhaustive] pub struct TargetDescriptor { @@ -133,8 +149,41 @@ impl TargetDescriptor { ), ("producible".to_string(), FfiValue::Bool(self.producible)), ("reason".to_string(), optional(&self.reason)), + ( + "operators".to_string(), + FfiValue::Array( + target_operators(self) + .into_iter() + .map(|(op, index)| { + FfiValue::Object(vec![ + ("operator".to_string(), text(op.name())), + ("fidelity".to_string(), text(op.fidelity().name())), + ( + "arity".to_string(), + text(if op.takes_pair() { "pair" } else { "one" }), + ), + ("index".to_string(), text(index.key())), + ]) + }) + .collect(), + ), + ), ]) } + + /// The operators the engine answers for a field that names this type, + /// in [`OpSpec::ALL`]'s order: none for a type it does not produce or + /// that has no query twin, and otherwise each exact operator of one value + /// that one of the type's indexes answers for its plaintext kind + /// ([`OpSpec::answered_by`]). The query for each is the type's query + /// twin. A probabilistic operator and `between` are not answered through + /// a type yet: see [`query_term`](super::query_term)'s module. + pub fn operators(&self) -> Vec { + target_operators(self) + .into_iter() + .map(|(op, _)| op) + .collect() + } } /// Why a target field could not be resolved or run. @@ -604,7 +653,8 @@ mod tests { "query", "query_sql_domain", "producible", - "reason" + "reason", + "operators" ], "the se_targets entry's keys, in order" ); @@ -624,7 +674,7 @@ mod tests { assert!(matches!(&entries[6].1, FfiValue::Null)); assert_eq!( entries.len(), - 10, + 11, "an absent value is null, never a missing key" ); } diff --git a/packages/stack-encrypt/src/dynamic/term.rs b/packages/stack-encrypt/src/dynamic/term.rs index 8382e9dd4..961b52fda 100644 --- a/packages/stack-encrypt/src/dynamic/term.rs +++ b/packages/stack-encrypt/src/dynamic/term.rs @@ -343,7 +343,7 @@ impl AsRef<[u8]> for TermBytes { /// What a derived term is for: a query refuses match text that cuts no /// token, which a stored value keeps as the empty filter. #[derive(Clone, Copy, PartialEq, Eq)] -enum Use { +pub(super) enum Use { Query, Store, } @@ -357,7 +357,7 @@ enum Use { /// [`Error::Term`] if the value is not one scalar `kind` takes, and /// [`Error::Cipher`] for a query's match text with no token or text the /// index cannot normalize, as for [`term`]. -fn derive<'a, 'c, K: 'static, P, D>( +pub(super) fn derive<'a, 'c, K: 'static, P, D>( cipher: &'a KeysetCipher<'_, K>, value: P, kind: &IndexSpec, diff --git a/packages/stack-encrypt/src/sem/mod.rs b/packages/stack-encrypt/src/sem/mod.rs index bca0cb965..3532f4d37 100644 --- a/packages/stack-encrypt/src/sem/mod.rs +++ b/packages/stack-encrypt/src/sem/mod.rs @@ -810,14 +810,7 @@ impl crate::op::Probabilistic for MatchQuery { type Plaintext = str; fn verify(&self, value: &str) -> bool { - let options = O::options(); - // A value the index cannot normalize holds no token of any query. - let Ok(tokens) = tokenize::tokens(value, options.tokenizer, options.downcase) else { - return false; - }; - let tokens = Zeroizing::new(tokens); - let have: HashSet<&str> = tokens.iter().map(String::as_str).collect(); - !self.tokens.0.is_empty() && self.tokens.0.iter().all(|t| have.contains(t.as_str())) + self.tokens.verify(value, &O::options()) } /// `want` plus the false positives expected among the rows `stats` @@ -829,11 +822,13 @@ impl crate::op::Probabilistic for MatchQuery { /// neither, the answer is /// [`Limit::Unbounded`](crate::op::Limit::Unbounded). fn target_limit(&self, want: u64, stats: &crate::op::Stats) -> crate::op::Limit { - let options = O::options(); - let declared = O::expected_tokens() - .map(|tokens| crate::op::stats::expected_fill(tokens, options.k, options.m)); - let bits = u32::try_from(self.positions().len()).unwrap_or(u32::MAX); - stats.limit(want, stats.pass_rate(bits, options.m, declared)) + match_limit( + self.positions().len(), + &O::options(), + O::expected_tokens(), + want, + stats, + ) } } @@ -962,6 +957,38 @@ impl QueryTokens { } Ok(tokens) } + + /// Whether `value` holds every one of these tokens, cut under the same + /// `options` the tokens were: what [`MatchQuery`]'s `verify` asks, with + /// the options a value rather than a type, so a match index declared as + /// data verifies by the same code. No tokens verify nothing, and a value + /// the index cannot normalize holds no token of any query. + pub(crate) fn verify(&self, value: &str, options: &MatchOptions) -> bool { + let Ok(tokens) = tokenize::tokens(value, options.tokenizer, options.downcase) else { + return false; + }; + let tokens = Zeroizing::new(tokens); + let have: HashSet<&str> = tokens.iter().map(String::as_str).collect(); + !self.0.is_empty() && self.0.iter().all(|t| have.contains(t.as_str())) + } +} + +/// The fetch for `want` real matches of a match query whose filter sets +/// `bits` positions, under `options` and the index's `expected_tokens` +/// prior: what [`MatchQuery`]'s `target_limit` answers, with the options a +/// value rather than a type, so a match index declared as data sizes its +/// fetch by the same code. +pub(crate) fn match_limit( + bits: usize, + options: &MatchOptions, + expected_tokens: Option, + want: u64, + stats: &crate::op::Stats, +) -> crate::op::Limit { + let declared = + expected_tokens.map(|tokens| crate::op::stats::expected_fill(tokens, options.k, options.m)); + let bits = u32::try_from(bits).unwrap_or(u32::MAX); + stats.limit(want, stats.pass_rate(bits, options.m, declared)) } /// A [`PrfVisitor`] that folds a sequence of per-token PRF blocks into diff --git a/packages/stack-encrypt/tests/fixtures/README.md b/packages/stack-encrypt/tests/fixtures/README.md index c48fd835f..440e4c962 100644 --- a/packages/stack-encrypt/tests/fixtures/README.md +++ b/packages/stack-encrypt/tests/fixtures/README.md @@ -86,3 +86,55 @@ by hand. `label_segments.json` beside this file is a different fixture: the one rule for a plain label segment, read by the Rust, derive and Go tests. + +# Query fixture: `query_operators.json` + +Queries named by operator through the data path +(`stack_encrypt::dynamic::query_term`, what a language binding calls), each +term equal to the one the typed query chain (`cipher.query(op::Lt, +&40u32).on(&plan)`) derives for the same query, with a page `verify` +answered and fetch limits `target_limit` sized (ADR-0009; ADR-0007, amended +2026-10-16). `tests/query_operators.rs` derives every entry and checks it +against the typed path; the Go SDK's `encrypt` tests derive, verify and size +the same queries through the guest's `se_query_term`, `se_verify` and +`se_target_limit` exports and compare. No entry depends on a nonce, so the +file changes only when a derivation does: regenerate it with +`STACK_ENCRYPT_UPDATE_FIXTURES=1 cargo test -p stack-encrypt --all-features +--test query_operators`, and do not edit it by hand. + +```json +{ + "_comment": "what this file is", + "key_source": { "kind": "deterministic-sha256", "seed": "<32 bytes, hex>" }, + "keyset_id": "", + "plan": { "": { "context": [...], "outputs": [...], "type": "" }, ... }, + "queries": [ + { "field": "", "operator": "", "input": , + "term": { "operator": "", "fidelity": "exact" | "probabilistic", + "index": "", "domain": null, "term": "" | ["", ""] } } + ], + "verify": { "field": "", "operator": "matches", "input": "", + "values": [, ...], "verified": [, ...] }, + "target_limit": [ + { "field": "", "operator": "matches", "input": "", "term": "", + "want": , "stats": { "rows": , "histogram": [[, ], ...], + "confidence": }, "limit": } + ] +} +``` + +- **`key_source`**, **`keyset_id`.** As in the record fixture: terms derive + under `FakeDataKeySource`'s index key for the nil keyset, whatever the + seed, so the Go tests run the deterministic guest built from the same + source. +- **`queries`.** Each query as a binding sends it, and its term in the wire + form `QueryTerm::to_value` writes, the bytes in hex: one term, or for + `between` the low bound's then the high's. An input is JSON, read as the + field's declared type, so `40` queries the `uint32` field `age`. +- **`verify`.** A page of decrypted values, null for an empty field, and the + boolean `verify` answers for each, which is what the typed `MatchQuery` + answers. +- **`target_limit`.** A match term's bytes, the statistics in the wire form + `Stats::from_value` reads (an absent histogram is the declared prior), and + the limit, null when no finite fetch can be promised, which is what the + typed `MatchQuery` answers. diff --git a/packages/stack-encrypt/tests/fixtures/query_operators.json b/packages/stack-encrypt/tests/fixtures/query_operators.json new file mode 100644 index 000000000..26c80d601 --- /dev/null +++ b/packages/stack-encrypt/tests/fixtures/query_operators.json @@ -0,0 +1,283 @@ +{ + "_comment": "Query terms derived by operator through the data path (stack_encrypt::dynamic::query_term), each equal to the typed query chain's, with a verified page and fetch limits. Read by tests/query_operators.rs and the Go SDK's tests; the schema is in README.md beside this file. Regenerate with STACK_ENCRYPT_UPDATE_FIXTURES=1; do not edit by hand.", + "key_source": { + "kind": "deterministic-sha256", + "seed": "737461636b2d656e6372797074207265636f7264206669787475726520763120" + }, + "keyset_id": "00000000-0000-0000-0000-000000000000", + "plan": { + "age": { + "context": [ + "users", + "age" + ], + "outputs": [ + "c", + "eq", + "ore" + ], + "type": "uint32" + }, + "email": { + "context": [ + "users", + "email" + ], + "outputs": [ + "c", + "eq" + ], + "type": "string" + }, + "name": { + "context": [ + "users", + "name" + ], + "outputs": [ + "c", + "match" + ], + "type": "string" + } + }, + "queries": [ + { + "field": "email", + "input": "bob@example.com", + "operator": "eq", + "term": { + "domain": null, + "fidelity": "exact", + "index": "eq", + "operator": "eq", + "term": "0cf708c07acaaa169e84e4adf7081f0433d87760bd7e9c63bd7a58c000b8d33b" + } + }, + { + "field": "age", + "input": 34, + "operator": "eq", + "term": { + "domain": null, + "fidelity": "exact", + "index": "eq", + "operator": "eq", + "term": "cc854fd5c7e3f4bb28cb15cae47d866b65b7012c06df43b4b9a6c51adff4710a" + } + }, + { + "field": "age", + "input": 40, + "operator": "lt", + "term": { + "domain": null, + "fidelity": "exact", + "index": "ore", + "operator": "lt", + "term": "7ca54e16d7944c0a9d8577d87f3864fdf79e99fea5956ba956a126a606d6d336" + } + }, + { + "field": "age", + "input": 40, + "operator": "lte", + "term": { + "domain": null, + "fidelity": "exact", + "index": "ore", + "operator": "lte", + "term": "7ca54e16d7944c0a9d8577d87f3864fdf79e99fea5956ba956a126a606d6d336" + } + }, + { + "field": "age", + "input": 18, + "operator": "gt", + "term": { + "domain": null, + "fidelity": "exact", + "index": "ore", + "operator": "gt", + "term": "7ca54e16d7944c0a9d8577d87f3864fdf79e99fea5956ba956a1256421f2eba6" + } + }, + { + "field": "age", + "input": 18, + "operator": "gte", + "term": { + "domain": null, + "fidelity": "exact", + "index": "ore", + "operator": "gte", + "term": "7ca54e16d7944c0a9d8577d87f3864fdf79e99fea5956ba956a1256421f2eba6" + } + }, + { + "field": "age", + "input": [ + 18, + 65 + ], + "operator": "between", + "term": { + "domain": null, + "fidelity": "exact", + "index": "ore", + "operator": "between", + "term": [ + "7ca54e16d7944c0a9d8577d87f3864fdf79e99fea5956ba956a1256421f2eba6", + "7ca54e16d7944c0a9d8577d87f3864fdf79e99fea5956ba956a20bc4002878be" + ] + } + }, + { + "field": "name", + "input": "alice", + "operator": "matches", + "term": { + "domain": null, + "fidelity": "probabilistic", + "index": "match", + "operator": "matches", + "term": "2800300053006d00890095009d00c100fe00" + } + }, + { + "field": "name", + "input": "ali", + "operator": "matches", + "term": { + "domain": null, + "fidelity": "probabilistic", + "index": "match", + "operator": "matches", + "term": "30009d00c100" + } + } + ], + "target_limit": [ + { + "field": "name", + "input": "ali", + "limit": 53954, + "operator": "matches", + "stats": { + "histogram": [ + [ + 40, + 500000 + ], + [ + 120, + 500000 + ] + ], + "rows": 1000000 + }, + "term": "30009d00c100", + "want": 10 + }, + { + "field": "name", + "input": "ali", + "limit": 3825, + "operator": "matches", + "stats": { + "confidence": 0.5, + "histogram": [ + [ + 40, + 1000000 + ] + ], + "rows": 1000000 + }, + "term": "30009d00c100", + "want": 10 + }, + { + "field": "name", + "input": "ali", + "limit": null, + "operator": "matches", + "stats": { + "rows": 1000000 + }, + "term": "30009d00c100", + "want": 10 + }, + { + "field": "name", + "input": "alice", + "limit": 611, + "operator": "matches", + "stats": { + "histogram": [ + [ + 40, + 500000 + ], + [ + 120, + 500000 + ] + ], + "rows": 1000000 + }, + "term": "2800300053006d00890095009d00c100fe00", + "want": 10 + }, + { + "field": "name", + "input": "alice", + "limit": 10, + "operator": "matches", + "stats": { + "confidence": 0.5, + "histogram": [ + [ + 40, + 1000000 + ] + ], + "rows": 1000000 + }, + "term": "2800300053006d00890095009d00c100fe00", + "want": 10 + }, + { + "field": "name", + "input": "alice", + "limit": null, + "operator": "matches", + "stats": { + "rows": 1000000 + }, + "term": "2800300053006d00890095009d00c100fe00", + "want": 10 + } + ], + "verify": { + "field": "name", + "input": "alice", + "operator": "matches", + "values": [ + "Alice Smith", + "alicia", + "Bob", + "malice", + "ALICE", + null + ], + "verified": [ + true, + false, + false, + true, + true, + false + ] + } +} diff --git a/packages/stack-encrypt/tests/query_operators.rs b/packages/stack-encrypt/tests/query_operators.rs new file mode 100644 index 000000000..d8e133690 --- /dev/null +++ b/packages/stack-encrypt/tests/query_operators.rs @@ -0,0 +1,354 @@ +//! The query fixture: a query a binding names by operator derives the term +//! the typed query chain derives, and a probabilistic term's two questions +//! answer as the typed `MatchQuery` answers them (ADR-0009; ADR-0007, +//! amended 2026-10-16). `tests/fixtures/query_operators.json` records each +//! query's term, a page of verified values and a few fetch limits, so the +//! Go SDK's tests can hold the guest's `se_query_term`, `se_verify` and +//! `se_target_limit` exports to the same results; its README gives the +//! schema. +//! +//! Every query derives under the deterministic key source the record +//! fixture uses, whose index key is `FakeDataKeySource`'s, so nothing in the +//! file depends on a nonce: a run that derives other bytes fails, and +//! `STACK_ENCRYPT_UPDATE_FIXTURES=1 cargo test --test query_operators +//! --all-features` rewrites the file when a derivation changes on purpose. +#![cfg(feature = "dynamic")] + +mod common; + +use std::path::PathBuf; + +use common::{deterministic_cipher, DeterministicSource}; +use serde_json::{json, Value as Json}; +use stack_encrypt::dynamic::{self, record, FfiValue, NoTargets, OpSpec, Operand, QueryTerm}; +use stack_encrypt::op::{self, Limit, Probabilistic, Stats}; +use stack_encrypt::{Equality, KeysetCipher, Match, Ore, Plan, StackCipher}; + +const SEED: [u8; 32] = *b"stack-encrypt record fixture v1 "; + +/// The declaration every query runs on, in the data grammar. +fn plan_json() -> Json { + json!({ + "email": { "context": ["users", "email"], "outputs": ["c", "eq"], "type": "string" }, + "age": { "context": ["users", "age"], "outputs": ["c", "eq", "ore"], "type": "uint32" }, + "name": { "context": ["users", "name"], "outputs": ["c", "match"], "type": "string" } + }) +} + +/// Each query: a field, an operator and its input, as a binding sends them. +fn queries() -> Vec<(&'static str, OpSpec, Json)> { + vec![ + ("email", OpSpec::Eq, json!("bob@example.com")), + ("age", OpSpec::Eq, json!(34)), + ("age", OpSpec::Lt, json!(40)), + ("age", OpSpec::Lte, json!(40)), + ("age", OpSpec::Gt, json!(18)), + ("age", OpSpec::Gte, json!(18)), + ("age", OpSpec::Between, json!([18, 65])), + ("name", OpSpec::Matches, json!("alice")), + ("name", OpSpec::Matches, json!("ali")), + ] +} + +/// The page `verify` answers for `matches "alice"` on `name`. +fn page() -> Json { + json!(["Alice Smith", "alicia", "Bob", "malice", "ALICE", null]) +} + +/// The statistics each recorded limit is sized under. +fn stats_cases() -> Vec<(Json, Stats)> { + vec![ + ( + json!({ "rows": 1_000_000, "histogram": [[40, 500_000], [120, 500_000]] }), + Stats::measured(1_000_000, [(40, 500_000), (120, 500_000)]), + ), + ( + json!({ "rows": 1_000_000, "histogram": [[40, 1_000_000]], "confidence": 0.5 }), + Stats::measured(1_000_000, [(40, 1_000_000)]).with_confidence(0.5), + ), + (json!({ "rows": 1_000_000 }), Stats::declared(1_000_000)), + ] +} + +fn ffi_of_json(value: &Json) -> FfiValue { + match value { + Json::Null => FfiValue::Null, + Json::String(s) => FfiValue::String(s.as_str().into()), + Json::Array(items) => FfiValue::Array(items.iter().map(ffi_of_json).collect()), + Json::Object(entries) => FfiValue::Object( + entries + .iter() + .map(|(k, v)| (k.clone(), ffi_of_json(v))) + .collect(), + ), + Json::Number(n) => match n.as_u64() { + Some(n) => FfiValue::UInt64(n), + None => FfiValue::Float64(n.as_f64().expect("a number")), + }, + Json::Bool(b) => FfiValue::Bool(*b), + } +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +/// A query term as the fixture holds it: the wire form's keys, the bytes in +/// hex. +fn json_of_term(term: &QueryTerm) -> Json { + json!({ + "operator": term.operator().name(), + "fidelity": term.fidelity().name(), + "index": term.index().key(), + "domain": term.domain(), + "term": match term.operand() { + Operand::One(bytes) => json!(hex(bytes)), + Operand::Range { lo, hi } => json!([hex(lo), hex(hi)]), + }, + }) +} + +fn limit_json(limit: Limit) -> Json { + limit.rows().map_or(Json::Null, |rows| json!(rows)) +} + +/// The typed chain's term for the same query, as the operand's bytes: the +/// proof that the data path derives what a Rust caller does. +async fn typed_operand( + cipher: &StackCipher, + field: &str, + operator: OpSpec, + input: &Json, +) -> Operand { + let int = |value: &Json| u32::try_from(value.as_u64().expect("an integer")).expect("a u32"); + let text = |value: &Json| value.as_str().expect("text").to_owned(); + let age = Plan::context("users/age") + .with::((Equality, Ore)) + .build() + .expect("plan"); + let one = Operand::One; + match (field, operator) { + ("email", OpSpec::Eq) => { + let plan = Plan::context("users/email") + .with::(Equality) + .build() + .expect("plan"); + one(cipher + .query(op::Eq, &text(input)) + .on(&plan) + .await + .expect("eq") + .to_bytes()) + } + ("age", OpSpec::Eq) => one(cipher + .query(op::Eq, &int(input)) + .on(&age) + .await + .expect("eq") + .to_bytes()), + ("age", OpSpec::Lt) => one(cipher + .query(op::Lt, &int(input)) + .on(&age) + .await + .expect("lt") + .to_bytes()), + ("age", OpSpec::Lte) => one(cipher + .query(op::Lte, &int(input)) + .on(&age) + .await + .expect("lte") + .to_bytes()), + ("age", OpSpec::Gt) => one(cipher + .query(op::Gt, &int(input)) + .on(&age) + .await + .expect("gt") + .to_bytes()), + ("age", OpSpec::Gte) => one(cipher + .query(op::Gte, &int(input)) + .on(&age) + .await + .expect("gte") + .to_bytes()), + ("age", OpSpec::Between) => { + let range = cipher + .query(op::Between, &(int(&input[0]), int(&input[1]))) + .on(&age) + .await + .expect("between"); + Operand::Range { + lo: range.lo().to_bytes(), + hi: range.hi().to_bytes(), + } + } + ("name", OpSpec::Matches) => one(typed_match(cipher, &text(input)).await.to_bytes()), + other => panic!("no typed query for {other:?}"), + } +} + +async fn typed_match( + cipher: &StackCipher, + input: &str, +) -> stack_encrypt::sem::MatchQuery { + let plan = Plan::context("users/name") + .with::(Match::default()) + .build() + .expect("plan"); + cipher + .query(op::Matches, input) + .on(&plan) + .await + .expect("matches") +} + +async fn data_term( + keyset: &KeysetCipher<'_, DeterministicSource>, + plan: &record::Plan, + field: &str, + operator: OpSpec, + input: &Json, +) -> QueryTerm { + dynamic::query_term( + keyset, + plan, + field, + operator, + ffi_of_json(input), + &NoTargets, + ) + .expect("the query is answered") + .await + .expect("the term derives") +} + +/// The fixture as this run derives it, checked against the typed path as +/// it is built. +async fn derived() -> Json { + let cipher = deterministic_cipher(SEED).await; + let keyset = cipher.default_keyset(); + let plan = record::plan(ffi_of_json(&plan_json())).expect("the plan parses"); + + let mut terms = Vec::new(); + for (field, operator, input) in queries() { + let term = data_term(&keyset, &plan, field, operator, &input).await; + assert_eq!( + term.operand(), + &typed_operand(&cipher, field, operator, &input).await, + "{operator} on {field}: the data path derives the typed chain's term" + ); + terms.push(json!({ + "field": field, + "operator": operator.name(), + "input": input, + "term": json_of_term(&term), + })); + } + + let typed = typed_match(&cipher, "alice").await; + let values: Vec = page() + .as_array() + .expect("a list") + .iter() + .map(ffi_of_json) + .collect(); + let verified = dynamic::verify( + &plan, + "name", + OpSpec::Matches, + ffi_of_json(&json!("alice")), + &values, + &NoTargets, + ) + .expect("verify"); + let typed_verified: Vec = page() + .as_array() + .expect("a list") + .iter() + .map(|value| value.as_str().is_some_and(|text| typed.verify(text))) + .collect(); + assert_eq!(verified, typed_verified, "verify is the typed MatchQuery's"); + + let mut limits = Vec::new(); + for input in ["ali", "alice"] { + let term = data_term(&keyset, &plan, "name", OpSpec::Matches, &json!(input)).await; + let Operand::One(bytes) = term.operand() else { + panic!("a match term is one term"); + }; + let typed = typed_match(&cipher, input).await; + for (stats_json, stats) in stats_cases() { + let read = Stats::from_value(&ffi_of_json(&stats_json)).expect("stats read"); + assert_eq!(read, stats, "the wire form reads as the typed stats"); + let limit = + dynamic::target_limit(&plan, "name", OpSpec::Matches, bytes, 10, &read, &NoTargets) + .expect("a limit"); + assert_eq!( + limit, + typed.target_limit(10, &stats), + "target_limit is the typed MatchQuery's" + ); + limits.push(json!({ + "field": "name", + "operator": "matches", + "input": input, + "term": hex(bytes), + "want": 10, + "stats": stats_json, + "limit": limit_json(limit), + })); + } + } + + json!({ + "_comment": "Query terms derived by operator through the data path (stack_encrypt::dynamic::query_term), each equal to the typed query chain's, with a verified page and fetch limits. Read by tests/query_operators.rs and the Go SDK's tests; the schema is in README.md beside this file. Regenerate with STACK_ENCRYPT_UPDATE_FIXTURES=1; do not edit by hand.", + "key_source": { "kind": "deterministic-sha256", "seed": hex(&SEED) }, + "keyset_id": keyset.keyset_id().to_string(), + "plan": plan_json(), + "queries": terms, + "verify": { + "field": "name", + "operator": "matches", + "input": "alice", + "values": page(), + "verified": verified, + }, + "target_limit": limits, + }) +} + +fn fixture_path() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/query_operators.json") +} + +/// The committed fixture is what this run derives. With +/// `STACK_ENCRYPT_UPDATE_FIXTURES=1` it is written first. +#[tokio::test] +async fn the_committed_query_fixture_is_what_the_engine_derives() { + let now = derived().await; + let path = fixture_path(); + if std::env::var_os("STACK_ENCRYPT_UPDATE_FIXTURES").is_some() { + let mut text = serde_json::to_string_pretty(&now).expect("serialise"); + text.push('\n'); + std::fs::write(&path, text).expect("write the fixture"); + } + let committed: Json = + serde_json::from_str(&std::fs::read_to_string(&path).expect("the fixture is committed")) + .expect("the fixture is JSON"); + assert_eq!(committed, now, "the committed query fixture is today's"); +} + +/// The fixture pins more than one answer of each kind: a verified page +/// with both outcomes, and limits both finite and unbounded. +#[tokio::test] +async fn the_fixture_pins_both_outcomes() { + let now = derived().await; + let verified = now["verify"]["verified"].as_array().expect("a list"); + assert!(verified.contains(&json!(true)) && verified.contains(&json!(false))); + let limits: Vec<&Json> = now["target_limit"] + .as_array() + .expect("a list") + .iter() + .map(|case| &case["limit"]) + .collect(); + assert!(limits.iter().any(|limit| limit.is_u64())); + assert!(limits.iter().any(|limit| limit.is_null())); +} diff --git a/packages/stack-encrypt/tests/ui/op_at_path_inherits_fidelity.stderr b/packages/stack-encrypt/tests/ui/op_at_path_inherits_fidelity.stderr index 64fc20e2f..8463a62f4 100644 --- a/packages/stack-encrypt/tests/ui/op_at_path_inherits_fidelity.stderr +++ b/packages/stack-encrypt/tests/ui/op_at_path_inherits_fidelity.stderr @@ -4,9 +4,9 @@ error[E0277]: `EqualityTerm` cannot answer `AtPath` 9 | answers::, EqualityTerm>(); | ^^^^^^^^^^^^ this query term does not meet the operator's requirement | - = help: the trait `Probabilistic` is not implemented for `EqualityTerm` + = help: the trait `stack_encrypt::op::Probabilistic` is not implemented for `EqualityTerm` = note: a term that answers `op::Matches` must implement `op::Probabilistic` -help: the trait `Probabilistic` is implemented for `MatchQuery` +help: the trait `stack_encrypt::op::Probabilistic` is implemented for `MatchQuery` --> src/sem/mod.rs | | impl crate::op::Probabilistic for MatchQuery { diff --git a/packages/stack-encrypt/tests/ui/op_inexact_operator_needs_probabilistic.stderr b/packages/stack-encrypt/tests/ui/op_inexact_operator_needs_probabilistic.stderr index ceaa770be..248900eee 100644 --- a/packages/stack-encrypt/tests/ui/op_inexact_operator_needs_probabilistic.stderr +++ b/packages/stack-encrypt/tests/ui/op_inexact_operator_needs_probabilistic.stderr @@ -4,9 +4,9 @@ error[E0277]: `EqualityTerm` cannot answer `stack_encrypt::op::Matches` 24 | type Query = EqualityTerm; | ^^^^^^^^^^^^ this query term does not meet the operator's requirement | - = help: the trait `Probabilistic` is not implemented for `EqualityTerm` + = help: the trait `stack_encrypt::op::Probabilistic` is not implemented for `EqualityTerm` = note: a term that answers `op::Matches` must implement `op::Probabilistic` -help: the trait `Probabilistic` is implemented for `MatchQuery` +help: the trait `stack_encrypt::op::Probabilistic` is implemented for `MatchQuery` --> src/sem/mod.rs | | impl crate::op::Probabilistic for MatchQuery { diff --git a/packages/stack-encrypt/tests/ui/op_json_not_answered.stderr b/packages/stack-encrypt/tests/ui/op_json_not_answered.stderr index a2feec653..7446d7c95 100644 --- a/packages/stack-encrypt/tests/ui/op_json_not_answered.stderr +++ b/packages/stack-encrypt/tests/ui/op_json_not_answered.stderr @@ -7,12 +7,12 @@ error[E0277]: `Contains` has no default index given `std::string::String` = help: the trait `DefaultIndex` is not implemented for `Contains` = note: `op::Eq` defaults to `Equality`; `op::Matches` defaults to `Match` with the default configuration, which is case-insensitive and over text only; `op::Lt`, `Lte`, `Gt`, `Gte` and `Between` default to `Ore` over an integer, and `Between` takes a pair; name another index, such as `Ope`, with `.via(index)`; no index answers `op::Contains`, `op::ContainedBy` or `op::AtPath` yet, since this crate has no JSON index = help: the following other types implement trait `DefaultIndex`: - `Between` implements `DefaultIndex<(S, S)>` - `Gt` implements `DefaultIndex` - `Gte` implements `DefaultIndex` - `Lt` implements `DefaultIndex` - `Lte` implements `DefaultIndex` + `stack_encrypt::op::Between` implements `DefaultIndex<(S, S)>` `stack_encrypt::op::Eq` implements `DefaultIndex` + `stack_encrypt::op::Gt` implements `DefaultIndex` + `stack_encrypt::op::Gte` implements `DefaultIndex` + `stack_encrypt::op::Lt` implements `DefaultIndex` + `stack_encrypt::op::Lte` implements `DefaultIndex` `stack_encrypt::op::Matches<::Case>` implements `DefaultIndex` = note: required for `ValuePlan>` to implement `Chooses` diff --git a/packages/stack-encrypt/tests/ui/op_order_on_text.stderr b/packages/stack-encrypt/tests/ui/op_order_on_text.stderr index eb4df5737..43ad24d5d 100644 --- a/packages/stack-encrypt/tests/ui/op_order_on_text.stderr +++ b/packages/stack-encrypt/tests/ui/op_order_on_text.stderr @@ -16,5 +16,5 @@ error[E0277]: `std::string::String` is not ordered: the ordering operators are a u16 u32 and $N others - = note: required for `Lt` to implement `DefaultIndex` - = note: required for `ValuePlan>` to implement `Chooses` + = note: required for `stack_encrypt::op::Lt` to implement `DefaultIndex` + = note: required for `ValuePlan>` to implement `Chooses` From ffbf57b361a7f6b4a93ba59bf3d52a4f388aed06 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Fri, 16 Oct 2026 03:13:20 +1100 Subject: [PATCH 2/8] feat(golang): export se_query_term, se_verify and se_target_limit so the Go SDK can query by operator without a tokenizer of its own The encrypt guest derives a query term from a plan, a field, an operator name and its input, verifies a page of decrypted values in one call, and sizes a probabilistic query's fetch from its term and the table's statistics; the last two use no key and run with no cipher. se_targets entries carry the operators each EQL type answers, and the Go reader requires them. Cipher.QueryTerm, Cipher.Verify and Cipher.TargetLimit wrap the exports for generated code, and the Go tests hold them to the Rust query fixture through the deterministic guest. Part of #1188 --- languages/golang/encrypt/checker.go | 64 ++- languages/golang/encrypt/guest.go | 4 + languages/golang/encrypt/guest/src/abi.rs | 124 +++++- languages/golang/encrypt/guest/src/lib.rs | 4 +- languages/golang/encrypt/guest/src/ops.rs | 149 ++++++- languages/golang/encrypt/guest/tests/leak.rs | 97 +++++ .../golang/encrypt/guest/tests/native_ops.rs | 377 +++++++++++++++++- languages/golang/encrypt/guest_test.go | 2 +- languages/golang/encrypt/query.go | 302 ++++++++++++++ languages/golang/encrypt/query_test.go | 362 +++++++++++++++++ languages/golang/encrypt/records.go | 45 ++- languages/golang/encrypt/targets_test.go | 68 +++- languages/golang/internal/record/record.go | 36 +- 13 files changed, 1593 insertions(+), 41 deletions(-) create mode 100644 languages/golang/encrypt/query.go create mode 100644 languages/golang/encrypt/query_test.go diff --git a/languages/golang/encrypt/checker.go b/languages/golang/encrypt/checker.go index 7f3a99e25..807967325 100644 --- a/languages/golang/encrypt/checker.go +++ b/languages/golang/encrypt/checker.go @@ -165,6 +165,8 @@ func parseTarget(entry vcvalue.Object) (record.Target, error) { t.Producible = b case "reason": t.Reason, err = targetOptionalString(f) + case "operators": + t.Operators, err = targetOperators(f.Value) default: return t, fmt.Errorf("unknown key %q", f.Key) } @@ -175,7 +177,7 @@ func parseTarget(entry vcvalue.Object) (record.Target, error) { if t.Name == "" { return t, errors.New("no name") } - for _, key := range []string{"indexes", "producible"} { + for _, key := range []string{"indexes", "producible", "operators"} { if !seen[key] { return t, fmt.Errorf("no %s", key) } @@ -183,6 +185,66 @@ func parseTarget(entry vcvalue.Object) (record.Target, error) { return t, nil } +// targetOperators reads an entry's operators: a list of objects, each with +// exactly the keys operator, fidelity, arity and index, every value one the +// engine writes. +func targetOperators(value any) ([]record.TargetOperator, error) { + list, ok := value.([]any) + if !ok { + return nil, fmt.Errorf("operators is %T, not a list", value) + } + operators := make([]record.TargetOperator, 0, len(list)) + for i, item := range list { + entry, ok := item.(vcvalue.Object) + if !ok { + return nil, fmt.Errorf("operator %d is %T, not an object", i, item) + } + var op record.TargetOperator + seen := map[string]bool{} + for _, f := range entry { + if seen[f.Key] { + return nil, fmt.Errorf("operator %d: key %q twice", i, f.Key) + } + seen[f.Key] = true + s, err := targetString(f) + if err != nil { + return nil, fmt.Errorf("operator %d: %v", i, err) + } + switch f.Key { + case "operator": + op.Name = s + case "fidelity": + op.Fidelity = record.Fidelity(s) + if op.Fidelity != record.Exact && op.Fidelity != record.Probabilistic { + return nil, fmt.Errorf("operator %d: unknown fidelity %q", i, s) + } + case "arity": + switch s { + case "one": + case "pair": + op.Pair = true + default: + return nil, fmt.Errorf("operator %d: unknown arity %q", i, s) + } + case "index": + switch o := record.Output(s); o { + case record.Equality, record.Match, record.Ore, record.Ope: + op.Index = o + default: + return nil, fmt.Errorf("operator %d: unknown index %q", i, s) + } + default: + return nil, fmt.Errorf("operator %d: unknown key %q", i, f.Key) + } + } + if len(seen) != 4 || op.Name == "" { + return nil, fmt.Errorf("operator %d: not every key, or no name", i) + } + operators = append(operators, op) + } + return operators, nil +} + func targetString(f vcvalue.Field) (string, error) { s, ok := f.Value.(string) if !ok { diff --git a/languages/golang/encrypt/guest.go b/languages/golang/encrypt/guest.go index c5bbecbd4..78ef91c4d 100644 --- a/languages/golang/encrypt/guest.go +++ b/languages/golang/encrypt/guest.go @@ -85,6 +85,7 @@ type instance struct { exports guest.Exports cipherInit, shutdown, keyset api.Function term, query api.Function + queryTerm, verify, limit api.Function encryptRecord, decryptRecord api.Function planCheck, targets api.Function } @@ -172,6 +173,9 @@ func newInstance(ctx context.Context, wasm []byte, t *transport, policy guest.Lo "se_keyset": &inst.keyset, "se_term": &inst.term, "se_query": &inst.query, + "se_query_term": &inst.queryTerm, + "se_verify": &inst.verify, + "se_target_limit": &inst.limit, "se_encrypt_record": &inst.encryptRecord, "se_decrypt_record": &inst.decryptRecord, "se_plan_check": &inst.planCheck, diff --git a/languages/golang/encrypt/guest/src/abi.rs b/languages/golang/encrypt/guest/src/abi.rs index 7abbdd660..e608bb90b 100644 --- a/languages/golang/encrypt/guest/src/abi.rs +++ b/languages/golang/encrypt/guest/src/abi.rs @@ -6,8 +6,10 @@ //! //! - **Every export handed plaintext wipes that buffer in place before it //! returns**, rather than leaving it for `se_dealloc`: [`se_cipher_init`] -//! (the config carries the client key), and [`se_term`], [`se_query`] and -//! [`se_encrypt_record`] (their value/source buffers). The host's plaintext therefore lives no longer +//! (the config carries the client key), and [`se_term`], [`se_query`], +//! [`se_query_term`], [`se_verify`] and [`se_encrypt_record`] (their +//! value, source and input buffers, and [`se_verify`]'s page of decrypted +//! values). The host's plaintext therefore lives no longer //! than the call, instead of until the host gets round to releasing it. //! **A host must not read a plaintext input buffer back after the call, or //! pass the same buffer to two calls** — it will be zeros. Option, context, @@ -53,7 +55,8 @@ //! otherwise): each is a [`stack_encrypt::NonEmpty`] from the moment it is //! parsed, and the sealing and opening sides bind that one value. //! [`se_plan_check`] and [`se_targets`] answer the Go generator's questions -//! and need no cipher. +//! and need no cipher; nor do [`se_verify`] and [`se_target_limit`], a +//! probabilistic query's two questions, which use no key. //! //! Every export decodes and validates *all* of its inputs — the operation //! payload, the plan or context, the term kind, the value against the @@ -500,6 +503,121 @@ pub unsafe extern "C" fn se_query( }) } +/// Derive the query term for one plan field, an operator and its input +/// under the keyset `opts` selects ([`ops::query_term`]): a codec-encoded +/// input (one value, or for `between` a list of two), the codec-encoded +/// plan, and the field's and the operator's names (UTF-8 bytes) in; the +/// term's codec-encoded object out. An operator the field does not answer +/// is `STATUS_ENCODING` naming the field, as a mistyped value is, before +/// the cipher is consulted; the refusals the typed path makes before a term +/// exists (a range the wrong way round, match text that cuts no token or +/// does not normalize) are `STATUS_TERM`, also before the cipher. The input +/// buffer is wiped in place. +/// +/// # Safety +/// +/// As for [`se_term`]. +#[no_mangle] +pub unsafe extern "C" fn se_query_term( + val_ptr: *mut u8, + val_len: u32, + plan_ptr: *const u8, + plan_len: u32, + field_ptr: *const u8, + field_len: u32, + op_ptr: *const u8, + op_len: u32, + opt_ptr: *const u8, + opt_len: u32, +) -> u64 { + export(|| { + let value = unsafe { take_plaintext(val_ptr, val_len)? }; + let value = value.as_slice(); + // SAFETY: host-owned ranges the export was handed; the borrows end + // before it returns and before any wipe of an overlapping range. + let plan = unsafe { input(plan_ptr, plan_len)? }; + let field = unsafe { input(field_ptr, field_len)? }; + let operator = unsafe { input(op_ptr, op_len)? }; + let opts = unsafe { input(opt_ptr, opt_len)? }; + ops::validate::query_term(value, plan, field, operator)?; + with_keyset(opts, |keyset| { + block_on(ops::query_term(keyset, value, plan, field, operator)) + }) + }) +} + +/// Verify a page of decrypted values against a probabilistic query +/// ([`ops::verify`]): the query's codec-encoded input and a codec-encoded +/// list of values, the codec-encoded plan, and the field's and the +/// operator's names in; a codec-encoded list of booleans out, one per value. +/// Both plaintext buffers, the input and the values, are wiped in place. An +/// exact operator has nothing to verify and is `STATUS_ENCODING`. Needs no +/// cipher: it works in every instance state, as [`se_plan_check`] does. +/// +/// # Safety +/// +/// As for [`se_term`]. +#[no_mangle] +pub unsafe extern "C" fn se_verify( + in_ptr: *mut u8, + in_len: u32, + vals_ptr: *mut u8, + vals_len: u32, + plan_ptr: *const u8, + plan_len: u32, + field_ptr: *const u8, + field_len: u32, + op_ptr: *const u8, + op_len: u32, +) -> u64 { + export(|| { + let query = unsafe { take_plaintext(in_ptr, in_len)? }; + let values = unsafe { take_plaintext(vals_ptr, vals_len)? }; + // SAFETY: host-owned ranges the export was handed; the borrows end + // before it returns. + let plan = unsafe { input(plan_ptr, plan_len)? }; + let field = unsafe { input(field_ptr, field_len)? }; + let operator = unsafe { input(op_ptr, op_len)? }; + ops::verify(query.as_slice(), values.as_slice(), plan, field, operator) + }) +} + +/// How many rows a probabilistic query should fetch for `want` of them to +/// verify ([`ops::target_limit`]): the term's bytes as [`se_query_term`] +/// returned them, the codec-encoded statistics, the codec-encoded plan, +/// and the field's and the operator's names in, with `want`; the +/// codec-encoded limit out, an unsigned integer or null. An exact operator +/// has no fetch to size and is `STATUS_ENCODING`. Needs no cipher. +/// +/// # Safety +/// +/// As for [`se_term`]. +#[no_mangle] +pub unsafe extern "C" fn se_target_limit( + term_ptr: *const u8, + term_len: u32, + stats_ptr: *const u8, + stats_len: u32, + plan_ptr: *const u8, + plan_len: u32, + field_ptr: *const u8, + field_len: u32, + op_ptr: *const u8, + op_len: u32, + want: u64, +) -> u64 { + export(|| { + // SAFETY: host-owned ranges the export was handed; the borrows end + // before it returns. + let term = unsafe { input(term_ptr, term_len)? }; + let stats = unsafe { input(stats_ptr, stats_len)? }; + let plan = unsafe { input(plan_ptr, plan_len)? }; + let field = unsafe { input(field_ptr, field_len)? }; + let operator = unsafe { input(op_ptr, op_len)? }; + ops::target_limit(term, stats, plan, field, operator, want) + }) +} + /// The EQL types this build knows, as a codec-encoded /// `{"targets": [...]}` ([`ops::targets`]): the catalog in the `eql` build, /// nothing in the other. Needs no cipher. diff --git a/languages/golang/encrypt/guest/src/lib.rs b/languages/golang/encrypt/guest/src/lib.rs index e402e14d0..c22d3a2dc 100644 --- a/languages/golang/encrypt/guest/src/lib.rs +++ b/languages/golang/encrypt/guest/src/lib.rs @@ -46,7 +46,9 @@ //! //! The exports are the record path (`se_encrypt_record`, `se_decrypt_record`), //! per-field term derivation (`se_term`), a target field's EQL query value -//! (`se_query`), keyset resolution (`se_keyset`), the generator's two +//! (`se_query`), a query named by operator and a probabilistic query's two +//! questions (`se_query_term`, `se_verify`, `se_target_limit`; ADR-0007, +//! amended 2026-10-16), keyset resolution (`se_keyset`), the generator's two //! questions (`se_plan_check`, `se_targets`) and the lifetime pair //! (`se_cipher_init`, `se_shutdown`). There is no whole-value export: the //! Go SDK seals every value under a declaration (ADR-0007). diff --git a/languages/golang/encrypt/guest/src/ops.rs b/languages/golang/encrypt/guest/src/ops.rs index 747296da2..78c036e5c 100644 --- a/languages/golang/encrypt/guest/src/ops.rs +++ b/languages/golang/encrypt/guest/src/ops.rs @@ -237,6 +237,132 @@ where .map_err(|e| fail_error(&e)) } +// ============================================================================= +// Queries by operator +// ============================================================================= + +/// Derive the query term for one plan field, an operator and its input +/// (ADR-0009; ADR-0007, amended 2026-10-16): a codec-encoded input, the +/// codec-encoded plan, the field's name and the operator's name (both UTF-8) +/// in, the term's codec-encoded object out ([`dynamic::QueryTerm::to_value`]): +/// +/// ```text +/// { "operator": "matches", "fidelity": "probabilistic", "index": "match", +/// "domain": null, "term": } +/// ``` +/// +/// The input is one value for every operator but `between`, whose input is +/// a list of two, low then high. It is read as the field's declared kind +/// first, so a host's float `34.0` queries a `uint64` field. A field with +/// indexes derives the term of the index that answers the operator, as +/// bytes; `between` derives one per bound, as a list of two. A field that +/// names an EQL type runs the type's query twin through this build's +/// resolver, and `term` holds the twin's JSON bytes and `domain` its +/// PostgreSQL domain. No data key is minted. +pub async fn query_term( + cipher: &KeysetCipher<'_, K>, + input: &[u8], + plan: &[u8], + field: &[u8], + operator: &[u8], +) -> Result, u32> +where + K: DataKeySource + Sync + 'static, +{ + let (plan, field, operator) = parse_query(plan, field, operator)?; + let term = dynamic::query_term( + cipher, + &plan, + field, + operator, + decode_value(input)?, + &resolver(), + ) + .map_err(|e| fail_dynamic(&e))? + .await + .map_err(|e| fail_error(&e))?; + encode_value(term.to_value()) +} + +/// Verify a page of decrypted values against a probabilistic query: the +/// query's codec-encoded input and a codec-encoded list of values, with the +/// plan, the field's name and the operator's name, in; a codec-encoded list +/// of booleans out, one per value in order ([`dynamic::verify`]). The query +/// is named as it was derived, and its tokens are cut again from the input: +/// they are plaintext, so no term carries them. A null value is `false`. +/// Needs no cipher and no key. +pub fn verify( + input: &[u8], + values: &[u8], + plan: &[u8], + field: &[u8], + operator: &[u8], +) -> Result, u32> { + let (plan, field, operator) = parse_query(plan, field, operator)?; + let FfiValue::Array(values) = decode_value(values)? else { + return Err(last_error::malformed("the values to verify are not a list")); + }; + let verified = dynamic::verify( + &plan, + field, + operator, + decode_value(input)?, + &values, + &resolver(), + ) + .map_err(|e| fail_dynamic(&e))?; + encode_value(FfiValue::Array( + verified.into_iter().map(FfiValue::Bool).collect(), + )) +} + +/// How many rows a probabilistic query should fetch for `want` of them to +/// verify: the term's bytes as [`query_term`] returned them (not a frame), +/// the codec-encoded statistics +/// ([`Stats::from_value`](stack_encrypt::op::Stats::from_value)), the plan, +/// the field's name and the operator's name in; the codec-encoded limit out, +/// an unsigned 64-bit integer or null for no finite fetch +/// ([`dynamic::limit_to_value`]). Needs no cipher and no key. +pub fn target_limit( + term: &[u8], + stats: &[u8], + plan: &[u8], + field: &[u8], + operator: &[u8], + want: u64, +) -> Result, u32> { + let (plan, field, operator) = parse_query(plan, field, operator)?; + let stats = stack_encrypt::op::Stats::from_value(&decode_value(stats)?) + .map_err(|e| fail_dynamic(&e))?; + let limit = dynamic::target_limit(&plan, field, operator, term, want, &stats, &resolver()) + .map_err(|e| fail_dynamic(&e))?; + encode_value(dynamic::limit_to_value(limit)) +} + +/// The plan, field name and operator a query names: the plan parsed against +/// this build's resolver, the field's name UTF-8, and the operator one the +/// engine knows ([`dynamic::OpSpec::parse`]); an unknown operator is +/// refused naming the field, as an operator the field does not answer is. +fn parse_query<'f>( + plan: &[u8], + field: &'f [u8], + operator: &[u8], +) -> Result<(dynamic::record::Plan, &'f str, dynamic::OpSpec), u32> { + let plan = parse_plan(plan)?; + let field = std::str::from_utf8(field) + .map_err(|_| last_error::malformed("the field name is not UTF-8"))?; + let operator = std::str::from_utf8(operator) + .ok() + .and_then(dynamic::OpSpec::parse) + .ok_or_else(|| { + fail_dynamic(&dynamic::Error::Source { + field: Some(field.to_owned()), + reason: dynamic::Reason::UnknownOperator, + }) + })?; + Ok((plan, field, operator)) +} + /// A codec-encoded plan, parsed against this build's resolver: a target /// name this build cannot run is refused here, the same way at every /// export that takes a plan. @@ -284,6 +410,11 @@ pub fn plan_check(plan: &[u8]) -> Result<(), u32> { /// | `query_sql_domain` | string or null | the Postgres domain of the query value | /// | `producible` | bool | whether this build produces the type | /// | `reason` | string or null | why not, when `producible` is false | +/// | `operators` | list of objects | the operators the engine answers for the type, each `{"operator", "fidelity", "arity", "index"}` ([`TargetDescriptor`](stack_encrypt::dynamic::TargetDescriptor#wire-format)) | +/// +/// The first ten keys are eql-bindings' row; `operators` is the engine's, +/// computed from the row, so a generator asks the engine which operators a +/// type answers and how far each answer can be trusted. pub fn targets() -> Result, u32> { use stack_encrypt::dynamic::TargetResolver as _; encode_value(FfiValue::Object(vec![( @@ -362,6 +493,17 @@ pub mod validate { Ok(()) } + /// A query's inputs, as [`query_term`](super::query_term) takes them: + /// the plan parses, the field is one of its fields, the operator is one + /// the engine knows and the field answers, and the input reads as the + /// field's kind and passes the refusals the typed path makes before a + /// term exists ([`dynamic::check_query`]). + pub fn query_term(input: &[u8], plan: &[u8], field: &[u8], operator: &[u8]) -> Result<(), u32> { + let (plan, field, operator) = parse_query(plan, field, operator)?; + dynamic::check_query(&plan, field, operator, decode_value(input)?, &resolver()) + .map_err(|e| fail_dynamic(&e)) + } + /// A record tree against its plan, as [`decrypt_record`] takes them: /// the plan parses, the tree decodes with well-formed leaves, every /// ciphertext-bearing field has a `"c"` node that is not a passthrough, @@ -699,10 +841,15 @@ mod tests { "query", "query_sql_domain", "producible", - "reason" + "reason", + "operators" ] ); assert!(matches!(text_eq[8].1, FfiValue::Bool(true))); + assert!( + matches!(&text_eq[10].1, FfiValue::Array(operators) if operators.len() == 1), + "TextEq answers one operator" + ); } #[test] diff --git a/languages/golang/encrypt/guest/tests/leak.rs b/languages/golang/encrypt/guest/tests/leak.rs index 46cb0816d..64dd85a31 100644 --- a/languages/golang/encrypt/guest/tests/leak.rs +++ b/languages/golang/encrypt/guest/tests/leak.rs @@ -646,6 +646,103 @@ fn scenarios() -> Vec<(&'static str, FfiValue)> { ), ), ]; + // -- queries by operator ----------------------------------------------- + let marker = || FfiValue::Bytes(vitaminc_protected::Protected::new(PLAINTEXT.into())); + let match_plan = || { + encode(obj(vec![( + "name", + field(label("name"), &["c", "match"], Some("string")), + )])) + }; + let source = &["stack_encrypt::dynamic_source"]; + out.extend([ + ( + "an operator the field does not answer", + failure("not answered", source, || { + block_on(ops::query_term( + &keyset, + &encode(s(PLAINTEXT)), + &plan_under(label("email")), + b"email", + b"lt", + )) + }), + ), + ( + "a query input of another type", + failure("query input", source, || { + block_on(ops::query_term( + &keyset, + &encode(marker()), + &plan_under(label("email")), + b"email", + b"eq", + )) + }), + ), + ( + "an operator name the engine does not know", + failure("unknown operator", source, || { + block_on(ops::query_term( + &keyset, + &encode(s(PLAINTEXT)), + &plan_under(label("email")), + b"email", + PLAINTEXT.as_bytes(), + )) + }), + ), + ( + "verifying an exact operator", + failure("verify exact", source, || { + ops::verify( + &encode(s(PLAINTEXT)), + &encode(FfiValue::Array(vec![s(PLAINTEXT)])), + &plan_under(label("email")), + b"email", + b"eq", + ) + }), + ), + ( + "verifying a value of another type", + failure("verify value", source, || { + ops::verify( + &encode(s(PLAINTEXT)), + &encode(FfiValue::Array(vec![marker()])), + &match_plan(), + b"name", + b"matches", + ) + }), + ), + ( + "a fetch sized from bytes that are not a term", + failure("limit term", source, || { + ops::target_limit( + PLAINTEXT.as_bytes(), + &encode(obj(vec![("rows", FfiValue::UInt64(1))])), + &match_plan(), + b"name", + b"matches", + 10, + ) + }), + ), + ( + "statistics that are not statistics", + failure("limit stats", source, || { + ops::target_limit( + &[1, 0], + &encode(obj(vec![("rows", s(PLAINTEXT))])), + &match_plan(), + b"name", + b"matches", + 10, + ) + }), + ), + ]); // An EQL value whose stored bytes are a JSON string where an object // belongs: serde_json's message would quote the string. #[cfg(feature = "eql")] diff --git a/languages/golang/encrypt/guest/tests/native_ops.rs b/languages/golang/encrypt/guest/tests/native_ops.rs index 56acfb085..410d1a9ef 100644 --- a/languages/golang/encrypt/guest/tests/native_ops.rs +++ b/languages/golang/encrypt/guest/tests/native_ops.rs @@ -26,7 +26,7 @@ use stack_encrypt::{ }; use stack_encrypt_guest::ops::{self, TERM_EQUALITY, TERM_MATCH, TERM_OPE, TERM_ORE}; use stack_encrypt_guest::status::{ - STATUS_AUTH, STATUS_CONTEXT_MISMATCH, STATUS_ENCODING, STATUS_FOREIGN_KEYSET, + STATUS_AUTH, STATUS_CONTEXT_MISMATCH, STATUS_ENCODING, STATUS_FOREIGN_KEYSET, STATUS_TERM, }; use stack_kms::{ DataKey, DataKeySource, DataKeyWithTag, FakeDataKeySource, GenerateKeyPayload, IndexKeySource, @@ -1383,6 +1383,381 @@ fn targets_lists_what_the_build_holds() { ); } +// ============================================================================= +// Queries by operator +// ============================================================================= + +/// The entries of a query term's object, by key. +fn query_term_entries(out: &[u8]) -> Vec<(String, FfiValue)> { + let FfiValue::Object(entries) = decode(out) else { + panic!("a query term is an object"); + }; + entries +} + +fn entry<'a>(entries: &'a [(String, FfiValue)], key: &str) -> &'a FfiValue { + &entries + .iter() + .find(|(k, _)| k == key) + .unwrap_or_else(|| panic!("no {key}")) + .1 +} + +fn bytes_of(value: &FfiValue) -> Vec { + match value { + FfiValue::Bytes(b) => b.risky_ref().to_vec(), + other => panic!("expected bytes, got {}", kind(other)), + } +} + +/// A query by operator derives, through the guest, the bytes the term +/// export derives for the index that answers it, under the field's +/// context: the plan picks the index, and the derivation is the one a +/// stored term came from. +#[test] +fn a_query_term_is_the_term_of_the_index_that_answers_its_operator() { + let cipher = cipher(); + let keyset = cipher.default_keyset(); + let term_of = |value: FfiValue, field: &str, kind: u32| { + block_on(ops::term( + &keyset, + &encode(value), + &encode(label(field)), + kind, + )) + .expect("term") + }; + let query = |input: FfiValue, field: &str, operator: &str| { + let input = encode(input); + assert_eq!( + ops::validate::query_term(&input, &plan(), field.as_bytes(), operator.as_bytes()), + Ok(()), + "{operator} on {field} validates" + ); + query_term_entries( + &block_on(ops::query_term( + &keyset, + &input, + &plan(), + field.as_bytes(), + operator.as_bytes(), + )) + .expect("query term"), + ) + }; + + let eq = query(FfiValue::UInt32(34), "age", "eq"); + assert_eq!(text(entry(&eq, "operator")), "eq"); + assert_eq!(text(entry(&eq, "fidelity")), "exact"); + assert_eq!(text(entry(&eq, "index")), "eq"); + assert!(matches!(entry(&eq, "domain"), FfiValue::Null)); + assert_eq!( + bytes_of(entry(&eq, "term")), + term_of(FfiValue::UInt32(34), "age", TERM_EQUALITY) + ); + + // A host's float reads as the field's integer. + let lt = query(FfiValue::Float64(40.0), "age", "lt"); + assert_eq!(text(entry(<, "index")), "ore"); + assert_eq!( + bytes_of(entry(<, "term")), + term_of(FfiValue::UInt32(40), "age", TERM_ORE) + ); + + let range = query( + FfiValue::Array(vec![FfiValue::UInt32(18), FfiValue::UInt32(65)]), + "age", + "between", + ); + let FfiValue::Array(bounds) = entry(&range, "term") else { + panic!("a range is a list of two terms"); + }; + assert_eq!(bounds.len(), 2); + assert_eq!( + bytes_of(&bounds[0]), + term_of(FfiValue::UInt32(18), "age", TERM_ORE) + ); + assert_eq!( + bytes_of(&bounds[1]), + term_of(FfiValue::UInt32(65), "age", TERM_ORE) + ); + + let matches = query(s("alice"), "name", "matches"); + assert_eq!(text(entry(&matches, "fidelity")), "probabilistic"); + assert_eq!( + bytes_of(entry(&matches, "term")), + term_of(s("alice"), "name", TERM_MATCH) + ); +} + +/// The validator refuses what the operation refuses, with the same status, +/// before a keyset is resolved: an operator the field does not answer, or +/// one the engine does not know, as a mistyped value is refused; the typed +/// path's own query refusals as `STATUS_TERM`. +#[test] +fn query_term_validation_refuses_what_the_query_op_refuses() { + let cipher = cipher(); + let keyset = cipher.default_keyset(); + for (what, input, field, operator, status) in [ + ( + "unknown operator", + FfiValue::UInt32(1), + "age", + "like", + STATUS_ENCODING, + ), + ("not answered", s("ali"), "age", "matches", STATUS_ENCODING), + ( + "not answered by a match index", + s("ali"), + "name", + "eq", + STATUS_ENCODING, + ), + ( + "no such field", + FfiValue::UInt32(1), + "height", + "eq", + STATUS_ENCODING, + ), + ("mistyped input", s("34"), "age", "eq", STATUS_ENCODING), + ( + "not a range", + FfiValue::UInt32(1), + "age", + "between", + STATUS_ENCODING, + ), + ( + "inverted range", + FfiValue::Array(vec![FfiValue::UInt32(65), FfiValue::UInt32(18)]), + "age", + "between", + STATUS_TERM, + ), + ("no token", s("al"), "name", "matches", STATUS_TERM), + ] { + let input = encode(input); + assert_eq!( + ops::validate::query_term(&input, &plan(), field.as_bytes(), operator.as_bytes()), + Err(status), + "{what}: validation" + ); + assert_eq!( + block_on(ops::query_term( + &keyset, + &input, + &plan(), + field.as_bytes(), + operator.as_bytes() + )), + Err(status), + "{what}: the op" + ); + } + assert_eq!( + ops::validate::query_term(&encode(FfiValue::UInt32(1)), &plan(), b"\xff", b"eq"), + Err(STATUS_ENCODING), + "a field name that is not UTF-8" + ); +} + +/// `verify` answers one boolean per value, in order, with no cipher; an +/// exact operator has nothing to verify. +#[test] +fn verify_answers_a_page_in_one_call() { + let values = encode(FfiValue::Array(vec![ + s("Alice Smith"), + s("Bob"), + FfiValue::Null, + s("malice"), + ])); + let out = + ops::verify(&encode(s("alice")), &values, &plan(), b"name", b"matches").expect("verify"); + let FfiValue::Array(verified) = decode(&out) else { + panic!("a list"); + }; + let verified: Vec = verified + .iter() + .map(|v| match v { + FfiValue::Bool(b) => *b, + other => panic!("a boolean, not {}", kind(other)), + }) + .collect(); + assert_eq!( + verified, + [true, false, false, true], + "malice holds every 3-gram of alice" + ); + + for (what, input, values, field, operator) in [ + ( + "an exact operator", + FfiValue::UInt32(1), + FfiValue::Array(vec![]), + "age", + "eq", + ), + ( + "values not a list", + s("alice"), + s("Alice"), + "name", + "matches", + ), + ( + "a value of another type", + s("alice"), + FfiValue::Array(vec![FfiValue::UInt32(1)]), + "name", + "matches", + ), + ( + "an unknown operator", + s("alice"), + FfiValue::Array(vec![]), + "name", + "match", + ), + ] { + assert_eq!( + ops::verify( + &encode(input), + &encode(values), + &plan(), + field.as_bytes(), + operator.as_bytes() + ), + Err(STATUS_ENCODING), + "{what}" + ); + } + assert_eq!( + ops::verify( + &encode(s("al")), + &encode(FfiValue::Array(vec![])), + &plan(), + b"name", + b"matches" + ), + Err(STATUS_TERM), + "a query that cuts no token verifies nothing" + ); +} + +/// `target_limit` reads the term's bytes and the statistics, with no +/// cipher: a measured histogram sizes a fetch, the declared prior a data +/// plan has none of answers null. +#[test] +fn target_limit_sizes_a_fetch_from_the_terms_bytes() { + let cipher = cipher(); + let keyset = cipher.default_keyset(); + let out = block_on(ops::query_term( + &keyset, + &encode(s("ali")), + &plan(), + b"name", + b"matches", + )) + .expect("query term"); + let term = bytes_of(entry(&query_term_entries(&out), "term")); + let pair = |bits: u32, rows: u64| { + FfiValue::Array(vec![FfiValue::UInt32(bits), FfiValue::UInt64(rows)]) + }; + let measured = encode(obj(vec![ + ("rows", FfiValue::UInt64(1_000_000)), + ( + "histogram", + FfiValue::Array(vec![pair(40, 500_000), pair(120, 500_000)]), + ), + ])); + let limit = |stats: &[u8], term: &[u8], field: &str, operator: &str| { + ops::target_limit( + term, + stats, + &plan(), + field.as_bytes(), + operator.as_bytes(), + 10, + ) + }; + let rows = decode(&limit(&measured, &term, "name", "matches").expect("a limit")); + assert!( + matches!(rows, FfiValue::UInt64(n) if n > 10), + "{}", + kind(&rows) + ); + let declared = encode(obj(vec![("rows", FfiValue::UInt64(1_000_000))])); + assert!(matches!( + decode(&limit(&declared, &term, "name", "matches").expect("a limit")), + FfiValue::Null + )); + + assert_eq!( + limit(&measured, &[1], "name", "matches"), + Err(STATUS_ENCODING), + "half a position" + ); + assert_eq!( + limit(&encode(s("x")), &term, "name", "matches"), + Err(STATUS_ENCODING), + "not stats" + ); + assert_eq!( + limit(&measured, &term, "age", "eq"), + Err(STATUS_ENCODING), + "an exact operator" + ); +} + +/// The `eql` build's catalog lists, per type, the operators the engine +/// answers: `TextEq` answers `eq`, exactly, and a type the engine does not +/// produce answers nothing. +#[test] +fn the_catalog_lists_each_types_operators() { + if !stack_encrypt_guest::targets::HOLDS_EQL { + return; + } + let FfiValue::Object(entries) = decode(&ops::targets().expect("targets")) else { + panic!("an object"); + }; + let FfiValue::Array(items) = &entries[0].1 else { + panic!("a list"); + }; + for item in items { + let FfiValue::Object(fields) = item else { + panic!("an object"); + }; + let name = text(entry(fields, "name")).to_owned(); + let FfiValue::Array(operators) = entry(fields, "operators") else { + panic!("{name}: operators is a list"); + }; + let producible = matches!(entry(fields, "producible"), FfiValue::Bool(true)); + if name == "TextEq" { + assert_eq!(operators.len(), 1, "TextEq answers eq alone"); + let FfiValue::Object(eq) = &operators[0] else { + panic!("an object"); + }; + let eq: Vec<(&str, &str)> = eq.iter().map(|(k, v)| (k.as_str(), text(v))).collect(); + assert_eq!( + eq, + [ + ("operator", "eq"), + ("fidelity", "exact"), + ("arity", "one"), + ("index", "eq") + ] + ); + } else if !producible { + assert!( + operators.is_empty(), + "{name} is not produced, so answers nothing" + ); + } + } +} + // ============================================================================= // A plan whose context is a field of the record // ============================================================================= diff --git a/languages/golang/encrypt/guest_test.go b/languages/golang/encrypt/guest_test.go index 11a21dfdb..166d6d57d 100644 --- a/languages/golang/encrypt/guest_test.go +++ b/languages/golang/encrypt/guest_test.go @@ -145,7 +145,7 @@ func TestImportSurfaceIsWASIPlusTransport(t *testing.T) { if !reflect.DeepEqual(transportImports, want) { t.Fatalf("transport imports = %v, want %v", transportImports, want) } - for name := range map[string]bool{"se_alloc": true, "se_dealloc": true, "se_cipher_init": true, "se_shutdown": true, "se_keyset": true, "se_term": true, "se_encrypt_record": true, "se_decrypt_record": true, "se_plan_check": true, "se_targets": true} { + for name := range map[string]bool{"se_alloc": true, "se_dealloc": true, "se_cipher_init": true, "se_shutdown": true, "se_keyset": true, "se_term": true, "se_query": true, "se_query_term": true, "se_verify": true, "se_target_limit": true, "se_encrypt_record": true, "se_decrypt_record": true, "se_plan_check": true, "se_targets": true} { if _, ok := compiled.ExportedFunctions()[name]; !ok { t.Errorf("guest does not export %s", name) } diff --git a/languages/golang/encrypt/query.go b/languages/golang/encrypt/query.go new file mode 100644 index 000000000..21c96acfa --- /dev/null +++ b/languages/golang/encrypt/query.go @@ -0,0 +1,302 @@ +package encrypt + +import ( + "context" + "fmt" + + "github.com/cipherstash/stack/languages/golang/internal/record" + "github.com/cipherstash/vitaminc/bindings/go/vcffi" + "github.com/cipherstash/vitaminc/bindings/go/vcvalue" +) + +// Queries named by operator (ADR-0009; ADR-0007, amended 2026-10-16): the +// guest's se_query_term, se_verify and se_target_limit exports, for +// generated code. A query names a field of a declaration and an operator, +// and the engine picks the index that answers it, or refuses an operator +// the field does not answer as it refuses a mistyped value. + +// QueryTerm is the query term the engine derives for one field, an operator +// and its input. +type QueryTerm struct { + // Operator is the operator the term answers: eq, lt, lte, gt, gte, + // between or matches. + Operator string + // Fidelity is how far the server's answer can be trusted. A + // [record.Probabilistic] term's rows are checked with [Cipher.Verify] + // after decrypt, and its fetch sized with [Cipher.TargetLimit]. + Fidelity record.Fidelity + // Index is the field's index that answers the operator. + Index record.Output + // Domain is the PostgreSQL domain the term is cast to, for a field + // that names an EQL type; "" for a field with indexes, whose terms are + // bytes. + Domain string + // Terms is the term: one, or for between two, the low bound's then the + // high's. For a field that names an EQL type it is the query value's + // JSON. + Terms [][]byte +} + +// Bucket is one bar of a histogram of the stored match filters' set-bit +// counts: Rows rows whose filter sets Bits bits. +type Bucket struct { + Bits uint32 + Rows uint64 +} + +// Stats is what [Cipher.TargetLimit] knows of the table: how many rows the +// query scans, how full their stored filters are, and the confidence to +// size the fetch at. None of it is secret: the server holds the filters and +// the catalog holds the row count. +type Stats struct { + // Rows is the rows the query scans, such as greatest(reltuples, 0) + // from pg_class. + Rows uint64 + // Histogram is a measured histogram of the stored filters' set-bit + // counts. With none, the engine uses the index's declared prior, and + // a field declared as data has none, so the limit is unbounded. + Histogram []Bucket + // Confidence is the probability that the fetch holds the rows wanted + // when the table has that many; 0 is the engine's default, 99%. + Confidence float64 +} + +// wire is the statistics as se_target_limit reads them. +func (s Stats) wire() vcvalue.Object { + out := vcvalue.Object{{Key: "rows", Value: s.Rows}} + if s.Histogram != nil { + buckets := make([]any, len(s.Histogram)) + for i, b := range s.Histogram { + buckets[i] = []any{b.Bits, b.Rows} + } + out = append(out, vcvalue.Field{Key: "histogram", Value: buckets}) + } + if s.Confidence != 0 { + out = append(out, vcvalue.Field{Key: "confidence", Value: s.Confidence}) + } + return out +} + +// Limit is how many rows a probabilistic query should fetch: a number, or +// unbounded when the statistics say nothing of the stored filters or the +// confidence is not below 1, in which case a caller pages with a cursor. +type Limit struct { + rows uint64 + bounded bool +} + +// Rows is the rows to fetch, and false when no finite fetch can be +// promised. +func (l Limit) Rows() (uint64, bool) { return l.rows, l.bounded } + +// FetchOr is the rows to fetch, or page when no finite fetch can be +// promised, for a caller that pages either way. +func (l Limit) FetchOr(page uint64) uint64 { + if l.bounded { + return l.rows + } + return page +} + +// QueryTerm derives the query term for one field of the plan, an operator +// and its input under this cipher's keyset: one value for every operator +// but between, whose input is a two-element slice, low then high. The +// engine reads the input as the field's declared type, finds the index +// that answers the operator, and derives that index's term under the +// field's context; for a field that names an EQL type it runs the type's +// query. An operator the field does not answer, or one the engine does not +// know, is [ErrEncoding] naming the field, as a mistyped value is; a range +// the wrong way round, and match text that cuts no token, are [ErrTerm]. +// No data key is minted. For generated code. +func (cph *Cipher) QueryTerm(ctx context.Context, plan *record.Plan, field, operator string, input any) (QueryTerm, error) { + p, err := cph.queryPlan(plan, field) + if err != nil { + return QueryTerm{}, err + } + encodedInput, err := vcffi.Marshal(input) + if err != nil { + return QueryTerm{}, fmt.Errorf("%w: %v", ErrEncoding, err) + } + // The input is plaintext: its transport copy is wiped once it is in the + // guest. + defer wipe(encodedInput) + encodedPlan, err := vcffi.Marshal(p.Wire()) + if err != nil { + return QueryTerm{}, fmt.Errorf("%w: %v", ErrEncoding, err) + } + opts, err := vcffi.Marshal(options(cph.keyset, "")) + if err != nil { + return QueryTerm{}, err + } + out, err := cph.client.call(ctx, func(inst *instance) ([]byte, error) { + return inst.call(ctx, inst.queryTerm, buf(encodedInput), buf(encodedPlan), buf([]byte(field)), buf([]byte(operator)), buf(opts)) + }) + if err != nil { + return QueryTerm{}, err + } + decoded, err := vcffi.Unmarshal(out) + if err != nil { + return QueryTerm{}, fmt.Errorf("%w: %v", ErrInternal, err) + } + term, err := parseQueryTerm(decoded) + if err != nil { + return QueryTerm{}, fmt.Errorf("%w: se_query_term: %v", ErrInternal, err) + } + return term, nil +} + +// parseQueryTerm reads se_query_term's object: exactly the keys operator, +// fidelity, index, domain (a string or null) and term (bytes, or a list of +// two for between). +func parseQueryTerm(decoded any) (QueryTerm, error) { + obj, ok := decoded.(vcvalue.Object) + if !ok { + return QueryTerm{}, fmt.Errorf("came back as %T, not an object", decoded) + } + var t QueryTerm + seen := map[string]bool{} + for _, f := range obj { + if seen[f.Key] { + return QueryTerm{}, fmt.Errorf("key %q twice", f.Key) + } + seen[f.Key] = true + var err error + switch f.Key { + case "operator": + t.Operator, err = targetString(f) + case "fidelity": + var s string + s, err = targetString(f) + t.Fidelity = record.Fidelity(s) + if err == nil && t.Fidelity != record.Exact && t.Fidelity != record.Probabilistic { + err = fmt.Errorf("unknown fidelity %q", s) + } + case "index": + var s string + s, err = targetString(f) + t.Index = record.Output(s) + case "domain": + t.Domain, err = targetOptionalString(f) + case "term": + switch v := f.Value.(type) { + case []byte: + t.Terms = [][]byte{v} + case []any: + if len(v) != 2 { + return QueryTerm{}, fmt.Errorf("a range of %d terms", len(v)) + } + for _, bound := range v { + b, ok := bound.([]byte) + if !ok { + return QueryTerm{}, fmt.Errorf("a bound is %T, not bytes", bound) + } + t.Terms = append(t.Terms, b) + } + default: + return QueryTerm{}, fmt.Errorf("term is %T", f.Value) + } + default: + return QueryTerm{}, fmt.Errorf("unknown key %q", f.Key) + } + if err != nil { + return QueryTerm{}, err + } + } + if len(seen) != 5 { + return QueryTerm{}, fmt.Errorf("%d keys, not 5", len(seen)) + } + return t, nil +} + +// Verify reports, for each of a page of decrypted values, whether it really +// matches a probabilistic query: one call into the guest for the page. The +// query is named as it was derived, by the field, the operator and the +// input; the engine cuts the input's tokens again with the field's own +// tokenizer, so nothing here holds a copy of it. A nil value is false. An +// exact operator has nothing to verify and is [ErrEncoding]. Uses no key. +// For generated code. +func (cph *Cipher) Verify(ctx context.Context, plan *record.Plan, field, operator string, input any, values []any) ([]bool, error) { + p, err := cph.plan(plan) + if err != nil { + return nil, err + } + encodedInput, err := vcffi.Marshal(input) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrEncoding, err) + } + defer wipe(encodedInput) + if values == nil { + values = []any{} + } + encodedValues, err := vcffi.Marshal(values) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrEncoding, err) + } + // The values are decrypted plaintext: wiped once they are in the guest. + defer wipe(encodedValues) + encodedPlan, err := vcffi.Marshal(p.Wire()) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrEncoding, err) + } + out, err := cph.client.call(ctx, func(inst *instance) ([]byte, error) { + return inst.call(ctx, inst.verify, buf(encodedInput), buf(encodedValues), buf(encodedPlan), buf([]byte(field)), buf([]byte(operator))) + }) + if err != nil { + return nil, err + } + decoded, err := vcffi.Unmarshal(out) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrInternal, err) + } + list, ok := decoded.([]any) + if !ok || len(list) != len(values) { + return nil, fmt.Errorf("%w: se_verify returned %T for %d values", ErrInternal, decoded, len(values)) + } + verified := make([]bool, len(list)) + for i, v := range list { + b, ok := v.(bool) + if !ok { + return nil, fmt.Errorf("%w: se_verify returned %T for value %d", ErrInternal, v, i) + } + verified[i] = b + } + return verified, nil +} + +// TargetLimit is how many rows to fetch so that, at the confidence stats +// names, want of them verify: term is the probabilistic term's bytes as +// [Cipher.QueryTerm] returned them. An exact operator has no fetch to size +// and is [ErrEncoding], as are bytes the field's index would not derive. +// Uses no key. For generated code. +func (cph *Cipher) TargetLimit(ctx context.Context, plan *record.Plan, field, operator string, term []byte, want uint64, stats Stats) (Limit, error) { + p, err := cph.plan(plan) + if err != nil { + return Limit{}, err + } + encodedStats, err := vcffi.Marshal(stats.wire()) + if err != nil { + return Limit{}, fmt.Errorf("%w: %v", ErrEncoding, err) + } + encodedPlan, err := vcffi.Marshal(p.Wire()) + if err != nil { + return Limit{}, fmt.Errorf("%w: %v", ErrEncoding, err) + } + out, err := cph.client.call(ctx, func(inst *instance) ([]byte, error) { + return inst.call(ctx, inst.limit, buf(term), buf(encodedStats), buf(encodedPlan), buf([]byte(field)), buf([]byte(operator)), scalar(want)) + }) + if err != nil { + return Limit{}, err + } + decoded, err := vcffi.Unmarshal(out) + if err != nil { + return Limit{}, fmt.Errorf("%w: %v", ErrInternal, err) + } + switch v := decoded.(type) { + case nil: + return Limit{}, nil + case uint64: + return Limit{rows: v, bounded: true}, nil + default: + return Limit{}, fmt.Errorf("%w: se_target_limit returned %T", ErrInternal, decoded) + } +} diff --git a/languages/golang/encrypt/query_test.go b/languages/golang/encrypt/query_test.go new file mode 100644 index 000000000..2deb0cd8f --- /dev/null +++ b/languages/golang/encrypt/query_test.go @@ -0,0 +1,362 @@ +package encrypt_test + +import ( + "context" + "encoding/hex" + "encoding/json" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/cipherstash/stack/languages/golang/encrypt" + "github.com/cipherstash/stack/languages/golang/internal/record" +) + +// The query fixture holds the guest's query exports to the Rust results: +// packages/stack-encrypt/tests/fixtures/query_operators.json records, for one +// declaration, the term each query by operator derives (each equal to the +// typed Rust query chain's), a page of verified values and a few fetch +// limits. These tests derive, verify and size the same queries through the +// deterministic test build of the guest and compare. Skipped when that build +// is absent. + +type queryFixture struct { + KeySource struct { + Seed string `json:"seed"` + } `json:"key_source"` + Queries []struct { + Field string `json:"field"` + Operator string `json:"operator"` + Input json.RawMessage `json:"input"` + Term struct { + Operator string `json:"operator"` + Fidelity string `json:"fidelity"` + Index string `json:"index"` + Domain *string `json:"domain"` + Term json.RawMessage `json:"term"` + } `json:"term"` + } `json:"queries"` + Verify struct { + Field string `json:"field"` + Operator string `json:"operator"` + Input string `json:"input"` + Values []*string `json:"values"` + Verified []bool `json:"verified"` + } `json:"verify"` + TargetLimit []struct { + Field string `json:"field"` + Operator string `json:"operator"` + Term string `json:"term"` + Want uint64 `json:"want"` + Stats struct { + Rows uint64 `json:"rows"` + Histogram [][2]uint64 `json:"histogram"` + Confidence float64 `json:"confidence"` + } `json:"stats"` + Limit *uint64 `json:"limit"` + } `json:"target_limit"` +} + +func readQueryFixture(t *testing.T) queryFixture { + t.Helper() + raw, err := os.ReadFile(filepath.Join("..", "..", "..", "packages", "stack-encrypt", "tests", "fixtures", "query_operators.json")) + if err != nil { + t.Fatal(err) + } + var f queryFixture + if err := json.Unmarshal(raw, &f); err != nil { + t.Fatal(err) + } + return f +} + +// queryPlan is the fixture's declaration: an email with equality, an age +// with equality and ORE, a name with match, all under users. +var queryPlan = &record.Plan{Context: []string{"users"}, Fields: []record.Field{ + {Name: "email", Kind: record.String, Outputs: []record.Output{record.Ciphertext, record.Equality}}, + {Name: "age", Kind: record.Uint32, Outputs: []record.Output{record.Ciphertext, record.Equality, record.Ore}}, + {Name: "name", Kind: record.String, Outputs: []record.Output{record.Ciphertext, record.Match}}, +}} + +func queryFixtureClient(t *testing.T, f queryFixture) *encrypt.Client { + t.Helper() + seed, err := hex.DecodeString(f.KeySource.Seed) + if err != nil || len(seed) != 32 { + t.Fatalf("seed: %v (%d bytes)", err, len(seed)) + } + c, err := encrypt.NewDeterministicClient(context.Background(), [32]byte(seed)) + if errors.Is(err, encrypt.ErrDeterministicGuestNotBuilt) || errors.Is(err, encrypt.ErrGuestNotBuilt) { + t.Skip(err) + } + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = c.Close() }) + return c +} + +// goInput is a fixture input as Go code passes it: text as a string, an +// age as the field's uint32, a range as two of them. +func goInput(t *testing.T, field string, raw json.RawMessage) any { + t.Helper() + if field != "age" { + var s string + if err := json.Unmarshal(raw, &s); err != nil { + t.Fatal(err) + } + return s + } + var n uint32 + if err := json.Unmarshal(raw, &n); err == nil { + return n + } + var pair [2]uint32 + if err := json.Unmarshal(raw, &pair); err != nil { + t.Fatal(err) + } + return []any{pair[0], pair[1]} +} + +func fixtureTerms(t *testing.T, raw json.RawMessage) [][]byte { + t.Helper() + var one string + if err := json.Unmarshal(raw, &one); err == nil { + b, err := hex.DecodeString(one) + if err != nil { + t.Fatal(err) + } + return [][]byte{b} + } + var pair []string + if err := json.Unmarshal(raw, &pair); err != nil { + t.Fatal(err) + } + out := make([][]byte, len(pair)) + for i, s := range pair { + b, err := hex.DecodeString(s) + if err != nil { + t.Fatal(err) + } + out[i] = b + } + return out +} + +func TestQueryTermsAreTheRustFixtures(t *testing.T) { + f := readQueryFixture(t) + c := queryFixtureClient(t, f) + ctx := context.Background() + cipher := c.DefaultKeyset() + if len(f.Queries) == 0 { + t.Fatal("the fixture holds no query") + } + for _, q := range f.Queries { + got, err := cipher.QueryTerm(ctx, queryPlan, q.Field, q.Operator, goInput(t, q.Field, q.Input)) + if err != nil { + t.Fatalf("%s on %s: %v", q.Operator, q.Field, err) + } + want := encrypt.QueryTerm{ + Operator: q.Term.Operator, + Fidelity: record.Fidelity(q.Term.Fidelity), + Index: record.Output(q.Term.Index), + Terms: fixtureTerms(t, q.Term.Term), + } + if q.Term.Domain != nil { + want.Domain = *q.Term.Domain + } + if got.Operator != want.Operator || got.Fidelity != want.Fidelity || got.Index != want.Index || got.Domain != want.Domain || len(got.Terms) != len(want.Terms) { + t.Fatalf("%s on %s: %+v, want %+v", q.Operator, q.Field, got, want) + } + for i := range want.Terms { + if hex.EncodeToString(got.Terms[i]) != hex.EncodeToString(want.Terms[i]) { + t.Fatalf("%s on %s: term %d = %x, want %x", q.Operator, q.Field, i, got.Terms[i], want.Terms[i]) + } + } + } + + // The engine reads the input as the field's type: a Go int queries a + // uint32 field as the uint32 it stored. + asUint32, err := cipher.QueryTerm(ctx, queryPlan, "age", "lt", uint32(40)) + if err != nil { + t.Fatal(err) + } + asInt, err := cipher.QueryTerm(ctx, queryPlan, "age", "lt", 40) + if err != nil { + t.Fatal(err) + } + if hex.EncodeToString(asInt.Terms[0]) != hex.EncodeToString(asUint32.Terms[0]) { + t.Fatal("an int input derived another term than the field's uint32") + } +} + +func TestVerifyIsTheRustFixtures(t *testing.T) { + f := readQueryFixture(t) + c := queryFixtureClient(t, f) + values := make([]any, len(f.Verify.Values)) + for i, v := range f.Verify.Values { + if v != nil { + values[i] = *v + } + } + verified, err := c.DefaultKeyset().Verify(context.Background(), queryPlan, f.Verify.Field, f.Verify.Operator, f.Verify.Input, values) + if err != nil { + t.Fatal(err) + } + if len(verified) != len(f.Verify.Verified) { + t.Fatalf("Verify = %v, want %v", verified, f.Verify.Verified) + } + for i := range verified { + if verified[i] != f.Verify.Verified[i] { + t.Fatalf("Verify = %v, want %v", verified, f.Verify.Verified) + } + } + empty, err := c.DefaultKeyset().Verify(context.Background(), queryPlan, "name", "matches", "alice", nil) + if err != nil || len(empty) != 0 { + t.Fatalf("an empty page: %v %v", empty, err) + } +} + +func TestTargetLimitIsTheRustFixtures(t *testing.T) { + f := readQueryFixture(t) + c := queryFixtureClient(t, f) + if len(f.TargetLimit) == 0 { + t.Fatal("the fixture holds no limit") + } + for i, l := range f.TargetLimit { + term, err := hex.DecodeString(l.Term) + if err != nil { + t.Fatal(err) + } + stats := encrypt.Stats{Rows: l.Stats.Rows, Confidence: l.Stats.Confidence} + for _, b := range l.Stats.Histogram { + stats.Histogram = append(stats.Histogram, encrypt.Bucket{Bits: uint32(b[0]), Rows: b[1]}) + } + got, err := c.DefaultKeyset().TargetLimit(context.Background(), queryPlan, l.Field, l.Operator, term, l.Want, stats) + if err != nil { + t.Fatalf("case %d: %v", i, err) + } + rows, bounded := got.Rows() + switch { + case l.Limit == nil && bounded: + t.Fatalf("case %d: %d rows, want unbounded", i, rows) + case l.Limit != nil && (!bounded || rows != *l.Limit): + t.Fatalf("case %d: %d (bounded %v), want %d", i, rows, bounded, *l.Limit) + } + if l.Limit == nil && got.FetchOr(50) != 50 { + t.Fatalf("case %d: an unbounded limit fetches the page", i) + } + } +} + +// A query the field does not answer is refused as a mistyped value is: an +// encoding error that names the field and the rule. The typed path's own +// refusals are term errors. +func TestAQueryTheFieldDoesNotAnswerIsRefused(t *testing.T) { + f := readQueryFixture(t) + c := queryFixtureClient(t, f) + ctx := context.Background() + cipher := c.DefaultKeyset() + for _, tc := range []struct { + field, operator string + input any + reason string + }{ + {"age", "matches", "ali", "operator_not_answered"}, + {"name", "lt", "ali", "operator_not_answered"}, + {"age", "like", uint32(1), "unknown_operator"}, + {"age", "between", uint32(1), "not_a_range"}, + {"age", "eq", "34", "field_type"}, + } { + _, err := cipher.QueryTerm(ctx, queryPlan, tc.field, tc.operator, tc.input) + if !errors.Is(err, encrypt.ErrEncoding) { + t.Fatalf("%s on %s: %v, want ErrEncoding", tc.operator, tc.field, err) + } + var d *encrypt.Diagnostic + if !errors.As(err, &d) || d.Field() != tc.field || d.Reason() != tc.reason { + t.Fatalf("%s on %s: %v, want field %q and reason %q", tc.operator, tc.field, err, tc.field, tc.reason) + } + } + if _, err := cipher.QueryTerm(ctx, queryPlan, "age", "between", []any{uint32(65), uint32(18)}); !errors.Is(err, encrypt.ErrTerm) { + t.Fatalf("an inverted range: %v, want ErrTerm", err) + } + if _, err := cipher.QueryTerm(ctx, queryPlan, "name", "matches", "al"); !errors.Is(err, encrypt.ErrTerm) { + t.Fatalf("text that cuts no token: %v, want ErrTerm", err) + } + + _, err := cipher.Verify(ctx, queryPlan, "email", "eq", "bob@example.com", []any{"bob@example.com"}) + var d *encrypt.Diagnostic + if !errors.Is(err, encrypt.ErrEncoding) || !errors.As(err, &d) || d.Reason() != "exact_operator" { + t.Fatalf("Verify on an exact operator: %v", err) + } + _, err = cipher.TargetLimit(ctx, queryPlan, "age", "lt", []byte{1, 0}, 10, encrypt.Stats{Rows: 1}) + if !errors.Is(err, encrypt.ErrEncoding) || !errors.As(err, &d) || d.Reason() != "exact_operator" { + t.Fatalf("TargetLimit on an exact operator: %v", err) + } + _, err = cipher.TargetLimit(ctx, queryPlan, "name", "matches", []byte{1}, 10, encrypt.Stats{Rows: 1}) + if !errors.Is(err, encrypt.ErrEncoding) || !errors.As(err, &d) || d.Reason() != "not_a_query_term" { + t.Fatalf("TargetLimit on half a position: %v", err) + } +} + +// Verifying and sizing use no key, so a guest never given one answers them, +// as it answers the generator's questions. +func TestVerifyAndTargetLimitNeedNoCipher(t *testing.T) { + wasm, err := encrypt.EmbeddedGuest() + if errors.Is(err, encrypt.ErrGuestNotBuilt) { + t.Skip(err) + } + if err != nil { + t.Fatal(err) + } + c := encrypt.RawClient(t, wasm) + ctx := context.Background() + cipher := c.DefaultKeyset() + verified, err := cipher.Verify(ctx, queryPlan, "name", "matches", "alice", []any{"Alice Smith", "Bob"}) + if err != nil || len(verified) != 2 || !verified[0] || verified[1] { + t.Fatalf("Verify = %v, %v", verified, err) + } + limit, err := cipher.TargetLimit(ctx, queryPlan, "name", "matches", []byte{1, 0, 2, 0}, 10, encrypt.Stats{Rows: 100, Histogram: []encrypt.Bucket{{Bits: 10, Rows: 100}}}) + if rows, ok := limit.Rows(); err != nil || !ok || rows < 10 { + t.Fatalf("TargetLimit = %d %v, %v", rows, ok, err) + } + if _, err := cipher.QueryTerm(ctx, queryPlan, "age", "eq", uint32(1)); !errors.Is(err, encrypt.ErrState) { + t.Fatalf("a query term needs the cipher: %v, want ErrState", err) + } +} + +// The catalog reports, per EQL type, the operators the engine answers: +// TextEq answers eq, exactly, through its equality index. +func TestTheCatalogListsEachTypesOperators(t *testing.T) { + wasm, err := encrypt.EmbeddedGuest() + if errors.Is(err, encrypt.ErrGuestNotBuilt) { + t.Skip(err) + } + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + checker, err := encrypt.NewCheckerOver(ctx, wasm) + if err != nil { + t.Fatal(err) + } + defer func() { _ = checker.Close() }() + targets, err := checker.Targets(ctx) + if err != nil { + t.Fatal(err) + } + if len(targets) == 0 { + t.Skip("this build holds no EQL types: import encrypt/eql") + } + for _, target := range targets { + switch { + case target.Name == "TextEq": + want := []record.TargetOperator{{Name: "eq", Fidelity: record.Exact, Index: record.Equality}} + if len(target.Operators) != 1 || target.Operators[0] != want[0] { + t.Fatalf("TextEq answers %+v, want %+v", target.Operators, want) + } + case !target.Producible && len(target.Operators) != 0: + t.Fatalf("%s is not produced and answers %+v", target.Name, target.Operators) + } + } +} diff --git a/languages/golang/encrypt/records.go b/languages/golang/encrypt/records.go index 24feb3163..d20d57565 100644 --- a/languages/golang/encrypt/records.go +++ b/languages/golang/encrypt/records.go @@ -118,25 +118,10 @@ func (cph *Cipher) Open(ctx context.Context, plan *record.Plan, records []record // with [Cipher.Context], and a cipher that names none is refused as // [ErrEncoding]. For generated code. func (cph *Cipher) Derive(ctx context.Context, plan *record.Plan, field string, output record.Output, value any) ([]byte, error) { - p, err := cph.plan(plan) + p, err := cph.queryPlan(plan, field) if err != nil { return nil, err } - if p.ContextField != "" { - if cph.context == "" { - return nil, fmt.Errorf("%w: field %q: the type takes its context from its field %q, so a query names it with Cipher.Context", ErrEncoding, field, p.ContextField) - } - // The label a query compares against is /, as - // the record stored it: the plan under the named context, with no - // context field of its own. Validate has checked the label. - segments, err := record.ParseContext(cph.context) - if err != nil { - return nil, fmt.Errorf("%w: %v", ErrEncoding, err) - } - under := *p - under.Context, under.ContextField = segments, "" - p = &under - } f := p.Field(field) if f == nil { return nil, fmt.Errorf("%w: the plan has no field %q", ErrEncoding, field) @@ -172,6 +157,34 @@ func (cph *Cipher) Derive(ctx context.Context, plan *record.Plan, field string, }) } +// queryPlan is the plan a query on field derives under: the declaration's +// plan with this cipher's extension and, for a plan that takes its context +// from a field, the context named with [Cipher.Context] in that field's +// place. A cipher that names none is refused as [ErrEncoding]: a term +// derives under a context, and such a plan has none until a record names it. +func (cph *Cipher) queryPlan(plan *record.Plan, field string) (*record.Plan, error) { + p, err := cph.plan(plan) + if err != nil { + return nil, err + } + if p.ContextField == "" { + return p, nil + } + if cph.context == "" { + return nil, fmt.Errorf("%w: field %q: the type takes its context from its field %q, so a query names it with Cipher.Context", ErrEncoding, field, p.ContextField) + } + // The label a query compares against is /, as the + // record stored it: the plan under the named context, with no context + // field of its own. Validate has checked the label. + segments, err := record.ParseContext(cph.context) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrEncoding, err) + } + under := *p + under.Context, under.ContextField = segments, "" + return &under, nil +} + // Query derives the EQL query value of a field that names an EQL type, for // one value: the operand that matches stored values of the field, as the // JSON bytes the field's eql_v4.query_* domain takes. The engine runs the diff --git a/languages/golang/encrypt/targets_test.go b/languages/golang/encrypt/targets_test.go index 50f883ae6..0faf938ac 100644 --- a/languages/golang/encrypt/targets_test.go +++ b/languages/golang/encrypt/targets_test.go @@ -19,20 +19,33 @@ func TestParseTargetsReadsEachEntry(t *testing.T) { vcvalue.Field{Key: "plaintext", Value: "string"}, vcvalue.Field{Key: "sql_domain", Value: "eql_v4_text_eq"}, vcvalue.Field{Key: "indexes", Value: []any{"eq"}}, vcvalue.Field{Key: "query", Value: "TextEqQuery"}, vcvalue.Field{Key: "query_sql_domain", Value: "eql_v4.query_text_eq"}, vcvalue.Field{Key: "producible", Value: true}, vcvalue.Field{Key: "reason", Value: nil}, + vcvalue.Field{Key: "operators", Value: []any{operator("eq", "exact", "one", "eq")}}, ) notYet := entry( vcvalue.Field{Key: "name", Value: "JSON"}, vcvalue.Field{Key: "family", Value: "JSON"}, vcvalue.Field{Key: "suffix", Value: ""}, vcvalue.Field{Key: "plaintext", Value: nil}, vcvalue.Field{Key: "sql_domain", Value: "eql_v4_json"}, vcvalue.Field{Key: "indexes", Value: []any{"json"}}, vcvalue.Field{Key: "query", Value: nil}, vcvalue.Field{Key: "query_sql_domain", Value: nil}, vcvalue.Field{Key: "producible", Value: false}, vcvalue.Field{Key: "reason", Value: "the JSON index is a new operation in the engine"}, + vcvalue.Field{Key: "operators", Value: []any{}}, ) - got, err := parseTargets(vcvalue.Object{{Key: "targets", Value: []any{textEq, notYet}}}) + ordered := entry( + vcvalue.Field{Key: "name", Value: "IntegerOrdOre"}, vcvalue.Field{Key: "plaintext", Value: "int64"}, + vcvalue.Field{Key: "indexes", Value: []any{"ore"}}, vcvalue.Field{Key: "producible", Value: true}, + vcvalue.Field{Key: "operators", Value: []any{operator("lt", "exact", "one", "ore"), operator("between", "exact", "pair", "ore"), operator("matches", "probabilistic", "one", "match")}}, + ) + got, err := parseTargets(vcvalue.Object{{Key: "targets", Value: []any{textEq, notYet, ordered}}}) if err != nil { t.Fatal(err) } want := []record.Target{ - {Name: "TextEq", Family: "Text", Suffix: "Eq", Plaintext: record.String, SQLDomain: "eql_v4_text_eq", Indexes: []record.Output{record.Equality}, Query: "TextEqQuery", QuerySQLDomain: "eql_v4.query_text_eq", Producible: true}, - {Name: "JSON", Family: "JSON", Plaintext: record.Untyped, SQLDomain: "eql_v4_json", Indexes: []record.Output{"json"}, Reason: "the JSON index is a new operation in the engine"}, + {Name: "TextEq", Family: "Text", Suffix: "Eq", Plaintext: record.String, SQLDomain: "eql_v4_text_eq", Indexes: []record.Output{record.Equality}, Query: "TextEqQuery", QuerySQLDomain: "eql_v4.query_text_eq", Producible: true, + Operators: []record.TargetOperator{{Name: "eq", Fidelity: record.Exact, Index: record.Equality}}}, + {Name: "JSON", Family: "JSON", Plaintext: record.Untyped, SQLDomain: "eql_v4_json", Indexes: []record.Output{"json"}, Reason: "the JSON index is a new operation in the engine", Operators: []record.TargetOperator{}}, + {Name: "IntegerOrdOre", Plaintext: record.Int64, Indexes: []record.Output{record.Ore}, Producible: true, Operators: []record.TargetOperator{ + {Name: "lt", Fidelity: record.Exact, Index: record.Ore}, + {Name: "between", Fidelity: record.Exact, Pair: true, Index: record.Ore}, + {Name: "matches", Fidelity: record.Probabilistic, Index: record.Match}, + }}, } if !reflect.DeepEqual(got, want) { t.Fatalf("parseTargets =\n%+v\nwant\n%+v", got, want) @@ -42,22 +55,45 @@ func TestParseTargetsReadsEachEntry(t *testing.T) { t.Fatalf("empty list: %v %v", empty, err) } for name, bad := range map[string]any{ - "not an object": "targets", - "another key": vcvalue.Object{{Key: "types", Value: []any{}}}, - "list not a list": vcvalue.Object{{Key: "targets", Value: "x"}}, - "entry not object": vcvalue.Object{{Key: "targets", Value: []any{"TextEq"}}}, - "numeric plaintext": vcvalue.Object{{Key: "targets", Value: []any{entry(vcvalue.Field{Key: "name", Value: "TextEq"}, vcvalue.Field{Key: "plaintext", Value: int64(1)}, vcvalue.Field{Key: "indexes", Value: []any{}}, vcvalue.Field{Key: "producible", Value: true})}}}, - "misspelled kind": vcvalue.Object{{Key: "targets", Value: []any{entry(vcvalue.Field{Key: "name", Value: "TextEq"}, vcvalue.Field{Key: "plaintext", Value: "strng"}, vcvalue.Field{Key: "indexes", Value: []any{}}, vcvalue.Field{Key: "producible", Value: true})}}}, - "misspelled index": vcvalue.Object{{Key: "targets", Value: []any{entry(vcvalue.Field{Key: "name", Value: "TextEq"}, vcvalue.Field{Key: "indexes", Value: []any{"equality"}}, vcvalue.Field{Key: "producible", Value: true})}}}, - "unknown key": vcvalue.Object{{Key: "targets", Value: []any{entry(vcvalue.Field{Key: "name", Value: "TextEq"}, vcvalue.Field{Key: "kind", Value: "string"}, vcvalue.Field{Key: "indexes", Value: []any{}}, vcvalue.Field{Key: "producible", Value: true})}}}, - "no name": vcvalue.Object{{Key: "targets", Value: []any{entry(vcvalue.Field{Key: "indexes", Value: []any{}}, vcvalue.Field{Key: "producible", Value: true})}}}, - "no producible": vcvalue.Object{{Key: "targets", Value: []any{entry(vcvalue.Field{Key: "name", Value: "TextEq"}, vcvalue.Field{Key: "indexes", Value: []any{}})}}}, - "index not string": vcvalue.Object{{Key: "targets", Value: []any{entry(vcvalue.Field{Key: "name", Value: "TextEq"}, vcvalue.Field{Key: "indexes", Value: []any{int64(1)}}, vcvalue.Field{Key: "producible", Value: true})}}}, - "producible string": vcvalue.Object{{Key: "targets", Value: []any{entry(vcvalue.Field{Key: "name", Value: "TextEq"}, vcvalue.Field{Key: "indexes", Value: []any{}}, vcvalue.Field{Key: "producible", Value: "yes"})}}}, - "key twice": vcvalue.Object{{Key: "targets", Value: []any{entry(vcvalue.Field{Key: "name", Value: "A"}, vcvalue.Field{Key: "name", Value: "B"}, vcvalue.Field{Key: "indexes", Value: []any{}}, vcvalue.Field{Key: "producible", Value: true})}}}, + "not an object": "targets", + "another key": vcvalue.Object{{Key: "types", Value: []any{}}}, + "list not a list": vcvalue.Object{{Key: "targets", Value: "x"}}, + "entry not object": vcvalue.Object{{Key: "targets", Value: []any{"TextEq"}}}, + "numeric plaintext": vcvalue.Object{{Key: "targets", Value: []any{entry(vcvalue.Field{Key: "name", Value: "TextEq"}, vcvalue.Field{Key: "plaintext", Value: int64(1)}, vcvalue.Field{Key: "indexes", Value: []any{}}, vcvalue.Field{Key: "producible", Value: true})}}}, + "misspelled kind": vcvalue.Object{{Key: "targets", Value: []any{entry(vcvalue.Field{Key: "name", Value: "TextEq"}, vcvalue.Field{Key: "plaintext", Value: "strng"}, vcvalue.Field{Key: "indexes", Value: []any{}}, vcvalue.Field{Key: "producible", Value: true})}}}, + "misspelled index": vcvalue.Object{{Key: "targets", Value: []any{entry(vcvalue.Field{Key: "name", Value: "TextEq"}, vcvalue.Field{Key: "indexes", Value: []any{"equality"}}, vcvalue.Field{Key: "producible", Value: true})}}}, + "unknown key": vcvalue.Object{{Key: "targets", Value: []any{entry(vcvalue.Field{Key: "name", Value: "TextEq"}, vcvalue.Field{Key: "kind", Value: "string"}, vcvalue.Field{Key: "indexes", Value: []any{}}, vcvalue.Field{Key: "producible", Value: true})}}}, + "no name": vcvalue.Object{{Key: "targets", Value: []any{entry(vcvalue.Field{Key: "indexes", Value: []any{}}, vcvalue.Field{Key: "producible", Value: true})}}}, + "no producible": vcvalue.Object{{Key: "targets", Value: []any{entry(vcvalue.Field{Key: "name", Value: "TextEq"}, vcvalue.Field{Key: "indexes", Value: []any{}})}}}, + "index not string": vcvalue.Object{{Key: "targets", Value: []any{entry(vcvalue.Field{Key: "name", Value: "TextEq"}, vcvalue.Field{Key: "indexes", Value: []any{int64(1)}}, vcvalue.Field{Key: "producible", Value: true})}}}, + "producible string": vcvalue.Object{{Key: "targets", Value: []any{entry(vcvalue.Field{Key: "name", Value: "TextEq"}, vcvalue.Field{Key: "indexes", Value: []any{}}, vcvalue.Field{Key: "producible", Value: "yes"})}}}, + "key twice": vcvalue.Object{{Key: "targets", Value: []any{entry(vcvalue.Field{Key: "name", Value: "A"}, vcvalue.Field{Key: "name", Value: "B"}, vcvalue.Field{Key: "indexes", Value: []any{}}, vcvalue.Field{Key: "producible", Value: true})}}}, + "no operators": vcvalue.Object{{Key: "targets", Value: []any{entry(vcvalue.Field{Key: "name", Value: "TextEq"}, vcvalue.Field{Key: "indexes", Value: []any{}}, vcvalue.Field{Key: "producible", Value: true})}}}, + "operators not list": withOperators("eq"), + "operator not object": withOperators([]any{"eq"}), + "unknown fidelity": withOperators([]any{operator("eq", "approximate", "one", "eq")}), + "unknown arity": withOperators([]any{operator("eq", "exact", "three", "eq")}), + "unknown op index": withOperators([]any{operator("eq", "exact", "one", "json")}), + "no op name": withOperators([]any{operator("", "exact", "one", "eq")}), + "op key missing": withOperators([]any{vcvalue.Object{{Key: "operator", Value: "eq"}}}), + "op unknown key": withOperators([]any{append(operator("eq", "exact", "one", "eq"), vcvalue.Field{Key: "case", Value: "insensitive"})}), + "op key twice": withOperators([]any{append(operator("eq", "exact", "one", "eq"), vcvalue.Field{Key: "index", Value: "eq"})}), + "op value not text": withOperators([]any{vcvalue.Object{{Key: "operator", Value: int64(1)}, {Key: "fidelity", Value: "exact"}, {Key: "arity", Value: "one"}, {Key: "index", Value: "eq"}}}), } { if _, err := parseTargets(bad); !errors.Is(err, ErrInternal) { t.Errorf("%s: %v, want ErrInternal", name, err) } } } + +// operator is one se_targets operators entry. +func operator(name, fidelity, arity, index string) vcvalue.Object { + return vcvalue.Object{{Key: "operator", Value: name}, {Key: "fidelity", Value: fidelity}, {Key: "arity", Value: arity}, {Key: "index", Value: index}} +} + +// withOperators is a well-formed TextEq entry whose operators are value. +func withOperators(value any) vcvalue.Object { + return vcvalue.Object{{Key: "targets", Value: []any{vcvalue.Object{ + {Key: "name", Value: "TextEq"}, {Key: "indexes", Value: []any{}}, {Key: "producible", Value: true}, {Key: "operators", Value: value}, + }}}} +} diff --git a/languages/golang/internal/record/record.go b/languages/golang/internal/record/record.go index 0123bb3dc..70ad1582c 100644 --- a/languages/golang/internal/record/record.go +++ b/languages/golang/internal/record/record.go @@ -417,7 +417,9 @@ type Sealed = map[string]Outputs // format, written by eql-bindings' serialiser and read by [ParseTargets]: // name, family, suffix, plaintext (a ValueKind name, or null), sql_domain, // indexes (eq | match | ore | ope | json), query (or null), query_sql_domain -// (or null), producible and reason (or null). +// (or null), producible, reason (or null), and operators, which the engine +// adds: the operators it answers for the type, each an object of operator, +// fidelity (exact | probabilistic), arity (one | pair) and index. type Target struct { // Name is the Go type name, and the value of encrypt_into: TextEq. Name string @@ -439,4 +441,36 @@ type Target struct { // Reason says why not when it does not. Producible bool Reason string + // Operators are the operators the engine answers for a field that names + // the type, in the engine's order; none for a type it does not produce. + Operators []TargetOperator +} + +// Fidelity says how far the server's answer to an operator can be trusted: +// [Exact], or [Probabilistic], whose rows a caller verifies after decrypt. +type Fidelity string + +// The fidelities an operator has. +const ( + // Exact is an answer the server gives whole: nothing to verify. + Exact Fidelity = "exact" + // Probabilistic is an answer that can hold false positives and no false + // negatives: each decrypted row is verified, and the fetch is sized for + // the false positives. + Probabilistic Fidelity = "probabilistic" +) + +// TargetOperator is one operator the engine answers for an EQL type, as +// se_targets lists it. +type TargetOperator struct { + // Name is the operator's name, as a query names it: eq, lt, lte, gt, + // gte, between or matches. + Name string + // Fidelity is how far the server's answer can be trusted. + Fidelity Fidelity + // Pair says the operator takes two values, low then high, as between + // does, where every other takes one. + Pair bool + // Index is the type's index that answers it. + Index Output } From 9071b05734244bca9e442f6059373b5f75494955 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Fri, 16 Oct 2026 03:13:58 +1100 Subject: [PATCH 3/8] docs(stack-encrypt): amend ADR-0007 for queries named by operator, proposed until ADR-0009 is accepted The data grammar gains the operator and its input, the term comes back with its query domain, verify and target_limit become engine exports, and the catalog reports each EQL type's operators and their fidelity. The amendment rests on ADR-0009, which is proposed, and takes effect when it is accepted. Part of #1188 --- ...-through-a-plan-never-a-second-executor.md | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/packages/stack-encrypt/docs/adr/0007-bindings-enter-through-a-plan-never-a-second-executor.md b/packages/stack-encrypt/docs/adr/0007-bindings-enter-through-a-plan-never-a-second-executor.md index 36ed1f64d..4fa765af6 100644 --- a/packages/stack-encrypt/docs/adr/0007-bindings-enter-through-a-plan-never-a-second-executor.md +++ b/packages/stack-encrypt/docs/adr/0007-bindings-enter-through-a-plan-never-a-second-executor.md @@ -2,6 +2,7 @@ status: accepted date: 2026-10-04 extends: ADR-0003, ADR-0004 +amended: 2026-10-16, proposed with ADR-0009 and in effect when ADR-0009 is accepted --- # Bindings enter the engine through a plan lowered from data, never through a second executor @@ -29,6 +30,14 @@ extends: ADR-0003, ADR-0004 > Go SDK, as the `context_field` tag word. The typed parts that stay > Rust-only are the typed verb `encrypt_into`, the picker and the one-value > start. +> +> **Amended 2026-10-16**, by #1188, **proposed**. This amendment depends on +> ADR-0009, which is proposed, and takes effect when ADR-0009 is accepted; +> until then it is proposed with it, and changes with it. The decision is +> unchanged. Added, under its last bullet: the operator and its input in the +> data grammar, the query term and its query domain back, `verify` and +> `target_limit` as engine exports, and the operators each EQL type answers, +> with their fidelity, in the catalog. Stack Encrypt has one execution engine: the `Encryption` and `Decryption` descriptions in `target/` and the batched `Pending` they produce. ADR-0003 @@ -188,3 +197,99 @@ to Go (G1: one engine). The reasons recorded against this on 2026-10-04 were guest size and an EQL-free guest for programs that never store into EQL. Two guest builds answer the second, and the first is measured before the SDK ships one build or two (G7). + +## Amended 2026-10-16 (#1188), proposed with ADR-0009 + +**Status: proposed.** ADR-0009 is proposed, and this amendment rests on it: +it takes effect when ADR-0009 is accepted, and is revised or withdrawn with +it. The code it describes lands first, as ADR-0009's Rust side has +(#1182 to #1186), because Stack Encrypt has no stored data and no released +binding a change could break. + +ADR-0009 makes the operator, not the index, what a query names, and gives a +probabilistic term `verify` and `target_limit`. The data grammar a binding +lowered named a field and, for a query, took the term the field's EQL type +chose. It had no operator and no operator input, and no export could verify +a decrypted value or size a fetch. The last bullet of the decision says what +to do with a capability the grammar cannot express: add it to the grammar, +once, so every author has it. This amendment does that for the query side. +ADR-0009's appendix B is the sketch it was amended from. + +**A query names a field, an operator and the operator's input.** The +operator crosses as its name: `eq`, `lt`, `lte`, `gt`, `gte`, `between` or +`matches`. The input is one value, or for `between` a list of two, low then +high. The engine reads the input as the field's declared type first +(`dynamic::read`), so a host's float `34.0` queries a `uint64` field as the +`u64` the field stored. It then finds the index that answers the operator by +the table the typed path's `DefaultIndex` impls state, written over a type +and the indexes a field declares (`dynamic::OpSpec::answered_by`): `eq` +through an equality index, `matches` through a match index over text, the +orderings through an ORE index over an integer, or OPE when the field +declares no ORE. A field that does not answer the operator is refused with +the same shape of error as a mistyped field: an input error naming the field, +with the reason `operator_not_answered` (or `unknown_operator`, +`not_a_range`). The refusals the typed path makes before a term exists, a +range the wrong way round and match text that cuts no token, are refused the +same way here, before the cipher is consulted. + +**The term comes back with its query domain.** The guest's +`se_query_term(input, plan, field, operator, options)` returns one object: +the operator, its fidelity, the index that answered, the PostgreSQL domain +the term is cast to, and the term. A field with indexes derives that index's +term under the field's context, by the derivation its stored terms came +from, as bytes, and `between` derives one term per bound under the one +context, as a list of two. A field that names an EQL type runs the type's +query twin, and the term is the twin's JSON and the domain its +`query_sql_domain`. One query domain per operator is EQL v4's (ADR-0009, +decision 8, #1179); until then a type's twin carries every term of the type. + +**`verify` and `target_limit` are engine exports.** `se_verify(input, values, +plan, field, operator)` answers one boolean per decrypted value, a page in +one call. `se_target_limit(term, stats, plan, field, operator, want)` answers +the rows to fetch, or null for no finite fetch. Neither uses a key, so each +runs in a guest that holds no cipher, as `se_plan_check` does. `verify` names +the query as it was derived, by the field, the operator and the input, and +the engine cuts the input's tokens again with the field's own tokenizer and +options. The tokens are plaintext, so no term carries them across the +binding. Sending the input and a page of decrypted values back in is no new +exposure: the input came from the host, and the values came out of the +guest as a decrypt. The alternative, a tokenizer in each language, is the +"lives twice" cost this ADR accepts only for an encoder, and a tokenizer that +drifts from the one that built the stored filter verifies the wrong thing. +`target_limit` takes the term's bytes, because the number of bits the query +sets is a property of the term the engine built, and the fill formula +belongs beside the index configuration. Both refuse an exact operator, which +has nothing to verify and no fetch to size, as the typed path does not +compile the call. + +**The catalog reports the operators each EQL type answers, and their +fidelity.** Each `se_targets` entry gains `operators`: one object per +operator the engine answers for a field that names the type, with its name, +its fidelity (`exact` or `probabilistic`), its arity (`one` or `pair`) and +the type's index that answers it. The engine computes the list from the +type's row by the same table every plan field is held to, so a generator +asks the engine it embeds rather than holding a copy of the rules, as the +2026-10-06 amendment asks of every rule. ADR-0009 rejects a fidelity value +on a term, because Rust says it with a trait. A generator asks before the +program runs, so it needs the answer as data, and the code it generates +carries `Verify` only where the answer is `probabilistic`, which rebuilds +decision 4 of ADR-0009 in a language without traits. Today `TextEq` answers +`eq` and no other type is produced. A type answers no probabilistic operator +and no `between` yet: the catalog names a type's match index but not its +options, which `verify` must cut tokens by, and a twin is one value, not a +range. Both wait for the EQL v4 types (#1179). + +What does not change: a binding still reaches the engine through a plan, and +nothing here derives a term outside the engine's own derivation, so a query +from Go and the typed query chain derive the same bytes. A cross-language +fixture holds them together (`tests/fixtures/query_operators.json`): the Rust +side checks each recorded term against the typed chain, and the Go SDK's +tests derive, verify and size the same queries through the guest. +`se_query`, which derives a target field's twin with no operator, stays until +the generated Go code names operators (#1189), and is removed then. +`se_term`, a term by index under an explicit context, stays for the generated +code of fields declared with indexes until that code names operators too. + +A data plan's match index declares no `expected_tokens` prior, so +`target_limit` on a field declared as data is unbounded without a measured +histogram. A data form for the prior is later work. From abb99fc8990479a63876028fd4d894186fa91aef Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Sat, 17 Oct 2026 00:37:24 +1100 Subject: [PATCH 4/8] docs(stack-encrypt): say TextMatch is produced and answers no operator yet, and pin it Review follow-ups. On top of next the EQL v4 catalog produces Integer, Bigint, Text, TextEq, TextMatch, Boolean, Real and Double, so the ADR-0007 amendment's "today TextEq answers eq and no other type is produced", and its "both wait for the EQL v4 types", were wrong. TextMatch is produced and has a query twin but answers no operator, because the catalog does not carry the match options `verify` cuts tokens by; the amendment and the `dynamic::query` module docs now say so, and say the gate is that catalog work. Nothing held that in place: the catalog tests checked TextEq and the types not produced, so a produced type starting to answer `matches` would have passed. The guest and Go tests now require every type but TextEq to answer nothing, and a guest test refuses `matches` and `eq` on a TextMatch field as not answered. The module docs no longer call the dynamic ordering rule the typed path's default: the typed default is ORE, and a field declaring only OPE is answered through it here, which the typed path spells `.via(Ope)`. The `Reason` docs count five inputs, not four. --- .../golang/encrypt/guest/tests/native_ops.rs | 46 +++++++++++++++++-- languages/golang/encrypt/query_test.go | 7 ++- ...-through-a-plan-never-a-second-executor.md | 13 ++++-- packages/stack-encrypt/src/dynamic/mod.rs | 2 +- packages/stack-encrypt/src/dynamic/query.rs | 10 ++-- 5 files changed, 61 insertions(+), 17 deletions(-) diff --git a/languages/golang/encrypt/guest/tests/native_ops.rs b/languages/golang/encrypt/guest/tests/native_ops.rs index 410d1a9ef..abc5f069a 100644 --- a/languages/golang/encrypt/guest/tests/native_ops.rs +++ b/languages/golang/encrypt/guest/tests/native_ops.rs @@ -1749,15 +1749,51 @@ fn the_catalog_lists_each_types_operators() { ("index", "eq") ] ); - } else if !producible { - assert!( - operators.is_empty(), - "{name} is not produced, so answers nothing" - ); + } else { + // A type the engine does not produce answers nothing, and no + // produced type but TextEq answers anything yet: TextMatch has a + // twin, but the catalog does not carry the match options a + // probabilistic operator's `verify` cuts tokens by. + let why = if producible { + "is produced but answers no operator yet" + } else { + "is not produced, so answers nothing" + }; + assert!(operators.is_empty(), "{name} {why}"); } } } +/// `TextMatch` is produced and has a query twin, but answers no operator +/// yet: a `matches` (or any) query on a field typed with it is refused as +/// not answered, never sent as the twin with no `verify` to check its rows. +#[test] +fn a_text_match_field_answers_no_operator_yet() { + if !stack_encrypt_guest::targets::HOLDS_EQL { + return; + } + let cipher = cipher(); + let keyset = cipher.default_keyset(); + let plan = encode(obj(vec![( + "name", + obj(vec![("context", label("name")), ("target", s("TextMatch"))]), + )])); + for operator in ["matches", "eq"] { + assert_eq!( + block_on(ops::query_term( + &keyset, + &encode(s("alice")), + &plan, + b"name", + operator.as_bytes(), + )) + .map(|_| ()), + Err(STATUS_ENCODING), + "{operator} on a TextMatch field" + ); + } +} + // ============================================================================= // A plan whose context is a field of the record // ============================================================================= diff --git a/languages/golang/encrypt/query_test.go b/languages/golang/encrypt/query_test.go index 2deb0cd8f..8550adc98 100644 --- a/languages/golang/encrypt/query_test.go +++ b/languages/golang/encrypt/query_test.go @@ -355,8 +355,11 @@ func TestTheCatalogListsEachTypesOperators(t *testing.T) { if len(target.Operators) != 1 || target.Operators[0] != want[0] { t.Fatalf("TextEq answers %+v, want %+v", target.Operators, want) } - case !target.Producible && len(target.Operators) != 0: - t.Fatalf("%s is not produced and answers %+v", target.Name, target.Operators) + case len(target.Operators) != 0: + // No produced type but TextEq answers anything yet: TextMatch has + // a twin, but the catalog does not carry the match options its + // Verify would cut tokens by. + t.Fatalf("%s (producible %v) answers %+v, want none", target.Name, target.Producible, target.Operators) } } } diff --git a/packages/stack-encrypt/docs/adr/0007-bindings-enter-through-a-plan-never-a-second-executor.md b/packages/stack-encrypt/docs/adr/0007-bindings-enter-through-a-plan-never-a-second-executor.md index 4fa765af6..1988c01fd 100644 --- a/packages/stack-encrypt/docs/adr/0007-bindings-enter-through-a-plan-never-a-second-executor.md +++ b/packages/stack-encrypt/docs/adr/0007-bindings-enter-through-a-plan-never-a-second-executor.md @@ -241,7 +241,8 @@ from, as bytes, and `between` derives one term per bound under the one context, as a list of two. A field that names an EQL type runs the type's query twin, and the term is the twin's JSON and the domain its `query_sql_domain`. One query domain per operator is EQL v4's (ADR-0009, -decision 8, #1179); until then a type's twin carries every term of the type. +decision 8); until that lands in the catalog a type's twin carries every term +of the type. **`verify` and `target_limit` are engine exports.** `se_verify(input, values, plan, field, operator)` answers one boolean per decrypted value, a page in @@ -274,10 +275,12 @@ on a term, because Rust says it with a trait. A generator asks before the program runs, so it needs the answer as data, and the code it generates carries `Verify` only where the answer is `probabilistic`, which rebuilds decision 4 of ADR-0009 in a language without traits. Today `TextEq` answers -`eq` and no other type is produced. A type answers no probabilistic operator -and no `between` yet: the catalog names a type's match index but not its -options, which `verify` must cut tokens by, and a twin is one value, not a -range. Both wait for the EQL v4 types (#1179). +`eq` and no other produced type answers anything: `TextMatch` is produced and +has a query twin, but answers no operator, and the storage-only types have no +twin. A type answers no probabilistic operator and no `between` yet: the +catalog names a type's match index but not its options, which `verify` must +cut tokens by, and a twin is one value, not a range. Both wait for that +catalog work, not for the types themselves. What does not change: a binding still reaches the engine through a plan, and nothing here derives a term outside the engine's own derivation, so a query diff --git a/packages/stack-encrypt/src/dynamic/mod.rs b/packages/stack-encrypt/src/dynamic/mod.rs index fbaef53dc..80f32dd53 100644 --- a/packages/stack-encrypt/src/dynamic/mod.rs +++ b/packages/stack-encrypt/src/dynamic/mod.rs @@ -460,7 +460,7 @@ reasons! { /// What was wrong with a context, plan, source, stored record or query: the /// reason a dynamic input error carries beside the field it names. /// - /// One vocabulary for all four, since several reasons apply to more than + /// One vocabulary for all five, since several reasons apply to more than /// one (a field given twice is a misfit in a source and in a stored /// record). [`as_str`](Self::as_str) is the `snake_case` name a binding /// reports in its payload's `reason` field; `Display` is the phrase the diff --git a/packages/stack-encrypt/src/dynamic/query.rs b/packages/stack-encrypt/src/dynamic/query.rs index b2100281f..0bcc9e7be 100644 --- a/packages/stack-encrypt/src/dynamic/query.rs +++ b/packages/stack-encrypt/src/dynamic/query.rs @@ -32,9 +32,10 @@ //! declared kind: `eq` through an equality index, `matches` through a match //! index over text, and the orderings (`lt`, `lte`, `gt`, `gte`, //! `between`) through an ORE index over an integer, or an OPE index when -//! the field declares no ORE. Those are the typed path's defaults: an -//! ordering over text, bytes or a float is not answered, as `op::Ordered` -//! covers the integers only. +//! the field declares no ORE. The typed path's default for an ordering is +//! ORE, and OPE is the typed path's `.via(Ope)`; here a field that declares +//! only OPE is answered through it. An ordering over text, bytes or a float +//! is not answered, as `op::Ordered` covers the integers only. //! //! A field that names an EQL type answers what its //! [`TargetDescriptor::operators`] lists: the exact operators of a single @@ -43,7 +44,8 @@ //! the type; one query domain per operator is EQL v4's (ADR-0009, decision //! 8). A probabilistic operator is not answered through a target yet: the //! descriptor names the type's match index but not its options, and -//! `verify` must cut tokens as the stored filter did. +//! `verify` must cut tokens as the stored filter did. So `TextMatch`, which +//! is produced and has a twin, answers no operator. //! //! # Wire format //! From 58718af5b67022a890e15bc89a1822d685929bed Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Sat, 17 Oct 2026 00:54:00 +1100 Subject: [PATCH 5/8] fix(stack-encrypt, guest): refuse unnormalizable equality text before a cipher, wipe both verify buffers, and name the field on bad stats Copilot review. An `eq` query on text with a code point unassigned in Unicode 16 passed `check_query`, which validated ranges and match text but not equality normalization, so `se_query_term` consulted the cipher (and could load a keyset) before derivation refused it; a guest with no cipher reported STATUS_STATE where the refusal is a term failure. `check_input` now runs the equality normalization the derivation runs, keylessly; a test in the engine and one at the guest boundary pin both the check and the op. `se_verify` took the query and the values buffers one after the other with `?`, so a bad query pointer returned before the page of decrypted values was taken and wiped. Both are taken before either result is used. (The Go host wipes every staged buffer after the call regardless.) Statistics are parsed with no field to hand, so a malformed `stats` in `se_target_limit` was refused naming no field; it now names the query's, as every other query refusal does. --- languages/golang/encrypt/guest/src/abi.rs | 8 +++-- languages/golang/encrypt/guest/src/ops.rs | 4 ++- .../golang/encrypt/guest/tests/native_ops.rs | 29 +++++++++++++++++++ packages/stack-encrypt/src/dynamic/query.rs | 7 +++++ packages/stack-encrypt/src/sem/mod.rs | 2 +- 5 files changed, 46 insertions(+), 4 deletions(-) diff --git a/languages/golang/encrypt/guest/src/abi.rs b/languages/golang/encrypt/guest/src/abi.rs index e608bb90b..da04e43d7 100644 --- a/languages/golang/encrypt/guest/src/abi.rs +++ b/languages/golang/encrypt/guest/src/abi.rs @@ -571,8 +571,12 @@ pub unsafe extern "C" fn se_verify( op_len: u32, ) -> u64 { export(|| { - let query = unsafe { take_plaintext(in_ptr, in_len)? }; - let values = unsafe { take_plaintext(vals_ptr, vals_len)? }; + // Both buffers are taken, and so wiped, before either result is + // acted on: a bad query pointer must not leave the page of + // decrypted values unwiped behind an early return. + let query = unsafe { take_plaintext(in_ptr, in_len) }; + let values = unsafe { take_plaintext(vals_ptr, vals_len) }; + let (query, values) = (query?, values?); // SAFETY: host-owned ranges the export was handed; the borrows end // before it returns. let plan = unsafe { input(plan_ptr, plan_len)? }; diff --git a/languages/golang/encrypt/guest/src/ops.rs b/languages/golang/encrypt/guest/src/ops.rs index 78c036e5c..4cdf228db 100644 --- a/languages/golang/encrypt/guest/src/ops.rs +++ b/languages/golang/encrypt/guest/src/ops.rs @@ -332,8 +332,10 @@ pub fn target_limit( want: u64, ) -> Result, u32> { let (plan, field, operator) = parse_query(plan, field, operator)?; + // Statistics are read without a field to hand; the error names the + // query's, as every other refusal of a query does. let stats = stack_encrypt::op::Stats::from_value(&decode_value(stats)?) - .map_err(|e| fail_dynamic(&e))?; + .map_err(|e| fail_dynamic(&e.in_field(field)))?; let limit = dynamic::target_limit(&plan, field, operator, term, want, &stats, &resolver()) .map_err(|e| fail_dynamic(&e))?; encode_value(dynamic::limit_to_value(limit)) diff --git a/languages/golang/encrypt/guest/tests/native_ops.rs b/languages/golang/encrypt/guest/tests/native_ops.rs index abc5f069a..d2845e311 100644 --- a/languages/golang/encrypt/guest/tests/native_ops.rs +++ b/languages/golang/encrypt/guest/tests/native_ops.rs @@ -1494,6 +1494,35 @@ fn a_query_term_is_the_term_of_the_index_that_answers_its_operator() { /// before a keyset is resolved: an operator the field does not answer, or /// one the engine does not know, as a mistyped value is refused; the typed /// path's own query refusals as `STATUS_TERM`. +/// Equality text that does not normalize (a code point unassigned in +/// Unicode 16) is refused by the cipher-free check as a term failure, as +/// derivation would refuse it, so a guest with no cipher reports +/// STATUS_TERM rather than STATUS_STATE, and the query op agrees. +#[test] +fn equality_text_that_does_not_normalize_is_a_term_failure_before_any_cipher() { + let plan = encode(obj(vec![( + "email", + obj(vec![ + ("context", label("email")), + ("outputs", FfiValue::Array(vec![s("c"), s("eq")])), + ("type", s("string")), + ]), + )])); + let input = encode(s("a\u{0378}bc")); + assert_eq!( + ops::validate::query_term(&input, &plan, b"email", b"eq"), + Err(STATUS_TERM), + "validation" + ); + let cipher = cipher(); + let keyset = cipher.default_keyset(); + assert_eq!( + block_on(ops::query_term(&keyset, &input, &plan, b"email", b"eq")).map(|_| ()), + Err(STATUS_TERM), + "the op" + ); +} + #[test] fn query_term_validation_refuses_what_the_query_op_refuses() { let cipher = cipher(); diff --git a/packages/stack-encrypt/src/dynamic/query.rs b/packages/stack-encrypt/src/dynamic/query.rs index 0bcc9e7be..6b622428b 100644 --- a/packages/stack-encrypt/src/dynamic/query.rs +++ b/packages/stack-encrypt/src/dynamic/query.rs @@ -464,6 +464,12 @@ fn check_input(index: &IndexSpec, input: &Input) -> Result<(), Error> { (IndexSpec::Match(options), Input::One(value)) => { query_tokens(value, options).map(drop)?; } + // Equality text the scheme cannot normalize (a code point + // unassigned in Unicode 16) is refused here, before a term or a + // keyset, as derivation would refuse it after one. + (IndexSpec::Equality, Input::One(value)) => { + crate::sem::normalized(value).map(drop).map_err(refused)?; + } _ => {} } Ok(()) @@ -1224,6 +1230,7 @@ mod tests { ("age", OpSpec::Between, pair(65, 18), "inverted"), ("name", OpSpec::Matches, text("al"), "empty"), ("name", OpSpec::Matches, text("\u{0378}abc"), "normalize"), + ("email", OpSpec::Eq, text("a\u{0378}bc"), "normalize"), ]; for (name, op, input, what) in cases { for error in [ diff --git a/packages/stack-encrypt/src/sem/mod.rs b/packages/stack-encrypt/src/sem/mod.rs index 3532f4d37..d13937a33 100644 --- a/packages/stack-encrypt/src/sem/mod.rs +++ b/packages/stack-encrypt/src/sem/mod.rs @@ -466,7 +466,7 @@ fn equality_of_normalized( /// is one scalar with an equality domain, and [`TermError::Normalize`] for a /// scalar the sink refuses (text with a code point unassigned in Unicode 16, /// a value not valid for its logical type). -fn normalized(value: T) -> Result { +pub(crate) fn normalized(value: T) -> Result { let mut terms = terms(value).map_err(|error| match error.term() { Some(refused) => TermError::Normalize(refused.error()), // A shape refusal or a source failure: not one scalar the sink could From 66f4f76a353fadb085a4c582c4690cb4dae5081b Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Sat, 17 Oct 2026 01:02:01 +1100 Subject: [PATCH 6/8] chore(biome): leave the generated query_operators.json fixture alone The fixture is written by serde_json's pretty printer when STACK_ENCRYPT_UPDATE_FIXTURES is set, and the test compares the file with what the engine derives, so Biome's JSON formatting would make every regeneration a diff. It joins record_lowering.json and the other generated fixtures in biome.json's excludes. --- biome.json | 1 + 1 file changed, 1 insertion(+) diff --git a/biome.json b/biome.json index 37dab459e..5aee2763b 100644 --- a/biome.json +++ b/biome.json @@ -11,6 +11,7 @@ "!languages/typescript/packages/protect-ffi/target", "!languages/typescript/packages/protect-ffi/src/eql-v3-types", "!packages/stack-encrypt/tests/fixtures/record_lowering.json", + "!packages/stack-encrypt/tests/fixtures/query_operators.json", "!packages/eql/tests/encryption/fixtures/ste_vec_selectors.json", "!packages/eql/tests/encryption/fixtures/v4_rows.json", "!packages/eql/crates/eql-bindings/bindings", From 451d452c77c0d604ef4f2e37c29a7efaf86d5327 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Sat, 17 Oct 2026 03:21:32 +1100 Subject: [PATCH 7/8] fix(stack-encrypt): refuse invalid match options before a query cuts tokens FieldPlan::new is public and does not check the MatchOptions of a match output, so a plan built by hand can carry an n-gram length of zero. The term derivation validates them, but check_query and query_term reach the tokenizer first through check_input, and verify calls it directly, so such a plan panicked in the tokenizer rather than returning InvalidOptions. query_tokens now validates the options first. --- packages/stack-encrypt/src/dynamic/query.rs | 52 +++++++++++++++++++-- 1 file changed, 48 insertions(+), 4 deletions(-) diff --git a/packages/stack-encrypt/src/dynamic/query.rs b/packages/stack-encrypt/src/dynamic/query.rs index 6b622428b..3f2cf7d13 100644 --- a/packages/stack-encrypt/src/dynamic/query.rs +++ b/packages/stack-encrypt/src/dynamic/query.rs @@ -490,10 +490,13 @@ fn integer_order(lo: &FfiValue, hi: &FfiValue) -> Option { /// /// # Errors /// -/// [`Error::Cipher`] with [`TermError::Normalize`] for text the index cannot -/// normalize, and [`TermError::EmptyTermText`] for text that cuts no token, -/// whose filter would match every row. [`Error::Internal`] for a value that -/// is not text, which [`read_input`] has ruled out. +/// [`Error::Cipher`] with [`TermError::InvalidOptions`] for options the +/// index cannot use (a [`FieldPlan`] built by hand is not +/// checked, and an n-gram length of zero would panic in the tokenizer), +/// [`TermError::Normalize`] for text the index cannot normalize, and +/// [`TermError::EmptyTermText`] for text that cuts no token, whose filter +/// would match every row. [`Error::Internal`] for a value that is not text, +/// which [`read_input`] has ruled out. fn query_tokens(value: &FfiValue, options: &MatchOptions) -> Result { let FfiValue::String(text) = value else { return Err(Error::Internal); @@ -1408,6 +1411,47 @@ mod tests { ); } + /// Match options on a field plan built by hand are not checked when the + /// plan is, so the query chain refuses them before it cuts a token: an + /// n-gram length of zero would otherwise panic in the tokenizer. + #[test] + fn invalid_match_options_are_refused_before_tokenising() { + let options = MatchOptions { + tokenizer: crate::sem::Tokenizer::Ngram { length: 0 }, + ..MatchOptions::default() + }; + let context = crate::dynamic::context(FfiValue::Array(vec![text("users"), text("name")])) + .expect("context"); + let field = FieldPlan::new( + "name", + context, + vec![record::Output::Term(IndexSpec::Match(options))], + ) + .and_then(|field| field.with_type(ValueKind::String)) + .expect("the field"); + let plan = Plan::new(vec![field]).expect("the plan"); + let invalid = |error: Error| { + assert!( + matches!(term_error(&error), Some(TermError::InvalidOptions(_))), + "{error:?}" + ); + }; + invalid( + check_query(&plan, "name", OpSpec::Matches, text("alice"), &NoTargets).unwrap_err(), + ); + invalid( + verify( + &plan, + "name", + OpSpec::Matches, + text("alice"), + &[text("alice")], + &NoTargets, + ) + .unwrap_err(), + ); + } + /// `target_limit` is the typed `MatchQuery::target_limit` for the term /// `query_term` returned, read from its bytes. #[tokio::test] From 60591a97a5eb03a0ea0040a84f70a5f8823e92e8 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Sat, 17 Oct 2026 12:18:48 +1100 Subject: [PATCH 8/8] fix(golang): address the second review of the operator exports - The catalog reader and se_query_term's reader share one parser each for an operator's name, fidelity and index. A name is one of eq, lt, lte, gt, gte, between and matches, the set the engine's OpSpec::parse accepts. The catalog took any non-empty name, so "like" or "Eq" parsed and failed later as unknown_operator from Cipher.QueryTerm, and se_query_term took any operator and any index, which the catalog refused. Both now refuse the same values, and a test feeds every name the engine writes through both. - The query fixture test checks that query_operators.json was recorded under queryPlan, so a fixture regenerated under another plan fails by name rather than as mismatched term bytes. --- languages/golang/encrypt/checker.go | 49 ++++++++++++++----- languages/golang/encrypt/query.go | 19 ++++--- languages/golang/encrypt/query_test.go | 32 ++++++++++++ languages/golang/encrypt/targets_test.go | 34 +++++++++++++ .../tests/ui/op_at_path_not_a_leaf.stderr | 20 ++++---- .../tests/ui/op_between_takes_a_pair.stderr | 8 +-- .../tests/ui/op_json_not_answered.stderr | 8 +-- 7 files changed, 133 insertions(+), 37 deletions(-) diff --git a/languages/golang/encrypt/checker.go b/languages/golang/encrypt/checker.go index 807967325..5cf5aee0f 100644 --- a/languages/golang/encrypt/checker.go +++ b/languages/golang/encrypt/checker.go @@ -212,12 +212,9 @@ func targetOperators(value any) ([]record.TargetOperator, error) { } switch f.Key { case "operator": - op.Name = s + op.Name, err = parseOperatorName(s) case "fidelity": - op.Fidelity = record.Fidelity(s) - if op.Fidelity != record.Exact && op.Fidelity != record.Probabilistic { - return nil, fmt.Errorf("operator %d: unknown fidelity %q", i, s) - } + op.Fidelity, err = parseFidelity(s) case "arity": switch s { case "one": @@ -227,15 +224,13 @@ func targetOperators(value any) ([]record.TargetOperator, error) { return nil, fmt.Errorf("operator %d: unknown arity %q", i, s) } case "index": - switch o := record.Output(s); o { - case record.Equality, record.Match, record.Ore, record.Ope: - op.Index = o - default: - return nil, fmt.Errorf("operator %d: unknown index %q", i, s) - } + op.Index, err = parseOperatorIndex(s) default: return nil, fmt.Errorf("operator %d: unknown key %q", i, f.Key) } + if err != nil { + return nil, fmt.Errorf("operator %d: %v", i, err) + } } if len(seen) != 4 || op.Name == "" { return nil, fmt.Errorf("operator %d: not every key, or no name", i) @@ -245,6 +240,38 @@ func targetOperators(value any) ([]record.TargetOperator, error) { return operators, nil } +// parseOperatorName reads an operator's name as the engine writes it: one +// of eq, lt, lte, gt, gte, between and matches, the names the engine's +// OpSpec::parse accepts. The catalog (se_targets) and se_query_term are +// read through this, parseFidelity and parseOperatorIndex, so the two +// readers refuse the same values. +func parseOperatorName(s string) (string, error) { + switch s { + case "eq", "lt", "lte", "gt", "gte", "between", "matches": + return s, nil + } + return "", fmt.Errorf("unknown operator %q", s) +} + +// parseFidelity reads an operator's fidelity: exact or probabilistic. +func parseFidelity(s string) (record.Fidelity, error) { + switch f := record.Fidelity(s); f { + case record.Exact, record.Probabilistic: + return f, nil + } + return "", fmt.Errorf("unknown fidelity %q", s) +} + +// parseOperatorIndex reads the index that answers an operator: eq, match, +// ore or ope. +func parseOperatorIndex(s string) (record.Output, error) { + switch o := record.Output(s); o { + case record.Equality, record.Match, record.Ore, record.Ope: + return o, nil + } + return "", fmt.Errorf("unknown index %q", s) +} + func targetString(f vcvalue.Field) (string, error) { s, ok := f.Value.(string) if !ok { diff --git a/languages/golang/encrypt/query.go b/languages/golang/encrypt/query.go index 21c96acfa..575af95af 100644 --- a/languages/golang/encrypt/query.go +++ b/languages/golang/encrypt/query.go @@ -147,7 +147,8 @@ func (cph *Cipher) QueryTerm(ctx context.Context, plan *record.Plan, field, oper // parseQueryTerm reads se_query_term's object: exactly the keys operator, // fidelity, index, domain (a string or null) and term (bytes, or a list of -// two for between). +// two for between). The operator, fidelity and index take only the values +// the engine writes, as the catalog reader's do. func parseQueryTerm(decoded any) (QueryTerm, error) { obj, ok := decoded.(vcvalue.Object) if !ok { @@ -163,18 +164,20 @@ func parseQueryTerm(decoded any) (QueryTerm, error) { var err error switch f.Key { case "operator": - t.Operator, err = targetString(f) + var s string + if s, err = targetString(f); err == nil { + t.Operator, err = parseOperatorName(s) + } case "fidelity": var s string - s, err = targetString(f) - t.Fidelity = record.Fidelity(s) - if err == nil && t.Fidelity != record.Exact && t.Fidelity != record.Probabilistic { - err = fmt.Errorf("unknown fidelity %q", s) + if s, err = targetString(f); err == nil { + t.Fidelity, err = parseFidelity(s) } case "index": var s string - s, err = targetString(f) - t.Index = record.Output(s) + if s, err = targetString(f); err == nil { + t.Index, err = parseOperatorIndex(s) + } case "domain": t.Domain, err = targetOptionalString(f) case "term": diff --git a/languages/golang/encrypt/query_test.go b/languages/golang/encrypt/query_test.go index 8550adc98..607f10b4d 100644 --- a/languages/golang/encrypt/query_test.go +++ b/languages/golang/encrypt/query_test.go @@ -7,10 +7,12 @@ import ( "errors" "os" "path/filepath" + "reflect" "testing" "github.com/cipherstash/stack/languages/golang/encrypt" "github.com/cipherstash/stack/languages/golang/internal/record" + "github.com/cipherstash/vitaminc/bindings/go/vcvalue" ) // The query fixture holds the guest's query exports to the Rust results: @@ -25,6 +27,7 @@ type queryFixture struct { KeySource struct { Seed string `json:"seed"` } `json:"key_source"` + Plan json.RawMessage `json:"plan"` Queries []struct { Field string `json:"field"` Operator string `json:"operator"` @@ -68,9 +71,38 @@ func readQueryFixture(t *testing.T) queryFixture { if err := json.Unmarshal(raw, &f); err != nil { t.Fatal(err) } + // The terms are derived under queryPlan: a fixture recorded under any + // other plan fails here, by name, rather than as mismatched bytes. + var recorded any + if err := json.Unmarshal(f.Plan, &recorded); err != nil { + t.Fatal(err) + } + if want := wireJSON(queryPlan.Wire()); !reflect.DeepEqual(recorded, want) { + t.Fatalf("the fixture's plan is not queryPlan; update queryPlan to match it:\nfixture: %v\nqueryPlan: %v", recorded, want) + } return f } +// wireJSON is a plan's wire form as encoding/json decodes the same JSON: +// objects as maps, lists as []any. +func wireJSON(v any) any { + switch v := v.(type) { + case vcvalue.Object: + out := make(map[string]any, len(v)) + for _, f := range v { + out[f.Key] = wireJSON(f.Value) + } + return out + case []any: + out := make([]any, len(v)) + for i, item := range v { + out[i] = wireJSON(item) + } + return out + } + return v +} + // queryPlan is the fixture's declaration: an email with equality, an age // with equality and ORE, a name with match, all under users. var queryPlan = &record.Plan{Context: []string{"users"}, Fields: []record.Field{ diff --git a/languages/golang/encrypt/targets_test.go b/languages/golang/encrypt/targets_test.go index 0faf938ac..03b1d5198 100644 --- a/languages/golang/encrypt/targets_test.go +++ b/languages/golang/encrypt/targets_test.go @@ -75,6 +75,8 @@ func TestParseTargetsReadsEachEntry(t *testing.T) { "unknown arity": withOperators([]any{operator("eq", "exact", "three", "eq")}), "unknown op index": withOperators([]any{operator("eq", "exact", "one", "json")}), "no op name": withOperators([]any{operator("", "exact", "one", "eq")}), + "unknown operator": withOperators([]any{operator("like", "exact", "one", "eq")}), + "operator case": withOperators([]any{operator("Eq", "exact", "one", "eq")}), "op key missing": withOperators([]any{vcvalue.Object{{Key: "operator", Value: "eq"}}}), "op unknown key": withOperators([]any{append(operator("eq", "exact", "one", "eq"), vcvalue.Field{Key: "case", Value: "insensitive"})}), "op key twice": withOperators([]any{append(operator("eq", "exact", "one", "eq"), vcvalue.Field{Key: "index", Value: "eq"})}), @@ -97,3 +99,35 @@ func withOperators(value any) vcvalue.Object { {Key: "name", Value: "TextEq"}, {Key: "indexes", Value: []any{}}, {Key: "producible", Value: true}, {Key: "operators", Value: value}, }}}} } + +// se_query_term's object is read with the catalog's own parsers: an +// operator, fidelity or index the catalog would refuse is refused here too, +// and every name the engine writes is read by both. +func TestParseQueryTermTakesOnlyWhatTheEngineWrites(t *testing.T) { + term := func(operator, fidelity, index string) vcvalue.Object { + return vcvalue.Object{ + {Key: "operator", Value: operator}, {Key: "fidelity", Value: fidelity}, {Key: "index", Value: index}, + {Key: "domain", Value: nil}, {Key: "term", Value: []byte{1, 2}}, + } + } + for _, name := range []string{"eq", "lt", "lte", "gt", "gte", "between", "matches"} { + got, err := parseQueryTerm(term(name, "exact", "ore")) + if err != nil || got.Operator != name || got.Fidelity != record.Exact || got.Index != record.Ore { + t.Errorf("%s: %+v, %v", name, got, err) + } + if _, err := parseTargets(withOperators([]any{operator(name, "exact", "one", "ore")})); err != nil { + t.Errorf("catalog %s: %v", name, err) + } + } + for name, bad := range map[string]vcvalue.Object{ + "unknown operator": term("like", "exact", "eq"), + "operator case": term("Eq", "exact", "eq"), + "unknown fidelity": term("eq", "approximate", "eq"), + "unknown index": term("eq", "exact", "json"), + "misspelled index": term("eq", "exact", "equality"), + } { + if _, err := parseQueryTerm(bad); err == nil { + t.Errorf("%s: read, want refused", name) + } + } +} diff --git a/packages/stack-encrypt/tests/ui/op_at_path_not_a_leaf.stderr b/packages/stack-encrypt/tests/ui/op_at_path_not_a_leaf.stderr index 6519c4a31..f9b886cc1 100644 --- a/packages/stack-encrypt/tests/ui/op_at_path_not_a_leaf.stderr +++ b/packages/stack-encrypt/tests/ui/op_at_path_not_a_leaf.stderr @@ -9,12 +9,12 @@ error[E0277]: `Contains` is not a leaf operator, so it cannot be applied at a pa = help: the trait `Leaf` is not implemented for `Contains` = note: `op::AtPath` applies `op::Eq`, `Lt`, `Lte`, `Gt`, `Gte`, `Between` or `Matches` at a path; `op::Contains` and `op::ContainedBy` compare whole documents, and an accessor does not nest = help: the following other types implement trait `Leaf`: - Between - Gt - Gte - Lt - Lte + stack_encrypt::op::Between stack_encrypt::op::Eq + stack_encrypt::op::Gt + stack_encrypt::op::Gte + stack_encrypt::op::Lt + stack_encrypt::op::Lte stack_encrypt::op::Matches = note: required for `AtPath` to implement `Operator` note: required by a bound in `operator` @@ -34,12 +34,12 @@ error[E0277]: `AtPath` is not a leaf operator, so it cann = help: the trait `Leaf` is not implemented for `AtPath` = note: `op::AtPath` applies `op::Eq`, `Lt`, `Lte`, `Gt`, `Gte`, `Between` or `Matches` at a path; `op::Contains` and `op::ContainedBy` compare whole documents, and an accessor does not nest = help: the following other types implement trait `Leaf`: - Between - Gt - Gte - Lt - Lte + stack_encrypt::op::Between stack_encrypt::op::Eq + stack_encrypt::op::Gt + stack_encrypt::op::Gte + stack_encrypt::op::Lt + stack_encrypt::op::Lte stack_encrypt::op::Matches = note: required for `AtPath>` to implement `Operator` note: required by a bound in `operator` diff --git a/packages/stack-encrypt/tests/ui/op_between_takes_a_pair.stderr b/packages/stack-encrypt/tests/ui/op_between_takes_a_pair.stderr index f14672a61..6d40d14b6 100644 --- a/packages/stack-encrypt/tests/ui/op_between_takes_a_pair.stderr +++ b/packages/stack-encrypt/tests/ui/op_between_takes_a_pair.stderr @@ -22,7 +22,7 @@ note: required by a bound in `plan::chain::>::query` | F: Input, | ^^^^^^^^^^^^^^^^ required by this bound in `plan::chain::>::query` -error[E0599]: the method `on` exists for struct `QueryBuilder<'_, Between, u32, FakeDataKeySource>`, but its trait bounds were not satisfied +error[E0599]: the method `on` exists for struct `QueryBuilder<'_, stack_encrypt::op::Between, u32, FakeDataKeySource>`, but its trait bounds were not satisfied --> tests/ui/op_between_takes_a_pair.rs:12:47 | 12 | let _ = cipher.query(op::Between, &40u32).on(&age_plan); @@ -39,7 +39,7 @@ error[E0271]: type mismatch resolving `<(u32, {integer}) as Input>::Plainte | = note: expected type `u32` found tuple `(u32, {integer})` - = note: required for `ValuePlan>` to implement `Chooses` + = note: required for `ValuePlan>` to implement `Chooses` error[E0277]: `(u32, {integer})` is not ordered: the ordering operators are answered over the integer types only --> tests/ui/op_between_takes_a_pair.rs:13:48 @@ -59,5 +59,5 @@ error[E0277]: `(u32, {integer})` is not ordered: the ordering operators are answ u16 u32 and $N others - = note: required for `Lt` to implement `DefaultIndex<(u32, {integer})>` - = note: required for `ValuePlan>` to implement `Chooses` + = note: required for `stack_encrypt::op::Lt` to implement `DefaultIndex<(u32, {integer})>` + = note: required for `ValuePlan>` to implement `Chooses` diff --git a/packages/stack-encrypt/tests/ui/op_json_not_answered.stderr b/packages/stack-encrypt/tests/ui/op_json_not_answered.stderr index 7446d7c95..0d2ea514f 100644 --- a/packages/stack-encrypt/tests/ui/op_json_not_answered.stderr +++ b/packages/stack-encrypt/tests/ui/op_json_not_answered.stderr @@ -16,7 +16,7 @@ error[E0277]: `Contains` has no default index given `std::string::String` `stack_encrypt::op::Matches<::Case>` implements `DefaultIndex` = note: required for `ValuePlan>` to implement `Chooses` -error[E0277]: `(&str, &str)` is not what a query of arity `Pathed` takes +error[E0277]: `(&str, &str)` is not what a query of arity `Pathed` takes --> tests/ui/op_json_not_answered.rs:15:36 | 15 | .query(op::AtPath(op::Eq), &("$.address.city", "Sydney")) @@ -25,13 +25,13 @@ error[E0277]: `(&str, &str)` is not what a query of arity `Pathed` takes | required by a bound introduced by this call | = note: an operator of arity `One` takes one value, `op::Between` (arity `Pair`) takes a pair, `&(lo, hi)`, and `op::AtPath` (arity `Pathed`) takes no input yet: it arrives with the JSON index -help: the trait `Input>` is not implemented for `(&str, &str)` +help: the trait `Input>` is not implemented for `(&str, &str)` but trait `Input` is implemented for it --> src/op/mod.rs | | impl Input for (S, S) { | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - = help: for that trait implementation, expected `Pair`, found `Pathed` + = help: for that trait implementation, expected `Pair`, found `Pathed` note: required by a bound in `plan::chain::>::query` --> src/plan/chain.rs | @@ -51,4 +51,4 @@ error[E0599]: the method `on` exists for struct `QueryBuilder<'_, AtPath>` + `(&str, &str): Input>`