From 67274c7a8bd718c0b3dafa735a3af83e7277068b Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Wed, 14 Oct 2026 17:49:12 +1100 Subject: [PATCH 1/4] feat(stack-encrypt): store a token-less match value as the empty filter, refuse only the query A match value that cuts no token under the index's tokenizer (empty or separator-only text, or text shorter than the n-gram length) used to fail term derivation on both the stored and the query side, so one two-letter name failed a whole row. ADR-0009 splits the two: the stored term is the empty filter, which no query matches, and only a query of such text is refused, because its filter would match every row. `match_term` now derives the empty filter for such text, with no PRF call, so `MatchTerms` as a target, `KeysetCipher::match_terms` and the `dynamic` record path all store it, and the value seals and opens as any other. The refusal moves to the query paths: `op::Supports::check` takes the query's value, the `Match` index refuses token-less text there with `TermError::EmptyTermText` when the query is awaited, before any term is derived or keyset loaded, and `dynamic::term` refuses it for a match kind. The error keeps its code and is reworded for the query. The Go module builds the stack-encrypt guest by path, so its round-trip test of the old refusal is rewritten to the new behaviour: a short `TextMatch` value seals, opens and matches no query, and a short match query is still `ErrTerm`. The package docs and the term-failure message follow. Two tests that used token-less text as a leaf that fails before any I/O now use a match configuration with invalid options, which still fails during the synchronous build. Closes #1185 Claude-Session: https://claude.ai/code/session_01VYKAWbd361xDLmqBdLoUcB --- languages/golang/encrypt/doc.go | 11 +- languages/golang/encrypt/records.go | 12 +- languages/golang/encrypt/roundtrip_test.go | 61 ++++++---- packages/stack-encrypt/CHANGELOG.md | 16 +++ ...and-a-query-term-is-not-the-stored-term.md | 9 +- packages/stack-encrypt/src/dynamic/term.rs | 53 ++++++++- packages/stack-encrypt/src/op/mod.rs | 44 ++++--- packages/stack-encrypt/src/plan/chain.rs | 21 ++-- packages/stack-encrypt/src/sem/mod.rs | 65 +++++++---- packages/stack-encrypt/src/sem/tokenize.rs | 11 +- packages/stack-encrypt/tests/derive.rs | 44 +++++-- packages/stack-encrypt/tests/op.rs | 109 +++++++++++++++++- packages/stack-encrypt/tests/sem_terms.rs | 57 +++++---- packages/stack-encrypt/tests/target.rs | 19 ++- ...op_inexact_operator_needs_probabilistic.rs | 2 +- 15 files changed, 410 insertions(+), 124 deletions(-) diff --git a/languages/golang/encrypt/doc.go b/languages/golang/encrypt/doc.go index f29c861bf..ecfd9de3f 100644 --- a/languages/golang/encrypt/doc.go +++ b/languages/golang/encrypt/doc.go @@ -116,12 +116,11 @@ // [Match], [Ore] and [Ope]; [JSON] is declared and refused until the engine // derives it. Every stored type implements driver.Valuer and sql.Scanner. // -// A match index needs text with at least one token: the engine derives no -// match term for an empty string, separator-only text, or text shorter than -// the n-gram length (3 characters), because an empty term would match every -// row. Encrypt then fails for the whole batch with [ErrTerm], naming the row, -// the field and the index, so an optional or short value does not belong -// under match. +// A match index stores text with no token (an empty string, separator-only +// text, or text shorter than the n-gram length of 3 characters) as the empty +// filter: a valid value that no query matches, so an optional or short value +// can live under match. A match *query* of such text is refused with +// [ErrTerm], because an empty filter as a query would match every row. // // # Transport and auth // diff --git a/languages/golang/encrypt/records.go b/languages/golang/encrypt/records.go index 443279e35..8e1f03129 100644 --- a/languages/golang/encrypt/records.go +++ b/languages/golang/encrypt/records.go @@ -470,11 +470,11 @@ func (c *Client) open(ctx context.Context, sel KeysetSelector, context string, p // locateTermFailure names the row, field and index behind an ErrTerm from a // record call. The guest's error says what was wrong with the term but not // which value or field it came from, so the host asks the engine again, one -// term at a time, which costs no key request: a term derives locally. The engine defines no match term for text that yields no -// token (empty, separator-only, or shorter than the n-gram), because an -// empty term would match every row; a program hands such a field a value or -// drops the match index. p is the plan without this cipher's extension, -// which Derive adds. +// term at a time, which costs no key request: a term derives locally. A +// stored value is never refused for its text (text with no token is stored +// as the empty filter), so a term failure on a record is the scheme refusing +// the value's kind under that index. p is the plan without this cipher's +// extension, which Derive adds. func (cph *Cipher) locateTermFailure(ctx context.Context, p *record.Plan, rows []record.Source, err error) error { for i, row := range rows { for _, f := range p.Fields { @@ -483,7 +483,7 @@ func (cph *Cipher) locateTermFailure(ctx context.Context, p *record.Plan, rows [ continue } if _, derr := cph.Derive(ctx, p, f.Name, o, row[f.Name]); errors.Is(derr, ErrTerm) { - return fmt.Errorf("%w: value %d, field %q: the engine derives no %s term for this value (a match index needs text with at least one token; an empty or separator-only string has none)", err, i, f.Name, indexWord(o)) + return fmt.Errorf("%w: value %d, field %q: the engine derives no %s term for this value", err, i, f.Name, indexWord(o)) } } } diff --git a/languages/golang/encrypt/roundtrip_test.go b/languages/golang/encrypt/roundtrip_test.go index b9a1522b3..c1658108b 100644 --- a/languages/golang/encrypt/roundtrip_test.go +++ b/languages/golang/encrypt/roundtrip_test.go @@ -260,37 +260,56 @@ func TestATamperedRecordDoesNotOpen(t *testing.T) { } } -// A match index needs text that yields a token: the engine defines no match -// term for an empty or separator-only string, because an empty term would -// match every row, and the typed Rust path refuses it the same way. The Go -// error names the row, the field and the index. Every other zero value -// seals: a zero integer under ore and equality, an empty string with no -// match index. -func TestAMatchIndexNeedsText(t *testing.T) { +// A match index stores text that yields no token as the empty filter: an +// empty or separator-only string, or one shorter than the n-gram, is a valid +// value that no query matches, and it seals and opens like any other. Only a +// match *query* of such text is refused, because an empty filter as a query +// would match every row. Every other zero value seals too: a zero integer +// under ore and equality, an empty string with no match index. +func TestAMatchIndexStoresShortText(t *testing.T) { c := deterministicClient(t) ctx := context.Background() cipher := c.DefaultKeyset() for name, u := range map[string]testusers.User{ - "the zero value": {}, - "an empty email": {ID: 1, Age: 34, Notes: "x"}, - "a separator email": {ID: 1, Age: 34, Email: " \t", Notes: "x"}, + "the zero value": {}, + "an empty email": {ID: 1, Age: 34, Notes: "x"}, + "a separator email": {ID: 1, Age: 34, Email: " \t", Notes: "x"}, + "a two-letter email": {ID: 1, Age: 34, Email: "ab", Notes: "x"}, } { - _, err := testusers.Encrypt(ctx, cipher, []testusers.User{{ID: 9, Age: 1, Email: "ok@example.com", Notes: "x"}, u}) - if !errors.Is(err, encrypt.ErrTerm) { - t.Fatalf("%s: %v, want ErrTerm", name, err) + rows := []testusers.User{{ID: 9, Age: 1, Email: "ok@example.com", Notes: "x"}, u} + encrypted, err := testusers.Encrypt(ctx, cipher, rows) + if err != nil { + t.Fatalf("%s: %v, want the row to seal", name, err) + } + if positions, err := encrypted[1].Email.Match.Positions(); err != nil || len(positions) != 0 { + t.Errorf("%s: the stored match term is %v %v, want the empty filter", name, positions, err) + } + back, err := testusers.Decrypt(ctx, cipher, encrypted) + if err != nil { + t.Fatalf("%s: decrypt: %v", name, err) } - for _, want := range []string{`value 1`, `field "email"`, "no match term"} { - if !strings.Contains(err.Error(), want) { - t.Errorf("%s: error %q does not name %s", name, err, want) - } + if back[1].Email != u.Email { + t.Errorf("%s: email opened as %q, want %q", name, back[1].Email, u.Email) + } + // No query matches the empty filter: a query has at least one + // position, and the empty filter holds none of them. + q, err := testusers.Fields.Email.Match(ctx, cipher, "example") + if err != nil { + t.Fatal(err) + } + if positions, _ := q.Positions(); len(positions) == 0 { + t.Fatalf("%s: a query with a token sets no bit", name) } } - // The field entry says the same for one value. - if _, err := testusers.Fields.Email.Encrypt(ctx, cipher, ""); !errors.Is(err, encrypt.ErrTerm) || !strings.Contains(err.Error(), `field "email"`) { + // The field entry stores one such value the same way. + if _, err := testusers.Fields.Email.Encrypt(ctx, cipher, ""); err != nil { t.Fatalf("Fields.Email.Encrypt(\"\"): %v", err) } - if _, err := testusers.Fields.Email.Match(ctx, cipher, ""); !errors.Is(err, encrypt.ErrTerm) { - t.Fatalf("Fields.Email.Match(\"\"): %v", err) + // A match query with no token is the one value the engine refuses. + for _, text := range []string{"", " \t", "ab"} { + if _, err := testusers.Fields.Email.Match(ctx, cipher, text); !errors.Is(err, encrypt.ErrTerm) { + t.Fatalf("Fields.Email.Match(%q): %v, want ErrTerm", text, err) + } } // Zero values the engine does seal. encrypted, err := testusers.Encrypt(ctx, cipher, []testusers.User{{Email: "zero@example.com"}}) diff --git a/packages/stack-encrypt/CHANGELOG.md b/packages/stack-encrypt/CHANGELOG.md index c72a6a08b..21b1ab36a 100644 --- a/packages/stack-encrypt/CHANGELOG.md +++ b/packages/stack-encrypt/CHANGELOG.md @@ -96,6 +96,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 under it. `FieldPlan::with_target`, `FieldPlan::target()`, `Plan::new_with`. +### Changed + +- **A match value with no tokens is stored as the empty filter; only a query + with none is refused.** Text that cuts no token under the match index's + tokenizer (empty or separator-only text, or text shorter than the n-gram + length) used to fail term derivation on both sides, so one two-letter name + failed a whole row. Now the stored term is the empty filter (`bf: []`), + which no query matches, and the value seals and opens as any other; the + `dynamic` record path stores it the same way. A *query* of such text is + still refused, with `TermError::EmptyTermText` as before, because its + filter would match every row: `cipher.query(op::Matches, ..)` refuses it + when the query is awaited, before a term exists and before any keyset is + loaded (`op::Supports::check` now takes the query's value), and + `dynamic::term` refuses it for a match kind. `KeysetCipher::match_terms` + derives the stored filter and no longer fails on such text (ADR-0009). + ### Breaking - **The query chain starts with the operator.** `cipher.query(op, v)` diff --git a/packages/stack-encrypt/docs/adr/0009-operators-select-the-index-and-a-query-term-is-not-the-stored-term.md b/packages/stack-encrypt/docs/adr/0009-operators-select-the-index-and-a-query-term-is-not-the-stored-term.md index 820eb9aad..f4e7aac4e 100644 --- a/packages/stack-encrypt/docs/adr/0009-operators-select-the-index-and-a-query-term-is-not-the-stored-term.md +++ b/packages/stack-encrypt/docs/adr/0009-operators-select-the-index-and-a-query-term-is-not-the-stored-term.md @@ -273,11 +273,14 @@ times `want`, so it takes nothing else. `verify` is infallible. It runs on a value that is already decrypted and already the type the index was declared over, so the only question it can answer is "does this row match", and every other outcome is "no": a JSON -path absent from the document, text the index would have refused to store. +path absent from the document, text too short to hold the query's tokens. A `bool` is what `Iterator::filter` takes; an error would have no recovery except dropping the row, which is `false` by another name. The one input the -library refuses, a needle too short to tokenise, is refused at -`cipher.query`, before a term exists. +library refuses, a query value too short to tokenise, is refused when the query +is awaited, before a term exists. The stored side refuses nothing: a value +too short to tokenise is stored as the empty filter, which no query matches, +so a two-letter name is a valid value in a match column and `verify` is +never asked about it. `Plaintext` is the decrypted value the term is defined over, and that is not always the value the user wrote. For `Matches` it is the text. For diff --git a/packages/stack-encrypt/src/dynamic/term.rs b/packages/stack-encrypt/src/dynamic/term.rs index d02b272c0..3b75fe7b8 100644 --- a/packages/stack-encrypt/src/dynamic/term.rs +++ b/packages/stack-encrypt/src/dynamic/term.rs @@ -20,7 +20,9 @@ use vitaminc_protected::{Controlled, OpaqueDebug, Protected}; use zeroize::Zeroizing; use super::{utf8, Error, Reason, Value}; -use crate::sem::{CllwOpeEncrypt, CllwOreEncrypt, DefaultMatch, MatchOptions, Tokenizer}; +use crate::sem::{ + CllwOpeEncrypt, CllwOreEncrypt, DefaultMatch, MatchOptions, QueryTokens, TermError, Tokenizer, +}; use crate::target::{chosen, CallerContext, ConsumeSource, Encryption, Index, IndexSpec, Pending}; use crate::{IntoPrfContext, KeysetCipher, NonEmpty}; @@ -293,7 +295,11 @@ impl Scalar { /// [`Error::Term`] if the scheme defines no such term for the scalar /// ([`IndexSpec::supports`] is the table, and checking it first is how a /// binding turns this into a boundary rejection). [`Error::Cipher`] if the -/// derivation itself fails. +/// derivation itself fails, or for the one query value the engine refuses: +/// match text that cuts no token under the index's options +/// ([`TermError::EmptyTermText`]), whose filter would match every row. That +/// refusal is the query path's alone; the record path stores such a value +/// as the empty filter, which no query matches. pub async fn term<'c, K, D>( cipher: &KeysetCipher<'_, K>, scalar: Scalar, @@ -304,6 +310,11 @@ where K: DataKeySource + Sync + 'static, D: IntoPrfContext<'c>, { + if let (IndexSpec::Match(options), Scalar::Text(text)) = (kind, &scalar) { + if QueryTokens::under(text, options).is_empty() { + return Err(Error::Cipher(TermError::EmptyTermText.into())); + } + } scalar_term(cipher, scalar, kind, context)? .await .map(TermBytes::into_bytes) @@ -613,6 +624,44 @@ mod tests { } } + /// The query path refuses match text with no tokens; the record path, + /// which runs the same dispatch as an index of a `Value`, stores it as + /// the empty filter. Same scalar, same options, two answers, because a + /// query with no tokens would match every row and a stored value with + /// none is matched by no query. + #[tokio::test] + async fn match_text_with_no_tokens_is_refused_as_a_query_and_stored_empty() { + let cipher = cipher().await; + let keyset = cipher.default_keyset(); + let ctx = context(s("users/bio")).expect("context"); + for text in ["", " ", "hi"] { + let scalar = Scalar::of(&s(text), &default_match()).expect("a scalar"); + let refused = term(&keyset, scalar, &default_match(), ctx.clone()).await; + assert!( + matches!( + refused, + Err(Error::Cipher(crate::Error::Term(TermError::EmptyTermText))) + ), + "{text:?} as a query: {refused:?}" + ); + + let stored: TermBytes = keyset + .run( + Index::::operation::( + &default_match(), + ), + Value::new(s(text)), + CallerContext::from(nonempty!("users/bio")), + ) + .await + .unwrap_or_else(|error| panic!("{text:?} stores: {error}")); + assert!(stored.as_bytes().is_empty(), "{text:?} is the empty filter"); + } + // Text with a token is a query like any other. + let scalar = Scalar::of(&s("hello"), &default_match()).expect("a scalar"); + assert!(term(&keyset, scalar, &default_match(), ctx).await.is_ok()); + } + /// A `TermBytes` reads as the typed term's bytes, through every /// accessor: `as_bytes`, `as_ref` and `into_bytes` give the same /// 32 PRF bytes an `EqualityTerm` holds, for a term derived through diff --git a/packages/stack-encrypt/src/op/mod.rs b/packages/stack-encrypt/src/op/mod.rs index 34e8dcfc9..e2f5b451e 100644 --- a/packages/stack-encrypt/src/op/mod.rs +++ b/packages/stack-encrypt/src/op/mod.rs @@ -45,7 +45,13 @@ //! [`query_index`](crate::StackCipher::query_index), before any key is //! requested. //! So is a match configuration whose [`MatchCase`] disagrees with its -//! options. +//! options, and so is the one query value the library refuses: text that +//! cuts no token under the match index's tokenizer, whose filter would +//! match every row ([`TermError::EmptyTermText`]). The stored side has no +//! such refusal: a value with no tokens is stored as the empty filter, +//! which no query matches. +//! +//! [`TermError::EmptyTermText`]: crate::sem::TermError::EmptyTermText //! //! The module is meant to be used by path (`op::Eq`): an imported `Eq` //! would shadow the prelude's `Eq` trait. @@ -60,9 +66,11 @@ use stack_kms::MaybeSend; pub(crate) mod stats; pub use stats::{Limit, Stats, DEFAULT_CONFIDENCE, MAX_ROWS}; -use crate::sem::{DefaultMatch, EqualityTerm, MatchConfig, MatchQuery, MatchTerms, QueryTokens}; +use crate::sem::{ + DefaultMatch, EqualityTerm, MatchConfig, MatchQuery, MatchTerms, QueryTokens, TermError, +}; use crate::target::{Equality, Index, Match}; -use crate::PlanError; +use crate::{Error, PlanError}; /// A query operator: a unit type naming what a query asks of the server. /// @@ -294,9 +302,13 @@ pub trait Supports: Index { /// its own text holds all three. The filter cannot give the tokens back, /// so without the verifier a Bloom false positive would reach the caller. type Verifier: MaybeSend + 'static; - /// Why this index cannot answer `Op` as configured, if it cannot: a - /// match index whose [`MatchCase`] disagrees with its options. - fn check(&self) -> Result<(), PlanError>; + /// Why this index cannot answer `Op` as configured, or over `value`, if + /// it cannot: a match index whose [`MatchCase`] disagrees with its + /// options ([`PlanError::MatchCase`]), or a match query of text that + /// cuts no token ([`TermError::EmptyTermText`]), whose filter would + /// match every row. Run when the query is awaited, before any key is + /// requested; an index that refuses nothing returns `Ok(())`. + fn check(&self, value: &S) -> Result<(), Error>; /// The verifier of the query's plaintext `value`. fn verifier(&self, value: &S) -> Self::Verifier; /// The query term, from the verifier and the term the index derived for @@ -321,7 +333,7 @@ impl QueryToken { impl Supports for Equality { type Query = EqualityTerm; type Verifier = (); - fn check(&self) -> Result<(), PlanError> { + fn check(&self, _: &S) -> Result<(), Error> { Ok(()) } fn verifier(&self, _: &S) {} @@ -337,16 +349,22 @@ where { type Query = MatchQuery; type Verifier = QueryTokens; - fn check(&self) -> Result<(), PlanError> { + fn check(&self, value: &S) -> Result<(), Error> { let folds = O::options().downcase; - if folds == O::Case::FOLDS { - Ok(()) - } else { - Err(PlanError::MatchCase { + if folds != O::Case::FOLDS { + return Err(PlanError::MatchCase { declared: O::Case::NAME, folds, - }) + } + .into()); + } + // A query with no tokens would match every row. The stored side + // keeps such a value as the empty filter; the query side refuses it + // here, before any term is derived. + if QueryTokens::of::(value.as_ref()).is_empty() { + return Err(TermError::EmptyTermText.into()); } + Ok(()) } fn verifier(&self, value: &S) -> QueryTokens { QueryTokens::of::(value.as_ref()) diff --git a/packages/stack-encrypt/src/plan/chain.rs b/packages/stack-encrypt/src/plan/chain.rs index 0526a0504..d91b3738f 100644 --- a/packages/stack-encrypt/src/plan/chain.rs +++ b/packages/stack-encrypt/src/plan/chain.rs @@ -958,15 +958,9 @@ pub struct QueryOp<'a, F, Op, I, K> { impl<'a, F, Op: Operator, I: Supports, K: 'static> QueryOp<'a, F, Op, I, K> { /// The chain for `inner`'s index, refused when awaited if the index - /// cannot answer `Op` as it is configured. A refusal the plan already - /// gave (an index it does not declare, or no context) is kept: it is - /// the more basic fault, and the one to fix first. - fn new(mut inner: QueryIndex<'a, F, I, K>, _op: Op) -> Self { - if inner.label.is_ok() { - if let Err(error) = inner.index.check() { - inner.label = Err(error); - } - } + /// cannot answer `Op` as it is configured or over the query's value + /// ([`Supports::check`]). + fn new(inner: QueryIndex<'a, F, I, K>, _op: Op) -> Self { Self { inner, op: PhantomData, @@ -1005,7 +999,8 @@ where self.inner.take_keyset() } fn check(&self) -> Result<(), Error> { - self.inner.check() + self.inner.check()?; + self.inner.index.check(&self.inner.value) } fn prepare<'p>( self, @@ -1015,6 +1010,12 @@ where where 'a: 'p, { + // `check` ran before any keyset was resolved; a chain prepared on + // its own is held to the same refusal, so no term is derived from a + // value the index refuses. + if let Err(error) = self.inner.index.check(&self.inner.value) { + return Pending::failed(keyset, error); + } let verifier = self.inner.index.verifier(&self.inner.value); self.inner .prepare(keyset, scoped) diff --git a/packages/stack-encrypt/src/sem/mod.rs b/packages/stack-encrypt/src/sem/mod.rs index 8068a73e9..a081f159c 100644 --- a/packages/stack-encrypt/src/sem/mod.rs +++ b/packages/stack-encrypt/src/sem/mod.rs @@ -187,18 +187,20 @@ pub enum TermError { #[error("invalid match options: {0}")] #[diagnostic(code(stack_encrypt::invalid_match_options))] InvalidOptions(&'static str), - /// The text produced no tokens under the configured tokenizer — empty or - /// separator-only text, or (for n-grams, as in the v1 match indexer) text - /// shorter than the n-gram length. Rejected at generation time for both - /// the write and query paths: an empty term used as a query would - /// vacuously match every stored row, and a sub-gram-length probe could - /// never match anything (a silent false negative). + /// A match query's text produced no tokens under the index's tokenizer: + /// empty or separator-only text, or (for n-grams, as in the v1 match + /// indexer) text shorter than the n-gram length. A query with no tokens + /// would match every row, so the query chain refuses it when the query + /// is awaited, before a term exists. A *stored* value with no tokens is + /// not refused: it is stored as the empty filter, which no query + /// matches, so a short or empty value is a valid thing to hold in a + /// match column (ADR-0009). #[error( - "text produces no match tokens (empty, separator-only, or shorter than the n-gram length)" + "the query's text produces no match tokens (empty, separator-only, or shorter than the n-gram length)" )] #[diagnostic( code(stack_encrypt::empty_term_text), - help("A match term needs text with at least one token: a query with none would match every row, and one shorter than the n-gram length could match none. Match on longer text, or skip the match condition for this value.") + help("A match query needs text with at least one token: a query with none would match every row, and one shorter than the n-gram length could match none. Query on longer text, or leave the match condition out.") )] EmptyTermText, /// Term bytes do not decode under the term kind's frozen encoding — see @@ -556,9 +558,11 @@ impl MatchTerms { /// /// An empty `query` returns `false`: containment of zero positions is /// vacuously true, which would turn an empty query into a match-every-row - /// query. Term generation already refuses to build such a term - /// ([`TermError::EmptyTermText`]); this guards any other - /// (e.g. deserialized) source of an empty term. + /// query. A query chain already refuses to build such a query + /// ([`TermError::EmptyTermText`]); this guards any other (e.g. + /// deserialized) source of an empty one. The stored side is the other + /// way round: a value with no tokens is stored as the empty filter, and + /// holds no query's positions, so no query matches it. pub fn matches(&self, query: &MatchQuery) -> bool { self.contains(query.filter()) } @@ -782,13 +786,23 @@ pub struct QueryTokens(Zeroizing>); impl QueryTokens { /// The tokens of `text` under `O`'s tokenizer and case folding. pub(crate) fn of(text: &str) -> Self { - let options = O::options(); + Self::under(text, &O::options()) + } + + /// The tokens of `text` under `options`' tokenizer and case folding. + pub(crate) fn under(text: &str, options: &MatchOptions) -> Self { Self(Zeroizing::new(tokenize::tokenize( text, options.tokenizer, options.downcase, ))) } + + /// Whether the text cut no token at all: the one query the library + /// refuses, because its filter would match every row. + pub(crate) fn is_empty(&self) -> bool { + self.0.is_empty() + } } /// A [`PrfVisitor`] that folds a sequence of per-token PRF blocks into @@ -840,6 +854,13 @@ impl PrfVisitor<[u8; 32], P> for BloomVisitor { /// Derive a match term. Synchronous — see [`equality`]: validation and /// tokenization run before the PRF, the visitor folds blocks into positions. +/// +/// Text that cuts no token (empty, separator-only, or shorter than the +/// n-gram length) is the empty filter: no position set, no PRF call. Stored, +/// it is a value no query matches, since a query always has at least one +/// token and the empty filter holds none of its positions. This is the +/// *stored* side's answer; a query of such text is refused by the query +/// chain before it gets here ([`TermError::EmptyTermText`]). fn match_term( prf: &HmacSha256Prf, text: &str, @@ -849,7 +870,7 @@ fn match_term( let mask = options.validate()?; let tokens = tokenize::tokenize(text, options.tokenizer, options.downcase); if tokens.is_empty() { - return Err(TermError::EmptyTermText); + return Ok(MatchTerms::normalised(Vec::new())); } let context = Context::pae(&[MATCH_DOMAIN, context.as_bytes()]); @@ -1278,15 +1299,15 @@ impl KeysetCipher<'_, K> { /// to the target-directed path for the same `O`. For custom options, /// define a marker type implementing [`MatchConfig`]. /// - /// The same call serves both write time (index the stored text) and query - /// time (index the probe text, then test [`MatchTerms::contains`] - /// server-side). - /// - /// Fails with [`TermError::EmptyTermText`] — as - /// [`Error::Term`], once awaited — when the text - /// yields no tokens: empty or separator-only text, or an n-gram probe - /// shorter than the gram length (which could never match; see - /// [`Tokenizer::Ngram`]). + /// This derives the *stored* filter. Text that yields no tokens (empty + /// or separator-only text, or text shorter than the n-gram length; see + /// [`Tokenizer::Ngram`]) is the empty filter, which no query matches, + /// so a short value can be stored. A query is the other type, + /// [`MatchQuery`], and comes from the query chain + /// (`cipher.query(op::Matches, ..)`), which refuses a query of such + /// text ([`TermError::EmptyTermText`]) because its filter would match + /// every row; this method does not know which it is deriving and + /// refuses nothing. pub fn match_terms<'c, O>( &self, text: &str, diff --git a/packages/stack-encrypt/src/sem/tokenize.rs b/packages/stack-encrypt/src/sem/tokenize.rs index 9e5457907..25a59f951 100644 --- a/packages/stack-encrypt/src/sem/tokenize.rs +++ b/packages/stack-encrypt/src/sem/tokenize.rs @@ -17,11 +17,12 @@ use zeroize::Zeroizing; pub enum Tokenizer { /// Sliding character n-grams of the given length over the whole text /// (whitespace included). Text shorter than `length` yields **no tokens**, - /// exactly like the v1 match indexer — so a probe shorter than the gram - /// length is rejected by - /// [`match_terms`](crate::KeysetCipher::match_terms) rather than - /// silently never matching. This is the default, matching the existing - /// match indexer's 3-gram configuration. + /// exactly like the v1 match indexer — so a query shorter than the gram + /// length is refused by the query chain + /// ([`TermError::EmptyTermText`](crate::sem::TermError::EmptyTermText)) + /// rather than silently never matching, and a stored value shorter than + /// it is the empty filter, which no query matches. This is the default, + /// matching the existing match indexer's 3-gram configuration. Ngram { length: usize }, /// Split on the v1 match indexer's separator set — space, comma, /// semicolon, colon and exclamation mark — one token per run of diff --git a/packages/stack-encrypt/tests/derive.rs b/packages/stack-encrypt/tests/derive.rs index d5b87a823..5c4de8c96 100644 --- a/packages/stack-encrypt/tests/derive.rs +++ b/packages/stack-encrypt/tests/derive.rs @@ -10,7 +10,7 @@ use std::sync::atomic::Ordering as AtomicOrdering; use cllw_ore::CllwOreEncrypt; use common::{counting_cipher, recording_cipher, stack_cipher}; -use stack_encrypt::sem::{EqualityTerm, MatchTerms, OreTerm}; +use stack_encrypt::sem::{EqualityTerm, MatchConfig, MatchOptions, MatchTerms, OreTerm, TermError}; use stack_encrypt::target::{AeadContext, DecryptFrom, EncryptInto, Encrypted}; use stack_encrypt::{ nonempty, ContextPiece, DecryptField, DecryptInto, Decryptable, EncryptFrom, Error, @@ -436,20 +436,50 @@ async fn a_ciphertext_only_record_accepts_an_aead_only_context_like_the_leaf_doe assert!(matches!(result, Err(Error::Aead)), "{result:?}"); } +/// A match configuration whose options fail the scheme's bounds, so its +/// leaf fails during the synchronous build. +struct TooManyHashes; +impl MatchConfig for TooManyHashes { + fn options() -> MatchOptions { + MatchOptions { + k: 17, + ..Default::default() + } + } +} + +#[derive(EncryptFrom, DecryptInto)] +struct BadlyIndexedText { + c: StackCipherText, + m: MatchTerms, +} + #[tokio::test] async fn a_failed_field_fails_the_derived_record_before_any_io() { let (cipher, generates, _) = counting_cipher().await; let keyset = cipher.default_keyset(); - // Text that yields no match tokens fails that leaf during the - // synchronous build; the derived record is the zip of its fields, so it - // fails the same way and never mints the data key its ciphertext field - // would have wanted. - let result: Result = String::new() + // Invalid match options fail that leaf during the synchronous build; + // the derived record is the zip of its fields, so it fails the same way + // and never mints the data key its ciphertext field would have wanted. + let result: Result = "alice" + .to_string() .encrypt_into_with_context(&keyset, nonempty!("users/name")) .await; - assert!(matches!(result, Err(Error::Term(_)))); + assert!(matches!( + result, + Err(Error::Term(TermError::InvalidOptions(_))) + )); assert_eq!(generates.load(AtomicOrdering::SeqCst), 0); + + // Text with no match tokens is not a failure: the leaf is the empty + // filter, and the record seals as any other. + let stored: SearchableText = String::new() + .encrypt_into_with_context(&keyset, nonempty!("users/name")) + .await + .unwrap(); + assert!(stored.m.positions().is_empty()); + assert_eq!(generates.load(AtomicOrdering::SeqCst), 1); } // --- Structs: each field from one field of the plaintext, under its own context diff --git a/packages/stack-encrypt/tests/op.rs b/packages/stack-encrypt/tests/op.rs index 5f5d975f1..fee9be2c1 100644 --- a/packages/stack-encrypt/tests/op.rs +++ b/packages/stack-encrypt/tests/op.rs @@ -8,7 +8,9 @@ use common::{loads_counting_cipher, stack_cipher}; use stack_encrypt::kms::FakeDataKeySource; use stack_encrypt::op::{self, Limit, MatchCase, Probabilistic, Stats}; use stack_encrypt::plan::{Field, FieldSchema, Fields, PlanError}; -use stack_encrypt::sem::{EqualityTerm, MatchConfig, MatchOptions, MatchQuery, MatchTerms}; +use stack_encrypt::sem::{ + EqualityTerm, MatchConfig, MatchOptions, MatchQuery, MatchTerms, TermError, +}; use stack_encrypt::target::{Encrypted, IndexSpec}; use stack_encrypt::{Equality, Error, Match, Plan}; @@ -831,3 +833,108 @@ async fn target_limit_without_a_prior_needs_a_measured_column() { "10 wanted, λ = {lambda}: {rows}" ); } + +/// A value too short to cut a token (two letters under 3-grams, or nothing +/// at all) is a valid thing to store in a match column. ADR-0009: the +/// stored term is the empty filter, which no query matches, and the value +/// itself seals and opens as any other. +#[tokio::test] +async fn a_short_match_value_stores_as_the_empty_filter_and_no_query_finds_it() { + let cipher = stack_cipher().await; + let plan = Plan::context("users/name") + .with::((Equality, Match)) + .build() + .unwrap(); + let query = cipher.query(op::Matches, "alice").on(&plan).await.unwrap(); + + for short in ["Al", "", " "] { + let stored: Encrypted<(EqualityTerm, MatchTerms)> = cipher + .encrypt(&short.to_owned()) + .using(&plan) + .await + .unwrap_or_else(|error| panic!("{short:?} stores: {error}")); + assert!( + stored.terms.1.positions().is_empty(), + "{short:?} is stored as the empty filter" + ); + assert!( + !stored.terms.1.matches(&query), + "{short:?}: the empty filter holds none of a query's positions" + ); + assert!(!query.verify(short)); + let back: String = cipher.open(stored).using(&plan).await.unwrap(); + assert_eq!(back, short, "{short:?} opens as written"); + } + + // The equality term still finds it: only the match term is empty. + let stored: Encrypted<(EqualityTerm, MatchTerms)> = + cipher.encrypt(&"Al".to_owned()).using(&plan).await.unwrap(); + let eq = cipher.query(op::Eq, "Al").on(&plan).await.unwrap(); + assert_eq!(eq, stored.terms.0); +} + +/// A query with no tokens would match every row, so it is the one query +/// value the library refuses: when the query is awaited, before a term +/// exists and before any keyset is loaded. +#[tokio::test] +async fn a_match_query_with_no_tokens_is_refused_before_any_term_or_keyset() { + let (cipher, loads) = loads_counting_cipher().await; + let plan: Plan = Plan::context("people") + .fields() + .encrypt_index::("email", (Equality, Match)) + .encrypt_index::("name", Match::::new()) + .build() + .unwrap(); + let email_plan = plan.field("email").unwrap(); + + for text in ["", " ", "bo"] { + let refused = cipher + .query(op::Matches, text) + .on(&email_plan) + .keyset("tenant") + .await; + assert!( + matches!(refused, Err(Error::Term(TermError::EmptyTermText))), + "{text:?}: {refused:?}" + ); + } + // Through `via`, under another configuration, and in a batch. + let refused = cipher + .query(op::Matches.case_sensitive(), "Al") + .via(Match::::new()) + .on(&plan.field("name").unwrap()) + .await; + assert!(matches!( + refused, + Err(Error::Term(TermError::EmptyTermText)) + )); + let refused = stack_encrypt::all(( + cipher.query(op::Eq, "bob@example.com").on(&email_plan), + cipher.query(op::Matches, "bo").on(&email_plan), + )) + .await; + assert!(matches!( + refused, + Err(Error::Term(TermError::EmptyTermText)) + )); + assert_eq!( + loads.load(std::sync::atomic::Ordering::SeqCst), + 0, + "refused before the named keyset is loaded" + ); + assert_eq!( + TermError::EmptyTermText.to_string(), + "the query's text produces no match tokens (empty, separator-only, or shorter than the n-gram length)" + ); + + // The same text is a fine equality query, and a fine stored value. + assert!(cipher.query(op::Eq, "").on(&email_plan).await.is_ok()); + let row = cipher + .encrypt(&Person { + email: "bo".into(), + name: "Al".into(), + }) + .using(&plan) + .await; + assert!(row.is_ok(), "short values store: {row:?}"); +} diff --git a/packages/stack-encrypt/tests/sem_terms.rs b/packages/stack-encrypt/tests/sem_terms.rs index 1ce6b799f..e22d62d98 100644 --- a/packages/stack-encrypt/tests/sem_terms.rs +++ b/packages/stack-encrypt/tests/sem_terms.rs @@ -286,38 +286,47 @@ async fn match_rejects_invalid_options() { } #[tokio::test] -async fn match_rejects_text_that_yields_no_tokens() { - use stack_encrypt::sem::TermError; - +async fn match_stores_text_that_yields_no_tokens_as_the_empty_filter() { let cipher = generator().await; let gen = cipher.default_keyset(); + let query = gen + .match_terms::("hello", nonempty!("users/bio")) + .await + .unwrap(); - // An empty term used as a query would vacuously match every stored row. - for text in ["", " "] { + // Empty or separator-only text, and text shorter than the n-gram, cut + // no token. The stored side keeps such a value as the empty filter: a + // query always has a token, and the empty filter holds none of its + // positions, so no query matches the value. (A query of such text is + // refused by the query chain instead; `tests/op.rs` holds that.) + for text in ["", " ", "hi"] { + let stored = gen + .match_terms::(text, nonempty!("users/bio")) + .await + .unwrap_or_else(|error| panic!("{text:?} stores: {error}")); assert!( - matches!( - gen.match_terms::(text, nonempty!("users/bio")) - .await, - Err(Error::Term(TermError::EmptyTermText)) - ), - "{text:?} must be rejected" + stored.positions().is_empty(), + "{text:?} is the empty filter" + ); + assert!(stored.to_bytes().is_empty()); + assert!(!stored.contains(&query), "{text:?}: no query matches it"); + assert!( + !query.contains(&stored), + "{text:?}: and the empty filter, as a query, matches nothing" + ); + assert_eq!( + MatchTerms::::from_bytes(&stored.to_bytes()).unwrap(), + stored, + "{text:?}: the empty filter round-trips through its bytes" ); } - // A probe shorter than the n-gram length could never match a stored gram - // (v1 indexer semantics) — rejected instead of a silent false negative. - assert!(matches!( - gen.match_terms::("hi", nonempty!("users/bio")) - .await, - Err(Error::Term(TermError::EmptyTermText)) - )); - // Separator-only text under the Standard tokenizer. - assert!(matches!( - gen.match_terms::(" ,;:! ", nonempty!("users/bio")) - .await, - Err(Error::Term(TermError::EmptyTermText)) - )); + let stored = gen + .match_terms::(" ,;:! ", nonempty!("users/bio")) + .await + .unwrap(); + assert!(stored.positions().is_empty()); } #[tokio::test] diff --git a/packages/stack-encrypt/tests/target.rs b/packages/stack-encrypt/tests/target.rs index dba4be098..14e12f98b 100644 --- a/packages/stack-encrypt/tests/target.rs +++ b/packages/stack-encrypt/tests/target.rs @@ -178,6 +178,19 @@ async fn match_leaf_supports_containment_queries() { assert!(stored.contains(&query)); } +/// A match configuration whose options fail the scheme's bounds, so a term +/// under it fails during the synchronous build, before any I/O. +struct TooManyHashes; + +impl MatchConfig for TooManyHashes { + fn options() -> MatchOptions { + MatchOptions { + k: 17, + ..Default::default() + } + } +} + /// A custom type-level match configuration. struct SmallFilter; @@ -706,13 +719,13 @@ async fn a_column_stops_declaring_at_the_first_refused_row() { #[tokio::test] async fn a_failed_field_fails_the_record_before_any_kms_call() { // One failed field must not cause the record's other fields to mint - // data keys that are then thrown away. A match term over text that - // yields no tokens fails during the synchronous build, before any I/O. + // data keys that are then thrown away. A match term under invalid + // options fails during the synchronous build, before any I/O. let (cipher, generates, _) = counting_cipher().await; let cipher = cipher.default_keyset(); let b = "b".to_string(); - let zipped = MatchTerms::::encrypt_from(&"", &cipher, nonempty!("users/x")) + let zipped = MatchTerms::::encrypt_from(&"abc", &cipher, nonempty!("users/x")) .zip(StackCipherText::encrypt_from( &b, &cipher, diff --git a/packages/stack-encrypt/tests/ui/op_inexact_operator_needs_probabilistic.rs b/packages/stack-encrypt/tests/ui/op_inexact_operator_needs_probabilistic.rs index 2f58d682b..22ea7d599 100644 --- a/packages/stack-encrypt/tests/ui/op_inexact_operator_needs_probabilistic.rs +++ b/packages/stack-encrypt/tests/ui/op_inexact_operator_needs_probabilistic.rs @@ -23,7 +23,7 @@ impl Index for Shortcut { impl op::Supports for Shortcut { type Query = EqualityTerm; type Verifier = (); - fn check(&self) -> Result<(), stack_encrypt::PlanError> { + fn check(&self, _: &String) -> Result<(), stack_encrypt::Error> { Ok(()) } fn verifier(&self, _: &String) {} From a9e4857403b31d9128d6bfaa890dc345117b13f7 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Fri, 16 Oct 2026 22:44:09 +1100 Subject: [PATCH 2/4] fix(stack-encrypt): address the review of the empty match filter match_terms is no longer a query. In 0.2.0 the same call served write time and query time and refused text that cuts no token; it now returns the empty filter for that text, and EQL treats an empty filter as contained by every stored filter. A caller who still sends its output as a query gets every row and no error, so the CHANGELOG entry moves to Breaking and says so plainly, naming cipher.query(op::Matches, ..) as the query. The query side validated the match options only when it derived the term, after the new no-token check had tokenised the text, so an n-gram length of zero panicked in the tokenizer instead of returning InvalidOptions. Both query prechecks, op::Supports::check and dynamic::term, now validate first, as the stored path always has. Go's term-failure locator asks the query path for each term, and the query path refuses short match text that a record stores as the empty filter. It could stop at an earlier row with a short email and name the wrong row. It now skips that one refusal by its code, and a test pins it with a short email in row 0. ADR-0009 and the Go package doc say that a stored empty filter tells a reader of the column the value is empty, separator-only or shorter than the n-gram length. The tokenizer's doc no longer says match_terms rejects that text. --- languages/golang/encrypt/doc.go | 6 ++- languages/golang/encrypt/locate_test.go | 41 ++++++++++++++++++ languages/golang/encrypt/records.go | 20 +++++++-- packages/stack-encrypt/CHANGELOG.md | 36 +++++++++------- ...and-a-query-term-is-not-the-stored-term.md | 5 ++- packages/stack-encrypt/src/dynamic/term.rs | 30 +++++++++++++ packages/stack-encrypt/src/op/mod.rs | 4 ++ packages/stack-encrypt/src/sem/tokenize.rs | 8 ++-- packages/stack-encrypt/tests/op.rs | 42 ++++++++++++++++++- 9 files changed, 166 insertions(+), 26 deletions(-) create mode 100644 languages/golang/encrypt/locate_test.go diff --git a/languages/golang/encrypt/doc.go b/languages/golang/encrypt/doc.go index ecfd9de3f..f612e2cf8 100644 --- a/languages/golang/encrypt/doc.go +++ b/languages/golang/encrypt/doc.go @@ -119,8 +119,10 @@ // A match index stores text with no token (an empty string, separator-only // text, or text shorter than the n-gram length of 3 characters) as the empty // filter: a valid value that no query matches, so an optional or short value -// can live under match. A match *query* of such text is refused with -// [ErrTerm], because an empty filter as a query would match every row. +// can live under match. The empty filter tells anyone who can read the +// column that the value is one of those. A match *query* of such text is +// refused with [ErrTerm], because an empty filter as a query would match every +// row. // // # Transport and auth // diff --git a/languages/golang/encrypt/locate_test.go b/languages/golang/encrypt/locate_test.go new file mode 100644 index 000000000..04413a1f9 --- /dev/null +++ b/languages/golang/encrypt/locate_test.go @@ -0,0 +1,41 @@ +package encrypt + +import ( + "context" + "errors" + "fmt" + "testing" + + "github.com/cipherstash/stack/languages/golang/internal/record" +) + +// A record stores match text that cuts no token as the empty filter, so the +// term-failure locator must not blame a row for it. Derive asks the query +// path, which refuses that text; the locator skips that refusal and goes on +// to the rows after it. +func TestLocateTermFailureSkipsShortMatchText(t *testing.T) { + ctx := context.Background() + c, err := NewDeterministicClient(ctx, [32]byte{}) + if errors.Is(err, ErrDeterministicGuestNotBuilt) { + t.Skip(err) + } + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = c.Close() }) + cph := c.DefaultKeyset() + plan := usersPlan() + + // The query path refuses the short email, so without the skip the + // locator would name row 0 for it. + _, derr := cph.Derive(ctx, plan, "email", record.Match, "ab") + if !queryOnlyRefusal(record.Match, derr) { + t.Fatalf("Derive of a short email: %v, want the empty-term-text refusal", derr) + } + + sentinel := fmt.Errorf("%w: from the record call", ErrTerm) + rows := []record.Source{usersRow(1, "ab"), usersRow(2, "alice@example.com")} + if got := cph.locateTermFailure(ctx, plan, rows, sentinel); got != sentinel { + t.Fatalf("locateTermFailure = %v, want the record call's error unchanged", got) + } +} diff --git a/languages/golang/encrypt/records.go b/languages/golang/encrypt/records.go index 8e1f03129..24feb3163 100644 --- a/languages/golang/encrypt/records.go +++ b/languages/golang/encrypt/records.go @@ -473,8 +473,10 @@ func (c *Client) open(ctx context.Context, sel KeysetSelector, context string, p // term at a time, which costs no key request: a term derives locally. A // stored value is never refused for its text (text with no token is stored // as the empty filter), so a term failure on a record is the scheme refusing -// the value's kind under that index. p is the plan without this cipher's -// extension, which Derive adds. +// the value's kind under that index. Derive asks the query path, which does +// refuse text with no token, so that one refusal is skipped here: it cannot +// be why the record failed. p is the plan without this cipher's extension, +// which Derive adds. func (cph *Cipher) locateTermFailure(ctx context.Context, p *record.Plan, rows []record.Source, err error) error { for i, row := range rows { for _, f := range p.Fields { @@ -482,7 +484,7 @@ func (cph *Cipher) locateTermFailure(ctx context.Context, p *record.Plan, rows [ if !o.IsTerm() { continue } - if _, derr := cph.Derive(ctx, p, f.Name, o, row[f.Name]); errors.Is(derr, ErrTerm) { + if _, derr := cph.Derive(ctx, p, f.Name, o, row[f.Name]); errors.Is(derr, ErrTerm) && !queryOnlyRefusal(o, derr) { return fmt.Errorf("%w: value %d, field %q: the engine derives no %s term for this value", err, i, f.Name, indexWord(o)) } } @@ -491,6 +493,18 @@ func (cph *Cipher) locateTermFailure(ctx context.Context, p *record.Plan, rows [ return err } +// queryOnlyRefusal reports whether derr is the one term refusal the query +// path makes and the record path does not: match text that cuts no token, +// which a record stores as the empty filter. +func queryOnlyRefusal(o record.Output, derr error) bool { + var d *Diagnostic + return o == record.Match && errors.As(derr, &d) && d.Code == emptyTermTextCode +} + +// emptyTermTextCode is the stable code of the engine's refusal of match text +// that cuts no token. +const emptyTermTextCode = "stack_encrypt::empty_term_text" + func indexWord(o record.Output) string { switch o { case record.Equality: diff --git a/packages/stack-encrypt/CHANGELOG.md b/packages/stack-encrypt/CHANGELOG.md index 21b1ab36a..a41ef4161 100644 --- a/packages/stack-encrypt/CHANGELOG.md +++ b/packages/stack-encrypt/CHANGELOG.md @@ -96,24 +96,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 under it. `FieldPlan::with_target`, `FieldPlan::target()`, `Plan::new_with`. -### Changed - -- **A match value with no tokens is stored as the empty filter; only a query - with none is refused.** Text that cuts no token under the match index's - tokenizer (empty or separator-only text, or text shorter than the n-gram - length) used to fail term derivation on both sides, so one two-letter name - failed a whole row. Now the stored term is the empty filter (`bf: []`), - which no query matches, and the value seals and opens as any other; the - `dynamic` record path stores it the same way. A *query* of such text is - still refused, with `TermError::EmptyTermText` as before, because its - filter would match every row: `cipher.query(op::Matches, ..)` refuses it - when the query is awaited, before a term exists and before any keyset is - loaded (`op::Supports::check` now takes the query's value), and - `dynamic::term` refuses it for a match kind. `KeysetCipher::match_terms` - derives the stored filter and no longer fails on such text (ADR-0009). - ### Breaking +- **A match value with no tokens is stored as the empty filter, so + `KeysetCipher::match_terms` is no longer a query.** In 0.2.0 the same call + served write time and query time, and refused text that cuts no token + under the match index's tokenizer (empty or separator-only text, or text + shorter than the n-gram length) with `TermError::EmptyTermText`. Now it + derives the stored filter only, and returns the empty filter (`bf: []`) + for such text with no error. **Do not send the output of `match_terms` as + a query:** EQL treats an empty filter as contained by every stored filter, + and its query domains do not refuse one, so that query returns every row + with no error anywhere. A query comes from `cipher.query(op::Matches, ..)`, + which still refuses such text with `TermError::EmptyTermText`, when the + query is awaited, before a term exists and before any keyset is loaded + (`op::Supports::check` now takes the query's value); `dynamic::term` + refuses it for a match kind the same way. The stored side is the change a + caller sees: one two-letter name no longer fails a whole row, the value + seals and opens as any other, and the `dynamic` record path stores it the + same way. Storing the empty filter tells a reader of the database that the + value is empty, separator-only or shorter than the n-gram length + (ADR-0009). + - **The query chain starts with the operator.** `cipher.query(op, v)` replaces `cipher.query(v)`: `cipher.query(op::Eq, v).on(&plan)` replaces `cipher.query(v).using(&plan).equality()`, and `on` replaces `using` for diff --git a/packages/stack-encrypt/docs/adr/0009-operators-select-the-index-and-a-query-term-is-not-the-stored-term.md b/packages/stack-encrypt/docs/adr/0009-operators-select-the-index-and-a-query-term-is-not-the-stored-term.md index f4e7aac4e..c2987bfd8 100644 --- a/packages/stack-encrypt/docs/adr/0009-operators-select-the-index-and-a-query-term-is-not-the-stored-term.md +++ b/packages/stack-encrypt/docs/adr/0009-operators-select-the-index-and-a-query-term-is-not-the-stored-term.md @@ -280,7 +280,10 @@ library refuses, a query value too short to tokenise, is refused when the query is awaited, before a term exists. The stored side refuses nothing: a value too short to tokenise is stored as the empty filter, which no query matches, so a two-letter name is a valid value in a match column and `verify` is -never asked about it. +never asked about it. The empty filter is visible to anyone who can read the +column: it tells them the value is empty, separator-only or shorter than the +n-gram length. A filter's fill already leaks its value's approximate length; +this is an exact class, and the price of storing such values at all. `Plaintext` is the decrypted value the term is defined over, and that is not always the value the user wrote. For `Matches` it is the text. For diff --git a/packages/stack-encrypt/src/dynamic/term.rs b/packages/stack-encrypt/src/dynamic/term.rs index 3b75fe7b8..db5da5081 100644 --- a/packages/stack-encrypt/src/dynamic/term.rs +++ b/packages/stack-encrypt/src/dynamic/term.rs @@ -311,6 +311,11 @@ where D: IntoPrfContext<'c>, { if let (IndexSpec::Match(options), Scalar::Text(text)) = (kind, &scalar) { + // Options the index cannot use are refused before the text is + // tokenised, as the record path refuses them. + let _mask = options + .validate() + .map_err(|error| Error::Cipher(error.into()))?; if QueryTokens::under(text, options).is_empty() { return Err(Error::Cipher(TermError::EmptyTermText.into())); } @@ -662,6 +667,31 @@ mod tests { assert!(term(&keyset, scalar, &default_match(), ctx).await.is_ok()); } + /// Options the index cannot use are refused before the query's text is + /// tokenised: an n-gram length of zero is `InvalidOptions`, not a panic + /// in the tokenizer. + #[tokio::test] + async fn a_match_query_under_invalid_options_is_refused_not_tokenised() { + let cipher = cipher().await; + let keyset = cipher.default_keyset(); + let ctx = context(s("users/bio")).expect("context"); + let zero = IndexSpec::Match(MatchOptions { + tokenizer: Tokenizer::Ngram { length: 0 }, + ..MatchOptions::default() + }); + let scalar = Scalar::of(&s("hello"), &zero).expect("a scalar"); + let refused = term(&keyset, scalar, &zero, ctx).await; + assert!( + matches!( + refused, + Err(Error::Cipher(crate::Error::Term( + TermError::InvalidOptions(_) + ))) + ), + "{refused:?}" + ); + } + /// A `TermBytes` reads as the typed term's bytes, through every /// accessor: `as_bytes`, `as_ref` and `into_bytes` give the same /// 32 PRF bytes an `EqualityTerm` holds, for a term derived through diff --git a/packages/stack-encrypt/src/op/mod.rs b/packages/stack-encrypt/src/op/mod.rs index e2f5b451e..a060f4584 100644 --- a/packages/stack-encrypt/src/op/mod.rs +++ b/packages/stack-encrypt/src/op/mod.rs @@ -358,6 +358,10 @@ where } .into()); } + // Options the index cannot use are refused before the text is + // tokenised, as term derivation refuses them: an n-gram length of + // zero would otherwise reach the tokenizer. + let _mask = O::options().validate()?; // A query with no tokens would match every row. The stored side // keeps such a value as the empty filter; the query side refuses it // here, before any term is derived. diff --git a/packages/stack-encrypt/src/sem/tokenize.rs b/packages/stack-encrypt/src/sem/tokenize.rs index 25a59f951..d583fe18d 100644 --- a/packages/stack-encrypt/src/sem/tokenize.rs +++ b/packages/stack-encrypt/src/sem/tokenize.rs @@ -44,10 +44,12 @@ impl Default for Tokenizer { /// are case-insensitive. /// /// May return no tokens (empty text, separator-only text, or text shorter -/// than the n-gram length); [`match_terms`] rejects that case so an empty -/// term can never reach a query. +/// than the n-gram length). A stored value with none is the empty filter; a +/// query with none is refused before a term exists, because its filter would +/// match every row (ADR-0009). /// -/// [`match_terms`]: crate::StackCipher::match_terms +/// The tokenizer must be valid: an n-gram length of zero panics. Callers +/// validate the [`MatchOptions`](super::MatchOptions) first. /// /// An n-gram length of zero yields no tokens rather than panicking, so a /// caller that tokenizes before validating the options (a query's verifier) diff --git a/packages/stack-encrypt/tests/op.rs b/packages/stack-encrypt/tests/op.rs index fee9be2c1..be289eabe 100644 --- a/packages/stack-encrypt/tests/op.rs +++ b/packages/stack-encrypt/tests/op.rs @@ -9,7 +9,7 @@ use stack_encrypt::kms::FakeDataKeySource; use stack_encrypt::op::{self, Limit, MatchCase, Probabilistic, Stats}; use stack_encrypt::plan::{Field, FieldSchema, Fields, PlanError}; use stack_encrypt::sem::{ - EqualityTerm, MatchConfig, MatchOptions, MatchQuery, MatchTerms, TermError, + EqualityTerm, MatchConfig, MatchOptions, MatchQuery, MatchTerms, TermError, Tokenizer, }; use stack_encrypt::target::{Encrypted, IndexSpec}; use stack_encrypt::{Equality, Error, Match, Plan}; @@ -29,6 +29,20 @@ impl MatchCase for Exact { type Case = op::CaseSensitive; } +/// A match configuration the index cannot use: an n-gram length of zero. +struct ZeroGram; +impl MatchConfig for ZeroGram { + fn options() -> MatchOptions { + MatchOptions { + tokenizer: Tokenizer::Ngram { length: 0 }, + ..Default::default() + } + } +} +impl MatchCase for ZeroGram { + type Case = op::CaseInsensitive; +} + #[derive(Clone)] struct Person { email: String, @@ -938,3 +952,29 @@ async fn a_match_query_with_no_tokens_is_refused_before_any_term_or_keyset() { .await; assert!(row.is_ok(), "short values store: {row:?}"); } + +/// A match query under options the index cannot use is refused with +/// `InvalidOptions`, as storing under them is, and never reaches the +/// tokenizer: an n-gram length of zero would panic there. +#[tokio::test] +async fn a_match_query_under_invalid_options_is_refused_not_tokenised() { + let cipher = stack_cipher().await; + let plan = Plan::context("users/name") + .with::(Match::::new()) + .build() + .unwrap(); + let refused = cipher + .query(op::Matches, "alice") + .via(Match::::new()) + .on(&plan) + .await; + assert!( + matches!(refused, Err(Error::Term(TermError::InvalidOptions(_)))), + "{refused:?}" + ); + let stored = cipher.encrypt(&"alice".to_owned()).using(&plan).await; + assert!( + matches!(stored, Err(Error::Term(TermError::InvalidOptions(_)))), + "{stored:?}" + ); +} From 9dbafcd4c5ba080b9a8bd922aa126fe7357e1d57 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Sat, 17 Oct 2026 00:02:15 +1100 Subject: [PATCH 3/4] fix(eql-bindings): refuse match text with no tokens for EQL v4 TextMatch, stored or queried Rebased onto next, the EQL v4 TextMatch filter met this PR's change to match_terms. eql-bindings builds the stored TextMatch filter and the TextMatchQuery operand from one type, BloomFilter, through the stored-side match_terms. Before this PR that refused text that cuts no token, so both refused it. After it, match_terms returns the empty filter for that text, so a TextMatchQuery of "" or "al" became `bf: []`, which EQL treats as contained by every stored filter: a query matching every row, with no error, through Rust's encrypt_as and through the Go SDK's generated Query. BloomFilter's visitor now refuses an empty filter with TermError::EmptyTermText, for the stored value and the query alike, which is what EQL v4 TextMatch did before and what its docs, the Rust tests and the Go test already say. Storing a short value as the empty filter while refusing it as a query needs the query twin to have a filter type of its own, which is a codegen change left for later. The Go SDK README and the stashgen README still said a plain `match` index refuses short text; they now say it stores the empty filter and refuses only the query, and that an EQL TextMatch field is stricter. --- languages/golang/cmd/stashgen/README.md | 6 +++--- languages/golang/encrypt/README.md | 13 +++++++------ packages/eql/crates/eql-bindings/src/encryption.rs | 13 ++++++++++++- packages/eql/tests/encryption/tests/v4_types.rs | 10 +++++++--- 4 files changed, 29 insertions(+), 13 deletions(-) diff --git a/languages/golang/cmd/stashgen/README.md b/languages/golang/cmd/stashgen/README.md index a281d0a4c..0473b4591 100644 --- a/languages/golang/cmd/stashgen/README.md +++ b/languages/golang/cmd/stashgen/README.md @@ -96,9 +96,9 @@ A query on such a type derives its term under the context the cipher names, so i > To update one column, use its `Fields` entry, such as `Fields.Email.Encrypt`. The index names are `equality`, `match`, `ore`, `ope` and `json`. -A `match` index needs text with at least one token: the engine derives no match term for an empty string, separator-only text, or text shorter than the n-gram length (3 characters), because an empty term would match every row. -`Encrypt` then fails for the whole batch, with an error naming the row, the field and the index. -So an optional or short value does not belong under `match`: give the field `equality` alone, or make the value required. +A `match` index stores text with no token (an empty string, separator-only text, or text shorter than the n-gram length of 3 characters) as the empty filter, which no query matches, so an optional or short value can live under `match`; the empty filter tells a reader of the column that the value is one of those. +A match query of such text is refused, because an empty filter as a query would match every row. +An EQL `TextMatch` field is stricter: see below. An index takes its options in parentheses after its name, separated by commas: `index=equality;match(k=3)`. This build refuses an index with options: a query term is derived with the default options only, so a stored term with other options would never match one. These words are the same as the Rust API's words for the same behaviour, with two that only Go has. diff --git a/languages/golang/encrypt/README.md b/languages/golang/encrypt/README.md index 26a3546ab..59eb1f283 100644 --- a/languages/golang/encrypt/README.md +++ b/languages/golang/encrypt/README.md @@ -202,12 +202,13 @@ and `Equality`, `Match`, `Ore` or `Ope` (the term types). A field with An `opaque` struct is one `Sealed` field. Every stored type implements `driver.Valuer` and `sql.Scanner`. -A `match` index needs text with at least one token. The engine derives no -match term for an empty string, separator-only text, or text shorter than the -n-gram length (3 characters), because an empty term would match every row. -`Encrypt` then fails for the whole batch with `ErrTerm`, naming the row, the -field and the index. So an optional or short value does not belong under -`match`: give such a field `equality` alone, or make the value required. +A `match` index stores text with no token (an empty string, separator-only +text, or text shorter than the n-gram length of 3 characters) as the empty +filter: a valid value that no query matches, so an optional or short value can +live under `match`. The empty filter tells anyone who can read the column that +the value is one of those. A match *query* of such text is refused with +`ErrTerm`, because an empty filter as a query would match every row. An EQL +`TextMatch` field is stricter, as "EQL columns" below says. Terms are byte-equal to the ones the Rust crate derives, so a term from `users.Fields.Email.Equality` compares against a stored term written from any diff --git a/packages/eql/crates/eql-bindings/src/encryption.rs b/packages/eql/crates/eql-bindings/src/encryption.rs index aeae19653..af87ceb33 100644 --- a/packages/eql/crates/eql-bindings/src/encryption.rs +++ b/packages/eql/crates/eql-bindings/src/encryption.rs @@ -84,7 +84,7 @@ pub mod targets; use base64::{engine::general_purpose::STANDARD, Engine as _}; #[allow(deprecated)] use stack_encrypt::sem::MatchTerm; -use stack_encrypt::sem::{EqualityTerm, MatchConfig, MatchOptions, Tokenizer}; +use stack_encrypt::sem::{EqualityTerm, MatchConfig, MatchOptions, TermError, Tokenizer}; use stack_encrypt::target::transcode::{Reader, Transcode, Visitor}; use stack_encrypt::target::{self, AeadContext, CallerContext}; use stack_encrypt::{ @@ -286,6 +286,14 @@ impl MatchConfig for EqlMatch { /// take it from the same type, so they always agree; matching /// cipherstash-client's parameters is not a goal, because cipherstash-client /// never writes an `eql_v4` column. +/// +/// Text that cuts no token (empty, separator-only, or shorter than a 3-gram) +/// is refused with [`TermError::EmptyTermText`], stored or queried. Stack +/// Encrypt stores such a value as the empty filter, but EQL treats an empty +/// filter as contained by every stored filter, and this one type builds both +/// the stored filter and the query operand, so it cannot tell a query from a +/// stored value: refusing both is the one way a query cannot match every +/// row. #[doc(hidden)] pub struct MatchVisitor; impl Visitor for MatchVisitor { @@ -295,6 +303,9 @@ impl Visitor for MatchVisitor { // tree: the one spelling that compiles against both. #[allow(deprecated)] fn matching(self, terms: MatchTerm) -> Result { + if terms.positions().is_empty() { + return Err(Error::Term(TermError::EmptyTermText)); + } terms .positions() .iter() diff --git a/packages/eql/tests/encryption/tests/v4_types.rs b/packages/eql/tests/encryption/tests/v4_types.rs index a2e673ae0..de19d5ec8 100644 --- a/packages/eql/tests/encryption/tests/v4_types.rs +++ b/packages/eql/tests/encryption/tests/v4_types.rs @@ -421,11 +421,15 @@ async fn text_match_filters_contain_their_substrings() { .await; assert_eq!(named.bf, stored.bf); - // Text shorter than a 3-gram has no tokens. Stack Encrypt 0.2 refuses it - // for a stored value as well as a query, so a short value cannot be - // written to a TextMatch column yet. + // Text shorter than a 3-gram has no tokens. eql-bindings refuses it for a + // stored value as well as a query: one filter type builds both, and an + // empty filter as a query would match every row. So a short value cannot + // be written to a TextMatch column yet. let short: Result = keyset.encrypt_as(&"ab".to_owned(), column()).await; assert!(short.is_err()); + let short_query: Result = + keyset.encrypt_as(&"ab".to_owned(), column()).await; + assert!(short_query.is_err(), "a short query would match every row"); } /// The filter bits for one query under the test key source. The From c8160460e8602578fbbe4de95959db6a399e5575 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Sat, 17 Oct 2026 12:19:02 +1100 Subject: [PATCH 4/4] fix(stack-encrypt): address the second review of the empty match filter - One rule for a match query, in one place. `QueryTokens::of_query` validates the options, tokenises the text and refuses text that cuts no token. The query chain and `dynamic::term` both call it, so the Rust chain and the Go guest cannot drift on what they refuse; `dynamic::term` no longer repeats the validate, tokenise and refuse steps by hand. - An awaited `op::Matches` query tokenises its text once fewer. `Supports` gains a provided `admit`, which checks and takes the carry together, and a match index overrides it to tokenise once for both. `prepare` calls `admit` instead of a second `check` and then `carry`. The term itself is still derived through the index's own operation, which tokenises again: that is the path a write takes, and sharing it is what makes the query's filter the stored filter's. --- packages/stack-encrypt/src/dynamic/term.rs | 17 ++++----- packages/stack-encrypt/src/op/mod.rs | 41 +++++++++++--------- packages/stack-encrypt/src/plan/chain.rs | 11 +++--- packages/stack-encrypt/src/sem/mod.rs | 44 ++++++++++++++++++++-- packages/stack-encrypt/tests/op.rs | 38 ------------------- 5 files changed, 76 insertions(+), 75 deletions(-) diff --git a/packages/stack-encrypt/src/dynamic/term.rs b/packages/stack-encrypt/src/dynamic/term.rs index db5da5081..9d5670e18 100644 --- a/packages/stack-encrypt/src/dynamic/term.rs +++ b/packages/stack-encrypt/src/dynamic/term.rs @@ -21,7 +21,7 @@ use zeroize::Zeroizing; use super::{utf8, Error, Reason, Value}; use crate::sem::{ - CllwOpeEncrypt, CllwOreEncrypt, DefaultMatch, MatchOptions, QueryTokens, TermError, Tokenizer, + CllwOpeEncrypt, CllwOreEncrypt, DefaultMatch, MatchOptions, QueryTokens, Tokenizer, }; use crate::target::{chosen, CallerContext, ConsumeSource, Encryption, Index, IndexSpec, Pending}; use crate::{IntoPrfContext, KeysetCipher, NonEmpty}; @@ -297,7 +297,7 @@ impl Scalar { /// binding turns this into a boundary rejection). [`Error::Cipher`] if the /// derivation itself fails, or for the one query value the engine refuses: /// match text that cuts no token under the index's options -/// ([`TermError::EmptyTermText`]), whose filter would match every row. That +/// ([`TermError::EmptyTermText`](crate::sem::TermError::EmptyTermText)), whose filter would match every row. That /// refusal is the query path's alone; the record path stores such a value /// as the empty filter, which no query matches. pub async fn term<'c, K, D>( @@ -311,14 +311,10 @@ where D: IntoPrfContext<'c>, { if let (IndexSpec::Match(options), Scalar::Text(text)) = (kind, &scalar) { - // Options the index cannot use are refused before the text is - // tokenised, as the record path refuses them. - let _mask = options - .validate() - .map_err(|error| Error::Cipher(error.into()))?; - if QueryTokens::under(text, options).is_empty() { - return Err(Error::Cipher(TermError::EmptyTermText.into())); - } + // The query chain's rule, from the same helper: options the index + // cannot use, then text that cuts no token. + let _tokens = + QueryTokens::of_query(text, options).map_err(|error| Error::Cipher(error.into()))?; } scalar_term(cipher, scalar, kind, context)? .await @@ -572,6 +568,7 @@ impl Index for IndexSpec { mod tests { use super::*; use crate::dynamic::{context, Output}; + use crate::sem::TermError; use crate::{nonempty, StackCipher}; use stack_kms::FakeDataKeySource; diff --git a/packages/stack-encrypt/src/op/mod.rs b/packages/stack-encrypt/src/op/mod.rs index a060f4584..9d77f0e6f 100644 --- a/packages/stack-encrypt/src/op/mod.rs +++ b/packages/stack-encrypt/src/op/mod.rs @@ -66,9 +66,7 @@ use stack_kms::MaybeSend; pub(crate) mod stats; pub use stats::{Limit, Stats, DEFAULT_CONFIDENCE, MAX_ROWS}; -use crate::sem::{ - DefaultMatch, EqualityTerm, MatchConfig, MatchQuery, MatchTerms, QueryTokens, TermError, -}; +use crate::sem::{DefaultMatch, EqualityTerm, MatchConfig, MatchQuery, MatchTerms, QueryTokens}; use crate::target::{Equality, Index, Match}; use crate::{Error, PlanError}; @@ -305,12 +303,21 @@ pub trait Supports: Index { /// Why this index cannot answer `Op` as configured, or over `value`, if /// it cannot: a match index whose [`MatchCase`] disagrees with its /// options ([`PlanError::MatchCase`]), or a match query of text that - /// cuts no token ([`TermError::EmptyTermText`]), whose filter would + /// cuts no token ([`TermError::EmptyTermText`](crate::sem::TermError::EmptyTermText)), whose filter would /// match every row. Run when the query is awaited, before any key is /// requested; an index that refuses nothing returns `Ok(())`. fn check(&self, value: &S) -> Result<(), Error>; /// The verifier of the query's plaintext `value`. fn verifier(&self, value: &S) -> Self::Verifier; + /// [`check`](Self::check), then [`verifier`](Self::verifier): the query's + /// verifier of `value`, or why the index refuses it. A query chain + /// calls this when it derives the term. An index whose check and verifier + /// do the same work overrides it to do that work once: a match index + /// tokenises the query's value for both. + fn admit(&self, value: &S) -> Result { + self.check(value)?; + Ok(self.verifier(value)) + } /// The query term, from the verifier and the term the index derived for /// the query's value. Only a query chain calls this: `QueryToken` /// cannot be made outside this crate, so no caller can pair one query's tokens @@ -350,6 +357,12 @@ where type Query = MatchQuery; type Verifier = QueryTokens; fn check(&self, value: &S) -> Result<(), Error> { + self.admit(value).map(drop) + } + fn verifier(&self, value: &S) -> QueryTokens { + QueryTokens::of::(value.as_ref()) + } + fn admit(&self, value: &S) -> Result { let folds = O::options().downcase; if folds != O::Case::FOLDS { return Err(PlanError::MatchCase { @@ -358,20 +371,12 @@ where } .into()); } - // Options the index cannot use are refused before the text is - // tokenised, as term derivation refuses them: an n-gram length of - // zero would otherwise reach the tokenizer. - let _mask = O::options().validate()?; - // A query with no tokens would match every row. The stored side - // keeps such a value as the empty filter; the query side refuses it - // here, before any term is derived. - if QueryTokens::of::(value.as_ref()).is_empty() { - return Err(TermError::EmptyTermText.into()); - } - Ok(()) - } - fn verifier(&self, value: &S) -> QueryTokens { - QueryTokens::of::(value.as_ref()) + // Options the index cannot use, checked before the text is + // tokenised (an n-gram length of zero would otherwise reach the + // tokenizer), then a query with no tokens, which would match every + // row: the stored side keeps such a value as the empty filter, the + // query side refuses it here, before any term is derived. + Ok(QueryTokens::of_query(value.as_ref(), &O::options())?) } fn query(tokens: QueryTokens, term: MatchTerms, _: QueryToken) -> MatchQuery { MatchQuery::new(term, tokens) diff --git a/packages/stack-encrypt/src/plan/chain.rs b/packages/stack-encrypt/src/plan/chain.rs index d91b3738f..6e38b5ed2 100644 --- a/packages/stack-encrypt/src/plan/chain.rs +++ b/packages/stack-encrypt/src/plan/chain.rs @@ -1012,11 +1012,12 @@ where { // `check` ran before any keyset was resolved; a chain prepared on // its own is held to the same refusal, so no term is derived from a - // value the index refuses. - if let Err(error) = self.inner.index.check(&self.inner.value) { - return Pending::failed(keyset, error); - } - let verifier = self.inner.index.verifier(&self.inner.value); + // value the index refuses. `admit` checks and takes the verifier in + // one pass over the value. + let verifier = match self.inner.index.admit(&self.inner.value) { + Ok(verifier) => verifier, + Err(error) => return Pending::failed(keyset, error), + }; self.inner .prepare(keyset, scoped) .map(move |term| I::query(verifier, term, QueryToken::new())) diff --git a/packages/stack-encrypt/src/sem/mod.rs b/packages/stack-encrypt/src/sem/mod.rs index a081f159c..96efc8d74 100644 --- a/packages/stack-encrypt/src/sem/mod.rs +++ b/packages/stack-encrypt/src/sem/mod.rs @@ -798,10 +798,21 @@ impl QueryTokens { ))) } - /// Whether the text cut no token at all: the one query the library - /// refuses, because its filter would match every row. - pub(crate) fn is_empty(&self) -> bool { - self.0.is_empty() + /// The tokens of a query's `text` under `options`, or why the query is + /// refused: options the index cannot use + /// ([`TermError::InvalidOptions`]), checked before the text is + /// tokenised, or text that cuts no token ([`TermError::EmptyTermText`]), + /// whose filter would match every row. The one rule for a match query: + /// the query chain and the `dynamic` path both apply it, so they cannot + /// disagree on what they refuse. The stored side keeps token-less text + /// as the empty filter instead. + pub(crate) fn of_query(text: &str, options: &MatchOptions) -> Result { + let _mask = options.validate()?; + let tokens = Self::under(text, options); + if tokens.0.is_empty() { + return Err(TermError::EmptyTermText); + } + Ok(tokens) } } @@ -1403,6 +1414,31 @@ mod tests { assert!(!crate::op::Probabilistic::verify(&query, "")); } + /// A match query's one rule: options the index cannot use are refused + /// before the text is tokenised, then text that cuts no token; text + /// that cuts one comes back as its tokens. + #[test] + fn a_query_is_refused_for_its_options_then_for_cutting_no_token() { + let zero = MatchOptions { + tokenizer: Tokenizer::Ngram { length: 0 }, + ..Default::default() + }; + assert!(matches!( + QueryTokens::of_query("", &zero), + Err(TermError::InvalidOptions(_)) + )); + let options = MatchOptions::default(); + assert!(matches!( + QueryTokens::of_query("al", &options), + Err(TermError::EmptyTermText) + )); + let tokens = QueryTokens::of_query("ALICE", &options).expect("cuts tokens"); + assert_eq!( + tokens.0.as_slice(), + QueryTokens::under("alice", &options).0.as_slice() + ); + } + /// The Bloom fold reads a token *stream*: handed a map-shaped PRF input /// it refuses, rather than folding the entries' blocks as if they were /// tokens. diff --git a/packages/stack-encrypt/tests/op.rs b/packages/stack-encrypt/tests/op.rs index be289eabe..ae3a87c60 100644 --- a/packages/stack-encrypt/tests/op.rs +++ b/packages/stack-encrypt/tests/op.rs @@ -483,44 +483,6 @@ async fn a_query_names_its_context_before_or_after_the_index() { assert_eq!(before, after); } -/// A configuration with an n-gram length of zero, which `MatchOptions` -/// refuses when a term is derived. -struct ZeroGram; -impl MatchConfig for ZeroGram { - fn options() -> MatchOptions { - MatchOptions { - tokenizer: stack_encrypt::sem::Tokenizer::Ngram { length: 0 }, - ..Default::default() - } - } -} -impl MatchCase for ZeroGram { - type Case = op::CaseInsensitive; -} - -#[tokio::test] -async fn a_match_query_under_invalid_options_is_refused_not_a_panic() { - let cipher = stack_cipher().await; - let plan = Plan::context("users/name") - .with::(Match::::new()) - .build() - .unwrap(); - let refused = cipher - .query(op::Matches, "Bob") - .via(Match::::new()) - .on(&plan) - .await; - assert!( - matches!( - refused, - Err(Error::Term(stack_encrypt::sem::TermError::InvalidOptions( - _ - ))) - ), - "{refused:?}" - ); -} - #[tokio::test] async fn a_refused_operator_query_loads_no_keyset() { let (cipher, loads) = loads_counting_cipher().await;