diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml index 1fae9ba39..8ef34de7d 100644 --- a/.cargo/mutants.toml +++ b/.cargo/mutants.toml @@ -72,6 +72,11 @@ exclude_re = [ 'stack-encrypt/src/cipher\.rs:\d+:\d+: replace client_key_provider -> EnvKeyProvider with Default::default\(\)$', 'stack-encrypt/src/target/pending\.rs:443:9: replace >::into_future -> Self::IntoFuture with Default::default\(\)$', # stack-encrypt — equivalent. + # An operator and an index are unit structs, whose `Default` is the value + # `clone` and `case_sensitive` already return. + 'stack-encrypt/src/op(/mod)?\.rs:\d+:\d+: replace >::clone -> Self with Default::default\(\)$', + 'stack-encrypt/src/op(/mod)?\.rs:\d+:\d+: replace Matches::case_sensitive -> Matches with Default::default\(\)$', + 'stack-encrypt/src/target/index\.rs:\d+:\d+: replace >::clone -> Self with Default::default\(\)$', 'stack-encrypt/src/cipher\.rs:\d+:\d+: replace StackCipher::builder -> StackCipherBuilder with Default::default\(\)$', 'stack-encrypt/src/sem/mod\.rs:\d+:\d+: replace ::options -> MatchOptions with Default::default\(\)$', # Both unit-context conversions explicitly return Self::default(). @@ -80,6 +85,28 @@ exclude_re = [ # the empty list (`dynamic::target::NoTargets`), which a test asserts on # and `resolve` turns into `TargetError::NoTargets`; `vec![]` is the body. 'stack-encrypt/src/dynamic/target\.rs:\d+:\d+: replace ::targets -> Vec with vec!\[\]$', + # A passthrough has no kind: `KindOf::passthrough` returns `Ok(None)`, + # which is `Ok(Default::default())`. `a_frame_names_its_kind` asserts it. + 'stack-encrypt/src/dynamic/value_frame\.rs:\d+:\d+: replace ::passthrough -> Result with Ok\(Default::default\(\)\)$', + # `model`'s `admits` guard on a date, decimal or undefined value is checked + # again by vitaminc on every path that reaches it: the frame writer behind + # `Modeled` and `ModelOnly` refuses an inadmissible logical value + # (`FrameWriteError::Invalid`, mapped to the same `Expected`), and + # `ValueFrame::check` only walks frames `Frame::parse` has validated. So + # `true` there refuses the same values with the same error. The guard stays + # so the model says what it holds without leaning on the writer behind it. + 'stack-encrypt/src/dynamic/value_frame\.rs:\d+:\d+: replace match guard logical\.admits\(&value\) with true in model$', + # A serde visitor's `expecting` writes only the "expected ..." half of a + # serde error's message, and no caller of these three reads that message: + # a context, statistics and a plan's root are each refused with one fixed + # `Reason` (`ContextKind`, `NotStats`, `NotAnObject`), and the plan's + # field-level reasons are read from the error's path, so writing nothing + # refuses the same input with the same error. + 'stack-encrypt/src/dynamic/(context|query|record)\.rs:\d+:\d+: replace for ContextVisitor|serde::Deserialize<\x27de> for Confidence>::deserialize:: for V|Deserialize<\x27de> for PlanWire>::deserialize:: for V)>::expecting -> fmt::Result with Ok\(Default::default\(\)\)$', + # The same for the visitor that reads a refused plan's keys back + # (`entry_key`): its error is discarded (`.ok()`), so its message is never + # read at all. + 'stack-encrypt/src/dynamic/record\.rs:\d+:\d+: replace entry_key:: for Keys>::deserialize:: for V>::expecting -> fmt::Result with Ok\(Default::default\(\)\)$', ] # Headroom over the measured baseline before a slow-but-correct mutant is diff --git a/.changeset/chacha20-0-10-2.md b/.changeset/chacha20-0-10-2.md new file mode 100644 index 000000000..266d137e4 --- /dev/null +++ b/.changeset/chacha20-0-10-2.md @@ -0,0 +1,6 @@ +--- +'@cipherstash/protect-ffi': patch +'@cipherstash/auth': patch +--- + +Build the native binaries against `chacha20` 0.10.2. Versions 0.10.0 and 0.10.1 were yanked: on an x86 CPU without AVX2, their SSE2 fallback for the random number generator used an SSE4.1 instruction it had not checked the CPU for. A CPU without SSE4.1 (one from before 2008, or a virtual machine using QEMU's default `qemu64` CPU model) would crash with an illegal-instruction error. Every other CPU produced correct random numbers, so randomness was never weakened. No API change. diff --git a/.changeset/eql-lints-own-domains.md b/.changeset/eql-lints-own-domains.md new file mode 100644 index 000000000..e8ed05994 --- /dev/null +++ b/.changeset/eql-lints-own-domains.md @@ -0,0 +1,5 @@ +--- +'@cipherstash/eql': patch +--- + +`eql_v3.lints()` now judges only EQL's own `public.eql_v3_*` domains. It used to take every `jsonb` domain in `public`, so a database with another set of encrypted domains installed beside EQL v3 got thousands of false `inlinability_*` errors from the self-check. diff --git a/.changeset/eql-plan-field-targets.md b/.changeset/eql-plan-field-targets.md index e1fd16c14..18b37b590 100644 --- a/.changeset/eql-plan-field-targets.md +++ b/.changeset/eql-plan-field-targets.md @@ -2,4 +2,4 @@ '@cipherstash/eql': minor --- -**The `eql-bindings` crate resolves an EQL type named as a string to its own Stack Encrypt plan** (`stack-encrypt` feature). `eql_bindings::encryption::targets` carries a catalog-generated table of every EQL type a data plan may name as a field target — its name across languages, family and suffix, the plaintext `ValueKind` it takes, the indexes it carries, its query twin, and whether the engine can produce it today with the reason when not — and `encrypt` / `decrypt` / `query` entry points that dispatch on the name and run the type's derived `EncryptFrom` / `DecryptInto`, resolving to the EQL value's JSON bytes through the engine's `Pending` so a guest batches it with the rest of a plan. This is the EQL half of EQL types as plan field targets (cipherstash/stack#1062): a Go data plan names `TextEq` and the guest returns the finished EQL value instead of assembling one. `TextEq` is the only producible type; every other name is refused with the plan's reason. The SQL surface and the TypeScript package are unchanged. +**The `eql-bindings` crate resolves an EQL type named as a string to its own Stack Encrypt plan** (`targets` feature, which turns on `stack-encrypt`). `eql_bindings::encryption::targets` carries a catalog-generated table of every EQL type a data plan may name as a field target — its name across languages, family and suffix, the plaintext `ValueKind` it takes, the indexes it carries, its query twin, and whether the engine can produce it today with the reason when not — and `encrypt` / `decrypt` / `query` entry points that dispatch on the name and run the type's derived `EncryptFrom` / `DecryptInto`, resolving to the EQL value's JSON bytes through the engine's `Pending` so a guest batches it with the rest of a plan. This is the EQL half of EQL types as plan field targets (cipherstash/stack#1062): a Go data plan names `TextEq` and the guest returns the finished EQL value instead of assembling one. Every target is an `eql_v4_*` domain. `TextEq` is the only producible type; every other name is refused with the plan's reason. The SQL surface and the TypeScript package are unchanged. diff --git a/.changeset/eql-target-match-options.md b/.changeset/eql-target-match-options.md new file mode 100644 index 000000000..49368aa91 --- /dev/null +++ b/.changeset/eql-target-match-options.md @@ -0,0 +1,5 @@ +--- +'@cipherstash/eql': minor +--- + +**`eql-bindings` says which match options an EQL type's filter is built under** (`targets` feature). `Target::match_options()` returns `EqlMatch`'s options for a type with a match index (`TextMatch`) and `None` otherwise, so Stack Encrypt can answer `matches` through the type's query twin: it cuts the query's tokens for `verify` exactly as the stored filter was cut. The SQL surface and the TypeScript package are unchanged. diff --git a/.changeset/eql-text-equality-transcoding.md b/.changeset/eql-text-equality-transcoding.md index 969aa9653..e3aae722e 100644 --- a/.changeset/eql-text-equality-transcoding.md +++ b/.changeset/eql-text-equality-transcoding.md @@ -2,4 +2,4 @@ '@cipherstash/eql': minor --- -**Rust `TextEq` and `TextEqQuery` support Stack Encrypt's target-directed API behind the `stack-encrypt` feature.** The stored table and column identifier supplies the encryption context; native ciphertext and equality terms are transcoded into EQL payloads while Vitamin C owns plaintext encoding. This is a new producer profile with exact string equality, independent of existing cipherstash-client ciphertext and terms. Query operands carry no recoverable ciphertext. +**Rust `eql_bindings::v4::text::{TextEq, TextEqQuery}` support Stack Encrypt's target-directed API behind the `stack-encrypt` feature.** Stack Encrypt writes EQL v4 only, so the `eql_v3` types carry no Stack Encrypt traits. The stored table and column identifier supplies the encryption context; native ciphertext and equality terms are transcoded into EQL payloads while Vitamin C owns plaintext encoding. This is a new producer profile whose string equality compares text in Unicode 16 NFC without folding case, independent of existing cipherstash-client ciphertext and terms. Text holding a code point unassigned in Unicode 16 is refused, when it is stored as well as when it is queried. Query operands carry no recoverable ciphertext. diff --git a/.changeset/eql-v4-bindings.md b/.changeset/eql-v4-bindings.md new file mode 100644 index 000000000..e92cee2ee --- /dev/null +++ b/.changeset/eql-v4-bindings.md @@ -0,0 +1,9 @@ +--- +'@cipherstash/eql': minor +--- + +**The `eql-bindings` crate has an `eql_bindings::v4` module, with the `stack-encrypt` feature: payload types for the EQL v4 domains.** EQL v4 is the v3 SQL installed under a second name (ADR-0002), for values written by Stack Encrypt, so the module exists only with that feature and its types derive Stack Encrypt's traits without a further gate. The module is generated from the same catalog as `v3`, with `eql_v4_*` domain names and `"v": 4`. `SchemaVersion` is now generic over the version it admits: `v3::SchemaVersion` is `SchemaVersion<3>` and `v4::SchemaVersion` is `SchemaVersion<4>`, each refusing the other's payloads, and plain `SchemaVersion` still means v3. Only v4 types derive Stack Encrypt's `EncryptFrom` and `DecryptInto`, and the Stack Encrypt target table (now `eql_bindings::v4::targets`) names v4 domains. Besides `TextEq`, Stack Encrypt now produces the storage-only types `Text`, `Integer`, `Bigint`, `Boolean`, `Real` and `Double`, and `TextMatch` (3-grams, `k = 6`, a 2048-bit filter, pinned as `eql_bindings::encryption::EqlMatch`; text shorter than three characters is refused, stored or queried, and only `stored @@ query` matches until #1171 is fixed), from the Rust primitives `i32`, `i64`, `bool`, `f32` and `f64`; the Go SDK's `encrypt/eql` table marks them producible. Stack Encrypt also produces the number equality types `IntegerEq`, `BigintEq`, `RealEq` and `DoubleEq`, whose `=` compares as PostgreSQL compares the plain types: an integer has one term whatever its width, and for floats `-0.0 = 0.0` and `NaN = NaN` (ADR-0002's semantic baseline). Every type it cannot produce yet names the issue that tracks its blocker. A Rust caller queries those types through Stack Encrypt's typed query chain: `cipher.query(op::Eq, v).on(&plan)` on a plan for `TextEq` or a number equality type returns the type's query twin, and `cipher.query(op::Matches, text).on(&plan)` on a plan for `TextMatch` returns a `MatchTwin` with `verify` and `target_limit`, the same bytes and answers as the data path the Go SDK uses (`eql_bindings::encryption::query`; an operator a type does not answer does not compile). The by-name target entry points (`eql_bindings::encryption::targets::encrypt`, `decrypt` and `query`) exchange the plaintext as Stack Encrypt's `dynamic::ValueFrame`, one value as its transport frame, rather than a value tree. They, and the generated `v4::targets` table, need the new `targets` feature, which turns on `stack-encrypt` and Stack Encrypt's `dynamic` feature. `stack-encrypt` alone no longer builds Stack Encrypt's `dynamic` layer, so a Rust program using the typed v4 types through `encrypt_as` doesn't compile it. + +**The EQL v4 SQL bundle has range query domains and `eql_v4.between`, so a `BETWEEN` query is one operand.** The v4 bundle (`cipherstash-encrypt-v4.sql`) is built from this source and reaches users as the `eql-v4-install.sql` attached to each Go release; neither the npm package nor the crate ships it, and the v3 SQL they ship does not change. Every ordered family has one range query domain per order term, beside its single-bound ones: `eql_v4.query_integer_range_ore` beside `eql_v4.query_integer_ord_ore`, `eql_v4.query_integer_range_ope` beside `eql_v4.query_integer_ord_ope`, and the same for `smallint`, `bigint`, `date`, `timestamp`, `numeric`, `text`, `real` and `double`. A range is `{"v": 4, "i": …, "lo": , "hi": }`: each bound is the single-bound domain's `ob` (ORE) or `op` (OPE) value, in a `{v, i, lo, hi}` envelope. Stack Encrypt does not build that envelope yet; it will need to when the ordered types become producible. `eql_v4.between(column, range)` is true when the column's order term lies between the bounds, both included, and has an overload for every ordered column domain with the range of the term it stores (`_ord_ore` and `_search_ore` take the ORE range; `_ord`, `_ord_ope` and `_search` the OPE one). It is an inlinable SQL function, so a btree on the column's order term (`eql_v4.ord_term_ore(col)` or `eql_v4.ord_term(col)`) serves it as one index range scan. Write it schema-qualified (`BETWEEN` is an SQL keyword, so a bare `between(...)` does not parse), and cast the range to its exact domain (`$1::eql_v4.query_integer_range_ore`): an uncast `$1`, a `jsonb` parameter, or a range of another term or family fails with SQLSTATE 42725 ("function … is not unique"), unlike the single-bound operators, which resolve an untyped `$1`. An ORE range scan needs the `ore_block_256` btree operator class, which the installer creates only as a superuser; on an install without it (Supabase and other non-superuser roles), the ORE range domains are disabled like the single-bound ORE domains, and a value cast to one raises `feature_not_supported` naming the family's OPE range. In the crate, `eql_bindings::v4` (`stack-encrypt` feature) has a type for each range domain (`IntegerRangeOreQuery`, `TextRangeOpeQuery`, …, listed by `v4::all_range_query()`). EQL v3 has no range domains: its SQL and types are unchanged (ADR-0009, decision 7). + +Nothing a released version shipped has changed. The v3 types, the v3 SQL and the TypeScript package are as they were; the `stack-encrypt` derives on the v3 text types and the `v3::targets` module were added after the last release and are moved to v4 here, not removed from a release. diff --git a/.changeset/eql-v4-block-ore-types.md b/.changeset/eql-v4-block-ore-types.md new file mode 100644 index 000000000..d5cb664f8 --- /dev/null +++ b/.changeset/eql-v4-block-ore-types.md @@ -0,0 +1,5 @@ +--- +'@cipherstash/eql': minor +--- + +**EQL v4 compares ore-rs's 6-bit block ORE, and Stack Encrypt produces the block ORE types `IntegerOrdOre`, `SmallintOrdOre`, `BigintOrdOre`, `RealOrdOre`, `DoubleOrdOre`, `TextOrdOre` and `TextSearchOre`.** A v4 `ob` term is one ore-rs ciphertext: the fixed-width scheme for numbers, the chained scheme for text (padded to a power-of-two length bucket of at least 16 bytes). The v4 bundle names its block ORE objects `ore_block_v2` (`eql_v4_internal.ore_block_v2_term`, `compare_ore_block_v2_term`) and compares them with a new comparator that checks each term's shape, raises on malformed or mismatched terms, and runs a constant operation count for terms of one shape. The EQL v3 bundle is unchanged, byte for byte: its `ob` is still cipherstash-client's 8-bit block ORE under `ore_block_256`. See [U-002](../../docs/upgrading/v3.1.md#u-002-the-block-ore-term-is-ore-rss-6-bit-scheme): the comparator refuses the 8-bit terms the previous `eql_v4` bundle compared, so rows or operands holding them must be written again through Stack Encrypt. diff --git a/.changeset/eql-v4-date-timestamp-numeric.md b/.changeset/eql-v4-date-timestamp-numeric.md new file mode 100644 index 000000000..96ffccffb --- /dev/null +++ b/.changeset/eql-v4-date-timestamp-numeric.md @@ -0,0 +1,5 @@ +--- +'@cipherstash/eql': minor +--- + +**Stack Encrypt produces the EQL v4 date, timestamp and numeric types `Date`, `DateEq`, `Timestamp`, `TimestampEq`, `Numeric` and `NumericEq`** (`eql-bindings`, `stack-encrypt` feature). Their Rust plaintexts are the crate's own `v4::plaintext::Date`, `Timestamp` and `Decimal`, which hold the integers the engine stores (days since 1970-01-01; Unix seconds and nanoseconds; a mantissa and a scale), are range checked when built and are wiped when dropped. The `chrono` and `rust_decimal` features convert to and from those crates' types. `=` on `NumericEq` normalises the scale (`1.5 = 1.50`) and on `TimestampEq` compares to the microsecond, truncating (never rounding) the nanoseconds below it; each value decrypts as it was stored. The ordered types of these families stay unproducible until the engine derives their order terms. The Go SDK's `encrypt/eql` table marks the six producible, from `encrypt.Date`, `time.Time` and `encrypt.Decimal`. diff --git a/.changeset/eql-v4-ordered-ope-types.md b/.changeset/eql-v4-ordered-ope-types.md new file mode 100644 index 000000000..27a8fb5a7 --- /dev/null +++ b/.changeset/eql-v4-ordered-ope-types.md @@ -0,0 +1,5 @@ +--- +'@cipherstash/eql': minor +--- + +**Stack Encrypt produces the OPE-ordered EQL v4 types `IntegerOrd`, `SmallintOrd`, `BigintOrd`, `RealOrd`, `DoubleOrd`, `TextOrd` and their `OrdOpe` twins** (`eql-bindings`, `stack-encrypt` feature). Their `op` term orders by ADR-0002's order key, so `<`, `<=`, `>`, `>=` and `ORDER BY eql_v4.ord_term(col)` compare integers by value, floats as PostgreSQL's `float8` does and text by its fold (case and accents folded). `=` on text reads `TextEq`'s equality term. `TextSearch` stays unproducible, with a reason: its query carries a match term that short text cannot make. diff --git a/.changeset/eql-v4-smallint.md b/.changeset/eql-v4-smallint.md new file mode 100644 index 000000000..5438f7c9e --- /dev/null +++ b/.changeset/eql-v4-smallint.md @@ -0,0 +1,5 @@ +--- +'@cipherstash/eql': minor +--- + +**Stack Encrypt produces the EQL v4 types `Smallint` and `SmallintEq`** (`eql-bindings`, `stack-encrypt` feature), from `i16`. `SmallintEq`'s equality term is the integer one, so a smallint and an integer of the same value under one column share it. The `date`, `timestamp` and `numeric` types now say they wait for a choice of Rust plaintext type, not for an engine kind. diff --git a/.changeset/eql-v4-text-search-optional-terms.md b/.changeset/eql-v4-text-search-optional-terms.md new file mode 100644 index 000000000..b9152c801 --- /dev/null +++ b/.changeset/eql-v4-text-search-optional-terms.md @@ -0,0 +1,5 @@ +--- +'@cipherstash/eql': minor +--- + +**An EQL v4 text search query carries only the terms its operator reads.** `eql_v4.query_text_search` and `eql_v4.query_text_search_ore` now require at least one of their terms (`hm`, `op` or `ob`, `bf`) instead of all of them, so `col = $1` and `col < $1` work on text too short to cut a 3-gram, which has no match term. Each extractor over such a query raises (`invalid_parameter_value`) wherever it reads the operand when the query lacks the term it reads, or holds it as JSON `null` or an empty array, naming the domain, the term and the operators that compare it: `=` without `hm`, `<` without `op`, `@@` without `bf`. So a missing term raises rather than matching no row silently, or every row; the raise is at evaluation, so a generic plan that never reaches the predicate returns no row and no error. Stored `eql_v4_text_search` values still carry every term. The `eql_v3` domains are unchanged: the v3 bundle is byte for byte what it was. With it, **Stack Encrypt produces `TextSearch`** (`eql-bindings`, `stack-encrypt` feature): `TextSearchQuery`'s terms are `Option`s, built per index through `TextSearchEqualityQuery`, `TextSearchOrderQuery` and `TextSearchMatchQuery`, and the stored value carries every term: text too short for a 3-gram stores the empty filter (`"bf": []`, the new `v4::terms::SearchBloomFilter` field type), so `=` and `<` find it and no `@@` query does, where `TextMatch` still refuses it. `TextSearchQuery`'s `Option` terms and `TextSearch`'s `SearchBloomFilter` change the fields of the unreleased `v4` module only; no published `eql-bindings` has it. See [U-001](../../docs/upgrading/v3.1.md#u-001-a-text-search-query-carries-only-the-terms-its-operator-reads) for what a hand-built query must carry, the raise, and the rollback. diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 000000000..f36468fef --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,20 @@ +# Copilot instructions for cipherstash/stack + +The project instructions are in `AGENTS.md`. When reviewing a pull request, +follow its "Review guidelines", in particular these known non-findings, +which are deliberate design decisions and not defects: + +- A sealed container's membership, order and length are not authenticated: + which entries a stored map holds, and a stored sequence's order, length + and duplicates, are not bound into any AAD, because stored values are + database rows and columns that come back in any order and in part. A + target that refuses a container's shape is a shape mismatch + (`Error::OpenedType`), not tampering (`Error::Aead`). +- A passthrough field is stored in the clear and is not authenticated, + except a plan's context field, which every other field is sealed under. +- Two leaves sealed under the same context are interchangeable between rows; + binding a value to its row is the caller's choice of context. + +A leaf that is not bound to its context, map key, role (single value or +sequence element), type tag or wire version is still a defect. See +`packages/stack-encrypt/docs/adr/0010-what-a-stored-value-authenticates.md`. diff --git a/.github/dependabot.yml b/.github/dependabot.yml index d3257df9b..3b16d3955 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -299,7 +299,7 @@ updates: - dependency-name: "recipher" - dependency-name: "cllw-ore" # vitaminc is pinned at 0.5.0 across the crates and the guests; the - # guests' `FfiValue: Decrypt` bound fails if two versions resolve. + # guests' `ValueFrame: Decrypt` bound fails if two versions resolve. - dependency-name: "vitaminc*" # Major bumps are reviewed and applied manually, not by Dependabot. - dependency-name: "*" diff --git a/.github/workflows/test-eql.yml b/.github/workflows/test-eql.yml index b3d0c2d60..e0b1f16be 100644 --- a/.github/workflows/test-eql.yml +++ b/.github/workflows/test-eql.yml @@ -597,6 +597,12 @@ jobs: rustup component add --toolchain "${active_rust_toolchain}" rustfmt clippy mise run test:crates + # The block ORE timing harness's ciphertext generator is a detached + # workspace with its own Cargo.lock that no job builds, so its lock is + # checked here rather than left to drift until someone runs the harness. + - name: Check the timing harness's Cargo.lock + run: cargo metadata --locked --format-version 1 --manifest-path tests/timing/ore_block_v2/ctgen/Cargo.toml > /dev/null + # The three encryption checks are mise tasks rather than raw cargo here, # so `eql-suite-ci.test.mjs` counts them among the cargo tasks a root # workflow must reach, and `eql-workflow-filters.test.mjs` derives the @@ -733,6 +739,8 @@ jobs: run: mise run test:installer_complete - name: Symbol-order cross-check (v3) run: mise run test:symbol_order_v3 + - name: v4 isolation gate and its self-test + run: mise run test:v4_isolation - name: Build-ordering helper unit tests run: mise run test:build_ordering_helpers diff --git a/.github/workflows/tests-golang.yml b/.github/workflows/tests-golang.yml index 9bfff334a..6ef27e601 100644 --- a/.github/workflows/tests-golang.yml +++ b/.github/workflows/tests-golang.yml @@ -36,6 +36,10 @@ on: # The eql guest build links eql-bindings. - packages/eql/crates/eql-bindings/** - packages/eql/crates/eql-domains/** + # The Go tests read the cross-language fixtures the EQL suite writes, + # which are derived under EQL's lockfile. + - packages/eql/tests/encryption/** + - packages/eql/Cargo.lock - languages/golang/** - scripts/check-wasm-imports.py - scripts/go-binding-test.sh @@ -61,6 +65,10 @@ on: # The eql guest build links eql-bindings. - packages/eql/crates/eql-bindings/** - packages/eql/crates/eql-domains/** + # The Go tests read the cross-language fixtures the EQL suite writes, + # which are derived under EQL's lockfile. + - packages/eql/tests/encryption/** + - packages/eql/Cargo.lock - languages/golang/** - scripts/check-wasm-imports.py - scripts/go-binding-test.sh @@ -103,6 +111,24 @@ jobs: # caches) the whole test toolset, which `mise run` would otherwise # install uncached. MISE_ENV: test + # The Go module's PostgreSQL test (encrypt/postgres_test.go) stores + # generated rows and runs the query methods against it through + # database/sql; it skips where DATABASE_URL is unset, as on the macOS + # and Windows jobs. + services: + postgres: + image: postgres:17 + env: + POSTGRES_PASSWORD: postgres + # Container port only: Docker picks the host port, and the Go step + # reads it from `job.services`, so the number is written once. + ports: + - 5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: @@ -133,6 +159,7 @@ jobs: . languages/golang/encrypt/guest languages/golang/auth/guest + packages/eql # The HTTP-free core compiles for wasm32-wasip1 with no JS-host backend # and no native HTTP/TLS stack in its graph: the invariant the wazero @@ -215,11 +242,28 @@ jobs: - name: Go binding env: STACK_ENCRYPT_TESTS_REQUIRE_LOCK: "1" + DATABASE_URL: "postgres://postgres:postgres@localhost:${{ job.services.postgres.ports['5432'] }}/postgres?sslmode=disable" run: | ulimit -l "$(ulimit -H -l)" echo "RLIMIT_MEMLOCK: $(ulimit -l) KiB" mise run go:test + # The Rust suite opens the committed v4_rows.json and + # v4_smallint_rows.json, whose ciphertexts the Go SDK sealed when each + # file was last regenerated. This also hands it the rows the Go SDK + # seals in this run, so a Go guest and a native Rust build that stop + # agreeing on today's ciphertexts fail here rather than when someone + # next regenerates a file. Both filters are exact, so each test named + # here is one that reads the rows handed to it. The toolchain is the + # root's pin, as for the guests. + - name: Rows the Go SDK seals today open in Rust + run: | + rows="$RUNNER_TEMP/v4_rows.json" + smallint_rows="$RUNNER_TEMP/v4_smallint_rows.json" + (cd languages/golang && CGO_ENABLED=0 EQL_V4_ROWS_OUT="$rows" EQL_V4_SMALLINT_ROWS_OUT="$smallint_rows" go test -count=1 -run '^(TestV4RowsMatchTheCommittedFixture|TestV4SmallintRowsMatchTheCommittedFixture)$' ./encrypt) + test -s "$rows" && test -s "$smallint_rows" + EQL_V4_ROWS="$rows" EQL_V4_SMALLINT_ROWS="$smallint_rows" cargo test --locked --manifest-path packages/eql/Cargo.toml -p eql-encryption-tests --test v4_types -- --exact rows_the_go_sdk_sealed_open_in_rust smallint_rows_the_go_sdk_sealed_open_in_rust + # What is encrypted is fixed before the program ships: every generated # file is committed, and this fails when `go generate` would change # one. It runs the real stashgen against the guest just built, so it diff --git a/AGENTS.md b/AGENTS.md index d828dd683..e25757555 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -695,8 +695,12 @@ exact pin would stop the suite sharing one `cts-common` with it. - **Three Cargo workspaces, not one.** The root workspace (the six stack-* crates and the three node binding crates), protect-ffi's and EQL's. The root `Cargo.toml` excludes the other two, and they pin their own vitaminc. The - three fuzz crates and the two Go guests are detached workspaces too, each - with its own `Cargo.lock`. + three fuzz crates, the two Go guests and the EQL v4 block ORE timing + harness's ciphertext generator + (`packages/eql/tests/timing/ore_block_v2/ctgen`) are detached workspaces + too, each with its own `Cargo.lock`. None of them has a Dependabot entry. + CI builds the generator nowhere, so `test-eql.yml` checks its lock with + `cargo metadata --locked`, and `run.sh` builds it with `--locked`. - **The toolchain is pinned in the root `mise.toml`:** Rust 1.94.1 (the `stack-encrypt` `tests/ui` trybuild snapshots record its diagnostics) and Go 1.26. EQL's `mise.toml` overrides the Rust pin in its folder. @@ -991,6 +995,39 @@ pnpm changeset:publish A skills-only change is **not** internal: `skills/` ships inside the `stash` tarball, so it needs a `stash` patch changeset. +## Review guidelines + +These apply to every agent that reviews a pull request here, human-prompted +or automated (Codex, Copilot, CodeRabbit, Claude Code's `/code-review`). + +### Known non-findings + +These are deliberate design decisions, recorded in ADRs. Do not report them +as defects or security issues, and do not change code to "fix" them. If a +change seems to need one reversed, say so and link the ADR. + +- **A sealed container's membership, order and length are not + authenticated.** Each leaf is bound to its context, its map key, its role + (single value or sequence element), its type tag and the wire version. But + which entries a stored map holds, and a stored sequence's order, length + and duplicates, are deliberately not bound: stored values are database + rows and columns, which come back in any order and in part. A target that + refuses a container's shape (a missing field, say) is a shape mismatch, + `Error::OpenedType`, not tampering, `Error::Aead`. Stricter authentication + will be opt-in, through named types. See + `packages/stack-encrypt/docs/adr/0010-what-a-stored-value-authenticates.md` + and vitaminc's ADR 0002. +- **A passthrough field is not authenticated.** It is stored in the clear, + by definition, and comes back as the store holds it. A plan's context + field is the exception: every other field is sealed under it, so a change + to it is caught. Same ADR. +- **Two leaves sealed under the same context are interchangeable between + rows.** Binding a value to its row is the caller's choice of context (a + row id or tenant in it). Same ADR. + +A finding that a *leaf* is not bound to its context, key, role, type or +version is a real defect: report it. + ## Useful Links - `README.md` for quickstart and feature overview diff --git a/Cargo.lock b/Cargo.lock index 6d6dce0f8..61991c907 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -27,6 +27,7 @@ dependencies = [ "cfg-if", "cipher", "cpufeatures 0.2.17", + "zeroize", ] [[package]] @@ -108,7 +109,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f50776554130342de4836ba542aa85a4ddb361690d7e8df13774d7284c3d5c2" dependencies = [ "include_dir", - "itertools 0.10.5", + "itertools", "proc-macro-error2", "proc-macro2", "quote", @@ -350,6 +351,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "block-modes" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e2211b0817f061502a8dd9f11a37e879e79763e3c698d2418cf824d8cb2f21e" + [[package]] name = "brotli" version = "8.0.2" @@ -383,6 +390,12 @@ version = "1.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.12.1" @@ -463,9 +476,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chacha20" -version = "0.10.0" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", "cpufeatures 0.3.0", @@ -509,15 +522,17 @@ dependencies = [ [[package]] name = "cllw-ore" -version = "0.5.0" +version = "0.5.2-rc.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "476f300d37a5029d3d9dd57145d4db50a23f40ae9b4d1374c44543978b906191" +checksum = "24eb4a774d070a6c2b52175b98273cd7193c61e1bda39c1a98b50b7a2d91375a" dependencies = [ "blake3", "hex", "subtle", "thiserror 1.0.69", "unicode-normalization", + "vitaminc-ore", + "vitaminc-protected 0.6.1", "zeroize", ] @@ -771,7 +786,7 @@ dependencies = [ "url", "utoipa", "uuid", - "vitaminc", + "vitaminc 0.5.1", ] [[package]] @@ -966,7 +981,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -1465,7 +1480,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.2", + "socket2 0.5.10", "tokio", "tower-service", "tracing", @@ -1708,15 +1723,6 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "1.0.17" @@ -2132,7 +2138,21 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", +] + +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", ] [[package]] @@ -2145,6 +2165,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.0" @@ -2160,6 +2189,27 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -2208,6 +2258,35 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "orderable-bytes" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7d30b72ffca73d9936fe5fc80a9c9520c73f5b4f64839a810211903d5ace3c" + +[[package]] +name = "ore-rs" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e1d6cb8c1249643266bd60269547c93d3cd263d5385f1262902bbb295ff9a5e" +dependencies = [ + "aes", + "block-modes", + "byteorder", + "hex", + "num", + "orderable-bytes", + "rand 0.8.6", + "rand_chacha 0.3.1", + "subtle-ng", + "thiserror 1.0.69", + "vitaminc-hmac", + "vitaminc-ore", + "vitaminc-prf", + "vitaminc-protected 0.6.1", + "zeroize", +] + [[package]] name = "parking_lot" version = "0.12.5" @@ -2328,6 +2407,15 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit 0.25.17+spec-1.1.0", +] + [[package]] name = "proc-macro-error-attr2" version = "2.0.0" @@ -2395,7 +2483,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.12.1", + "itertools", "proc-macro2", "quote", "syn 2.0.114", @@ -2420,7 +2508,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.6.2", + "socket2 0.5.10", "thiserror 2.0.18", "tokio", "tracing", @@ -2459,7 +2547,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.2", + "socket2 0.5.10", "tracing", "windows-sys 0.60.2", ] @@ -2763,7 +2851,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -2776,7 +2864,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2833,7 +2921,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3219,8 +3307,8 @@ dependencies = [ "tracing-subscriber", "url", "uuid", - "vitaminc", - "vitaminc-protected", + "vitaminc 0.6.1", + "vitaminc-protected 0.6.1", "web-time", "zeroize", "zerokms-protocol", @@ -3242,7 +3330,7 @@ dependencies = [ "tempfile", "tokio", "url", - "vitaminc-protected", + "vitaminc-protected 0.6.1", "zeroize", ] @@ -3267,11 +3355,12 @@ dependencies = [ [[package]] name = "stack-encrypt" -version = "0.2.0" +version = "0.3.0" dependencies = [ "base64ct", "cllw-ore", "miette", + "ore-rs", "serde", "serde_json", "stack-auth", @@ -3281,19 +3370,21 @@ dependencies = [ "tokio", "trybuild", "uuid", - "vitaminc-aead", - "vitaminc-aead-value", - "vitaminc-encrypt", + "vitaminc-aead 0.6.1", + "vitaminc-aead-frame", + "vitaminc-encrypt 0.6.1", "vitaminc-hmac", + "vitaminc-normalize", + "vitaminc-ore", "vitaminc-prf", - "vitaminc-protected", + "vitaminc-protected 0.6.1", "zeroize", "zerokms-protocol", ] [[package]] name = "stack-encrypt-derive" -version = "0.2.0" +version = "0.3.0" dependencies = [ "proc-macro2", "quote", @@ -3307,12 +3398,14 @@ dependencies = [ name = "stack-guest-abi" version = "0.0.0" dependencies = [ + "futures", "miette", "proptest", "serde_json", "thiserror 1.0.69", - "vitaminc-aead-value", - "vitaminc-protected", + "vitaminc-aead 0.6.1", + "vitaminc-aead-frame", + "vitaminc-protected 0.6.1", "zeroize", ] @@ -3344,8 +3437,8 @@ dependencies = [ "tracing", "url", "uuid", - "vitaminc", - "vitaminc-protected", + "vitaminc 0.6.1", + "vitaminc-protected 0.6.1", "zeroize", "zerokms-protocol", ] @@ -3392,6 +3485,12 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "subtle-ng" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "734676eb262c623cec13c3155096e08d1f8f29adce39ba17948b18dad1e54142" + [[package]] name = "syn" version = "2.0.114" @@ -3492,7 +3591,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix 1.1.5", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3680,7 +3779,7 @@ dependencies = [ "serde", "serde_spanned 0.6.9", "toml_datetime 0.6.11", - "toml_edit", + "toml_edit 0.22.27", ] [[package]] @@ -3730,6 +3829,18 @@ dependencies = [ "winnow 0.7.14", ] +[[package]] +name = "toml_edit" +version = "0.25.17+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3641d5bbb5349a79e1020a242d251efbc546ad8048d133958323ce9c40a9c9c" +dependencies = [ + "indexmap", + "toml_datetime 1.1.2+spec-1.1.0", + "toml_parser", + "winnow 1.0.4", +] + [[package]] name = "toml_parser" version = "1.1.5+spec-1.1.0" @@ -3910,6 +4021,12 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" +[[package]] +name = "unicode-general-category" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b993bddc193ae5bd0d623b49ec06ac3e9312875fdae725a975c51db1cc1677f" + [[package]] name = "unicode-ident" version = "1.0.23" @@ -4073,12 +4190,26 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35042fb7a81d72dc43c53fec835e61027f18b4d01cc0f7e12d6c6a6ffbf3bfb9" dependencies = [ - "vitaminc-aead", - "vitaminc-context", - "vitaminc-encrypt", - "vitaminc-protected", - "vitaminc-random", - "vitaminc-traits", + "vitaminc-aead 0.5.1", + "vitaminc-context 0.5.1", + "vitaminc-encrypt 0.5.1", + "vitaminc-protected 0.5.1", + "vitaminc-random 0.5.1", + "vitaminc-traits 0.5.1", +] + +[[package]] +name = "vitaminc" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac5c15b6c25efcb9a6111f7108ab8101947a7bf7e9af3ff6d6a73358a1ace0da" +dependencies = [ + "vitaminc-aead 0.6.1", + "vitaminc-context 0.6.1", + "vitaminc-encrypt 0.6.1", + "vitaminc-protected 0.6.1", + "vitaminc-random 0.6.1", + "vitaminc-traits 0.6.1", ] [[package]] @@ -4089,10 +4220,27 @@ checksum = "62cacd4dae485e98cfe5407e919187a8468dd933ea4234be5115b7d13a30f3c6" dependencies = [ "bytes", "serde", - "vitaminc-aead-derive", - "vitaminc-context", - "vitaminc-protected", - "vitaminc-random", + "vitaminc-aead-derive 0.5.1", + "vitaminc-context 0.5.1", + "vitaminc-protected 0.5.1", + "vitaminc-random 0.5.1", + "zeroize", +] + +[[package]] +name = "vitaminc-aead" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71b8c5c4ced92545dce5fd7cf04581d08210def174cc81475cf227fa2506c5b7" +dependencies = [ + "bytes", + "futures-util", + "serde", + "thiserror 2.0.18", + "vitaminc-aead-derive 0.6.1", + "vitaminc-context 0.6.1", + "vitaminc-protected 0.6.1", + "vitaminc-random 0.6.1", "zeroize", ] @@ -4108,13 +4256,24 @@ dependencies = [ ] [[package]] -name = "vitaminc-aead-value" -version = "0.5.1" +name = "vitaminc-aead-derive" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c546497a972d0f0498f2f8d5f537c514bad82df8662951c63d0c80023808900" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "vitaminc-aead-frame" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d78cdadbc6374c3bd7c19fb4d99abca693b96953b18fef8c608efbe092cf3ea" +checksum = "e35992a269bcaa5b201f1938e54bfbf167bfa6fe975d83cbdbdb54c2a363c5ef" dependencies = [ - "vitaminc-aead", - "vitaminc-protected", + "serde", + "vitaminc-aead 0.6.1", "zeroize", ] @@ -4125,7 +4284,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7df52501fe33cdb3ee9e155b5242fac184c037d147e58c96e5143ed011252a8" dependencies = [ "mutants", - "vitaminc-protected", + "vitaminc-protected 0.5.1", +] + +[[package]] +name = "vitaminc-context" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4c4bc2bd9811253eb427657565a45bd4826e2379e93ee1c4d00c808a769882" +dependencies = [ + "mutants", + "vitaminc-protected 0.6.1", ] [[package]] @@ -4136,35 +4305,79 @@ checksum = "9fbd0e1748055e381e02033e08af385de879c4dcf32e0f6739ac0c0cf0c6f13d" dependencies = [ "aes-gcm", "aws-lc-rs", - "vitaminc-aead", - "vitaminc-protected", - "vitaminc-random", + "vitaminc-aead 0.5.1", + "vitaminc-protected 0.5.1", + "vitaminc-random 0.5.1", + "zeroize", +] + +[[package]] +name = "vitaminc-encrypt" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0923694602855b33a296d9af473e282e8096c1cfe1b30b8695d3b6651717fc5" +dependencies = [ + "aes-gcm", + "aws-lc-rs", + "vitaminc-aead 0.6.1", + "vitaminc-protected 0.6.1", + "vitaminc-random 0.6.1", "zeroize", ] [[package]] name = "vitaminc-hmac" -version = "0.5.1" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15fe558a5e92d4f2b3f811c5b67d863d9255da5f66d08a71d4f0dd3d0011c563" +checksum = "bd361f626e95cbe770b9d012e4a86c43cfefe4aefb3d5b3ed9339a83085bdc53" dependencies = [ "hmac", "sha2 0.11.0", "vitaminc-prf", - "vitaminc-protected", + "vitaminc-protected 0.6.1", "zeroize", ] +[[package]] +name = "vitaminc-normalize" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "214b2a9d085924c3bc3c91f3ba74238e8848bf54990e5f48f170c71fbacea07a" +dependencies = [ + "thiserror 2.0.18", + "unicode-general-category", + "unicode-normalization", + "vitaminc-aead 0.6.1", + "vitaminc-protected 0.6.1", +] + +[[package]] +name = "vitaminc-ore" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed88c54b5f06f2177ecbfb7666812adf645406f6fb555f4e2f0fd7fa218e71c4" +dependencies = [ + "thiserror 2.0.18", + "unicode-general-category", + "unicode-normalization", + "vitaminc-aead 0.6.1", + "vitaminc-context 0.6.1", + "vitaminc-normalize", + "vitaminc-protected 0.6.1", +] + [[package]] name = "vitaminc-prf" -version = "0.5.1" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712d31d46d76b38c6b62deb35f516fc5636a35f90f39c0213d0ca9a052b4400b" +checksum = "c2715cca25006b4656223bc932ab4fd7f70d387aa4ec63a4e2ef33aa8cd2b3a5" dependencies = [ "mutants", "thiserror 2.0.18", - "vitaminc-context", - "vitaminc-protected", + "vitaminc-aead 0.6.1", + "vitaminc-context 0.6.1", + "vitaminc-normalize", + "vitaminc-protected 0.6.1", ] [[package]] @@ -4180,7 +4393,24 @@ dependencies = [ "serde_bytes", "subtle", "thiserror 2.0.18", - "vitaminc-protected-derive", + "vitaminc-protected-derive 0.5.1", + "zeroize", +] + +[[package]] +name = "vitaminc-protected" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1bb76dd473f5aa89b844f92018ebe3041deb8e5390e7061cca4283d959ff855" +dependencies = [ + "bitvec", + "digest 0.11.3", + "libc", + "serde", + "serde_bytes", + "subtle", + "thiserror 2.0.18", + "vitaminc-protected-derive 0.6.1", "zeroize", ] @@ -4195,6 +4425,18 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "vitaminc-protected-derive" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7478ed160c019059746e75fc613563dcdcd5266d155a34601d29377a1c4afc3" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 3.0.6", +] + [[package]] name = "vitaminc-random" version = "0.5.1" @@ -4205,8 +4447,23 @@ dependencies = [ "getrandom 0.4.2", "rand 0.10.1", "thiserror 2.0.18", - "vitaminc-protected", - "vitaminc-random-derives", + "vitaminc-protected 0.5.1", + "vitaminc-random-derives 0.5.1", + "zeroize", +] + +[[package]] +name = "vitaminc-random" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b477af7c01c3d9ef2647e46952ad3df7f246b3f806b1d4009ee83beecec613a" +dependencies = [ + "chacha20", + "getrandom 0.4.2", + "rand 0.10.1", + "thiserror 2.0.18", + "vitaminc-protected 0.6.1", + "vitaminc-random-derives 0.6.1", "zeroize", ] @@ -4221,6 +4478,17 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "vitaminc-random-derives" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab5a0de8da428c6789dc067a578a0462e411099721d616736cf29e1622230c35" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + [[package]] name = "vitaminc-traits" version = "0.5.1" @@ -4232,8 +4500,24 @@ dependencies = [ "rmp-serde", "serde", "thiserror 2.0.18", - "vitaminc-protected", - "vitaminc-random", + "vitaminc-protected 0.5.1", + "vitaminc-random 0.5.1", + "zeroize", +] + +[[package]] +name = "vitaminc-traits" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a900f0edcacd49c8557ab54722175869fec66926054a6fbdc53a05069db3c922" +dependencies = [ + "anyhow", + "bytes", + "rmp-serde", + "serde", + "thiserror 2.0.18", + "vitaminc-protected 0.6.1", + "vitaminc-random 0.6.1", "zeroize", ] @@ -4491,7 +4775,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4586,15 +4870,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets 0.52.6", -] - [[package]] name = "windows-sys" version = "0.60.2" @@ -4870,6 +5145,9 @@ name = "winnow" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] [[package]] name = "winreg" diff --git a/Cargo.toml b/Cargo.toml index 3da394ab4..50f5b3cb6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -81,7 +81,15 @@ stack-profile = { path = "./packages/stack-profile", version = "0.43.0" } cts-common = { version = "0.43.0", default-features = false } zerokms-protocol = "0.12.31" recipher = "=0.3.1" -cllw-ore = { version = "=0.5.0", default-features = false } +# cllw-ore's `vitaminc` feature implements vitaminc-ore's `OrderScheme`, +# which Stack Encrypt's order terms are encrypted through. It first ships +# in a pre-release, and Cargo resolves a pre-release only when named. +cllw-ore = { version = "=0.5.2-rc.1", default-features = false, features = ["vitaminc"] } +# ore-rs's `vitaminc` feature implements the same traits for block ORE: +# `Bit6BlockOre` (fixed width up to ten bytes) and `ChainedBlockOre` +# (variable length), the 6-bit schemes EQL v4's `_ord_ore` domains store. +# Both wire formats are frozen as of 0.9.0. ore-rs feeds only stack-encrypt. +ore-rs = { version = "0.9.0", default-features = false, features = ["vitaminc"] } # External dependencies, with the suite's feature lists. base64 = "0.22.0" @@ -120,13 +128,18 @@ web-time = "1.1" zeroize = { version = "1.8.1", features = ["derive"] } # This is needed because lock_api 0.4.6 was breaking things and the branch forces the use of 0.4.12 temp-env = { git = "https://github.com/cipherstash/temp-env", branch = "main" } -vitaminc = { version = "0.5.1", features = ["random", "protected", "encrypt"] } -vitaminc-aead = "0.5.1" -# The dynamic value model (`FfiValue`) and its FFI transport codec, shared by -# every language binding. Optional in stack-encrypt (the `dynamic` feature). -# 0.5.1 adds `ValueKind`, the field-type vocabulary a dynamic plan declares. -vitaminc-aead-value = "0.5.1" -vitaminc-encrypt = "0.5.1" -vitaminc-hmac = "0.5.1" -vitaminc-prf = "0.5.1" -vitaminc-protected = "0.5.1" +vitaminc = { version = "0.6.1", features = ["random", "protected", "encrypt"] } +vitaminc-aead = "0.6.1" +# The transport frame and ciphertext codec every language binding carries +# across its FFI boundary. Optional in stack-encrypt (the `dynamic` feature). +vitaminc-aead-frame = "0.6.1" +vitaminc-encrypt = "0.6.1" +vitaminc-hmac = "0.6.1" +# Normalized equality terms (Unicode 16 NFC text, one integer encoding for +# every width), which `vitaminc-prf`'s `normalize` feature derives over. +vitaminc-normalize = "0.6.1" +# The text order key: the system ADR-0002's fold, pinned to Unicode 16, +# which text ORE and OPE terms are derived over (ADR-0009). +vitaminc-ore = "0.6.1" +vitaminc-prf = { version = "0.6.1", features = ["normalize"] } +vitaminc-protected = "0.6.1" diff --git a/biome.json b/biome.json index 765be3b6a..02cdf6475 100644 --- a/biome.json +++ b/biome.json @@ -11,7 +11,12 @@ "!languages/typescript/packages/protect-ffi/target", "!languages/typescript/packages/protect-ffi/src/eql-v3-types", "!packages/stack-encrypt/tests/fixtures/record_lowering.json", + "!packages/stack-encrypt/tests/fixtures/query_operators.json", + "!packages/stack-encrypt/tests/fixtures/value_kinds.json", + "!packages/stack-encrypt/tests/fixtures/stored_rows_v1.json", "!packages/eql/tests/encryption/fixtures/ste_vec_selectors.json", + "!packages/eql/tests/encryption/fixtures/v4_rows.json", + "!packages/eql/tests/encryption/fixtures/stored_v4_rows_v1.json", "!packages/eql/crates/eql-bindings/bindings", "!packages/eql/crates/eql-bindings/schema", "!packages/eql/packages/eql/src/generated", diff --git a/docs/adr/0002-one-value-model-three-encodings-and-eql-v4-by-producer.md b/docs/adr/0002-one-value-model-three-encodings-and-eql-v4-by-producer.md new file mode 100644 index 000000000..ab2c4fff9 --- /dev/null +++ b/docs/adr/0002-one-value-model-three-encodings-and-eql-v4-by-producer.md @@ -0,0 +1,626 @@ +--- +status: accepted +date: 2026-10-08 +revised: 2026-10-12, amended 2026-10-18 and 2026-10-19 +relates-to: ADR-0001; stack-encrypt ADR-0003, ADR-0007 +--- + +# One value model, three encodings in vitaminc, and EQL v4 separated by producer + +> **Revised 2026-10-12.** This ADR now carries the decisions from +> [`docs/plans/2026-10-11-order-and-equality-terms-layering.md`](../plans/2026-10-11-order-and-equality-terms-layering.md): +> vitaminc sits at the bottom of the stack, equality and order terms take +> per-domain transforms over a shared semantic baseline, `vitaminc-prf` no +> longer depends on order encodings, and `orderable-bytes` is frozen. The plan +> keeps the steps; this document is the decision. +> +> **Amended 2026-10-18.** vitaminc 0.6 removed the shared value type rather +> than renaming `FfiValue` to `Value` (vitaminc ADR 0001: a value describes +> itself once, and sinks consume it). Stack Encrypt follows: +> +> - A plaintext whose type is known only at run time is held as its +> transport frame (`dynamic::ValueFrame`) and walked into vitaminc's sinks. +> - `FfiValue` is removed. Each kind of control data a binding exchanges +> with the engine (a plan, a context, options, statistics, a configuration, +> the engine's answers) gets a type of its own, read from and written to a +> frame. Credentials in a configuration, such as the client key, are held +> in `Protected` fields. Until each message has its type, `FfiValue` carries +> only control data, never plaintext. +> - The open decision on exhaustive handling is settled by vitaminc 0.6's +> design (see "Exhaustive handling of kinds"). +> +> The bullets below that this changes say so where they stand. +> +> **Amended 2026-10-19.** A timestamp's microsecond resolution truncates +> rather than rounding half to even, as vitaminc does for both the equality +> and the order term. The reason, and the rule, are under "Lossy transforms". + +Every value Stack Encrypt handles is encoded three times: as a **ciphertext** +(reversible and self-describing), as an **equality term** (a keyed hash that +must be unambiguous) and as an **order term** (bytes whose order is the +value's order). This ADR decides where each encoding lives, which kinds of +value exist, how each kind is normalised before a term is derived from it, +and how EQL keeps columns written by Stack Encrypt apart from columns written +by cipherstash-client. + +## The problem + +In October 2026 Stack Encrypt could produce one of EQL's 51 types, `TextEq`. +The number, date, timestamp and decimal families were blocked on encoding, +not code. (`Bool` was not: it already had a kind, and EQL's only boolean +type, `eql_v3_boolean`, stores a ciphertext and nothing else.) + +- **The three encodings disagreed about which types exist.** `i16` had an + equality domain in `vitaminc-prf` and an order encoding, but no ciphertext + tag, so it could not be a kind. Date, timestamp and decimal had order + encodings only. +- **The mapping from a kind to a Rust type lived in Stack Encrypt.** + `dynamic::term::Scalar` mirrored `FfiValue`'s variants one for one to + dispatch to the term crates, and `dynamic::Value` wrapped `FfiValue` only to + add `Clone`. Both restated vitaminc's vocabulary in a downstream crate. +- **Ordering was implemented twice.** `orderable-bytes` defines a canonical, + order-preserving encoding per type. cllw-ore uses it for chrono and decimal + but hand-rolls integers and floats, and differs on `-0.0`. Block ORE was not + derivable at all, so `TextOrdOre` and `TextSearchOre` were refused. +- **Order encodings were spread over three repositories.** Encodings lived in + ore.rs, schemes in ore.rs and cipherstash-suite, and term derivation in + vitaminc and Stack Encrypt. Every change needed releases in all of them, + and the scheme crates accepted raw bytes, so a hand-encoded integer + (little-endian, sign bit not flipped) encrypted without error and sorted + wrong. +- **The existing writer's encodings are inconsistent.** cipherstash-client + hashes a timestamp's milliseconds but orders by its nanoseconds, and hashes + a decimal with its scale while ordering ignores it. Its block ORE text is + ASCII-only, lowercased, maps every digit to one symbol and truncates to six + blocks. cllw-ore's text order form (`orderize_string`) decomposes to NFD and + then drops every character that is not alphanumeric, whitespace or ASCII + punctuation, which removes symbols, emoji and non-ASCII punctuation as well + as accents, and does not fold case. +- **Two producers wrote the same EQL domains.** A Stack Encrypt query term + never matches a cipherstash-client term for the same value, and the + `eql_v3_*` domains could not tell them apart. A query from one against a + column written by the other returned no rows, silently. + +## Options considered + +**Where the value model lives.** + +1. **Move `vitaminc-aead-value` into Stack Encrypt.** It looks like FFI + plumbing. But it is the value model for vitaminc's own `Cipher` traits: + `aead-napi` and vitaminc's Go binding encrypt with `Aes256Cipher` through + it, without Stack Encrypt. Moving it makes vitaminc Rust-only, makes + vitaminc depend on Stack Encrypt (which depends on six vitaminc crates), + or forks the frozen tag table into two copies. +2. **Keep it in vitaminc, and move term derivation there too.** All three + encodings then live in one repository, next to the value type they + encode. Chosen. How "a kind exists in every layer or in none" is enforced + is under "Exhaustive handling of kinds" below, with the conformance test + described under "Which kinds get which terms" as the backstop. + +**Where order schemes sit.** + +1. **`vitaminc-ore` adapts the scheme crates.** It depends on cllw-ore and + ore-rs and wraps each in its own trait, with `orderable-bytes` underneath. + This is the spread over three repositories described above. +2. **vitaminc at the bottom.** `vitaminc-ore` owns the order encodings and an + `OrderScheme` trait and depends on no scheme crate; cllw-ore and ore-rs + implement the trait. A scheme's input is a sealed type only `vitaminc-ore` + can build, so raw bytes cannot reach a scheme. Chosen. + +**How EQL separates the two producers.** The SQL for both is identical; only +the producer of the terms differs. + +1. **Name only.** Stack Encrypt payloads go in the `eql_v3_*` domains, + distinguished by the `stack-encrypt:1:` ciphertext prefix. Nothing in the + database stops a cross-producer comparison. +2. **A producer tag in every payload and term, checked by every operator.** + A runtime check on the hottest SQL paths, across about 24k lines of + hand-written SQL. +3. **One SQL source, emitted under two names.** The build writes the same + source out as `eql_v3` (cipherstash-client terms) and `eql_v4` (Stack + Encrypt terms). A column's domain names its producer: a payload written + to the other producer's domain fails its check on insert, and a typed + client sees two different types. Names alone do not stop a cross-producer + `=` in SQL; see Consequences. Chosen. + +**How v3 and v4 ship.** Bumping `main` to v4 and patching v3 from a branch +needs a maintenance release path for five lockstep artefacts that does not +exist, and the EQL publish script puts every stable release on `latest`, so a +3.x patch would move `latest` backwards. A second package doubles the +trusted-publishing and release surface. One package carrying both bundles +avoids both, at the cost of teaching the release tooling about a second +bundle (see Consequences). Chosen. + +## Decision + +### Kinds and the ciphertext + +- vitaminc 0.6.0 adds the kinds `Int8`, `UInt8`, `Int16`, `UInt16`, `Int128`, + `UInt128`, `Date`, `Timestamp` and `Decimal`, each with a `ValueKind` name, + a `Value` variant and a ciphertext tag, in one breaking release. Rust `i128` + and `u128` get `Encrypt` and `Decrypt` impls. +- ~~`FfiValue` is renamed `Value`, with `#[deprecated] pub type FfiValue = Value` + for one release.~~ *Amended 2026-10-18:* vitaminc 0.6 has no shared value + type to rename it to. Its ADR 0001 has a value describe itself once + (`Encrypt`) and every consumer be a sink: the encrypting sink, a term + deriver, a frame writer. A host binding walks its transport frame straight + into those sinks. So Stack Encrypt holds a dynamic plaintext as its frame, + `dynamic::ValueFrame`, in a wiped buffer, from the source a binding hands + over to the sealed leaves and from the opened leaves back to the frame it + returns, with no value tree in between. `FfiValue` is removed rather than + kept for control data: each control message gets its own type, read from + and written to a frame, and holds any credential in a `Protected` field. + `ValueKind`, the vocabulary a plan field declares, stays, and is + exhaustive. +- Every variant exists in every build. `Date`, `Timestamp` and `Decimal` use + vitaminc's own representation (the payloads in the table below), with + conversions to and from chrono and rust_decimal behind those features. A + variant that appeared only with a feature would break any exhaustive + handling outside `vitaminc-aead-value`, because features unify across a + build and a crate can check only its own. +- `Value` implements `Clone` as a deep copy that rebuilds every leaf into a + fresh `Protected`. A clone is under the same custody as its original. +- The sealed leaf tag table (`tags.rs`) stays frozen and contiguous. The + transport codec's framing tags move from `0x10`–`0x12` to `0xF0`–`0xF2`; + transport carries no compatibility commitment and every user of it updates + together. A leaf is `[tag] ++ payload`. The new leaves, which every binding + must produce byte for byte: + + | Tag | Kind | Payload | + |---|---|---| + | `0x0C`, `0x0D` | `Int8`, `UInt8` | 1 byte; two's complement for `Int8` | + | `0x0E`, `0x0F` | `Int16`, `UInt16` | 2 bytes, little-endian; two's complement for `Int16` | + | `0x10`, `0x11` | `Int128`, `UInt128` | 16 bytes, little-endian; two's complement for `Int128` | + | `0x12` | `Date` | `i32`, little-endian: the day number in the proleptic Gregorian calendar with **0001-01-01 as day 1** (chrono's `num_days_from_ce`). 0001-01-01 is `01 00 00 00`, not zero | + | `0x13` | `Timestamp` | `i64` Unix seconds, then `u32` nanoseconds, both little-endian, UTC. Nanoseconds are below 1,000,000,000. *Amended 2026-10-18 (#1144):* vitaminc 0.6 refuses a leap second: nanoseconds of 1,000,000,000 or more (chrono's leap-second representation, which the pre-ADR-0001 `Value` codec accepted when the seconds modulo 60 were 59) are refused, because a leap second has no count in any unit. The instant must also lie within chrono's `NaiveDateTime` range (vitaminc's `MIN_TIMESTAMP_NANOS` to `MAX_TIMESTAMP_NANOS`, years -262143 to 262142). Anything else is refused | + | `0x14` | `Decimal` | 16 bytes: a little-endian `u32` of flags (sign in bit 31, scale in bits 16–23 and at most 28, every other bit zero), then the low, middle and high little-endian `u32` words of the 96-bit mantissa. This is rust_decimal's `serialize()` and keeps the scale. There is no NaN or infinity | + +- `transport` stays a module of `vitaminc-aead-value`. + +### Semantic baseline and per-domain transforms + +The ciphertext keeps the value exactly as given: `1.50` decrypts as `1.50`, +and a timestamp keeps its nanoseconds. Terms are derived in two steps. + +**The semantic baseline** makes values that are equal under `Value`'s own +semantics encode the same. Every term applies it; it loses nothing a user +would call a different value. + +| Kind | Baseline | +|---|---| +| `Decimal` | scale normalised: `1`, `1.0` and `1.00` are equal | +| `Float32`, `Float64` | `-0.0` folded into `+0.0`; every NaN replaced by one positive quiet NaN, which sorts above +Infinity. This is Postgres's own `float8` semantics: `'NaN' = 'NaN'`, `'-0' = '0'` and `'NaN' > 'Infinity'` are all true | +| text | NFC | +| every other kind | none | + +**Lossy transforms** sit on top of the baseline, chosen by each term's +domain: timestamp resolution, text folding and prefix truncation. Each +transform and its version is named in the domain label, for example +`…/timestamp/day/v1`. The ones defined now: + +- **Timestamp resolution**: exact, microsecond or day. ~~Microsecond rounds + to the nearest microsecond, ties to even, on the signed microsecond count + since the epoch. That is what Postgres does when it parses a timestamp with + more digits than it keeps (`.1234565` becomes `.123456` and `.1234575` + becomes `.123458`, before and after 1970; checked on Postgres 17.5).~~ + *Amended 2026-10-19:* microsecond truncates: the nanoseconds below the + microsecond are dropped, never rounded, which is floor on the signed count + since the epoch (`.1234567` becomes `.123456`, and before 1970 an instant + moves earlier). vitaminc applies this rule to the equality term and to the + order term alike (vitaminc-ore's timestamp domain is + `vitaminc/ore/value/timestamp-micros/v1`, and both terms normalise through + `vitaminc_normalize::timestamp`), so `=` and the order agree on which + instants are one. Matching PostgreSQL's parse-time rounding is not a goal: + timestamps are stored in many places besides PostgreSQL, and a term must + mean the same thing whichever store holds it. A value that already has + microsecond precision is unchanged. Day takes the UTC calendar day. +- **Text fold, v1**: NFD, then full Unicode case folding (`CaseFolding.txt` + statuses C and F, so `ß` becomes `ss`), then NFD again, then removal of + every accent, then NFC. An accent is a mark (General_Category `Mn`, `Mc` + or `Me`) with a non-zero canonical combining class: the acute in `é`, the + umlaut in `ö`, a Devanagari virama. A mark of class zero is a letter of + its own in its script, such as a Thai or Devanagari vowel sign (U+0E31, + U+0901), and is kept. The order of the steps is part of the transform, + because folding can produce text that is no longer normalised and the + steps do not commute. + *Amended 2026-10-19:* this step said "every code point with + General_Category `Mn`", which would also drop class-zero vowel signs and + so tie different words. The fold that ships, `vitaminc-ore`'s + `vitaminc/ore/value/text-folded/unicode-16/v1`, keeps them, and this + bullet now describes it. +- **Prefix truncation**: a number of code points kept after any other + transform. Alphabet packing, if ever built, is a transform of the same + kind. + +Text transforms are pinned to a Unicode version, which is part of the label +(for example `text-fold/unicode-16/v1`), and the `unicode-normalization` and +case-folding tables are pinned to it. Code points unassigned in that version +pass through unchanged (canonical combining class 0, no decomposition, no +case mapping), so the transform is a fixed function and nothing is refused on +write. The cost is that two strings using characters added after the pin, +which a later Unicode would treat as equivalent, may not match. Moving the +pin is a new label: existing columns keep the label they were written under +and stay queryable, and a column moves only by re-encryption. +*Amended 2026-10-18:* vitaminc 0.6 refuses a code point unassigned in the +pinned version instead of passing it through. `vitaminc-normalize` (equality +and match terms) and `vitaminc-ore`'s fold (order terms) both return +`UnassignedCodePoint`, so text holding one has no term, and writing it to an +indexed column or querying with it is refused. The refusal is the same on +every path and in every language, so the transform is still a fixed +function of the text it accepts, and moving the pin is still a new label. + +Order terms fold case and accents because code-point order puts `é` after +`z`; a fixed fold approximates the first level of Unicode collation without +depending on ICU, whose sort keys change between versions. Equality on text +is not folded by default: a folded order term only produces ties, while a +folded equality term produces false matches. A case-insensitive equality is +its own domain. + +**Equality and order may take different transforms**, for example an exact +timestamp for equality and the day for order, or the same one ("locked"). +Stack's index spec carries the choice per column. Two rules bound it: + +- **A non-injective order transform needs an equality term beside it.** In + the EQL SQL, the numeric, date and timestamp `_ord`, `_ord_ope` and + `_ord_ore` domains store no `hm` and answer `=` with the order term + (`timestamp_ord_functions.sql`: `eq` is `ord_term(a) = ord_term(b)`). A + day-resolution order term there would make `=` match every row from that + day. So a lossy order transform is allowed only on a domain that also + stores an equality term and answers `=` through it, the shape `text_ord` + already has (`hm` and `op`). On the other domains the order transform is + injective: exact for numbers and dates, and for timestamps whatever + resolution the equality term would have used. +- **Labels separate domains; they do not make terms incomparable.** Terms + built with different transforms are unrelated bytes, so a probe built with + one matches nothing built with the other. Nothing in a stored term records + its label. Keeping a query on the same transforms as the column is the + index spec's job, and the EQL domain is what ties a column to its spec. + +A column's leakage is the combined leakage of all its terms. Truncating the +order term to the day hides the order of events within a day, but an exact +equality term on the same column still reveals which rows share a timestamp. + +### Which kinds get which terms + +- **Every scalar kind has a ciphertext.** Containers and null have no terms. +- **Equality:** every scalar kind except `Bool`, including floats over their + baseline bits. Stack Encrypt refused float equality until now, on the + grounds that equality on IEEE-754 values is a modelling error. The baseline + removes the two cases where bitwise equality and numeric equality disagree + (signed zero and NaN payloads), and what is left is exactly Postgres's + `float8 =`, which EQL's `real_eq` and `double_eq` domains already expose. + Whether a computed float is the value the caller meant is the caller's + problem, as it is for a plaintext float column. +- **Order:** every scalar kind except `Bool`, under all three schemes, with + one exception: block ORE for text, which waits for ore-rs's variable-length + block ORE (see Deferred). *Amended 2026-10-19:* the exception is gone. + ore-rs 0.9.0 released its chained scheme, and Stack Encrypt orders text + with it (see "EQL v4"). +- **`Bool` has a ciphertext only.** A keyed hash or an order term over a + domain of two values hides nothing: it splits the rows into two groups, and + an order term also says which group is `true`. Stack Encrypt stops + producing CLLW order terms for `Bool`, and `vitaminc-ore` has no `Bool` + encoding. cllw-ore keeps its `Bool` impl, so cipherstash-client and its + stored v3 terms are unaffected. + +These exceptions are an explicit, documented list kept next to vitaminc's +conformance test. The test fails for any kind that is missing from +`vitaminc-prf` or `vitaminc-ore` without an entry, and checks that values +equal under `Value` semantics produce equal terms in both. + +### Where each encoding lives + +- **Ciphertext:** `vitaminc-aead-value`. It does no term normalisation: its + `canonical` module and its dependency on `orderable-bytes` are removed. +- **Equality:** `vitaminc-prf`, which depends only on `vitaminc-aead-value`, + gains domains for every new kind, floats and `Decimal`, and applies the + baseline and its own transforms. It has no ordering dependency. The + value-derived PRF domains landed after vitaminc 0.5.1 and are unreleased, + so their bytes are still free to choose. +- **Order:** a new `vitaminc-ore` crate, released in lockstep with the other + vitaminc crates. + - It owns the order encodings (ported from `orderable-bytes`, kept in + `Protected`, pinned by golden vectors), the baseline, the order + transforms, and derivation of order terms from `&Value`. + - Its `OrderScheme` trait takes a sealed `OrderPlaintext`, fixed-length or + variable-length, that only `vitaminc-ore` can build, and returns a + scheme-defined term, roughly `type Term: Ord + AsRef<[u8]>`. Each scheme + keeps its own comparison and stored bytes. The trait also covers the key + type, domain separation (the label goes in as the salt) and typed errors. + - It depends on no scheme crate; its tests use a mock scheme. +- **Schemes:** cllw-ore (CLLW ORE and OPE) and ore-rs (block ORE) implement + `OrderScheme` behind an optional `vitaminc` feature, so cipherstash-client + takes no vitaminc dependency. Their typed APIs are unchanged and not + feature-gated. Block ORE through the trait uses each kind's natural width; + the 8-byte padding stays a detail of the typed API cipherstash-client uses. +- **`orderable-bytes` is frozen.** It stays as it is for cipherstash-client's + stored terms and is not part of the target design. +- **Term derivation** takes a `&Value` in `vitaminc-prf` and `vitaminc-ore` + and keeps leaves inside `Protected`. A pairing a layer does not support + returns a typed error from vitaminc. + +### Stack Encrypt + +- `dynamic::Value` and `dynamic::term::Scalar` are deleted. `dynamic::term` + takes any source that describes itself to vitaminc (amended 2026-10-17, + #1232: there is no `vitaminc_aead_value::Value`; a binding's value arrives + as a transport frame and stays one, `dynamic::ValueFrame`). `admits` keeps + only the rules that belong to Stack Encrypt, such as `Match` taking text + only. +- Order terms go through `vitaminc-ore`. The index for an ORE or OPE column + picks the concrete scheme (cllw-ore or ore-rs, through `OrderScheme`) and + its transforms; Stack Encrypt no longer calls cllw-ore's typed API. A block + ORE term kind joins CLLW ORE and OPE. +- `TextEq` normalises to NFC and moves to EQL v4. The Stack Encrypt targets on + `eql_v3_*` domains are deleted. The `stack-encrypt:1:` ciphertext prefix + stays: decryption does not pass through Postgres types, and the prefix is + what lets it reject the other producer's payload. + +### Host languages + +A binding picks the kind from the field's **declared EQL target**, never +from the host type. This keeps the rule `eql-bindings` already enforces: a +target accepts exactly one kind, and nothing below the binding converts. The +binding converts a host value into the target's kind when it can do so +without loss, and refuses it otherwise. + +| Kind | EQL family | | +|---|---|---| +| `Int16` | `smallint` | | +| `Int32` | `integer` | | +| `Int64` | `bigint` | | +| `Float32` | `real` | | +| `Float64` | `double` | | +| `Decimal` | `numeric` | | +| `Date` | `date` | | +| `Timestamp` | `timestamp` | | +| `String` | `text` | | +| `Bool` | `boolean` | | +| `Int8`, `UInt8`, `UInt16`, `UInt32`, `UInt64`, `Int128`, `UInt128`, `Bytes` | none | no EQL domain and no target name yet; a kind is selectable only once a target names it | + +- **Go.** Any Go integer type whose every value fits the target's kind is + accepted (`int8` into `smallint`, `int32` into `bigint`); a wider one is + range-checked and refused if the value does not fit. `int8`, `int16`, + `uint8` and `uint16` get their own kinds when a target names them; no + target does yet, so today they reach `Int16`, `Int32` or `Int64` through + the fits-losslessly rule. + *Amended 2026-10-18 (#1144):* a field with no EQL target (`encrypt`, + `index=`) has no target to pick a kind from, so Go picks it from the Go + type: each integer type is its own kind (`int8` is `Int8`, `int16` + `Int16`, `uint8` `UInt8`, `uint16` `UInt16`, `int` and `int64` `Int64`, + `uint` and `uint64` `UInt64`), and its order terms are that width's. A + field stored as an EQL target still takes the target's kind, by the rule + above. + `Int128` and `Uint128` are SDK value types. `time.Time` means `Timestamp`, + and `encrypt.Date{Year, Month, Day}` is a date. A field whose target is a + date family also accepts `time.Time` and takes its calendar day **in the + value's own location** (`t.Date()`), which is how Go code builds a date; + taking the UTC day would store the previous day for a local midnight east + of UTC. +- **JavaScript.** A `BigInt` takes the target's kind, and is refused if it + does not fit. It decodes as a `BigInt`. `Date` means `Timestamp`; a date + has its own wrapper. +- **Decimal.** A host decimal (Go's `shopspring/decimal`, a JavaScript + string) is refused if it is NaN or infinite, has more than 28 digits after + the point, or does not fit a 96-bit mantissa at its scale. Postgres + `numeric` holds values that `Decimal` cannot, and a binding refuses them + rather than rounding. +- **Every binding** runs vitaminc's shared test vectors: a host value, the + target, the kind it must map to, and the exact `[tag] ++ payload` leaf + bytes, including normalisation and refusal cases. The leaf is + deterministic even though the AEAD is not. + +### EQL v4 + +- **v4 is the v3 SQL with Stack Encrypt terms.** One SQL source is emitted + under two names. The hand-written SQL takes the schema as a build-time + placeholder, as eql-codegen's templates already do. Consistent with + ADR-0001, the data-bearing domains (`eql_v3_*`, `eql_v4_*`) live in + `public` and survive reinstall, and the implementation schemas + (`eql_v3`, `eql_v4` and their `_internal` schemas) stay disposable. The + second name separates producers; it is not a versioned upgrade mechanism of + the kind ADR-0001 rejects. +- **A v4 payload's envelope carries `"v": 4`**, and the `eql_v4_*` domains' + check constraints test it, as the `eql_v3_*` ones test `3`. The version is + one more build-time placeholder. A payload written to the other producer's + domain then fails on insert, rather than only matching nothing when it is + queried. Queries are a different matter; see Consequences. +- **New EQL targets are v4-only.** In v3 they stay refused, with a reason + that points to v4. A domain that a lossy order transform needs, with an + `hm` beside its `op`, is one of these, and is the first point where the v4 + SQL differs from the v3 SQL. +- **`@cipherstash/eql` 4.x ships both bundles** from `main`. A SQL fix lands in + both names in one release. +- **`--eql-version` comes back, on purpose.** It was removed when v3 became + the only installable generation, and the CLI now refuses it. It returns as + `--eql-version 3|4|all` on both `stash eql install` and + `stash eql migration`, so the migration-first paths (Drizzle, and Supabase + with a local `supabase/` directory, where a direct install does not survive + `supabase db reset`) can install v4 too. The migration they write carries + the chosen bundles, and `--supabase` grants on each bundle's schemas. The + default is 3, which is today's behaviour, and changes to 4, announced in + advance, when the TypeScript stack has moved to Stack Encrypt. `2` keeps + its current message pointing at the EQL 2.3.1 release, rather than + becoming an ordinary invalid value. +- **During the overlap**, expected to last a quarter or more, cipherstash-client + takes fixes only. New kinds and domains are produced through Stack Encrypt + and v4. An exception is a decision written down in its issue. +- *Amended 2026-10-19:* **a v4 search query carries only the terms its + operator reads.** The text search query domains (`eql_v4.query_text_search` + and `eql_v4.query_text_search_ore`, whose terms include the match term + `bf` beside others) require at least one of their terms rather than all + of them. Text shorter than one 3-gram has no match term, so a query that + had to carry every term could not express `=` or `<` on short text. + Stack Encrypt emits only the term of the index that answers the query's + operator. Every extractor over such an operand raises wherever it reads + the operand when the term it reads is absent, JSON `null` or an empty + array, naming the domain, the term and the operators that compare it, and + never the operand's contents, rather than reading it as `NULL` (no row) or + as an empty filter. The raise is at evaluation: a literal operand is + folded and raises at plan time, while a bound parameter under a generic + plan whose predicate is never reached returns no row and no error. A + stored search value carries every term, and text too short for a match + term stores the empty filter (`"bf": []`), the stored side's answer in + Stack Encrypt: `=` and `<` find it, and no `@@` query does, because a + query's filter is never empty (the extractor refuses `"bf": []`). That is + a claim about queries: `@@` between two stored values (a self-join + `a.name @@ b.name`, or `matches(col, )`) keeps + the shared guard, so two stored empty filters match each other there, as + they do in v3. No producer builds that predicate. A match + query of such text is refused, as it is for `TextMatch`, whose stored + value still refuses it too. The v3 domains are cipherstash-client's format + and keep requiring every term, byte for byte, so here the v4 SQL differs + from the v3 SQL. The one source holds both forms between version-block directive lines (`-- {{#v3}}` … `-- {{/v3}}`, + `-- {{#v4}}` … `-- {{/v4}}`), which the renderer keeps for their own + version and drops from the other, so the v3 bundle renders exactly as it + did before the block was written. +- *Amended 2026-10-19:* **a v4 `ob` term is ore-rs's 6-bit block ORE.** The + key name is v3's, but the format is not. Stack Encrypt writes one term per + value, the whole ore-rs ciphertext (left and right halves) as one hex + element: the fixed-width scheme (`Bit6BlockOre`, wire header `02 02`) for + the integers and floats, and the chained scheme (`ChainedBlockOre`, `02 + 03`, then the padding policy) for text, padded with NUL bytes to a + power-of-two bucket of at least 16 bytes. v3's `ob` is cipherstash-client's + 8-bit block ORE, up to six terms per text value. The two need different + comparators, so the v4 SQL holds its own (a version block in + `sem/ore_block_256/functions.sql`), and every name built on the block ORE + type is a placeholder too: `{{ore_block}}` renders as `ore_block_256` in + v3 and `ore_block_v2` (ore-rs's wire version 2) in v4, so no v4 object + carries the 8-bit format's name. The v3 bundle renders byte for byte as + before. + + The v4 comparator checks both terms' shape before it compares (header, + block count, exact length, padding policy, every permuted symbol below 64) + and raises, naming the length and never the bytes, on a malformed term, on + terms of different schemes, on fixed-width terms of different block counts + and on chained terms padded under different policies. For terms of one + shape it runs a constant operation count: one statement per block + evaluates both of its comparisons and latches the first difference + arithmetically, and the hash runs exactly once on every path + (`packages/eql/tests/timing/ore_block_v2/` measures it). + The hash is ore-rs's σ-MMO over AES under a fixed public key, which + pgcrypto computes. A query carries the full ciphertext too, not the left + half alone, because a btree index or `between` can pass the stored value + first, and the comparator reads its second operand's right half. + + A comparison of two chained terms reveals the length of their common + prefix in 6-bit blocks, and a chained term reveals its text's length + bucket. Both are ore-rs's documented leakage for the chained scheme. + +### Review + +A domain that locks equality and order together derives both from the same +transformed bytes, and that choice is frozen once data is stored under it. +Dan Draper signs off each such domain in writing before it ships. Domains +whose equality and order transforms are independent need no such sign-off. + +## Exhaustive handling of kinds + +`vitaminc-prf`, `vitaminc-ore` and every binding must handle every kind of +value, and adding a kind should fail to build until each one does. This was +left open between dropping `#[non_exhaustive]` (A) and a visitor trait with +one required method per kind (B). + +*Settled 2026-10-18 by vitaminc 0.6's design.* There is no value enum to +match. A sink receives each scalar as a physical value and an optional +logical type. The physical enums (`Physical`, `Int`, `Float`) are exhaustive, +so a new physical kind fails to build in every sink that matches them, which +is A for the bytes a value is made of. `Logical` is `#[non_exhaustive]`, so a +new logical type (a date, a timestamp) is additive, and a sink that does not +know it refuses it at run time. Stack Encrypt's `ValueKind` is exhaustive. +The conformance test stays the backstop for the logical types a sink reaches +through a wildcard. + +## Consequences + +- **The release order is fixed.** vitaminc 0.6.0 first, then cllw-ore and + ore-rs (additive releases implementing `OrderScheme`), then the Stack + Encrypt breaking release. Stack Encrypt must reach crates.io before + `eql-bindings` uses its new API, since `cargo publish` builds + `eql-bindings` against the registry. The EQL v4 bundle and the Go SDK + changes follow. +- **Adding a kind later** takes a tag, PRF and order encodings, a codec in + every binding, test vectors and a row in the kind-to-family table, or an + entry in the exceptions list. A new physical kind is a breaking change for + every sink. A new logical type is not. +- **Stack Encrypt's terms change** for `-0.0` and negative-sign NaN floats, + for non-NFC text equality, for `Bool` order terms, which are removed, and + for every text order term: `orderize_string` drops symbols and keeps case, + while the fold keeps symbols and folds case. None is deployed, so the + breaking release carries them without migration. +- **cipherstash-client's terms do not change.** `orderable-bytes`, cllw-ore's + typed impls (`Bool` included) and its block ORE padding are all kept, so + every stored v3 payload stays queryable. +- **Separate names do not stop a cross-producer comparison in SQL.** EQL's + domains are over `jsonb`, and when no operator is defined for an exact pair + of types, Postgres treats a domain as its base type. `v3col = v4term` then + runs as the built-in `jsonb = jsonb` and returns no rows, silently, as + `eql_v3_text_eq = eql_v3.query_integer_eq` already does within v3. A join + between a v3 and a v4 column does the same, and + `eql_v3.eq_term(v3col) = eql_v4.eq_term(v4term)` compares the two terms as + text. Three things do fail loudly: an insert of the wrong version (the + domain check), the function form `eql_v3.eq(v3col, v4term)` ("not unique"), + and `OPERATOR(public.=)` between a v3 and a v4 column ("not unique"). + Verified on Postgres 17 with both bundles installed in one database. +- **Tooling that recognises encrypted columns by the `eql_v3_` prefix must + learn `eql_v4_` before any v4 column can exist.** Today that is + `stash init`'s introspection (`cli/src/commands/init/lib/introspect.ts`), + the wizard's tools (`wizard/src/tools/wizard-tools.ts`) and + `classifyEqlDomain` (`migrate/src/version.ts`), which `stash status` and + `stash encrypt status` use. Getting this wrong shows an encrypted column as + plaintext, which invites encrypting it a second time. The installer's + ADR-0001 capture, rebuild and verify logic hard-codes the disposable + schemas as `('eql_v3', 'eql_v3_internal')` + (`installer/derived-search-index-restoration.ts`, + `installer/installation-state.ts`); without `eql_v4` and + `eql_v4_internal` there, a v4 reinstall would capture and restore none of + its own indexes. Both land with the first released v4 bundle. +- **The release tooling assumes one bundle.** The lockstep stamp is read from + `COMMENT ON SCHEMA eql_v3` (`scripts/sync-lockstep-versions.mjs`, + `packages/eql/tasks/release/prepare-bindings-assets.sh`); there is one + `release-manifest.json` per artefact, which the CLI's `bundle-digest.ts` + and `eql-sql-asset-freshness.test.mjs` check; `eql_bindings::sql` embeds + only the v3 files; and `_build-eql-sql.yml` uploads only them. The v4 + bundle needs its own stamp, manifest entries, freshness check, crate + constants and release upload before it ships, or v4 drift goes unnoticed + with every digest still verifying. The `postgres-eql` image states which + bundles it installs. +- **Moving a column from v3 to v4 means re-encrypting it.** The ciphertext and + every term change. How that migration runs belongs to the decision that + moves the TypeScript stack onto Stack Encrypt. +- **The CLI surface changes** (`--eql-version` on `stash eql install` and + `stash eql migration`), so `skills/stash-cli`, `skills/stash-indexing` and + `skills/stash-postgres` change in the same PR. +- **This replaces a definition in a plan.** `docs/plans/2026-10-04-plan-builder.md` + called "EQL v4" a name for a Stack Encrypt payload in the v3 envelope and + the `eql_v3_*` domains. That plan is updated to this ADR. + +## Deferred + +- ~~**Block ORE for text** waits for ore-rs's chained, variable-length block ORE + to be reviewed and released, then arrives in a minor release of + `vitaminc-ore`. Until then it is an entry in the exceptions list.~~ + *Amended 2026-10-19:* done. ore-rs 0.9.0's `vitaminc` feature implements + `OrderScheme` for its chained scheme, and Stack Encrypt produces + `TextOrdOre` and `TextSearchOre` with it (see "EQL v4"). +- **Block ORE for `Bool`.** Block ORE stored as right ciphertexts only is + fully randomised and semantically secure, so a two-value domain leaks + nothing through it. `Bool` could support that scheme alone, enforced by a + marker trait on schemes. Not built until needed. +- **Cross-version blocker operators.** Blockers for every v3/v4 pair of the + same type family and capability would turn the silent cross-producer `=` + into an error. About 2,300 operators, generated from the catalog like the + existing blockers. Not built yet; until then, a cross-producer query is + caught only by typed clients, the function form and `OPERATOR(public.=)`; + the plain `=` returns no rows. When they are built, + **the v4 bundle owns them**: it creates them in `eql_v4_internal`, against + whichever `public.eql_v3_*` domains exist, on every install. A v4 reinstall + drops them with its schema and creates them again. A v3 reinstall leaves + them alone, because they depend only on the `public` domains, which + survive, and not on anything in `eql_v3`. Installing v3 after v4 into a + database that had no v3 domains leaves them absent until v4 is reinstalled, + so `--eql-version all` installs v3 first. +- **Locale-aware collation**, as its own domain with the collation version in + its name. +- **ASCII-packed text domains**, until a customer's column sizes make the + case. diff --git a/docs/plans/2026-10-04-plan-builder.md b/docs/plans/2026-10-04-plan-builder.md index 88dd628fb..5037c67ad 100644 --- a/docs/plans/2026-10-04-plan-builder.md +++ b/docs/plans/2026-10-04-plan-builder.md @@ -662,9 +662,11 @@ In the `Text` family, the three `Ord` suffixes carry equality too. Each type has a query type, with `Query` after its name: `TextEqQuery`. The value of `encrypt_into` is the Go type name. -An EQL value has the EQL v3 envelope: its version field is `3`, and Postgres stores it in an `eql_v3` domain. +Once EQL v4 lands, an EQL value from the engine is an EQL v4 value: its version field is `4`, and Postgres stores it in an `eql_v4_*` domain. The ciphertext inside it is a Stack Encrypt ciphertext, which starts with `stack-encrypt:1:`. -"EQL v4" in this plan is the name of that form, and not a new envelope. +EQL v4 is the v3 SQL, emitted from the same source under a second name, so that a column written by Stack Encrypt and one written by cipherstash-client are different Postgres types ([ADR-0002](../adr/0002-one-value-model-three-encodings-and-eql-v4-by-producer.md)). +This replaces this plan's first definition, under which "EQL v4" named a Stack Encrypt payload in the v3 envelope and an `eql_v3` domain; Postgres could not tell the two producers apart, and a query from one against a column written by the other matched nothing. +The `TextEq` described below is still produced into an `eql_v3` domain until that move lands. The engine produces one EQL type today: `TextEq`. Status (2026-10-06, #1062): `TextEq` is producible through the data plan's target form. @@ -1157,9 +1159,10 @@ model rather than a strain: it puts key material in the database. ## EQL v4 types as field targets -Naming: **EQL v4** is the EQL form of a stack-encrypt payload. **EQL v3** is -the existing SQL bundle and its `eql_v3_*` domains, which this section does -not change. Depends on #971 (`TextEq` / `TextEqQuery` through stack-encrypt +Naming: **EQL v4** is the EQL form of a stack-encrypt payload: the v3 SQL +emitted under a second name, with `eql_v4_*` domains and `"v": 4` envelopes +(ADR-0002). **EQL v3** is the same SQL holding cipherstash-client payloads in +its `eql_v3_*` domains, which this section does not change. Depends on #971 (`TextEq` / `TextEqQuery` through stack-encrypt in `eql-bindings`, and `Identifier` as a two-segment `Label`). **The engine returns an EQL type only when a plan names it as a target.** diff --git a/docs/plans/2026-10-11-order-and-equality-terms-layering.md b/docs/plans/2026-10-11-order-and-equality-terms-layering.md new file mode 100644 index 000000000..aacd27a2d --- /dev/null +++ b/docs/plans/2026-10-11-order-and-equality-terms-layering.md @@ -0,0 +1,343 @@ +# Order and equality terms: vitaminc at the bottom + +> **Plan, not specification.** This document is an indicative sketch of the +> steps required, written before the work was done. It is not kept in step +> with the implementation and must not be used as a formal specification or +> as a reference for reviewing what was actually built: the code, its rustdoc +> and the tests are the source of truth. Where the two disagree, the code wins +> and this document is simply out of date. + +**Status:** planned. Decisions settled 2026-10-11, except the one listed +under "Open decision". +**Date:** 2026-10-11 +**Amends:** ADR-0002 (`docs/adr/0002-one-value-model-three-encodings-and-eql-v4-by-producer.md`). +The decisions below were folded into the ADR on 2026-10-12, and the ADR is +now the reference for them. This plan keeps the steps. +**Issues:** cipherstash/vitaminc#374 (`vitaminc-ore`), #1063 (index crates) + +## Why + +Starting on cipherstash/vitaminc#374 under ADR-0002 turned up four problems. + +- **Too many dependencies across repositories.** `vitaminc-ore` was to depend + on cllw-ore and ore-rs and adapt each to its `Scheme` trait, with + `orderable-bytes` underneath both. Order encodings, scheme traits and term + derivation ended up spread over three repositories, and every change needed + releases in all of them. +- **Raw bytes reach the schemes.** cllw-ore and ore-rs accept bytes that the + caller has already encoded. Bytes encoded by hand (little-endian integers, + an unflipped sign bit, `f64::to_bits`) encrypt without error and sort + wrong. Fixing that inside cllw-ore needed wrapper types in `orderable-bytes` + (cipherstash/ore.rs#103) and a feature split in cllw-ore + (cipherstash/cipherstash-suite#2308). Cargo features unify across a build, + so the feature split could not hold whenever anything else in the build + turned the feature on. +- **PRF depends on ordering.** `vitaminc-prf` reaches `orderable-bytes` + through `vitaminc-aead-value`'s `canonical` module, which reuses order + encodings as equality input. Equality needs a canonical form, not an order. +- **One canonical form for both term kinds is too rigid.** ADR-0002 builds + equality and order terms from the same canonical form. A leakage-tuned + domain needs them to differ, for example an exact timestamp for equality + and only the day for order. + +## Decisions + +### vitaminc is the bottom of the stack + +- `vitaminc-ore` owns: + - the order encodings; + - an `OrderScheme` trait; + - the derivation of order terms from `&Value`, generic over the scheme. + + It depends on no scheme crate. +- A scheme's input is a sealed type, `OrderPlaintext`, that only + `vitaminc-ore` can construct. It comes in fixed-length and variable-length + forms. A scheme can't be given bytes that skipped the encoding, so no + wrapper types or feature flags are needed to keep raw input out. +- `OrderScheme` returns a scheme-defined term, roughly + `type Term: Ord + AsRef<[u8]>`, whose length is only known at runtime. Each + scheme keeps its own comparison (CLLW ORE and block ORE don't compare byte + by byte) and its own stored bytes. Fixed-width types such as + `OreCllw8V1` stay as inherent APIs on the scheme crates, because stable + Rust can't name their width generically. +- The trait also covers the key type, domain separation (the term's domain + label goes in as the salt) and typed errors. +- cllw-ore and ore-rs depend on `vitaminc-ore` and implement `OrderScheme` + behind an optional `vitaminc` feature, so cipherstash-client doesn't take a + vitaminc dependency. Their existing typed APIs stay as they are, so + cipherstash-client's stored terms don't change. +- `vitaminc-ore` releases in lockstep with the other vitaminc crates. On each + vitaminc minor release, cllw-ore and ore-rs bump their optional + `vitaminc-ore` dependency before stack can upgrade. That is acceptable + while stack is the only user of the trait path. +- `vitaminc-ore`'s tests use a mock scheme defined inside the crate. A + dev-dependency on cllw-ore would form a cycle across repositories and pull + a second `vitaminc-ore` in from crates.io. + +### Equality and order are tuned per domain + +- There is no shared canonical form. Each kind has a **semantic baseline**, + which makes values that are equal under `Value`'s own semantics encode the + same: decimal scale (`1.0` = `1.00`), signed zero, NaN, NFC. +- On top of the baseline, each term's domain chooses its own **lossy + transforms**, such as timestamp resolution (exact, microsecond, day), + text folding (removing combining marks, case folding, with the Unicode + version pinned) and prefix truncation. +- Each transform is named in its term's domain label, for example + `…/timestamp/day/v1`, so terms built with different transforms are + domain-separated: a probe built with one matches nothing built with the + other. A stored term does not record its label, so they are not made + incomparable. Keeping a query on the column's transforms is the index + spec's job. +- Equality and order can be **independent** (an exact timestamp for + equality, the day for order) or **locked** (the same transform value + passed to both). Stack's index spec (#1063) carries the choice per column. +- A **lossy order transform needs an equality term beside it.** EQL's + numeric, date and timestamp `_ord` domains store no `hm` and answer `=` + with the order term, so a day-resolution order term there would make `=` + match every row from that day. A non-injective order transform is allowed + only on a domain that stores an equality term and answers `=` through it, + as `text_ord` does. Those domains are new, v4-only EQL targets. +- The semantic baseline lives in each term crate: `vitaminc-prf` and + `vitaminc-ore` each apply it. A conformance test checks that values equal + under `Value` semantics produce equal terms in both. There is no shared + canonicalisation crate. +- A column's leakage is the combined leakage of all its terms. Truncating the + order term to the day hides the order of events within a day, but an exact + equality term on the same column still reveals which rows share a + timestamp. Domain design must account for the combination. + +### PRF has no ordering dependency + +- `vitaminc-prf` depends only on `vitaminc-aead-value` and applies its own + baseline and transforms. +- `vitaminc-aead-value` does no term normalisation. Its `canonical` module and + its dependency on `orderable-bytes` are removed. +- The value-derived PRF domains (cipherstash/vitaminc#380) haven't been + released: they landed on 2026-10-08, after vitaminc 0.5.1. So `vitaminc-prf` + can choose its own canonical bytes for dates, timestamps and decimals + without affecting any stored term. + +### `orderable-bytes` is frozen + +- `orderable-bytes` stays as it is for cipherstash-client's stored terms. It + is not part of the target design. +- There is no 0.3. cipherstash/ore.rs#103 (binary wrapper types) is closed. +- 0.2.0 is not yanked: ore-rs 0.8.4 depends on it and nothing new needs it + gone. + +## Open decision: exhaustive matching over `Value` + +Once `canonical.rs` leaves `vitaminc-aead-value`, `vitaminc-prf` and +`vitaminc-ore` must each handle every kind of `Value`, and adding a kind must +fail to compile until each one does. Two approaches are on the table. + +### Prerequisite for either approach: variants that exist in every build + +`Value::Date` and `Value::Timestamp` exist only with `vitaminc-aead-value`'s +`chrono` feature, and `Value::Decimal` only with its `rust_decimal` feature. +Because features unify across a build, an exhaustive match outside +`vitaminc-aead-value` breaks: + +1. `vitaminc-prf` matches `Value` exhaustively, with its `Date` arm behind its + own `chrono` feature. +2. In a build where another crate turns on `vitaminc-aead-value/chrono` and + `vitaminc-prf`'s `chrono` is off, `Value::Date` exists and `vitaminc-prf` + has no arm for it. `vitaminc-prf` fails to compile in that build. + +A visitor fails the same way: a required `visit_date` that exists only with +`chrono` is a method an implementor can't know whether to write. A crate can +check its own features, never a dependency's. Today's design avoids this only +because the match sits inside `vitaminc-aead-value`, which checks its own +features. + +The fix is to make every variant exist in every build, using vitaminc's own +representation, with conversions to and from chrono and rust_decimal behind +the features: + +| Variant | Representation | +|---|---| +| `Date` | days from the start of the common era, `i32` | +| `Timestamp` | Unix seconds `i64`, nanoseconds `u32` | +| `Decimal` | mantissa and scale | + +The ciphertext wire format already uses these payloads (tags `0x12`–`0x14` +are independent of chrono), so stored ciphertexts don't change. The Rust +types do, which is a breaking change to `Value`. + +### The two approaches + +**A: drop `#[non_exhaustive]`.** `vitaminc-prf`, `vitaminc-ore` and the +bindings match `Value` exhaustively in their own crates. + +**B: a visitor trait, keeping `#[non_exhaustive]`.** `vitaminc-aead-value` +defines a `ValueVisitor` with one required method per kind and does the +dispatch. Crates that must handle every kind implement it. Other crates keep +matching with a `_ =>` arm. + +| | A: drop `#[non_exhaustive]` | B: visitor, keep `#[non_exhaustive]` | +|---|---|---| +| Who gets exhaustiveness | Every crate that matches `Value` | Crates that implement the visitor | +| Adding a kind | Breaking for every crate that matches | Breaking for visitor implementors, additive for others | +| Code to maintain | Nothing new | One trait, one dispatch function, a method per kind in each implementor | +| Grouping | Each match writes out every integer width | The trait can offer grouped hooks, such as one integer method that the per-width methods default to | +| Changing `Value`'s representation | Breaks every match | Hidden behind the method signatures | +| Main risk | Low | A default on a per-kind method lets a new kind compile without being handled. Per-kind methods need no defaults; only grouping hooks may have them. | + +**Evidence:** +- Outside `vitaminc-aead-value`, `Value` is matched in `vitaminc-prf` (through + `canonical`), in `aead-napi`'s `convert.rs` and `scalar.rs`, and in the hmac + tests. +- `aead-napi` already has `_ => Err("unsupported value kind")` + (`convert.rs`). That is the silent refusal `#[non_exhaustive]` invites. +- ADR-0002 requires every binding to handle every kind, so the bindings want + exhaustiveness as well. +- Under A they get it automatically. Under B each binding opts in by + implementing the visitor. + +**Leaning:** B. The decision waits on a closer look at the `Value` enum's +design, which may change the representation in ways that favour one +approach. + +## Current state + +Arrows point from a crate to what it depends on. cipherstash-client and +`vitaminc-protected` are left out. + +```mermaid +flowchart TB + subgraph stack + SE[stack-encrypt] + end + subgraph suite["cipherstash-suite"] + CLLW["cllw-ore 0.5
typed impls, own encodings"] + end + subgraph vitaminc + PRF[vitaminc-prf] + AV["vitaminc-aead-value
Value, canonical module"] + end + subgraph orers["ore.rs"] + ORERS["ore-rs 0.8"] + OB["orderable-bytes"] + end + SE --> PRF + SE --> AV + SE --> CLLW + PRF -- "canonical feature" --> AV + AV -- "git pin, pre-0.2" --> OB + CLLW -- "0.1, chrono and decimal" --> OB + ORERS -- "0.2" --> OB +``` + +- stack-encrypt calls cllw-ore directly for order terms. There is no block + ORE path. +- Equality bytes for dates, timestamps and decimals come from order + encodings, through `vitaminc-aead-value`. +- `orderable-bytes` is reached three ways, at three different versions. + +## Target state + +```mermaid +flowchart TB + subgraph stack + EQI["Equality index
picks transforms"] + ORI["Ore / Ope index
picks scheme and transforms"] + end + subgraph schemes["scheme crates"] + ORERS["ore-rs
block ORE"] + CLLW["cllw-ore
CLLW ORE and OPE"] + end + subgraph vitaminc + PRF["vitaminc-prf
equality terms, own baseline and transforms"] + VORE["vitaminc-ore
order encodings, transforms,
OrderScheme, sealed OrderPlaintext"] + AV["vitaminc-aead-value
Value"] + end + EQI --> PRF + ORI --> VORE + ORI --> ORERS + ORI --> CLLW + ORERS -- "impl OrderScheme" --> VORE + CLLW -- "impl OrderScheme" --> VORE + PRF --> AV + VORE --> AV +``` + +- vitaminc depends on nothing in the other repositories. +- The scheme crates implement `vitaminc-ore`'s trait. The index crates choose + a concrete scheme and the transforms for each column. +- `orderable-bytes` doesn't appear. It stays frozen for cipherstash-client. + +## Changes to ADR-0002 + +These have been made in the ADR itself. + +| ADR-0002 says | Now | +|---|---| +| "Equality and order terms are computed from one canonical form per kind, and both layers use the same one" | Each kind has a semantic baseline; lossy transforms are chosen per domain, independently or locked | +| Canonical forms table: timestamp truncated to microseconds, text order folded | These become per-domain transforms, named in each domain's label | +| Order: "The plaintext layer is `orderable-bytes`, which … gains a variable-length encoding" | `vitaminc-ore` owns the order encodings; `orderable-bytes` is frozen | +| "A `Scheme` trait, with block ORE (over ore-rs), CLLW ORE and CLLW OPE (over cllw-ore)" | `OrderScheme` lives in `vitaminc-ore`; ore-rs and cllw-ore implement it | +| "cllw-ore keeps its typed impls … behind a cargo feature that cipherstash-client enables, and gains a bytes-level entry point" | cllw-ore's typed API is unchanged and not feature-gated; it gains an `OrderScheme` impl behind an optional `vitaminc` feature | +| Review: "Using the canonical order bytes as the PRF input … Dan Draper signs it off" | PRF no longer uses order bytes; the review item applies only to domains that lock equality and order together | +| `Value` and `ValueKind` become `#[non_exhaustive]` | Open: see "Open decision" | +| Release order: "ore.rs (`orderable-bytes`) and cllw-ore first, then vitaminc 0.6.0" | vitaminc first, then cllw-ore and ore-rs, then stack | + +## Steps + +### Phase 1: tidy up work in flight + +1. Close cipherstash/ore.rs#103. Don't yank `orderable-bytes` 0.2.0. +2. Cut cipherstash/cipherstash-suite#2308 down to its first two commits: + known-answer tests that pin every cllw-ore encrypt path, and wiping the + plaintext bit buffer in `encrypt_ope_bits`. It becomes a cllw-ore patch + release. The known-answer tests then protect cipherstash-client's terms + through the rest of this work. + +### Phase 2: vitaminc + +3. Settle the open decision. If the `Value` changes are accepted, make every + variant exist in every build. +4. `vitaminc-aead-value` and `vitaminc-prf`: + - add the chosen exhaustiveness mechanism; + - remove `canonical.rs` and the `orderable-bytes` dependency from + `vitaminc-aead-value`; + - move the equality baseline and transforms into `vitaminc-prf`, with each + transform named in its domain label. +5. New crate `vitaminc-ore` (rewrite cipherstash/vitaminc#374 to match): + - `OrderScheme` and the sealed `OrderPlaintext`; + - order encodings ported from `orderable-bytes`, kept in `Protected`, + pinned by golden vectors, and checked against `orderable-bytes` where + they match byte for byte; + - per-domain transforms: timestamp resolution, text folding with a pinned + Unicode version, room for prefix truncation; + - order-term derivation from `&Value`, generic over the scheme; + - a mock scheme for tests; + - docs that state the combined-leakage rule. +6. A conformance test across `vitaminc-prf` and `vitaminc-ore`: every kind is + handled or listed as an exception, and values equal under `Value` + semantics produce equal terms. +7. Release vitaminc 0.6.0. + +### Phase 3: scheme crates + +8. cllw-ore: implement `OrderScheme` behind an optional `vitaminc` feature, + with known-answer tests for the trait path. This is an additive release. +9. ore-rs: the same. Block ORE uses each kind's natural width; + cipherstash-client keeps its 8-byte padding through the existing API. + Block ORE for text waits for cipherstash/ore.rs#95 and goes on the + exceptions list. +10. Release both. + +### Phase 4: stack + +11. Index crates (#1063): each index spec carries its scheme and transforms. + The Equality index calls `vitaminc-prf`; the Ore and Ope index builds the + concrete scheme and calls `vitaminc-ore`. stack-encrypt drops its direct + use of cllw-ore. +12. EQL v4 domain labels name their transforms. + +### Later + +- Block ORE for text, once cipherstash/ore.rs#95 is released. +- Retire cllw-ore's typed impls and `orderable-bytes` once cipherstash-client + is retired. diff --git a/languages/golang/CHANGELOG.md b/languages/golang/CHANGELOG.md new file mode 100644 index 000000000..68a877461 --- /dev/null +++ b/languages/golang/CHANGELOG.md @@ -0,0 +1,190 @@ +# Changelog + +Changes to the Go module `github.com/cipherstash/stack/languages/golang`: +the SDK (`encrypt`), the generator (`stashgen`, `cmd/stashgen`) and the +credential package (`auth`). + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +The module is pre-1.0, so a minor version may break the API; every break is +listed under **Breaking**. Releases are tagged `languages/golang/vX.Y.Z`. + +## [Unreleased] + +### Breaking + +- **An `int8`, `int16`, `uint8` or `uint16` sealed field seals at its own + width; run stashgen again.** It used to seal as a 32-bit integer. Values + stored before do not decrypt under the regenerated declaration: the + engine checks the stored leaf's kind against the declared one and + refuses an int32 leaf for an `int16` field. Equality terms are the same + at every width, order terms are not. To keep reading stored values, + declare the field `int32` or `uint32` (or re-encrypt them). +- **Run stashgen again.** A field's query methods are now one per operator + the engine answers on it (`Eq`, `Lt`, `Lte`, `Gt`, `Gte`, `Between`, + `Matches`), in place of one per index (`Equality`, `Match`, `Ore`, `Ope`, + `Query`, `JSON`). A file generated by the alpha's stashgen no longer + compiles against this library. +- **`Cipher.Query` and `Cipher.Derive` are gone**, with the guest's + `se_query` export. Generated methods derive terms through + `Cipher.QueryTerm`, named by operator. +- **`gensupport.Field`** loses `Equality`, `Match`, `Ore`, `Ope`, `Query` and + `JSON`, and gains `Term` and `Range`; `gensupport.Matches` and + `gensupport.EQLMatches` derive match queries. +- **`stashgen.Engine.Check` returns the operators each field answers**: + `Check(ctx, d) (map[string][]stashgen.Operator, error)`. An engine + implemented outside this module (a wrapper, or a fake in a generator's + tests) changes its `Check` to return them; `enginetest.Static` shows the + table. +- **An `int8`, `uint8` or `byte` field takes no `ore` or `ope` index; drop + the index and run stashgen again.** With 256 values, an order term ranks + every row among 256, which a few known rows resolve to the values, the + leak a `bool`'s order term is already refused for. The engine refuses the + declaration as it refuses one on a `bool`, so stashgen stops at it. The + field keeps `equality` and `Eq`; a value that must order declares a wider + type. +- **Every ORE and OPE term changes bytes; encrypt indexed rows again.** + Order terms now encrypt through cllw-ore's vitaminc schemes, for every + type: integers of every width, floats, text and bytes. A term v0.1.0-alpha.1 + stored does not compare with one this release derives, and nothing + refuses it: `Lt`, `Between` or an `ORDER BY` over a column holding the + alpha's terms silently misses or misplaces rows. Re-derive a column's + order terms (encrypt its rows again) before querying it. + +### Added + +- **The date, timestamp and numeric EQL v4 types are producible** (#1130): + `Date` and `DateEq` from an `encrypt.Date`, `Timestamp` and `TimestampEq` + from a `time.Time`, and `Numeric` and `NumericEq` from an + `encrypt.Decimal`, with `encrypt_into=`. The `Eq` types answer `Eq`: + `NumericEq` normalises the scale (`1.5 = 1.50`) and `TimestampEq` compares + to the microsecond (truncated, never rounded), while each value opens as it was sealed. A + `time.Time` seals its instant and opens in UTC. Their `Ord`, `OrdOpe` and + `OrdOre` types stay unproducible until the engine derives order terms + for these kinds (#1236). +- **Every integer width, 128-bit integers, dates, timestamps and decimals + as sealed fields** (#1144). Each integer type seals at its own width + (`int8` as an int8, `uint16` as a uint16), so `record.Kind` and + `gensupport` gain `Int8`, `Int16`, `Int128`, `Uint8`, `Uint16`, + `Uint128`, `Decimal`, `Date` and `Timestamp`. A `time.Time` field seals + as a timestamp and opens in UTC. `encrypt.Date`, `Int128`, `Uint128` and + `Decimal` are vitaminc's `vcvalue` types, with `encrypt.DateOf`, + `ParseInt128`, `ParseUint128` and `ParseDecimal` (which refuses NaN and + the infinities with `ErrNonFiniteDecimal`). Equality applies to all of + them, ORE and OPE to every integer width but 8 bits. stashgen refuses an + order index on a decimal, date or timestamp, naming #1236. The value-kinds fixture + test opens a record the Rust lowering sealed and derives the same terms. +- **Any built-in Go integer field takes an integer EQL type** (ADR-0002): + an `int16` or a `uint16` into `Integer`, an `int` into `Bigint`. The + field is sealed at the type's width and opens back into the field's + type. A type wider than the EQL type (an `int64` into `Integer`) is range + checked value by value, and `Encrypt` refuses a value that does not fit + with `ErrEncoding`. `encrypt.Int128` and `Uint128` are not converted. +- **Range query types in `encrypt/eql`**, one per ordered family and order + index: `eql.IntegerRangeOreQuery`, `eql.IntegerRangeOpeQuery`, + `eql.TextRangeOreQuery`, `eql.TextRangeOpeQuery`, and the same for + `smallint`, `bigint`, `date`, `timestamp`, `numeric`, `real` and `double`. + Each holds both bounds of one `BETWEEN` (each bound the single-bound + domain's `ob` or `op` value, in a `{v, i, lo, hi}` envelope) as the JSON + value `eql_v4.between(column, $1::eql_v4.query__range_)` + takes, so a range binds as one parameter. The cast to the exact range + domain is required, or the statement fails with SQLSTATE 42725. Each type + implements `driver.Valuer`, `sql.Scanner` and JSON like the other query + types. They are not in + `eql.Types`, which lists stored types, and no generated method returns one + yet: the ordered EQL types are not producible. +- **`encrypt.MatchQuery` and `encrypt.EQLMatchQuery`**: what a `Matches` + method returns. Each carries the term or EQL query value to bind, + `Verify` (check decrypted rows, since a match can return false positives) + and `TargetLimit` (size the fetch). A `TextMatch` field gets `Matches`, + returning an `EQLMatchQuery`. Neither prints its query text, through any + fmt verb, reflection or slog. The zero value of either returns + `ErrEncoding` from every method, `Value` included, so it never binds as + NULL. +- **`encrypt.Range`**: what `Between` returns, the low and high bound's + terms. +- **`IntegerEq`, `BigintEq`, `RealEq` and `DoubleEq`**: an `int32`, `int64`, + `float32` or `float64` field tagged `encrypt_into=` one of them gets `Eq`. + `=` compares as PostgreSQL compares the plain types: an integer has one + term whatever its width, and for floats `-0.0 = 0.0` and `NaN = NaN`. +- **`Smallint` and `SmallintEq`**: an `int16` field (or any Go integer + type, range checked as for `Integer`) tagged `encrypt_into=` one of them + seals at 16 bits; `SmallintEq` gets `Eq`, with the same equality term as + an integer of the same value under the same column. +- **`IntegerOrd`, `SmallintOrd`, `BigintOrd`, `RealOrd`, `DoubleOrd`, + `TextOrd` and their `OrdOpe` twins**: a field tagged `encrypt_into=` one of them gets `Lt`, + `Lte`, `Gt` and `Gte`, and a text one `Eq` too. They store an OPE term + that orders by ADR-0002's order key: floats as PostgreSQL's `float8` + sorts them, text by its fold. No `Between` yet: it needs EQL v4's range + query domains. +- **`IntegerOrdOre`, `SmallintOrdOre`, `BigintOrdOre`, `RealOrdOre`, + `DoubleOrdOre`, `TextOrdOre` and `TextSearchOre`**: a field tagged `encrypt_into=` one of + them gets `Lt`, `Lte`, `Gt` and `Gte` (and on text `Eq`, and on + `TextSearchOre` `Matches`), as the `Ord` types do. They store a block ORE + term (ore-rs's 6-bit schemes), compared by EQL v4's comparator; each + write and each query carries a fresh nonce, so two terms of one value + never share bytes. Text whose fold is longer than 1023 bytes or ends in + NUL is refused, stored or queried. The engine reports their ordering + operators through the index `ore_block` (`record.OreBlock`, + `stashgen.IndexOreBlock`), which a struct reaches only by naming one of + these types: it is not a tag word. +- **`TextSearch`**: a `string` field tagged `encrypt_into=TextSearch` gets + `Eq`, `Lt`, `Lte`, `Gt`, `Gte` and `Matches`, one column for all of them. + The stored value keeps all three terms. Text shorter than three + characters, an empty string included, is stored with an empty match + filter, where `TextMatch` refuses it with `ErrTerm`: `Eq`, `Lt`, `Lte`, + `Gt` and `Gte` find the row, and no `Matches` query does. Each query + carries only the term its operator reads, so `Eq` and `Lt` take a + two-letter value and `Matches` refuses one (`ErrTerm`). Needs the EQL v4 + SQL from this release, whose `eql_v4.query_text_search` accepts one term. +- **Ordering methods on text.** A string field with `index=ore` or + `index=ope` gets `Lt`, `Lte`, `Gt`, `Gte` and `Between`. Text orders by its + folded form (ADR-0002: case and accents folded, under Unicode 16), so + `Café` and `cafe` tie and `alice` orders before `Zoe`. +- **Ordering methods on floats.** A `float32` or `float64` field with + `index=ore` or `index=ope` gets `Lt`, `Lte`, `Gt`, `Gte` and `Between`. + Floats order as PostgreSQL's `float8` does (ADR-0002): `-0.0` ties `0.0`, + and every NaN is one value, above `+Inf`. A `float32` orders as the + `float64` it widens to. +- **stashgen notices an index no generated method queries**: an `ore` or + `ope` index on bytes (the engine answers the orderings over integers, + floats and text only), or the second of two order indexes on a field. + The index is still stored; the notice is on stderr and in the file. + +### Changed + +- **A call whose context ends fails on its own and leaves the client + open.** A cancelled or timed-out call used to close the `encrypt.Client`, + or the `auth` profile behind its strategy, for every later caller, and a + request whose deadline passed while it waited for the client closed it + before running. Now a call waiting for the client leaves the queue with + the context's error; a ZeroKMS request in flight is cancelled, and the + error matches both the context's error and `ErrTransport`; a device + session's refresh in flight finishes in the background, for up to 10 + seconds, so a rotated refresh token is never lost; and the client stays + open. A client or profile still closes itself (`ErrState`) when the + guest traps, or when a call runs on for 30 seconds after its context + ended. +- **A call no longer holds the client across a ZeroKMS round trip.** The + guest returns each ZeroKMS request to the client instead of performing it + through a blocking host import (`transport_send` is gone; the guest's one + import is `token_get`). The client performs the request with its lock + free, so other callers run meanwhile, then resumes the call. A call whose + context ends between those steps returns the context's error, matching + `ErrTransport` too, and the guest drops the paused call, wiping what it + held; the client stays open. + The token is fetched once per call, before its first step and outside + the lock. That includes a call the guest answers from its cache (a query + term on a loaded keyset, a cached keyset lookup), which never asked for + a token before, so a token source sees one request per such call; one + that fails is still reported only when the guest needs the token. At + most 64 calls that can reach ZeroKMS are in progress at once (the + guest's cap on paused calls); one past that waits for another to finish, + as it waited for the client before. This is the step loop an + edge runtime's host needs, where `fetch` cannot block. +- **stashgen refuses a match index on an index-only field** + (`stash:"name,index=match"` with no `encrypt`). A match query's rows are + checked by decrypting them, and such a field stores no ciphertext. +- A query with a nil cipher returns `ErrEncoding` from generated code as from + the library. +- stashgen's refusal of an EQL type it cannot produce names the issue that + tracks the blocker, in place of telling you to wait for vitaminc 0.6. diff --git a/languages/golang/auth/cancel_test.go b/languages/golang/auth/cancel_test.go new file mode 100644 index 000000000..1dda26166 --- /dev/null +++ b/languages/golang/auth/cancel_test.go @@ -0,0 +1,303 @@ +package auth + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/cipherstash/stack/languages/golang/internal/guest" + "github.com/tetratelabs/wazero/api" +) + +// A profile serves every strategy, and through them every encrypt client, +// in a process: a request cancelled while its token is fetched must fail +// on its own and leave the profile open for the next. + +// tokenServer answers the access-key exchange, holding each request until +// the client gives up while hold is set. +func tokenServer(t *testing.T, hold *atomic.Bool) *httptest.Server { + t.Helper() + var jwt string + // The server notices a client that hung up a moment after the client + // has gone: it reads the closed connection on a goroutine of its own. + // So the test's end waits for each held request to see its + // cancellation, for a bounded time, before the server is closed (which + // would end the request's context itself and hide a request the client + // never cancelled). + var held sync.WaitGroup + done := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if hold.Load() { + held.Add(1) + defer held.Done() + // The server notices a client that hung up only once the body + // is read. + _, _ = io.Copy(io.Discard, r.Body) + select { + case <-r.Context().Done(): + return + case <-done: + } + select { + case <-r.Context().Done(): + case <-time.After(5 * time.Second): + t.Error("a held request was never cancelled") + } + return + } + fmt.Fprintf(w, `{"accessToken":%q,"expiry":%d}`, jwt, time.Now().Add(time.Hour).Unix()) + })) + t.Cleanup(server.Close) + t.Cleanup(func() { + close(done) + held.Wait() + }) + jwt = testJWT(t, server.URL) + return server +} + +func TestATokenFetchCancelledFailsOnlyItself(t *testing.T) { + guestOrSkip(t) + var hold atomic.Bool + server := tokenServer(t, &hold) + profile, err := Open(context.Background(), t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer profile.Close() + strategy, err := profile.AccessKey(context.Background(), testCRN, "CSAKtestKeyId.testKeySecret", WithBaseURL(server.URL)) + if err != nil { + t.Fatal(err) + } + defer strategy.Close() + + hold.Store(true) + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + if _, err := strategy.Token(ctx); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("cancelled Token: %v, want the deadline", err) + } + hold.Store(false) + // The next Token waits for the abandoned one to finish. + if token, err := strategy.Token(context.Background()); err != nil || token == "" { + t.Fatalf("the next Token: %q, %v", token, err) + } + if profile.root.closed || profile.root.inst.module.IsClosed() { + t.Fatal("the profile was closed") + } +} + +// An access-key exchange rotates nothing, so it ends with its caller: the +// held request is cancelled at the caller's deadline, not run on for the +// refresh grace, and Close does not wait for it. +func TestAnAccessKeyExchangeEndsWithItsCaller(t *testing.T) { + guestOrSkip(t) + var hold atomic.Bool + server := tokenServer(t, &hold) + profile, err := Open(context.Background(), t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer profile.Close() + strategy, err := profile.AccessKey(context.Background(), testCRN, "CSAKtestKeyId.testKeySecret", WithBaseURL(server.URL)) + if err != nil { + t.Fatal(err) + } + + hold.Store(true) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond) + defer cancel() + if _, err := strategy.Token(ctx); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("cancelled Token: %v, want the deadline", err) + } + // The grace is 10 seconds; a Close that waited for it would take that. + start := time.Now() + if err := strategy.Close(); err != nil { + t.Fatal(err) + } + if waited := time.Since(start); waited > 2*time.Second { + t.Fatalf("Close took %v: the exchange ran on past its caller", waited) + } +} + +// A device session's refresh is the request that rotates a credential: the +// service answers with a new refresh token and retires the old one. A +// caller that gives up returns at once, but the refresh runs on, and what +// it brings is saved. The next Token is served from it with no second +// request, and the profile on disk holds the rotated refresh token. +func TestARefreshAnsweredAfterItsCallerGaveUpIsKept(t *testing.T) { + guestOrSkip(t) + dir, _ := expiredDeviceProfile(t) + var requests atomic.Int32 + answered := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if requests.Add(1) > 1 { + // The old refresh token is retired once the service answers. + w.WriteHeader(http.StatusBadRequest) + fmt.Fprint(w, `{"error":"invalid_grant"}`) + return + } + _, _ = io.Copy(io.Discard, r.Body) + time.Sleep(150 * time.Millisecond) + fmt.Fprint(w, `{"access_token":"fresh","token_type":"Bearer","expires_in":3600,"refresh_token":"refresh-2"}`) + close(answered) + })) + t.Cleanup(server.Close) + profile, err := Open(context.Background(), dir) + if err != nil { + t.Fatal(err) + } + defer profile.Close() + ws, err := profile.WorkspaceStore(context.Background(), wsA) + if err != nil { + t.Fatal(err) + } + strategy, err := ws.DeviceSession(context.Background(), WithBaseURL(server.URL)) + if err != nil { + t.Fatal(err) + } + defer strategy.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond) + defer cancel() + if _, err := strategy.Token(ctx); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("cancelled Token: %v, want the deadline", err) + } + select { + case <-answered: + t.Fatal("the cancelled Token waited for the service's answer") + default: + } + if token, err := strategy.Token(context.Background()); err != nil || token != "fresh" { + t.Fatalf("the next Token: %q, %v, want the refreshed token", token, err) + } + if n := requests.Load(); n != 1 { + t.Fatalf("%d refresh requests, want 1: the late answer was dropped", n) + } + data, err := os.ReadFile(filepath.Join(dir, "workspaces", wsA, "auth.json")) + if err != nil { + t.Fatal(err) + } + var stored map[string]any + if err := json.Unmarshal(data, &stored); err != nil { + t.Fatal(err) + } + if stored["refresh_token"] != "refresh-2" { + t.Fatalf("stored refresh token = %v, want the rotated one", stored["refresh_token"]) + } +} + +// A refresh runs on after its caller gives up, so the guest call under it +// can succeed after the caller's context ended. It still fails with the +// context's error; what the refresh brought is kept for the next call. +func TestARefreshThatSucceedsAfterItsContextEndsFailsWithIt(t *testing.T) { + guestOrSkip(t) + dir, _ := expiredDeviceProfile(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + cancel() + fmt.Fprint(w, `{"access_token":"fresh","token_type":"Bearer","expires_in":3600,"refresh_token":"refresh-2"}`) + })) + t.Cleanup(server.Close) + profile, err := Open(context.Background(), dir) + if err != nil { + t.Fatal(err) + } + defer profile.Close() + ws, err := profile.WorkspaceStore(context.Background(), wsA) + if err != nil { + t.Fatal(err) + } + strategy, err := ws.DeviceSession(context.Background(), WithBaseURL(server.URL)) + if err != nil { + t.Fatal(err) + } + defer strategy.Close() + + out, err := strategy.store.callArgs(withRefresh(ctx), func(i *instance) api.Function { return i.authRefresh }, guest.BufArg([]byte(strategy.handle))) + if !errors.Is(err, context.Canceled) || out != nil { + t.Fatalf("refresh: %q, %v, want no token and context.Canceled", out, err) + } + if token, err := strategy.Token(context.Background()); err != nil || token != "fresh" { + t.Fatalf("the next Token: %q, %v, want the refreshed token", token, err) + } +} + +func TestATokenRequestWhoseContextEndsInTheQueueNeverReachesTheGuest(t *testing.T) { + guestOrSkip(t) + var hold atomic.Bool + server := tokenServer(t, &hold) + profile, err := Open(context.Background(), t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer profile.Close() + strategy, err := profile.AccessKey(context.Background(), testCRN, "CSAKtestKeyId.testKeySecret", WithBaseURL(server.URL)) + if err != nil { + t.Fatal(err) + } + defer strategy.Close() + + for name, lock := range map[string]interface { + Acquire(context.Context) error + Unlock() + }{"the strategy's": &strategy.mu, "the profile's": &profile.root.mu} { + t.Run(name, func(t *testing.T) { + if err := lock.Acquire(context.Background()); err != nil { + t.Fatal(err) + } + released := make(chan struct{}) + time.AfterFunc(200*time.Millisecond, func() { lock.Unlock(); close(released) }) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + if _, err := strategy.Token(ctx); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("queued Token: %v, want the deadline", err) + } + <-released + }) + } + if token, err := strategy.Token(context.Background()); err != nil || token == "" { + t.Fatalf("the next Token: %q, %v", token, err) + } +} + +// The backstop: a call still running long after its caller gave up closes +// the profile. The caller has returned by then; the next call is ErrState. +func TestATokenFetchThatRunsOnAfterItsContextEndsClosesTheProfile(t *testing.T) { + guestOrSkip(t) + profile, err := Open(context.Background(), t.TempDir(), WithRoundTripper(roundTripFunc(func(*http.Request) (*http.Response, error) { + // A RoundTripper that ignores the request's context. + time.Sleep(300 * time.Millisecond) + return nil, errors.New("too late") + }))) + if err != nil { + t.Fatal(err) + } + defer profile.Close() + strategy, err := profile.AccessKey(context.Background(), testCRN, "CSAKtestKeyId.testKeySecret", WithBaseURL("https://cts.example.com")) + if err != nil { + t.Fatal(err) + } + defer strategy.Close() + profile.root.stuck = 20 * time.Millisecond + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + if _, err := strategy.Token(ctx); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("stuck Token: %v, want the deadline", err) + } + if _, err := strategy.Token(context.Background()); !errors.Is(err, ErrState) { + t.Fatalf("a Token after the close: %v, want ErrState", err) + } +} diff --git a/languages/golang/auth/doc.go b/languages/golang/auth/doc.go index b0bf4f16d..d61e99371 100644 --- a/languages/golang/auth/doc.go +++ b/languages/golang/auth/doc.go @@ -39,6 +39,18 @@ // [OAuth2TokenSource] adapts an existing golang.org/x/oauth2.TokenSource // into the OIDC provider interface. // +// A call whose context ends, [Strategy.Token] included, fails with the +// context's error and leaves the store and its strategies open: a request +// cancelled while its token is exchanged or refreshed does not stop the +// encrypt clients that read tokens from the same store. A request to the +// token service is cancelled with its caller, except a device session's +// refresh: the service rotates the refresh token when it answers one, so +// the caller returns at once while the refresh runs on, for up to 10 +// seconds, and the store keeps what it sends back for the next call. +// [Strategy.Close] waits for such a refresh. The store closes itself, and +// every later call is [ErrState], only when its guest traps or a call runs +// on for 30 seconds after its context ended. +// // # Why a second guest // // The crypto guest behind encrypt has no filesystem and no diff --git a/languages/golang/auth/guest.go b/languages/golang/auth/guest.go index 5c17ef261..69e395a3a 100644 --- a/languages/golang/auth/guest.go +++ b/languages/golang/auth/guest.go @@ -111,6 +111,9 @@ func newInstance(ctx context.Context, wasm []byte, hostDir string, policy guest. return nil, fmt.Errorf("%w: %s: %w", ErrNoProfile, hostDir, err) } } + // A call never runs under its caller's context (see guest.Detach), so + // this closes the module only for a call that ran on long after its + // caller gave up: ProfileStore.callArgs. config := wazero.NewRuntimeConfig(). WithCompilationCache(compilationCache()). WithCloseOnContextDone(true) @@ -183,8 +186,8 @@ func newInstance(ctx context.Context, wasm []byte, hostDir string, policy guest. // release runs the guest's shutdown — every buffer it still holds wiped — // and closes the runtime, which frees the linear memory through the -// allocator's wipe, then the directory handle the mount holds. A module an -// interrupted call or a trap already closed cannot run sa_shutdown; the +// allocator's wipe, then the directory handle the mount holds. A module a +// stuck call or a trap already closed cannot run sa_shutdown; the // runtime close still wipes and frees its memory. func (inst *instance) release() error { ctx := context.Background() diff --git a/languages/golang/auth/guest/Cargo.lock b/languages/golang/auth/guest/Cargo.lock index 6b36c6d82..ce1a5e7eb 100644 --- a/languages/golang/auth/guest/Cargo.lock +++ b/languages/golang/auth/guest/Cargo.lock @@ -419,7 +419,7 @@ dependencies = [ "url", "utoipa", "uuid", - "vitaminc", + "vitaminc 0.5.1", ] [[package]] @@ -1365,6 +1365,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro-error-attr2" version = "2.0.0" @@ -1838,8 +1847,8 @@ dependencies = [ "tracing", "url", "uuid", - "vitaminc", - "vitaminc-protected", + "vitaminc 0.6.1", + "vitaminc-protected 0.6.1", "web-time", "zeroize", "zerokms-protocol", @@ -1858,8 +1867,7 @@ dependencies = [ "stack-profile", "tempfile", "url", - "vitaminc-aead-value", - "vitaminc-protected", + "vitaminc-protected 0.6.1", "zeroize", ] @@ -1870,8 +1878,9 @@ dependencies = [ "miette", "serde_json", "thiserror 1.0.69", - "vitaminc-aead-value", - "vitaminc-protected", + "vitaminc-aead 0.6.1", + "vitaminc-aead-frame", + "vitaminc-protected 0.6.1", "zeroize", ] @@ -2066,6 +2075,36 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "toml_datetime" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b86d767906c6c42421dcba507eb9d203e779497710a47782a224bb871653053" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.17+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3641d5bbb5349a79e1020a242d251efbc546ad8048d133958323ce9c40a9c9c" +dependencies = [ + "indexmap", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.1.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baa693a8032d7e1cada7d0041e96126df243179ff061456783ac7f12bda4744c" +dependencies = [ + "winnow", +] + [[package]] name = "tracing" version = "0.1.44" @@ -2250,12 +2289,26 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35042fb7a81d72dc43c53fec835e61027f18b4d01cc0f7e12d6c6a6ffbf3bfb9" dependencies = [ - "vitaminc-aead", - "vitaminc-context", - "vitaminc-encrypt", - "vitaminc-protected", - "vitaminc-random", - "vitaminc-traits", + "vitaminc-aead 0.5.1", + "vitaminc-context 0.5.1", + "vitaminc-encrypt 0.5.1", + "vitaminc-protected 0.5.1", + "vitaminc-random 0.5.1", + "vitaminc-traits 0.5.1", +] + +[[package]] +name = "vitaminc" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac5c15b6c25efcb9a6111f7108ab8101947a7bf7e9af3ff6d6a73358a1ace0da" +dependencies = [ + "vitaminc-aead 0.6.1", + "vitaminc-context 0.6.1", + "vitaminc-encrypt 0.6.1", + "vitaminc-protected 0.6.1", + "vitaminc-random 0.6.1", + "vitaminc-traits 0.6.1", ] [[package]] @@ -2266,10 +2319,27 @@ checksum = "62cacd4dae485e98cfe5407e919187a8468dd933ea4234be5115b7d13a30f3c6" dependencies = [ "bytes", "serde", - "vitaminc-aead-derive", - "vitaminc-context", - "vitaminc-protected", - "vitaminc-random", + "vitaminc-aead-derive 0.5.1", + "vitaminc-context 0.5.1", + "vitaminc-protected 0.5.1", + "vitaminc-random 0.5.1", + "zeroize", +] + +[[package]] +name = "vitaminc-aead" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71b8c5c4ced92545dce5fd7cf04581d08210def174cc81475cf227fa2506c5b7" +dependencies = [ + "bytes", + "futures-util", + "serde", + "thiserror 2.0.20", + "vitaminc-aead-derive 0.6.1", + "vitaminc-context 0.6.1", + "vitaminc-protected 0.6.1", + "vitaminc-random 0.6.1", "zeroize", ] @@ -2285,14 +2355,23 @@ dependencies = [ ] [[package]] -name = "vitaminc-aead-value" -version = "0.5.1" +name = "vitaminc-aead-derive" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d78cdadbc6374c3bd7c19fb4d99abca693b96953b18fef8c608efbe092cf3ea" +checksum = "4c546497a972d0f0498f2f8d5f537c514bad82df8662951c63d0c80023808900" dependencies = [ - "vitaminc-aead", - "vitaminc-protected", - "zeroize", + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "vitaminc-aead-frame" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e35992a269bcaa5b201f1938e54bfbf167bfa6fe975d83cbdbdb54c2a363c5ef" +dependencies = [ + "vitaminc-aead 0.6.1", ] [[package]] @@ -2302,7 +2381,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7df52501fe33cdb3ee9e155b5242fac184c037d147e58c96e5143ed011252a8" dependencies = [ "mutants", - "vitaminc-protected", + "vitaminc-protected 0.5.1", +] + +[[package]] +name = "vitaminc-context" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4c4bc2bd9811253eb427657565a45bd4826e2379e93ee1c4d00c808a769882" +dependencies = [ + "mutants", + "vitaminc-protected 0.6.1", ] [[package]] @@ -2313,9 +2402,23 @@ checksum = "9fbd0e1748055e381e02033e08af385de879c4dcf32e0f6739ac0c0cf0c6f13d" dependencies = [ "aes-gcm", "aws-lc-rs", - "vitaminc-aead", - "vitaminc-protected", - "vitaminc-random", + "vitaminc-aead 0.5.1", + "vitaminc-protected 0.5.1", + "vitaminc-random 0.5.1", + "zeroize", +] + +[[package]] +name = "vitaminc-encrypt" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0923694602855b33a296d9af473e282e8096c1cfe1b30b8695d3b6651717fc5" +dependencies = [ + "aes-gcm", + "aws-lc-rs", + "vitaminc-aead 0.6.1", + "vitaminc-protected 0.6.1", + "vitaminc-random 0.6.1", "zeroize", ] @@ -2332,7 +2435,24 @@ dependencies = [ "serde_bytes", "subtle", "thiserror 2.0.20", - "vitaminc-protected-derive", + "vitaminc-protected-derive 0.5.1", + "zeroize", +] + +[[package]] +name = "vitaminc-protected" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1bb76dd473f5aa89b844f92018ebe3041deb8e5390e7061cca4283d959ff855" +dependencies = [ + "bitvec", + "digest 0.11.3", + "libc", + "serde", + "serde_bytes", + "subtle", + "thiserror 2.0.20", + "vitaminc-protected-derive 0.6.1", "zeroize", ] @@ -2347,6 +2467,18 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "vitaminc-protected-derive" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7478ed160c019059746e75fc613563dcdcd5266d155a34601d29377a1c4afc3" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 3.0.6", +] + [[package]] name = "vitaminc-random" version = "0.5.1" @@ -2357,8 +2489,23 @@ dependencies = [ "getrandom 0.4.3", "rand 0.10.3", "thiserror 2.0.20", - "vitaminc-protected", - "vitaminc-random-derives", + "vitaminc-protected 0.5.1", + "vitaminc-random-derives 0.5.1", + "zeroize", +] + +[[package]] +name = "vitaminc-random" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b477af7c01c3d9ef2647e46952ad3df7f246b3f806b1d4009ee83beecec613a" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand 0.10.3", + "thiserror 2.0.20", + "vitaminc-protected 0.6.1", + "vitaminc-random-derives 0.6.1", "zeroize", ] @@ -2373,6 +2520,17 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "vitaminc-random-derives" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab5a0de8da428c6789dc067a578a0462e411099721d616736cf29e1622230c35" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + [[package]] name = "vitaminc-traits" version = "0.5.1" @@ -2384,8 +2542,24 @@ dependencies = [ "rmp-serde", "serde", "thiserror 2.0.20", - "vitaminc-protected", - "vitaminc-random", + "vitaminc-protected 0.5.1", + "vitaminc-random 0.5.1", + "zeroize", +] + +[[package]] +name = "vitaminc-traits" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a900f0edcacd49c8557ab54722175869fec66926054a6fbdc53a05069db3c922" +dependencies = [ + "anyhow", + "bytes", + "rmp-serde", + "serde", + "thiserror 2.0.20", + "vitaminc-protected 0.6.1", + "vitaminc-random 0.6.1", "zeroize", ] @@ -2622,6 +2796,15 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] + [[package]] name = "wit-bindgen" version = "0.57.1" diff --git a/languages/golang/auth/guest/Cargo.toml b/languages/golang/auth/guest/Cargo.toml index d3928e995..df3ce1563 100644 --- a/languages/golang/auth/guest/Cargo.toml +++ b/languages/golang/auth/guest/Cargo.toml @@ -41,10 +41,6 @@ stack-guest-abi = { path = "../../../../packages/stack-guest-abi" } # hostname and the process id are native-only there, and creating a device # identity is CLI territory. stack-profile = { path = "../../../../packages/stack-profile" } -# The FFI codec for the structured results (a list of workspace ids, the -# fields of a token): the one codec every binding carries. Same vitaminc -# version the crypto guest builds against. -vitaminc-aead-value = "0.5.0" # The deserialized form of `secretkey.json` is wiped when it drops. zeroize = { version = "1", features = ["derive"] } @@ -52,7 +48,7 @@ zeroize = { version = "1", features = ["derive"] } serde_json = "1" tempfile = "3" # `Controlled::risky_ref`, to read a codec value back in the tests. -vitaminc-protected = "0.5.0" +vitaminc-protected = "0.6.0" [profile.release] # Smaller .wasm; the guest is IO-bound on the FFI copy and the file reads, diff --git a/languages/golang/auth/guest/src/lib.rs b/languages/golang/auth/guest/src/lib.rs index fb9986d4c..a95e5c511 100644 --- a/languages/golang/auth/guest/src/lib.rs +++ b/languages/golang/auth/guest/src/lib.rs @@ -58,8 +58,8 @@ //! //! Profile inputs are UTF-8 strings: the store directory (a guest path under //! the mount), a workspace id, a filename. Outputs are either a UTF-8 string -//! (an id, a path) or a value in vitaminc's FFI codec -//! (`vitaminc_aead_value::transport`): a list of ids, or the fields of +//! (an id, a path) or a vitaminc transport frame +//! ([`HostValue`](stack_guest_abi::value::HostValue)): a list of ids, or the fields of //! `secretkey.json`, `auth.json` or `device.json` as an object. The client //! key in `secretkey.json` crosses as the text the file holds; the Go side //! wraps it in its opaque `ClientKey` and wipes its transport copy. A diff --git a/languages/golang/auth/guest/src/ops.rs b/languages/golang/auth/guest/src/ops.rs index 6a32bfa5d..19e80580a 100644 --- a/languages/golang/auth/guest/src/ops.rs +++ b/languages/golang/auth/guest/src/ops.rs @@ -30,8 +30,8 @@ use serde::Deserialize; use stack_auth::Token; +use stack_guest_abi::value::HostValue; use stack_profile::{DeviceIdentity, ProfileStore}; -use vitaminc_aead_value::{transport as codec, FfiValue}; use zeroize::{Zeroize, ZeroizeOnDrop}; use stack_guest_abi::last_error::{internal, malformed}; @@ -76,18 +76,18 @@ fn text(bytes: &[u8]) -> Result<&str, u32> { std::str::from_utf8(bytes).map_err(|_| malformed("an input is not UTF-8")) } -fn string(value: impl Into) -> FfiValue { - FfiValue::String(value.into().into()) +fn string(value: impl Into) -> HostValue { + HostValue::text(value) } -fn optional(value: Option<&str>) -> FfiValue { - value.map_or(FfiValue::Null, string) +fn optional(value: Option<&str>) -> HostValue { + value.map_or(HostValue::Null, string) } -fn encode(value: FfiValue) -> Result, u32> { - let mut out = Vec::new(); - codec::encode_value(value, &mut out).map_err(|_| internal("an output did not encode"))?; - Ok(out) +fn encode(value: HostValue) -> Result, u32> { + value + .encode() + .map_err(|_| internal("an output did not encode")) } /// The current workspace id, as text. @@ -122,7 +122,7 @@ pub fn list_workspaces(dir: &[u8]) -> Result, u32> { let ids = store(dir)? .list_workspaces() .map_err(|e| fail_profile(&e))?; - encode(FfiValue::Array(ids.into_iter().map(string).collect())) + encode(HostValue::List(ids.into_iter().map(string).collect())) } /// The directory of the store scoped to workspace `id`, as text: what @@ -165,11 +165,11 @@ pub fn secret_key(dir: &[u8]) -> Result, u32> { // fields cannot be moved out of it directly. let client_id = std::mem::take(&mut file.client_id); let client_key = std::mem::take(&mut file.client_key); - encode(FfiValue::Object(vec![ + encode(HostValue::Map(vec![ ("client_id".to_string(), string(client_id)), ( "client_key".to_string(), - FfiValue::Bytes(client_key.into_bytes().into()), + HostValue::Bytes(client_key.into_bytes().into()), ), ])) } @@ -184,7 +184,7 @@ pub fn token(dir: &[u8]) -> Result, u32> { let token: Token = store(dir)? .load(AUTH_FILENAME) .map_err(|e| fail_profile(&e))?; - encode(FfiValue::Object(vec![ + encode(HostValue::Map(vec![ ( "access_token".to_string(), string(token.access_token().as_str()), @@ -192,7 +192,7 @@ pub fn token(dir: &[u8]) -> Result, u32> { ("token_type".to_string(), string(token.token_type())), ( "expires_at".to_string(), - FfiValue::UInt64(token.expires_at()), + HostValue::UInt(token.expires_at()), ), ("region".to_string(), optional(token.region())), ("client_id".to_string(), optional(token.client_id())), @@ -214,7 +214,7 @@ pub fn has_token(dir: &[u8]) -> Result, u32> { /// `{device_instance_id, device_name}`. Creating one is the CLI's. pub fn device_identity(dir: &[u8]) -> Result, u32> { let identity = DeviceIdentity::load(&store(dir)?).map_err(|e| fail_profile(&e))?; - encode(FfiValue::Object(vec![ + encode(HostValue::Map(vec![ ( "device_instance_id".to_string(), string(identity.device_instance_id.to_string()), @@ -235,12 +235,12 @@ mod tests { t.path().to_str().unwrap().as_bytes().to_vec() } - fn decode(bytes: &[u8]) -> FfiValue { - codec::decode_value(&mut codec::Reader::new(bytes)).unwrap() + fn decode(bytes: &[u8]) -> HostValue { + HostValue::decode(bytes).unwrap() } - fn field<'a>(value: &'a FfiValue, key: &str) -> &'a FfiValue { - let FfiValue::Object(entries) = value else { + fn field<'a>(value: &'a HostValue, key: &str) -> &'a HostValue { + let HostValue::Map(entries) = value else { panic!("not an object"); }; entries @@ -250,16 +250,16 @@ mod tests { .unwrap_or_else(|| panic!("no field {key}")) } - fn as_text(value: &FfiValue) -> String { - let FfiValue::String(s) = value else { + fn as_text(value: &HostValue) -> String { + let HostValue::Text(s) = value else { panic!("not a string"); }; - String::from_utf8(s.risky_ref().to_vec()).unwrap() + s.to_string() } - fn as_bytes(value: &FfiValue) -> Vec { + fn as_bytes(value: &HostValue) -> Vec { use vitaminc_protected::Controlled; - let FfiValue::Bytes(b) = value else { + let HostValue::Bytes(b) = value else { panic!("not bytes"); }; b.risky_ref().to_vec() @@ -314,7 +314,7 @@ mod tests { assert_eq!(current_workspace(&d).unwrap(), WS_A.as_bytes()); let listed = decode(&list_workspaces(&d).unwrap()); - let FfiValue::Array(items) = listed else { + let HostValue::List(items) = listed else { panic!("not an array"); }; let ids: Vec = items.iter().map(as_text).collect(); @@ -411,11 +411,11 @@ mod tests { assert_eq!(as_text(field(&tok, "token_type")), "Bearer"); assert!(matches!( field(&tok, "expires_at"), - FfiValue::UInt64(1_800_000_000) + HostValue::UInt(1_800_000_000) )); assert_eq!(as_text(field(&tok, "region")), "ap-southeast-2"); - assert!(matches!(field(&tok, "client_id"), FfiValue::Null)); - let FfiValue::Object(entries) = tok else { + assert!(matches!(field(&tok, "client_id"), HostValue::Null)); + let HostValue::Map(entries) = tok else { panic!("not an object"); }; assert!( diff --git a/languages/golang/auth/guest/tests/leak.rs b/languages/golang/auth/guest/tests/leak.rs index 6d08d4cf5..ae0083d80 100644 --- a/languages/golang/auth/guest/tests/leak.rs +++ b/languages/golang/auth/guest/tests/leak.rs @@ -25,7 +25,7 @@ use stack_auth::AuthError; use stack_auth_guest::ops; use stack_auth_guest::status::fail_auth; use stack_guest_abi::call; -use vitaminc_aead_value::{transport as codec, FfiValue}; +use stack_guest_abi::value::HostValue; const ACCESS_TOKEN: &str = "leak-marker-access-token"; const ACCESS_KEY: &str = "leak-marker-access-key"; @@ -43,18 +43,18 @@ fn failure( what: &str, expect: &[&str], step: impl FnOnce() -> Result, -) -> FfiValue { +) -> HostValue { let _ = call::run(step).expect_err(what); let bytes = call::take_last_error().unwrap_or_else(|| panic!("{what}: no error recorded")); - let error = codec::decode_value(&mut codec::Reader::new(&bytes)).expect("an error decodes"); + let error = HostValue::decode(&bytes).expect("an error decodes"); assert_eq!(codes(&error), expect, "{what}: the codes it reaches"); error } /// An error's code, then its causes' codes, in order. -fn codes(value: &FfiValue) -> Vec { +fn codes(value: &HostValue) -> Vec { let mut out: Vec = text(get(value, "code")).into_iter().collect(); - if let Some(FfiValue::Array(causes)) = get(value, "causes") { + if let Some(HostValue::List(causes)) = get(value, "causes") { out.extend(causes.iter().filter_map(|cause| text(get(cause, "code")))); } out @@ -71,19 +71,19 @@ fn check(text: &[u8], at: &str, found: &mut Vec) { } } -fn leaks(value: &FfiValue, path: &str, found: &mut Vec) { +fn leaks(value: &HostValue, path: &str, found: &mut Vec) { match value { - FfiValue::String(text) => check(text.risky_ref(), path, found), - FfiValue::Bytes(bytes) => { + HostValue::Text(text) => check(text.as_bytes(), path, found), + HostValue::Bytes(bytes) => { use vitaminc_protected::Controlled; check(bytes.risky_ref(), path, found); } - FfiValue::Array(items) => { + HostValue::List(items) => { for (at, item) in items.iter().enumerate() { leaks(item, &format!("{path}[{at}]"), found); } } - FfiValue::Object(entries) => { + HostValue::Map(entries) => { for (key, item) in entries { check(key.as_bytes(), &format!("{path} key"), found); leaks(item, &format!("{path}.{key}"), found); @@ -93,16 +93,16 @@ fn leaks(value: &FfiValue, path: &str, found: &mut Vec) { } } -fn get<'a>(value: &'a FfiValue, key: &str) -> Option<&'a FfiValue> { +fn get<'a>(value: &'a HostValue, key: &str) -> Option<&'a HostValue> { match value { - FfiValue::Object(entries) => entries.iter().find(|(k, _)| k == key).map(|(_, v)| v), + HostValue::Map(entries) => entries.iter().find(|(k, _)| k == key).map(|(_, v)| v), _ => None, } } -fn text(value: Option<&FfiValue>) -> Option { +fn text(value: Option<&HostValue>) -> Option { match value? { - FfiValue::String(s) => String::from_utf8(s.risky_ref().to_vec()).ok(), + HostValue::Text(s) => Some(s.to_string()), _ => None, } } @@ -111,7 +111,7 @@ fn dir(path: &Path) -> Vec { path.to_str().expect("utf8 temp dir").as_bytes().to_vec() } -fn scenarios() -> Vec<(&'static str, FfiValue)> { +fn scenarios() -> Vec<(&'static str, HostValue)> { let root = tempfile::tempdir().expect("temp dir"); let store = dir(root.path()); let workspace = root.path().join("workspaces").join("AAAAAAAAAAAAAAAA"); @@ -268,7 +268,7 @@ fn a_profile_json_error_points_at_the_line() { Some("stack_profile::json") ); let fields = get(&error, "fields").expect("fields"); - assert!(matches!(get(fields, "line"), Some(FfiValue::UInt64(2)))); + assert!(matches!(get(fields, "line"), Some(HostValue::UInt(2)))); assert!( text(get(&error, "help")).is_some(), "the help says how to fix it" diff --git a/languages/golang/auth/store.go b/languages/golang/auth/store.go index b5baea7a6..23b9b5915 100644 --- a/languages/golang/auth/store.go +++ b/languages/golang/auth/store.go @@ -10,7 +10,7 @@ import ( "path/filepath" "runtime" "strings" - "sync" + "time" "github.com/cipherstash/stack/languages/golang/internal/guest" "github.com/cipherstash/vitaminc/bindings/go/vcffi" @@ -53,6 +53,15 @@ func RequireLockedMemory() Option { // stores from one root share one guest instance over the one mounted // directory; Close on any of them releases it. It is safe for concurrent // use; calls are serialised, because a wasm instance is single-threaded. +// +// A call whose context ends fails with the context's error, and only that +// call: a request cancelled while its token is fetched or refreshed never +// closes the profile, or the strategies and encrypt clients that read +// tokens from it. A device session's refresh already sent finishes in the +// background, for up to 10 seconds, so a rotated refresh token is never +// lost (see refreshGrace). The profile closes itself, and every later call is +// [ErrState], only when the guest's state is unknown: it trapped, or it +// went on running long after its caller gave up, which is a bug. type ProfileStore struct { root *root // dir is the store's directory as the guest sees it: guestRoot, or a @@ -62,13 +71,17 @@ type ProfileStore struct { // root is the guest instance the stores of one profile share. type root struct { - mu sync.Mutex + mu guest.Lock inst *instance hostDir string - closed bool - // released is the runtime's own state, apart from closed: an - // interrupted call closes the module, and so the stores, while the - // runtime is still allocated. Close frees it even then. + // stuck is how long a call may run on after its context ends before + // the guest is treated as stuck and closed. guest.StuckAfter; tests + // shorten it. + stuck time.Duration + closed bool + // released is the runtime's own state, apart from closed: a stuck + // call closes the module, and so the stores, while the runtime is + // still allocated. Close frees it even then. released bool cleanup runtime.Cleanup } @@ -135,7 +148,7 @@ func open(ctx context.Context, hostDir string, opts []Option) (*ProfileStore, er if err != nil { return nil, err } - r := &root{inst: inst, hostDir: hostDir} + r := &root{inst: inst, hostDir: hostDir, stuck: guest.StuckAfter} // The cleanup takes the instance, not the root: a cleanup whose // argument reaches its object keeps that object alive forever. r.cleanup = runtime.AddCleanup(r, func(inst *instance) { _ = inst.release() }, inst) @@ -204,7 +217,7 @@ func (s *ProfileStore) Close() error { type export func(*instance) api.Function // call runs one export under the profile's lock, closing the profile if -// the guest trapped or an interrupted call took the module down. +// the guest trapped or a stuck call took the module down. func (s *ProfileStore) call(ctx context.Context, fn export, args ...string) ([]byte, error) { // Every profile export works on the mount; without one the guest would // report an I/O error that says less than this does. @@ -221,23 +234,32 @@ func (s *ProfileStore) call(ctx context.Context, fn export, args ...string) ([]b // callArgs is the shared checked call path for profile exports and auth // exports. The latter pass secret-bearing byte buffers and wipe host copies. +// +// The caller's context bounds the wait for the lock, as encrypt's +// Client.call does, and the guest runs under guest.Detach, so a caller +// that gives up never closes the profile. A refresh request runs on for +// refreshGrace, so the guest keeps the rotated token the service sends. func (s *ProfileStore) callArgs(ctx context.Context, fn export, args ...guest.Arg) ([]byte, error) { r := s.root - r.mu.Lock() + if err := r.mu.Acquire(ctx); err != nil { + return nil, err + } defer r.mu.Unlock() if r.closed || r.inst.module.IsClosed() { r.closed = true return nil, ErrState } growth := r.inst.mem.GrowthRefusal() - out, err := guest.Call(ctx, r.inst.mem, r.inst.module, r.inst.exports, fn(r.inst), args...) + run, done := guest.Detach(ctx, r.stuck) + out, err := guest.Call(run, r.inst.mem, r.inst.module, r.inst.exports, fn(r.inst), args...) + done() switch { case r.inst.module.IsClosed(): r.closed = true if err == nil { err = ErrState } - err = fmt.Errorf("%w: interrupted call closed the profile", err) + err = fmt.Errorf("%w: the guest ran on after its caller's context ended, and the profile is closed", err) case errors.Is(err, guest.ErrTrap): r.closed = true _ = r.inst.module.Close(context.Background()) @@ -253,8 +275,19 @@ func (s *ProfileStore) callArgs(ctx context.Context, fn export, args ...guest.Ar err = fmt.Errorf("%w (growth refused under RequireLockedMemory): %w", guest.MemoryLockError(g.Reason), err) } if err != nil { + if ctxErr := ctx.Err(); ctxErr != nil && !errors.Is(err, ctxErr) { + err = fmt.Errorf("%w: %w", ctxErr, err) + } return nil, err } + // A call that succeeded after its context ended fails too, as in + // encrypt's Client.call, and its output (a token, say) is wiped. What + // the guest wrote stays written: a refresh that ran on keeps the + // rotated token, and the next call reads it. + if ctxErr := ctx.Err(); ctxErr != nil { + guest.Wipe(out) + return nil, ctxErr + } return out, nil } diff --git a/languages/golang/auth/strategy.go b/languages/golang/auth/strategy.go index 5fadc7bce..51379cbdf 100644 --- a/languages/golang/auth/strategy.go +++ b/languages/golang/auth/strategy.go @@ -7,7 +7,6 @@ import ( "fmt" "os" "strconv" - "sync" "github.com/cipherstash/stack/languages/golang/internal/guest" "github.com/tetratelabs/wazero/api" @@ -58,7 +57,7 @@ type Strategy struct { handle string providerID uint32 device bool - mu sync.Mutex + mu guest.Lock closed bool } @@ -182,28 +181,63 @@ func (s *ProfileStore) hasToken(ctx context.Context) (bool, error) { // Token gets the current CTS bearer credential. Device sessions read a fresh // token without a file lock; only a refresh call takes the cross-process lock. func (s *Strategy) Token(ctx context.Context) (string, error) { - s.mu.Lock() - defer s.mu.Unlock() + // A request queued behind another's refresh leaves when its context + // ends, as at the profile's own lock. + if err := s.mu.Acquire(ctx); err != nil { + return "", err + } if s.closed { + s.mu.Unlock() return "", ErrState } - ctx, status := withAuthHTTPStatus(ctx) - token, err := s.token(ctx) - return token, status.wrap(err) + // The work runs on its own goroutine, which holds the lock until it is + // done, so a caller that gives up returns at once. A device session's + // refresh already sent finishes there, and the guest keeps what the + // service sends (see refreshGrace): a rotated refresh token is never + // lost. The token it brings is dropped; the next Token reads it. Any + // other request ends with the caller. + type result struct { + token string + err error + } + done := make(chan result, 1) + go func() { + defer s.mu.Unlock() + ctx, status := withAuthHTTPStatus(ctx) + token, err := s.token(ctx) + done <- result{token, status.wrap(err)} + }() + select { + case r := <-done: + return r.token, r.err + case <-ctx.Done(): + // A call that finished as the context ended still answers. + select { + case r := <-done: + return r.token, r.err + default: + return "", ctx.Err() + } + } } // token is Token under the strategy's lock: a read, then a locked refresh // for a device session that needs one. func (s *Strategy) token(ctx context.Context) (string, error) { - call := func(export func(*instance) api.Function) (string, error) { + call := func(ctx context.Context, export func(*instance) api.Function) (string, error) { out, err := s.store.callArgs(ctx, export, guest.BufArg([]byte(s.handle))) if err != nil { return "", err } defer guest.Wipe(out) + // A caller that has left gets no copy: a string cannot be wiped, + // and the next Token reads the guest's. + if err := ctx.Err(); err != nil { + return "", err + } return string(out), nil } - token, err := call(func(i *instance) api.Function { return i.authToken }) + token, err := call(ctx, func(i *instance) api.Function { return i.authToken }) if !s.device || !errors.Is(err, guest.ErrAuthRefreshRequired) { return token, err } @@ -213,7 +247,9 @@ func (s *Strategy) token(ctx context.Context) (string, error) { } err = withRefreshLock(ctx, path, func() error { var err error - token, err = call(func(i *instance) api.Function { return i.authRefresh }) + // The one request that rotates a credential: it runs on after its + // caller leaves (see refreshGrace). + token, err = call(withRefresh(ctx), func(i *instance) api.Function { return i.authRefresh }) return err }) return token, err @@ -229,6 +265,9 @@ func (s *Strategy) token(ctx context.Context) (string, error) { func (s *Strategy) MemoryLockError() error { return s.store.MemoryLockError() } // Close drops the strategy's cached token and unregisters its provider. +// It waits for a call in flight, including a device session's refresh that +// a cancelled Token left running: that refresh finishes, for up to 10 +// seconds, so the rotated refresh token it brings is saved rather than lost. func (s *Strategy) Close() error { s.mu.Lock() defer s.mu.Unlock() diff --git a/languages/golang/auth/transport.go b/languages/golang/auth/transport.go index 599f40dab..b2e18d89c 100644 --- a/languages/golang/auth/transport.go +++ b/languages/golang/auth/transport.go @@ -10,6 +10,7 @@ import ( "sort" "strings" "sync" + "time" "github.com/cipherstash/stack/languages/golang/internal/guest" "github.com/tetratelabs/wazero" @@ -70,7 +71,10 @@ func (s *authHTTPStatus) wrap(err error) error { } type authTransport struct { - rt http.RoundTripper + rt http.RoundTripper + // grace is how long a refresh request runs on after its caller gives + // up: refreshGrace, shortened by tests. + grace time.Duration mu sync.Mutex providers map[uint32]OIDCProvider nextProvider uint32 @@ -80,7 +84,7 @@ func newAuthTransport(rt http.RoundTripper) *authTransport { if rt == nil { rt = http.DefaultTransport } - return &authTransport{rt: rt, providers: make(map[uint32]OIDCProvider)} + return &authTransport{rt: rt, grace: refreshGrace, providers: make(map[uint32]OIDCProvider)} } func (t *authTransport) register(provider OIDCProvider) uint32 { @@ -101,6 +105,40 @@ func (t *authTransport) unregister(id uint32) { t.mu.Unlock() } +// refreshGrace is how long a device session's refresh request runs on after +// its caller's context ends. The token service rotates the refresh token +// when it answers a refresh, so a refresh abandoned after the service +// answered would lose the new token, and the next refresh would send one +// the service has retired. So a refresh is not cancelled with its caller: +// the caller returns at once (Strategy.Token), the request finishes, and +// the guest saves what came back. A refresh still running this long after +// its caller gave up fails as a transport failure, well inside +// guest.StuckAfter, so the profile stays open. +// +// Every other request to the token service (an access-key exchange, an +// OIDC exchange) rotates nothing, and is cancelled with its caller. +const refreshGrace = 10 * time.Second + +// refreshKey marks the context of a device session's refresh call +// (withRefresh), whose requests get refreshGrace. +type refreshKey struct{} + +// withRefresh marks ctx as a refresh call's. +func withRefresh(ctx context.Context) context.Context { + return context.WithValue(ctx, refreshKey{}, true) +} + +// requestContext is the context a request to the token service runs under: +// the caller's, so a caller that gives up cancels it, except for a refresh, +// which runs on for the grace (see refreshGrace). done must be called when +// the request is finished with. +func (t *authTransport) requestContext(ctx context.Context) (context.Context, func()) { + if refresh, _ := ctx.Value(refreshKey{}).(bool); refresh { + return guest.Detach(guest.Caller(ctx), t.grace) + } + return guest.Caller(ctx), func() {} +} + func (t *authTransport) instantiate(ctx context.Context, r wazero.Runtime) error { _, err := r.NewHostModuleBuilder("cipherstash_transport"). NewFunctionBuilder().WithFunc(t.send).Export("transport_send"). @@ -124,7 +162,9 @@ func (t *authTransport) send(ctx context.Context, m api.Module, return t.placeResponse(ctx, m, respHeadersPtrOut, respHeadersLenOut, respBodyPtrOut, respBodyLenOut, -1, nil, []byte("request buffer out of range")) } requestBody := newAuthRequestBody(body) - req, err := http.NewRequestWithContext(ctx, string(method), string(url), requestBody) + reqCtx, done := t.requestContext(ctx) + defer done() + req, err := http.NewRequestWithContext(reqCtx, string(method), string(url), requestBody) if err != nil { _ = requestBody.Close() return t.placeResponse(ctx, m, respHeadersPtrOut, respHeadersLenOut, respBodyPtrOut, respBodyLenOut, -1, nil, []byte(err.Error())) @@ -174,7 +214,7 @@ func (t *authTransport) oidcTokenGet(ctx context.Context, m api.Module, provider if source == nil { return 1 } - token, err := source.Token(ctx) + token, err := source.Token(guest.Caller(ctx)) if err != nil || token == "" || strings.ContainsAny(token, "\r\n\x00") { return 1 } diff --git a/languages/golang/cmd/stashgen/README.md b/languages/golang/cmd/stashgen/README.md index d3f38554e..2603cdf20 100644 --- a/languages/golang/cmd/stashgen/README.md +++ b/languages/golang/cmd/stashgen/README.md @@ -40,11 +40,11 @@ Your program calls those functions, and it never builds or names a plan. ```go encrypted, err := users.Encrypt(ctx, cipher, people) opened, err := users.Decrypt(ctx, cipher, encrypted) - query, err := users.Fields.Email.Query(ctx, cipher, "bob@example.com") + query, err := users.Fields.Email.Eq(ctx, cipher, "bob@example.com") ``` 6. Store the encrypted type. - Each `encrypt_into` field is one EQL column: `eql.TextEq` implements `driver.Valuer` and `sql.Scanner`, so a database library binds and scans it as the JSON a `public.eql_v3_text_eq` column holds. + Each `encrypt_into` field is one EQL column: `eql.TextEq` implements `driver.Valuer` and `sql.Scanner`, so a database library binds and scans it as the JSON a `public.eql_v4_text_eq` column holds. Stack Encrypt writes EQL v4; see the `encrypt` package README for the EQL types it produces and the Go field type each takes. 7. Run the generator again after each change to the struct or to a tag. A change to the fields of the struct stops the build until you do. @@ -96,9 +96,9 @@ A query on such a type derives its term under the context the cipher names, so i > To update one column, use its `Fields` entry, such as `Fields.Email.Encrypt`. The index names are `equality`, `match`, `ore`, `ope` and `json`. -A `match` index needs text with at least one token: the engine derives no match term for an empty string, separator-only text, or text shorter than the n-gram length (3 characters), because an empty term would match every row. -`Encrypt` then fails for the whole batch, with an error naming the row, the field and the index. -So an optional or short value does not belong under `match`: give the field `equality` alone, or make the value required. +A `match` index stores text with no token (an empty string, separator-only text, or text shorter than the n-gram length of 3 characters) as the empty filter, which no query matches, so an optional or short value can live under `match`; the empty filter tells a reader of the column that the value is one of those. +A match query of such text is refused, because an empty filter as a query would match every row. +An EQL `TextMatch` field is stricter: see below. An index takes its options in parentheses after its name, separated by commas: `index=equality;match(k=3)`. This build refuses an index with options: a query term is derived with the default options only, so a stored term with other options would never match one. These words are the same as the Rust API's words for the same behaviour, with two that only Go has. @@ -123,7 +123,12 @@ For `-type User`, the file `user_stash.go` holds: - `Encrypt` and `Decrypt`, which take a slice and return a slice, and send one ZeroKMS request for each 500 sealed values in it, plus one the first time a keyset is used. - `Fields`, with one entry for each sealed field. An opaque struct, or a struct with no sealed field outside an opaque one, gets no `Fields`: nothing in it is sealed on its own, so nothing can be queried. - An entry encrypts one value, and it has a query method only for what the field declares: `Query` for `encrypt_into`, and `Equality`, `Match`, `Ore` or `Ope` for `index=`. + An entry encrypts one value, and has one query method for each operator the engine answers on the field: `Eq`, `Lt`, `Lte`, `Gt`, `Gte`, `Between` or `Matches`. + stashgen asks the embedded engine which operators each field answers, and which index answers each one, so it holds no copy of those rules: an equality index answers `Eq`, a match index on a string `Matches`, and an ORE or OPE index on an integer, a float or a string the four orderings and `Between`, through ORE when the field declares both. A float orders as PostgreSQL's `float8` does (`-0.0` ties `0.0`, NaN sorts last), and a string by its folded form: case and accents folded, under Unicode 16. + `encrypt_into=TextEq` answers `Eq`, which returns the type's query value, `eql.TextEqQuery`. + `encrypt_into=TextMatch` answers `Matches`, which returns an `encrypt.EQLMatchQuery`: the `eql.TextMatchQuery` operand, with `Verify` and `TargetLimit`. + The query of `Matches` carries `Verify` and `TargetLimit`, because its answer can hold rows that do not match; an exact operator's query carries neither. + See the encrypt package's README for each method's result and the query with `database/sql`. - `String` and `LogValue` on `EncryptedUser`, which print the passthrough fields and hide the sealed ones. - A copy of the fields of `User`, so a change to them stops the build. @@ -149,7 +154,30 @@ The same input always gives the same file: fields keep their declared order, and `stashgen` checks each declaration with the engine, and holds no copy of the engine's rules: it runs the WASI guest the SDK embeds and asks it, one field at a time, so the error names the field. It asks the engine for the EQL types it holds: each name, its plaintext type, its indexes, its query form, and whether the engine produces it today. The command links the build of the engine that holds the EQL types (`encrypt/eql`), so it can answer for every type; a generated file imports `encrypt/eql` only when it names one. -The engine produces `TextEq` today; `encrypt_into` with any other type is refused with the type's name. +The engine produces these EQL types today, each from one Go field type: + +| `encrypt_into` | Go field | +|---|---| +| `TextEq`, `Text`, `TextMatch`, `TextOrd`, `TextOrdOpe`, `TextOrdOre`, `TextSearch`, `TextSearchOre` | `string` | +| `Smallint`, `SmallintEq`, `SmallintOrd`, `SmallintOrdOpe`, `SmallintOrdOre` | `int16`, or any Go integer type | +| `Integer`, `IntegerEq`, `IntegerOrd`, `IntegerOrdOpe`, `IntegerOrdOre` | `int32`, or any Go integer type | +| `Bigint`, `BigintEq`, `BigintOrd`, `BigintOrdOpe`, `BigintOrdOre` | `int64`, or any Go integer type | +| `Boolean` | `bool` | +| `Real`, `RealEq`, `RealOrd`, `RealOrdOpe`, `RealOrdOre` | `float32` | +| `Double`, `DoubleEq`, `DoubleOrd`, `DoubleOrdOpe`, `DoubleOrdOre` | `float64` | +| `Date`, `DateEq` | `encrypt.Date` | +| `Timestamp`, `TimestampEq` | `time.Time` | +| `Numeric`, `NumericEq` | `encrypt.Decimal` | + +The `Eq` types answer `Eq`; the `Ord`, `OrdOpe` and `OrdOre` types answer `Lt`, `Lte`, `Gt` and `Gte`, and on text `Eq` too; `TextMatch` answers `Matches`, and `TextSearch` and `TextSearchOre` all of `Eq`, `Lt`, `Lte`, `Gt`, `Gte` and `Matches`, each query carrying only the term its operator reads; `Text`, `Smallint`, `Integer`, `Bigint`, `Boolean`, `Real`, `Double`, `Date`, `Timestamp` and `Numeric` are stored and read only, with no query method. +The ordered date, timestamp and numeric types are refused: the engine derives no order term for these kinds yet. +An ordered type has no `Between` method yet, because a `BETWEEN` operand needs EQL v4's range query domains. + +Any built-in Go integer field takes an integer type (ADR-0002) and seals at the type's width: an `int16` field always fits an `Integer`, and an `int64` field is range checked value by value, `Encrypt` refusing a value that does not fit with `encrypt.ErrEncoding`. `encrypt.Int128` and `encrypt.Uint128` are not converted. A float's width must match: a `float64` field is not a `Real`. +A `TextMatch` field refuses text shorter than three characters, such as an empty string or a two-letter code, because it has no 3-grams to index; its query refuses such text too. +A `TextSearch` field stores such text with an empty match filter, so its `Eq`, `Lt`, `Lte`, `Gt` and `Gte` queries find it and no `Matches` query does; those queries take such text too, and its `Matches` query refuses it. +Put the column on the left of `@@` (`WHERE name @@ $1::eql_v4.query_text_match`): the other operand order finds nothing until #1171 is fixed. +`encrypt_into` with any other type is refused with the type's name and the reason it is not produced yet. Separate columns work today for four indexes: `equality`, `match`, `ore` and `ope`. A field with `encrypt_into` is stored under its table and column, which is what an EQL value records in its `i`. @@ -219,10 +247,13 @@ No warning, error or log line holds a plaintext value. ## What crosses the binding Generated code sends the engine every sealed field with its value, under the declaration lowered to data: each field's label (`/`), its outputs and its wire type (`int64`, `string`, `bytes`, ...), which `stashgen` chose from the field's Go type. -A passthrough field stays on the host: the engine does nothing to it a program could observe, and the FFI codec cannot carry every Go type a program stores beside a ciphertext. +A passthrough field stays on the host: the engine does nothing to it a program could observe, and the FFI codec cannot carry every Go type a program stores beside a ciphertext (a `driver.Valuer`, a `sql.NullTime`). An `opaque` struct crosses as one JSON document and is one column, decoded back into the exact Go types the struct declares; a field may be any type `encoding/json` carries both ways (a scalar or a type defined over one, `[]byte`, slices, maps with string or integer keys, pointers, structs whose fields are all exported, `time.Time`). A float in an opaque struct must be finite: `encoding/json` refuses NaN and the infinities, so `Encrypt` fails for the batch with `encrypt.ErrEncoding`. A sealed float field outside an opaque struct carries them. A sealed field outside an opaque struct is one scalar, or a type defined over one; a struct, slice or map field seals only inside an opaque struct. +The scalars are `string`, `[]byte`, `bool`, every integer type (each sealed at its own width, `int` and `uint` as 64-bit), `encrypt.Int128` and `encrypt.Uint128`, `float32`, `float64`, `encrypt.Decimal`, `encrypt.Date` and `time.Time` (a timestamp). The wire type `stashgen` writes for each is the field's own: `int8` is `gensupport.Int8`, `time.Time` is `gensupport.Timestamp`, and so on. +An order index (`ore`, `ope`) on a decimal, date or timestamp is refused: those order terms wait for new cllw-ore and ore-rs releases (#1236). +The zero `encrypt.Date` is not a date: a date field must be set, and `Encrypt` refuses a zero one with `encrypt.ErrEncoding`, naming the field. A sealed field cannot be a pointer (`*encrypt.Date` is refused as a composite), so an optional date has no sealed form yet: keep it in an opaque struct, or store a sentinel date. A nil `[]byte` in a sealed field, or a nil slice of a type defined over `[]byte`, comes back from `Decrypt` as an empty, non-nil slice; inside an opaque struct, nil comes back as nil. A protobuf message with a `oneof` cannot be generated from a policy: protoc-gen-go puts its members in wrapper types, not in the message struct, and `Generate` refuses it. diff --git a/languages/golang/cmd/stashgen/main_test.go b/languages/golang/cmd/stashgen/main_test.go index b832fcb4f..84ab1ddb9 100644 --- a/languages/golang/cmd/stashgen/main_test.go +++ b/languages/golang/cmd/stashgen/main_test.go @@ -142,7 +142,7 @@ func TestRunAsksTheEmbeddedEngine(t *testing.T) { t.Fatalf("the generated file does not name the EQL type:\n%s", written) } // A type the engine cannot produce yet: refused by name, nothing written. - unproducible := strings.Replace(userSource, "encrypt_into=TextEq", "encrypt_into=TextMatch", 1) + unproducible := strings.Replace(userSource, "encrypt_into=TextEq", "encrypt_into=DateOrdOre", 1) dir = writeModule(t, map[string]string{"model.go": unproducible}) stderr.Reset() if code := run([]string{"-type", "User"}, dir, &stdout, &stderr, stashgen.GuestEngine); code != 1 { @@ -150,7 +150,7 @@ func TestRunAsksTheEmbeddedEngine(t *testing.T) { } // Refused by the generator with the engine's reason, before the engine // is asked: se_targets lists the type as not producible. - if !strings.Contains(stderr.String(), "cannot produce the EQL type TextMatch yet") { + if !strings.Contains(stderr.String(), "cannot produce the EQL type DateOrdOre yet") { t.Fatalf("stderr = %q", stderr.String()) } if _, err := os.Stat(filepath.Join(dir, "user_stash.go")); !os.IsNotExist(err) { diff --git a/languages/golang/encrypt/README.md b/languages/golang/encrypt/README.md index 6f8f51904..d6a82d91c 100644 --- a/languages/golang/encrypt/README.md +++ b/languages/golang/encrypt/README.md @@ -31,11 +31,15 @@ The package reference is on [pkg.go.dev]; the generator's reference is type User struct { _ struct{} `stash:"context=users"` ID int64 `stash:"id,passthrough"` - Email string `stash:"email,encrypt,index=equality;match"` - Age uint32 `stash:"age,encrypt,index=equality;ore"` + Email string `stash:"email,encrypt_into=TextEq"` + Name string `stash:"name,encrypt_into=TextMatch"` + Age int32 `stash:"age,encrypt_into=Integer"` } ``` + `encrypt_into=` names the EQL type the field is stored as. The Go field + type must match it; see [EQL columns](#eql-columns). + 3. Run the generator. It writes `user_stash.go` beside the struct. ```sh @@ -56,17 +60,28 @@ The package reference is on [pkg.go.dev]; the generator's reference is encrypted, err := users.Encrypt(ctx, cipher, people) opened, err := users.Decrypt(ctx, cipher, encrypted) - term, err := users.Fields.Email.Equality(ctx, cipher, "bob@example.com") + term, err := users.Fields.Email.Eq(ctx, cipher, "bob@example.com") ``` `Encrypt` and `Decrypt` send one ZeroKMS request for each 500 sealed values in the batch, plus one request the first time a keyset is used. + A query derives its term locally and sends no request of its own; only + the first use of a keyset, by a query or anything else, loads it from + ZeroKMS. See [Queries](#queries). + +6. Store the encrypted type. Each `encrypt_into` field is one EQL v4 column: + `eql.TextEq` implements `driver.Valuer` and `sql.Scanner`, so a database + library binds and scans it as the JSON a `public.eql_v4_text_eq` column + holds. Install EQL v4 into PostgreSQL first, with the + `eql-v4-install.sql` attached to the release. A query casts its parameter + to the column's query type: + + ```sql + SELECT id, email, name, age FROM users WHERE email = $1::eql_v4.query_text_eq + ``` -6. Store the encrypted type. Each sealed field is one or more byte columns: - `e.Email.Ciphertext`, `e.Email.Equality`, `e.Email.Match`. `Ciphertext` - and each term type implement `driver.Valuer` and `sql.Scanner`, so a - database library binds and scans each column as bytes; map each one to - its own column. + A field tagged `encrypt` or `index=` stores its ciphertext and each term + as separate byte columns instead; see [What is stored](#what-is-stored). 7. Run the generator again after each change to the struct or to a tag. A change to the fields of the struct stops the build until you do. @@ -80,14 +95,18 @@ The package reference is on [pkg.go.dev]; the generator's reference is `git diff` sees only files git already tracks; the `git status` check also fails on a generated file that was never committed. -The rest of this file is the reference. [`example/`](example/) is the eight -steps as a program. +The rest of this file is the reference. [`example/`](example/) is a program +that stores separate byte columns (`index=`) against real ZeroKMS. ## Connect A `Client` is one ZeroKMS client: its client key, its default keyset, and the keysets it has loaded since. Make one per process and share it; it is safe -for concurrent use. +for concurrent use. The guest runs one call at a time, but never holds the +client across a ZeroKMS round trip: a call that needs ZeroKMS hands its +request to the client, which performs it with the client free for other +callers, and then carries on. A call whose context ends while its request +is in flight returns the context's error and leaves the client open. ```go client, err := encrypt.NewClient(ctx) @@ -102,6 +121,25 @@ ZeroKMS round trip `NewClient` makes. It has nothing to do with an *encryption* context, which is what a field is sealed under; that is the `context=` tag. +Every call takes a `ctx` too, and a call whose `ctx` is cancelled or passes +its deadline fails with the context's error. Only that call fails: one +request giving up never closes the shared client for the others. + +- A call waiting for the client (it runs one call at a time) leaves the + queue when its `ctx` ends, and never starts. +- A call waiting on ZeroKMS has that request cancelled. Its error matches + both `context.Canceled` or `context.DeadlineExceeded` and `ErrTransport`. +- A call waiting on its auth strategy for a token returns too, with the + context's error. If the strategy is refreshing a device session, the + refresh finishes in the background, for up to 10 seconds. The service + rotates the refresh token when it answers, and dropping that answer + would leave the strategy unable to refresh. + +A client closes itself, and every later call returns `ErrState`, only when +the guest's state is unknown: the guest trapped, or a call ran on for 30 +seconds after its `ctx` ended, which means a bug. A service that sees +`ErrState` from a client it did not close builds a new one. + Every other setting is a functional option with a default: ```go @@ -190,15 +228,56 @@ and `Equality`, `Match`, `Ore` or `Ope` (the term types). A field with An `opaque` struct is one `Sealed` field. Every stored type implements `driver.Valuer` and `sql.Scanner`. -A `match` index needs text with at least one token. The engine derives no -match term for an empty string, separator-only text, or text shorter than the -n-gram length (3 characters), because an empty term would match every row. -`Encrypt` then fails for the whole batch with `ErrTerm`, naming the row, the -field and the index. So an optional or short value does not belong under -`match`: give such a field `equality` alone, or make the value required. +A sealed field outside an opaque struct is one value of one of these types, +or a type defined over one (`type Email string`): + +| Go field | Seals as | Indexes it can store | Query methods | +|---|---|---|---| +| `string` | text | `equality`, `match`, `ore`, `ope` | `Eq`, `Matches`, `Lt`, `Lte`, `Gt`, `Gte`, `Between` | +| `[]byte` | bytes | `equality`, `ore`, `ope` | `Eq` | +| `bool` | a boolean | none | none | +| `int16`, `int32`, `int64`, `int` | an integer of the field's own width (`int` is 64-bit) | `equality`, `ore`, `ope` | `Eq`, `Lt`, `Lte`, `Gt`, `Gte`, `Between` | +| `uint16`, `uint32`, `uint64`, `uint` | the same, unsigned | `equality`, `ore`, `ope` | the same | +| `int8`, `uint8` (`byte`) | an 8-bit integer | `equality` | `Eq` | +| `encrypt.Int128`, `encrypt.Uint128` | a 128-bit integer | `equality`, `ore`, `ope` | the same | +| `float32`, `float64` | a float of its width | `equality`, `ore`, `ope` | `Eq`, `Lt`, `Lte`, `Gt`, `Gte`, `Between` | +| `encrypt.Decimal` | a decimal: a 96-bit coefficient and a scale up to 28 | `equality` | `Eq` | +| `encrypt.Date` | a calendar date | `equality` | `Eq` | +| `time.Time` | a timestamp, to the nanosecond, in UTC | `equality` | `Eq` | + +Each query method needs its index declared. An `ore` or `ope` index on bytes +stores its term, but the engine answers no ordering through it, so such a +field has no `Lt` (see [Queries](#queries)). + +`encrypt.Date`, `Int128`, `Uint128` and `Decimal` are vitaminc's `vcvalue` +types; build them with `encrypt.DateOf`, `ParseInt128`, `ParseUint128` and +`ParseDecimal` (which refuses NaN and the infinities). A `time.Time` opens in +UTC: compare it with `Equal`, not `==`. A decimal opens at the scale it was +sealed with, but a zero has no sign: `-0.00` opens as `0.00`. The zero +`encrypt.Date` is not a date, and `Encrypt` or a query refuses it with +`ErrEncoding`, naming the field; so is a decimal or a `time.Time` outside +vitaminc's range. A sealed field cannot be a pointer, so an optional date has +no sealed form yet: keep it in an opaque struct, or use a sentinel date. Each integer type seals at its own +width, and each width is passed to the order scheme as itself, so an `int16` +field's terms are an `int16`'s. An `int8` or `uint8` field has no order +index: with 256 values, an order term would rank every row among 256, which +a few known rows resolve to the values, as a `bool`'s would say which rows +are true. Equality follows vitaminc's normalisation: an +integer equals itself at every width, decimals compare at any scale +(`1.50 = 1.5`), and a timestamp compares to the microsecond, truncated, never rounded. An order index on +a decimal, date or timestamp is refused for now: the engine derives no +order term for those kinds yet. + +A `match` index stores text with no token (an empty string, separator-only +text, or text shorter than the n-gram length of 3 characters) as the empty +filter: a valid value that no query matches, so an optional or short value can +live under `match`. The empty filter tells anyone who can read the column that +the value is one of those. A match *query* of such text is refused with +`ErrTerm`, because an empty filter as a query would match every row. An EQL +`TextMatch` field is stricter, as "EQL columns" below says. Terms are byte-equal to the ones the Rust crate derives, so a term from -`users.Fields.Email.Equality` compares against a stored term written from any +`users.Fields.Email.Eq` compares against a stored term written from any language. `EqualityTerm.Equal` compares in constant time; `OreTerm.Compare` and `OpeTerm.Compare` order as the plaintexts; `MatchTerm.Positions` decodes the token positions. @@ -211,8 +290,122 @@ A database can compare some terms by itself: - `Ore`: do not compare in SQL. Plain byte order is almost always right, and sometimes wrong, with no error. Compare in Go with `OreTerm.Compare`, or use an EQL type (`encrypt_into=`), which the database compares correctly. -- `Match`: do not compare in SQL. Decode the positions in Go with - `MatchTerm.Positions`, or use an EQL type. +- `Match`: a row matches when its stored filter holds every position of the + query's term. Decode both into positions in SQL and compare with `@>`, as + [Queries](#queries) shows, or in Go with `MatchTerm.Positions`. + +## Queries + +Each sealed field's entry in `Fields` has one method for each operator the +engine answers on the field: `Eq`, `Lt`, `Lte`, `Gt`, `Gte`, `Between` and +`Matches`. A field has no method for an operator it does not answer, so such +a query does not compile. The generator asks the engine which operators each +field answers, and which index answers each one; it holds no copy of those +rules. + +| The field declares | Methods | Each returns | +|---|---|---| +| `index=equality` | `Eq` | `encrypt.EqualityTerm` | +| `index=ore` or `index=ope`, on an integer of 16 bits or wider or a string | `Lt`, `Lte`, `Gt`, `Gte` | `encrypt.OreTerm`, or `encrypt.OpeTerm` when the field declares no `ore` | +| the same | `Between(ctx, cipher, lo, hi)` | `encrypt.Range[encrypt.OreTerm]` or `encrypt.Range[encrypt.OpeTerm]`, the two bounds' terms | +| `index=match`, on a string | `Matches` | `encrypt.MatchQuery[T]` | +| `encrypt_into=TextEq` | `Eq` | `eql.TextEqQuery` | +| `encrypt_into=TextMatch` | `Matches` | `encrypt.EQLMatchQuery[eql.TextMatchQuery, T]` | + +A method takes the field's own Go type, so `Fields.Age.Lt` on a `uint32` +field takes a `uint32`. + +Text orders by its folded form (ADR-0002): case and accents are folded and +`ß` becomes `ss`, under Unicode 16. So `Café`, `cafe` and `CAFÉ` are one +value and tie, `alice` orders before `Zoe`, and `é` orders with `e`, not +after `z`. `Between` is inclusive at both ends and, as SQL's `BETWEEN`, +does not refuse a range whose low bound is above its high bound: such a +range matches nothing. So `Between("b", "A")` matches nothing, since `A` +folds to `a`, below `b`. Text holding a code point Unicode 16 does not assign has no +fold, and its query or write is refused with `encrypt.ErrTerm`. + +Floats order as PostgreSQL's `float8` does (ADR-0002): `-0.0` and `0.0` are +one value, and every NaN is one value, above `+Inf`, so `Between(math.NaN(), +math.Inf(1))` matches nothing. A `float32` orders as the `float64` it +widens to. + +An order index on bytes stores its term but answers no ordering +operator, so such a field has no `Lt`; nor does the `ope` index of a field +that declares `ore` too, since `ore` answers the orderings. stashgen prints a notice for each such index, on stderr and in +the generated file: the terms are stored, and only your own code can compare +them. A match index on an index-only field (`index=match` with no `encrypt`) +is refused, because `Verify` checks a match query's rows by decrypting them. + +Every query type implements `driver.Valuer`, so it binds as a parameter in +any library built on `database/sql` or pgx. A `Range` binds as two +parameters, `Low` and `High`. On an EQL column a range is one value instead: +package `encrypt/eql` has a range query type per ordered family and order +index (`eql.IntegerRangeOreQuery`, `eql.TextRangeOpeQuery`, ...), the one +operand of `eql_v4.between(column, $1::eql_v4.query_integer_range_ore)`. Keep +the cast to the exact range domain: an uncast `$1`, a `jsonb` parameter, or a +range of another term or family fails with SQLSTATE 42725 ("function ... is +not unique"), unlike the single-bound operators, which resolve an untyped +`$1`. No generated method returns one yet, because the ordered EQL types are +not producible. + +`Eq`, the orderings and `Between` are exact: the server's answer is the +answer. `Matches` is probabilistic: the server answers with every row whose +stored filter holds the query's bits, which can include rows that do not +hold the query's text. So `MatchQuery` carries two more methods, and an +exact query's type carries neither (on an `encrypt_into=TextMatch` field, +`Matches` returns an `encrypt.EQLMatchQuery`, the `eql.TextMatchQuery` +operand with the same two): + +- `TargetLimit(ctx, want, stats)` is how many rows to fetch for `want` of + them to verify. A field declared with tags states no expected number of + tokens, so with no measured histogram of the stored filters in + `encrypt.Stats` the limit is unbounded, and `Limit.FetchOr(page)` fetches + a page. +- `Verify(ctx, values)` says, for each decrypted value of a page, whether it + really holds the query's text: one call into the engine for the page, with + no key. + +The example below uses separate byte columns, as [`example/`](example/) +declares its fields (`Email` with `stash:"email,encrypt,index=equality;match"`, +`Age` with `stash:"age,encrypt,index=equality;ore"`), not the `encrypt_into` +`User` of the quick start: + +```go +// Positions of a stored filter or a match term, for @>. +const positions = `CREATE FUNCTION match_positions(term bytea) RETURNS int[] +LANGUAGE sql IMMUTABLE STRICT AS $$ + SELECT coalesce(array_agg(get_byte(term, 2 * i) | (get_byte(term, 2 * i + 1) << 8)), '{}') + FROM generate_series(0, length(term) / 2 - 1) AS i +$$` + +q, err := users.Fields.Email.Matches(ctx, cipher, "alice") +limit, err := q.TargetLimit(ctx, 10, encrypt.Stats{Rows: rowCount}) + +rows, err := db.QueryContext(ctx, ` + SELECT id, email, email_eq, email_match, age, age_eq, age_ore FROM users + WHERE match_positions(email_match) @> match_positions($1) + ORDER BY id LIMIT $2`, q, limit.FetchOr(50)) +// scan each row into a users.Encrypted, then +people, err := users.Decrypt(ctx, cipher, encrypted) +emails := make([]string, len(people)) +for i, p := range people { + emails[i] = p.Email +} +verified, err := q.Verify(ctx, emails) +hits := people[:0] +for i, p := range people { + if verified[i] { + hits = append(hits, p) + } +} +``` + +`Verify` cuts the query's text into tokens again in the engine, with the +field's own tokenizer, so a `MatchQuery` keeps that text for as long as the +program holds the query. Go cannot wipe a string: the text stays in memory +until the garbage collector reclaims it, as the string you passed in does. +A `MatchQuery` never prints it; `fmt`, `slog` and `%#v` show the field and +the size of the term only. ## Errors @@ -229,7 +422,11 @@ or for stored data. Check an error two ways. its field's context (ZeroKMS refuses the key, or the AEAD fails). - `ErrEncoding`: a stored value or a call that does not fit the declaration. - `ErrUnauthorized`, `ErrNotFound`, `ErrTransport`, `ErrKMS`: ZeroKMS. -- `ErrState`: a call on a closed client. `ErrMemoryLock`: see below. +- `ErrState`: a call on a closed client: you closed it, or the guest + trapped or ran on long after a call's context ended (see Connect). + `ErrMemoryLock`: see below. +- `context.Canceled`, `context.DeadlineExceeded`: the call's `ctx` ended. + The client is still open. `errors.As` with a `*Diagnostic` gives you the detail behind a failure the engine reports: a stable `Code` (`stack_encrypt::foreign_keyset`, @@ -276,12 +473,115 @@ package documentation for the full account. ## EQL columns A field tagged `encrypt_into=TextEq` is stored as one EQL value, the JSON -PostgreSQL holds in a `public.eql_v3_text_eq` column, as an `eql.TextEq` -from package `encrypt/eql`. The guest builds the value: the generated code +PostgreSQL holds in a `public.eql_v4_text_eq` column, as an `eql.TextEq` +from package `encrypt/eql`. Stack Encrypt writes EQL v4, the `eql_v4` install +of EQL's SQL; the `eql_v3` columns the TypeScript SDK writes hold +cipherstash-client's terms, which these never match. Install EQL v4 into +PostgreSQL first, with the `eql-v4-install.sql` attached to each Go release +(`psql -f eql-v4-install.sql`); `stash eql install` installs EQL v3 only, and +an `eql_v4_*` column needs the v4 domains. + +The EQL types the engine produces, and the Go field types each takes: + +| EQL type | Go field | Queries | +|---|---|---| +| `TextEq` | `string` | `=`, `<>` | +| `TextMatch` | `string` | `@@` (substring match, 3-grams, case-insensitive) | +| `TextOrd`, `TextOrdOpe`, `TextOrdOre` | `string` | `=`, `<>`, `<`, `<=`, `>`, `>=`, `ORDER BY` | +| `TextSearch`, `TextSearchOre` | `string` | `=`, `<>`, `<`, `<=`, `>`, `>=`, `ORDER BY`, `@@` | +| `Text` | `string` | none: stored and read only | +| `Smallint` | `int16`, or any Go integer type (see below) | none | +| `SmallintEq` | the same | `=`, `<>` | +| `SmallintOrd`, `SmallintOrdOpe`, `SmallintOrdOre` | the same | `=`, `<>`, `<`, `<=`, `>`, `>=`, `ORDER BY` | +| `Integer` | `int32`, or any Go integer type (see below) | none | +| `IntegerEq` | the same | `=`, `<>` | +| `IntegerOrd`, `IntegerOrdOpe`, `IntegerOrdOre` | the same | `=`, `<>`, `<`, `<=`, `>`, `>=`, `ORDER BY` | +| `Bigint` | `int64`, or any Go integer type | none | +| `BigintEq` | the same | `=`, `<>` | +| `BigintOrd`, `BigintOrdOpe`, `BigintOrdOre` | the same | `=`, `<>`, `<`, `<=`, `>`, `>=`, `ORDER BY` | +| `Boolean` | `bool` | none | +| `Real` | `float32` | none | +| `RealEq` | `float32` | `=`, `<>` | +| `RealOrd`, `RealOrdOpe`, `RealOrdOre` | `float32` | `=`, `<>`, `<`, `<=`, `>`, `>=`, `ORDER BY` | +| `Double` | `float64` | none | +| `DoubleEq` | `float64` | `=`, `<>` | +| `DoubleOrd`, `DoubleOrdOpe`, `DoubleOrdOre` | `float64` | `=`, `<>`, `<`, `<=`, `>`, `>=`, `ORDER BY` | +| `Date` | `encrypt.Date` | none | +| `DateEq` | `encrypt.Date` | `=`, `<>` | +| `Timestamp` | `time.Time` | none | +| `TimestampEq` | `time.Time` | `=`, `<>` (to the microsecond, truncated) | +| `Numeric` | `encrypt.Decimal` | none | +| `NumericEq` | `encrypt.Decimal` | `=`, `<>` (`1.5 = 1.50`) | + +Any built-in Go integer field takes an integer EQL type (ADR-0002): the field +is sealed at the type's width, and opens back into the field's type. A type +every value of which fits (an `int16` or a `uint16` field for `Integer`, an +`int8` for `Smallint`) +always seals; a wider one (an `int64` or a `uint32` field for `Integer`) is +range checked value by value, and `Encrypt` refuses a value that does not fit +with `ErrEncoding`, naming the field's kind and not the value. +`encrypt.Int128` and `encrypt.Uint128` are not converted, and floats take +exactly their own width. + +A `TextMatch` value must be at least three characters long: shorter text has +no 3-grams to index and is refused. A `TextSearch` value of such text, an +empty string included, is stored with an empty match filter: `Eq`, `Lt`, +`Lte`, `Gt` and `Gte` find it through its other terms, and no `Matches` +query does. A `TextSearch` query carries only the term its operator reads, +so `Eq` and `Lt` on two-letter text work and `Matches` on it is refused; EQL +v4 raises if a query lacks the term its operator compares. The number +equality types compare as +PostgreSQL compares the plain types: an integer equals itself whatever its +width, and for floats `-0.0 = 0.0` and `NaN = NaN`; the stored value still +decrypts to exactly what was written. The same holds for the date, +timestamp and numeric types: `NumericEq` normalises the scale, so `1.5` and +`1.50` are equal, and `TimestampEq` compares to the microsecond, truncating +(never rounding) the nanoseconds below it, so two instants in one +microsecond are equal; each still opens as it was written, a +decimal with its scale and a `time.Time` with its nanoseconds, in UTC. A +`time.Time` is sealed as its instant, whatever its location. A zero +`encrypt.Date`, a date or instant outside years -262143 to 262142 and a +decimal with more than 28 digits after the point are refused with +`ErrEncoding`. The ordered date, timestamp and numeric types are not +produced yet: the engine derives no order term for these kinds. + +The `Ord` and `OrdOpe` types store an OPE term, which EQL compares as bytes, +and order by ADR-0002's order key: integers by value, floats as PostgreSQL's +`float8` sorts them (`-0.0` ties `0.0`, every NaN is one value after +`+Infinity`), text by its fold (case and accents folded, under Unicode 16). +A field gets `Lt`, `Lte`, `Gt` and `Gte`, and a text field `Eq` too; sort +with `ORDER BY eql_v4.ord_term(col)`. `ord_term` returns a `bytea` domain, so +a plain functional btree index, `CREATE INDEX ON t (eql_v4.ord_term(col))`, +serves `<`, `>` and `ORDER BY` with no operator class. That is why these +types work on an install without superuser rights, where the ORE operator +class is skipped. On a number column SQL's `=` compares +the order term; stashgen writes no `Eq` method for one yet, since the engine +answers `Eq` through an equality term only. A text order type also stores +`TextEq`'s equality term, and `=` reads that, so `Café` and `cafe` are +neither `<` nor `>` each other and not `=` either. Text holding a code point +Unicode 16 does not assign has no order term and is refused. There is no +`Between` method yet: a `BETWEEN` needs EQL v4's range query domains. + +The `OrdOre` types and `TextSearchOre` order by the same key, through a block +ORE term (ore-rs's 6-bit schemes) in place of the OPE term. Each term +carries a random nonce, so EQL v4's own comparator orders them, not their +bytes: sort with `ORDER BY eql_v4.ord_term_ore(col)`, and index with +`CREATE INDEX ON t USING btree (eql_v4.ord_term_ore(col))`. That index needs +the ORE operator class, which only a superuser install creates; elsewhere +the installer disables these domains, and the `Ord` types are the ones to +use. Text whose fold is longer than 1023 bytes, or ends in NUL, has no block +ORE term and is refused, stored or queried. + +The rest of the catalog (ordered dates, timestamps and decimals, JSON) is listed in +`eql.Types` with the reason it is not produced yet, and `stashgen` refuses it +with that reason. The guest builds the value: the generated code imports `encrypt/eql`, which embeds the build of the engine that holds the EQL types and registers it on import, so the program runs that build and -stores what it returns. `Fields.Email.Query` returns the `eql.TextEqQuery` -the column's `eql_v3.query_text_eq` operand takes. `mise run +stores what it returns. `Fields.Email.Eq` returns the `eql.TextEqQuery` +the column's `eql_v4.query_text_eq` operand takes; `TextEq` answers `eq` +alone, and a `TextMatch` field's `Matches` returns an +`encrypt.EQLMatchQuery`: the `eql.TextMatchQuery` operand with `Verify` and +`TargetLimit`. `mise run wasm:guest:build:eql` builds that guest beside the other. A cipher extended with a tenant part refuses a struct with an `encrypt_into` field: an EQL value is stored under a table and a column, and the extended label has no diff --git a/languages/golang/encrypt/cancel_test.go b/languages/golang/encrypt/cancel_test.go new file mode 100644 index 000000000..0e7d7018a --- /dev/null +++ b/languages/golang/encrypt/cancel_test.go @@ -0,0 +1,484 @@ +package encrypt + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "math/rand/v2" + "net/http" + "net/http/httptest" + "runtime" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/cipherstash/stack/languages/golang/auth" + "github.com/cipherstash/stack/languages/golang/internal/guest" + "github.com/cipherstash/stack/languages/golang/internal/record" +) + +// One encrypt.Client serves every request in a service, so a caller's +// context ending must fail that caller's call and nothing else. These tests +// pin that contract, and the one case that still closes the client: a guest +// that runs on long after its caller gave up. + +// unauthorized answers every ZeroKMS request with a 401: a full round trip +// through the guest's network path that ends in a status the client can +// tell apart from ErrState. +func unauthorized(*http.Request) (*http.Response, error) { + return &http.Response{StatusCode: http.StatusUnauthorized, Body: io.NopCloser(strings.NewReader("nope")), Header: http.Header{}}, nil +} + +// switchable is a RoundTripper whose behaviour a test changes between calls. +type switchable struct { + mu sync.Mutex + rt roundTripFunc + sent atomic.Int32 +} + +func (s *switchable) set(rt roundTripFunc) { + s.mu.Lock() + defer s.mu.Unlock() + s.rt = rt +} + +func (s *switchable) RoundTrip(r *http.Request) (*http.Response, error) { + s.sent.Add(1) + s.mu.Lock() + rt := s.rt + s.mu.Unlock() + return rt(r) +} + +// cancelClient is a guest not yet given a key, whose ZeroKMS requests go to +// rt and whose token comes from token. Its se_cipher_init is the call the +// tests drive: it pauses on a load-keyset request, so it reaches the +// network between two steps. +func cancelClient(t *testing.T, rt http.RoundTripper, token tokenSource) *Client { + t.Helper() + tr := &transport{rt: rt, token: token} + inst, err := newInstance(context.Background(), guestOrSkip(t), tr, guest.BestEffort) + if err != nil { + t.Fatal(err) + } + c := newClient(inst, tr) + t.Cleanup(func() { _ = c.Close() }) + return c +} + +// cipherInit runs se_cipher_init on c under ctx. +func cipherInit(ctx context.Context, c *Client) error { + encoded, err := encodeConfig(testInit("http://zerokms.invalid")) + if err != nil { + return err + } + defer wipe(encoded) + _, err = c.callKMS(ctx, func(ctx context.Context, inst *instance) ([]byte, error) { + return inst.call(ctx, inst.cipherInit, buf(encoded)) + }) + return err +} + +// wantOpen fails the test unless c is still open: its next call reaches +// ZeroKMS and comes back with ZeroKMS's answer, not ErrState. +func wantOpen(t *testing.T, c *Client, rt *switchable) { + t.Helper() + if c.closed || c.inst.module.IsClosed() { + t.Fatal("the client was closed") + } + rt.set(unauthorized) + if err := cipherInit(context.Background(), c); !errors.Is(err, ErrUnauthorized) { + t.Fatalf("the next call: %v, want ErrUnauthorized from ZeroKMS", err) + } +} + +// The request runs between two steps, under the caller's context, with the +// guest paused: cancelling it ends the call and aborts the paused +// operation, and the client stays open. +func TestACallCancelledBetweenStepsFailsOnlyItself(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + rt := &switchable{rt: func(r *http.Request) (*http.Response, error) { + // The caller gives up once its request is in flight. + cancel() + <-r.Context().Done() + return nil, r.Context().Err() + }} + c := cancelClient(t, rt, staticToken("t")) + err := cipherInit(ctx, c) + if !errors.Is(err, context.Canceled) || !errors.Is(err, ErrTransport) { + t.Fatalf("cancelled call: %v, want context.Canceled and ErrTransport", err) + } + wantOpen(t, c, rt) +} + +// The token is fetched before the operation's first step: a caller that +// gives up waiting on it never starts the operation. +func TestACallCancelledDuringATokenFetchFailsOnlyItself(t *testing.T) { + rt := &switchable{rt: unauthorized} + var blocking atomic.Bool + blocking.Store(true) + token := tokenFunc(func(ctx context.Context) (string, error) { + if blocking.Load() { + <-ctx.Done() + return "", ctx.Err() + } + return "t", nil + }) + c := cancelClient(t, rt, token) + ctx, cancel := context.WithCancel(context.Background()) + time.AfterFunc(50*time.Millisecond, cancel) + err := cipherInit(ctx, c) + if !errors.Is(err, context.Canceled) || errors.Is(err, ErrState) { + t.Fatalf("cancelled call: %v, want context.Canceled", err) + } + if n := rt.sent.Load(); n != 0 { + t.Fatalf("%d request(s) sent without a token", n) + } + blocking.Store(false) + wantOpen(t, c, rt) +} + +func TestACallWhoseContextEndsInTheQueueNeverReachesTheGuest(t *testing.T) { + rt := &switchable{rt: unauthorized} + c := cancelClient(t, rt, staticToken("t")) + + t.Run("ends while waiting", func(t *testing.T) { + // Another call holds the client: the slow ZeroKMS round trip the + // queue is waiting behind. + if err := c.mu.Acquire(context.Background()); err != nil { + t.Fatal(err) + } + released := make(chan struct{}) + time.AfterFunc(200*time.Millisecond, func() { c.mu.Unlock(); close(released) }) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + err := cipherInit(ctx, c) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("queued call: %v, want the deadline", err) + } + // It left at its deadline, not when the lock came free. + select { + case <-released: + t.Error("the queued call waited past its deadline for the lock") + default: + } + <-released + }) + t.Run("already ended", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := cipherInit(ctx, c); !errors.Is(err, context.Canceled) { + t.Fatalf("call with an ended context: %v, want context.Canceled", err) + } + }) + if n := rt.sent.Load(); n != 0 { + t.Fatalf("%d request(s) reached ZeroKMS from calls whose context had ended", n) + } + wantOpen(t, c, rt) +} + +// The backstop: a step still running long after its caller gave up is a +// guest or host function that ignores cancellation. Its state is unknown, +// so the client closes, and says so. The step here waits out its caller's +// context and the backstop and only then enters the guest, as a guest +// stuck in a loop would still be running when the backstop fired. +func TestAStepThatRunsOnAfterItsContextEndsClosesTheClient(t *testing.T) { + rt := &switchable{rt: unauthorized} + c := cancelClient(t, rt, staticToken("t")) + c.stuck = 20 * time.Millisecond + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + _, err := c.call(ctx, func(run context.Context, inst *instance) ([]byte, error) { + cancel() + <-run.Done() + return inst.call(run, inst.targets) + }) + if err == nil || !strings.Contains(err.Error(), "the client is closed") { + t.Fatalf("stuck call: %v, want the client closed", err) + } + if !errors.Is(err, context.Canceled) { + t.Errorf("stuck call: %v, want it to match context.Canceled", err) + } + if err := cipherInit(context.Background(), c); !errors.Is(err, ErrState) { + t.Fatalf("a call after the close: %v, want ErrState", err) + } +} + +// A request that ignores its context holds up only its own call: the guest +// is paused while it runs, so nothing is stuck, and the client stays open. +func TestARequestThatIgnoresItsContextLeavesTheClientOpen(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + rt := &switchable{rt: func(*http.Request) (*http.Response, error) { + // The caller gives up once its request is in flight, and the + // request runs on past the backstop regardless. + cancel() + time.Sleep(100 * time.Millisecond) + return unauthorized(nil) + }} + c := cancelClient(t, rt, staticToken("t")) + c.stuck = 20 * time.Millisecond + if err := cipherInit(ctx, c); !errors.Is(err, context.Canceled) || !errors.Is(err, ErrTransport) { + t.Fatalf("call: %v, want context.Canceled and ErrTransport", err) + } + wantOpen(t, c, rt) +} + +// A call that finishes in time stops the backstop: the client stays open +// however long after the call its context ends. +func TestAFinishedCallStopsTheBackstop(t *testing.T) { + rt := &switchable{rt: unauthorized} + c := cancelClient(t, rt, staticToken("t")) + c.stuck = time.Millisecond + ctx, cancel := context.WithCancel(context.Background()) + if err := cipherInit(ctx, c); !errors.Is(err, ErrUnauthorized) { + t.Fatalf("call: %v, want ErrUnauthorized", err) + } + cancel() + time.Sleep(20 * time.Millisecond) + wantOpen(t, c, rt) +} + +// Work the guest does without the network runs on through a cancellation, +// so a call can succeed after its caller gave up. It still fails with the +// context's error, and the output nobody will read is wiped. +func TestACallThatSucceedsAfterItsContextEndsFailsWithIt(t *testing.T) { + rt := &switchable{rt: unauthorized} + c := cancelClient(t, rt, staticToken("t")) + ctx, cancel := context.WithCancel(context.Background()) + plaintext := []byte("plaintext") + out, err := c.call(ctx, func(context.Context, *instance) ([]byte, error) { + cancel() + return plaintext, nil + }) + if !errors.Is(err, context.Canceled) || out != nil { + t.Fatalf("call: %q, %v, want no output and context.Canceled", out, err) + } + if string(plaintext) != string(make([]byte, len(plaintext))) { + t.Error("the discarded output was not wiped") + } + wantOpen(t, c, rt) +} + +// A step that pauses after its caller gave up sends no request: the +// operation is aborted in the guest, so its handle is spent, and the call +// fails with the context's error. +func TestAStepThatPausesAfterItsContextEndsSendsNothing(t *testing.T) { + rt := &switchable{rt: unauthorized} + c := cancelClient(t, rt, staticToken("t")) + encoded, err := encodeConfig(testInit("http://zerokms.invalid")) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + _, err = c.callKMS(ctx, func(run context.Context, inst *instance) ([]byte, error) { + cancel() + return inst.call(run, inst.cipherInit, buf(encoded)) + }) + if !errors.Is(err, context.Canceled) || !errors.Is(err, ErrTransport) { + t.Fatalf("call: %v, want context.Canceled and ErrTransport", err) + } + if n := rt.sent.Load(); n != 0 { + t.Fatalf("%d request(s) sent for a caller that had given up", n) + } + if _, err := c.inst.call(context.Background(), c.inst.abort, scalar(1)); !errors.Is(err, ErrState) { + t.Fatalf("the paused operation's handle: %v, want ErrState (already dropped)", err) + } + wantOpen(t, c, rt) +} + +// A caller whose context ends while its resume waits for the client never +// resumes: the paused operation is still aborted in the guest once the +// client is free. +func TestAResumeWhoseContextEndsInTheQueueAbortsTheOperation(t *testing.T) { + stub := newHeldStub(t) + c := stepClient(t, staticToken("stub-token")) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { + _, err := c.callKMS(ctx, initStep(t, stub.URL)) + done <- err + }() + waitFor(t, stub.arrived, "the init's request") + // Another call holds the client when the response comes back. + if err := c.mu.Acquire(context.Background()); err != nil { + t.Fatal(err) + } + close(stub.release) + time.Sleep(50 * time.Millisecond) + cancel() + time.Sleep(20 * time.Millisecond) + c.mu.Unlock() + if err := <-done; !errors.Is(err, context.Canceled) { + t.Fatalf("call: %v, want context.Canceled", err) + } + if _, err := c.inst.call(context.Background(), c.inst.abort, scalar(1)); !errors.Is(err, ErrState) { + t.Fatalf("the paused operation's handle: %v, want ErrState (already dropped)", err) + } + if c.closed || c.inst.module.IsClosed() { + t.Fatal("giving up closed the client") + } +} + +// A closed module is ErrState, and Close still releases the runtime. +func TestAClosedModuleIsState(t *testing.T) { + ctx := context.Background() + c := rawInstance(t) + if err := c.inst.module.Close(ctx); err != nil { + t.Fatal(err) + } + if _, err := c.Keyset(KeysetName("k")).KeysetID(ctx); !errors.Is(err, ErrState) { + t.Fatalf("Keyset on a closed module: %v, want ErrState", err) + } + if err := c.Close(); err != nil { + t.Fatalf("Close after the module closed: %v", err) + } + // The close must reach the runtime. A stuck call closes the module and + // marks the client closed; a Close that treated that as "already done" + // would leave the runtime and the host module it carries allocated for + // the life of the process. + if !c.released { + t.Error("Close after the module closed left the runtime unreleased") + } + if m := c.inst.runtime.Module(transportModule); m != nil { + t.Errorf("host module %s is still registered after Close", transportModule) + } + // Still idempotent. + if err := c.Close(); err != nil { + t.Fatalf("second Close: %v", err) + } +} + +// Many callers share one client, as a service's requests do, and each call +// gets a deadline: none, already passed, or one that may pass in the queue +// or in the guest. The client never closes, every call whose context did +// not end succeeds, and nothing is left running afterwards. +func TestManyCallersCancellingAtRandomShareOneClient(t *testing.T) { + c, err := NewDeterministicClient(context.Background(), [32]byte{7}) + if errors.Is(err, ErrDeterministicGuestNotBuilt) { + t.Skip(err) + } + if err != nil { + t.Fatal(err) + } + defer c.Close() + cipher := c.DefaultKeyset() + plan := usersPlan() + before := runtime.NumGoroutine() + + const callers, calls = 16, 40 + var succeeded atomic.Int32 + var wg sync.WaitGroup + for g := range callers { + wg.Add(1) + go func() { + defer wg.Done() + r := rand.New(rand.NewPCG(uint64(g), 1)) + for i := range calls { + ctx, cancel := context.Background(), context.CancelFunc(func() {}) + switch r.IntN(4) { + case 0: // no deadline + case 1: + ctx, cancel = context.WithCancel(ctx) + cancel() + default: + ctx, cancel = context.WithTimeout(ctx, time.Duration(r.IntN(2000))*time.Microsecond) + } + rows := []record.Source{usersRow(uint32(i), "user@example.com")} + _, err := cipher.Seal(ctx, plan, rows) + ended := ctx.Err() != nil + cancel() + switch { + case errors.Is(err, ErrState): + t.Errorf("the client closed: %v", err) + return + case err == nil: + succeeded.Add(1) + case !ended: + t.Errorf("a call whose context did not end failed: %v", err) + case !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded): + t.Errorf("a cancelled call: %v, want the context's error", err) + } + } + }() + } + wg.Wait() + if succeeded.Load() == 0 { + t.Fatal("no call succeeded") + } + if _, err := cipher.Seal(context.Background(), plan, []record.Source{usersRow(1, "a@example.com")}); err != nil { + t.Fatalf("a call after the load: %v", err) + } + // Every backstop timer and context watcher has stopped. + deadline := time.Now().Add(2 * time.Second) + for runtime.NumGoroutine() > before && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + if n := runtime.NumGoroutine(); n > before { + t.Errorf("%d goroutines after the load, %d before", n, before) + } +} + +// The encrypt client and the auth profile fail together or not at all: a +// call cancelled while the auth strategy exchanges its access key for a +// token leaves both open, and the next call gets a token and reaches +// ZeroKMS. +func TestACallCancelledDuringATokenExchangeLeavesTheProfileOpen(t *testing.T) { + var hold atomic.Bool + done := make(chan struct{}) + var jwt string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + if hold.Load() { + select { + case <-r.Context().Done(): + case <-done: + t.Error("a held token request was never cancelled") + } + return + } + fmt.Fprintf(w, `{"accessToken":%q,"expiry":%d}`, jwt, time.Now().Add(time.Hour).Unix()) + })) + t.Cleanup(server.Close) + t.Cleanup(func() { close(done) }) + claims, err := json.Marshal(map[string]any{"iss": server.URL, "sub": "CS|test", "workspace": "ZVATKW3VHMFG27DY", "exp": time.Now().Add(time.Hour).Unix()}) + if err != nil { + t.Fatal(err) + } + jwt = "e30." + base64.RawURLEncoding.EncodeToString(claims) + ".c2ln" + + store, err := auth.OpenWithoutProfile(context.Background()) + if errors.Is(err, auth.ErrGuestNotBuilt) { + t.Skip(err) + } + if err != nil { + t.Fatal(err) + } + defer store.Close() + strategy, err := store.AccessKey(context.Background(), testCRN, testAccessKey, auth.WithBaseURL(server.URL)) + if err != nil { + t.Fatal(err) + } + defer strategy.Close() + + rt := &switchable{rt: unauthorized} + c := cancelClient(t, rt, strategy) + hold.Store(true) + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + if err := cipherInit(ctx, c); !errors.Is(err, context.DeadlineExceeded) || errors.Is(err, ErrState) { + t.Fatalf("call cancelled during the exchange: %v, want the deadline", err) + } + hold.Store(false) + wantOpen(t, c, rt) + if n := rt.sent.Load(); n != 1 { + t.Fatalf("%d ZeroKMS request(s), want the one after the token arrived", n) + } +} diff --git a/languages/golang/encrypt/checker.go b/languages/golang/encrypt/checker.go index 7f3a99e25..3ea4c81eb 100644 --- a/languages/golang/encrypt/checker.go +++ b/languages/golang/encrypt/checker.go @@ -57,7 +57,7 @@ func (k *Checker) engineCheck(ctx context.Context, plan *record.Plan) error { if err != nil { return fmt.Errorf("%w: %v", ErrEncoding, err) } - _, err = k.c.call(ctx, func(inst *instance) ([]byte, error) { + _, err = k.c.call(ctx, func(ctx context.Context, inst *instance) ([]byte, error) { return inst.call(ctx, inst.planCheck, buf(encoded)) }) return err @@ -66,7 +66,7 @@ func (k *Checker) engineCheck(ctx context.Context, plan *record.Plan) error { // Targets lists the EQL types this build of the engine produces: none until // the EQL target dispatch lands. func (k *Checker) Targets(ctx context.Context) ([]record.Target, error) { - out, err := k.c.call(ctx, func(inst *instance) ([]byte, error) { + out, err := k.c.call(ctx, func(ctx context.Context, inst *instance) ([]byte, error) { return inst.call(ctx, inst.targets) }) if err != nil { @@ -79,6 +79,37 @@ func (k *Checker) Targets(ctx context.Context) ([]record.Target, error) { return parseTargets(decoded) } +// Operators lists the operators one field of a plan answers, in the +// engine's order, each with its fidelity, its arity and the index that +// answers it: exactly the operators [Cipher.QueryTerm] derives a term for +// on that field, so stashgen writes one query method for each and holds no +// copy of which index answers which operator. A field the plan does not +// have is [ErrEncoding]. +func (k *Checker) Operators(ctx context.Context, plan *record.Plan, field string) ([]record.Operator, error) { + if err := plan.Validate(); err != nil { + return nil, fmt.Errorf("%w: %v", ErrEncoding, err) + } + encoded, err := vcffi.Marshal(plan.Wire()) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrEncoding, err) + } + out, err := k.c.call(ctx, func(ctx context.Context, inst *instance) ([]byte, error) { + return inst.call(ctx, inst.operators, buf(encoded), buf([]byte(field))) + }) + if err != nil { + return nil, err + } + decoded, err := vcffi.Unmarshal(out) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrInternal, err) + } + operators, err := parseOperators(decoded) + if err != nil { + return nil, fmt.Errorf("%w: se_operators: %v", ErrInternal, err) + } + return operators, nil +} + // parseTargets reads se_targets's decoded value: {"targets": [entry, ...]}, // each entry an object with exactly the keys record.Target documents. The // shape is a contract between eql-bindings' serialiser and this reader, so @@ -147,7 +178,7 @@ func parseTarget(entry vcvalue.Object) (record.Target, error) { switch o := record.Output(s); o { // "json" is the SteVec document index, which no plan output // carries. - case record.Equality, record.Match, record.Ore, record.Ope, "json": + case record.Equality, record.Match, record.Ore, record.Ope, record.OreBlock, "json": t.Indexes = append(t.Indexes, o) default: return t, fmt.Errorf("unknown index %q", s) @@ -165,6 +196,8 @@ func parseTarget(entry vcvalue.Object) (record.Target, error) { t.Producible = b case "reason": t.Reason, err = targetOptionalString(f) + case "operators": + t.Operators, err = parseOperators(f.Value) default: return t, fmt.Errorf("unknown key %q", f.Key) } @@ -175,7 +208,7 @@ func parseTarget(entry vcvalue.Object) (record.Target, error) { if t.Name == "" { return t, errors.New("no name") } - for _, key := range []string{"indexes", "producible"} { + for _, key := range []string{"indexes", "producible", "operators"} { if !seen[key] { return t, fmt.Errorf("no %s", key) } @@ -183,6 +216,93 @@ func parseTarget(entry vcvalue.Object) (record.Target, error) { return t, nil } +// parseOperators reads a list of operators, an se_targets entry's or +// se_operators's: a list of objects, each with exactly the keys operator, +// fidelity, arity and index, every value one the engine writes. +func parseOperators(value any) ([]record.Operator, error) { + list, ok := value.([]any) + if !ok { + return nil, fmt.Errorf("operators is %T, not a list", value) + } + operators := make([]record.Operator, 0, len(list)) + for i, item := range list { + entry, ok := item.(vcvalue.Object) + if !ok { + return nil, fmt.Errorf("operator %d is %T, not an object", i, item) + } + var op record.Operator + seen := map[string]bool{} + for _, f := range entry { + if seen[f.Key] { + return nil, fmt.Errorf("operator %d: key %q twice", i, f.Key) + } + seen[f.Key] = true + s, err := targetString(f) + if err != nil { + return nil, fmt.Errorf("operator %d: %v", i, err) + } + switch f.Key { + case "operator": + op.Name, err = parseOperatorName(s) + case "fidelity": + op.Fidelity, err = parseFidelity(s) + case "arity": + switch s { + case "one": + case "pair": + op.Pair = true + default: + return nil, fmt.Errorf("operator %d: unknown arity %q", i, s) + } + case "index": + op.Index, err = parseOperatorIndex(s) + default: + return nil, fmt.Errorf("operator %d: unknown key %q", i, f.Key) + } + if err != nil { + return nil, fmt.Errorf("operator %d: %v", i, err) + } + } + if len(seen) != 4 || op.Name == "" { + return nil, fmt.Errorf("operator %d: not every key, or no name", i) + } + operators = append(operators, op) + } + return operators, nil +} + +// parseOperatorName reads an operator's name as the engine writes it: one +// of eq, lt, lte, gt, gte, between and matches, the names the engine's +// OpSpec::parse accepts. The catalog (se_targets) and se_query_term are +// read through this, parseFidelity and parseOperatorIndex, so the two +// readers refuse the same values. +func parseOperatorName(s string) (string, error) { + switch s { + case "eq", "lt", "lte", "gt", "gte", "between", "matches": + return s, nil + } + return "", fmt.Errorf("unknown operator %q", s) +} + +// parseFidelity reads an operator's fidelity: exact or probabilistic. +func parseFidelity(s string) (record.Fidelity, error) { + switch f := record.Fidelity(s); f { + case record.Exact, record.Probabilistic: + return f, nil + } + return "", fmt.Errorf("unknown fidelity %q", s) +} + +// parseOperatorIndex reads the index that answers an operator: eq, match, +// ore, ope or ore_block. +func parseOperatorIndex(s string) (record.Output, error) { + switch o := record.Output(s); o { + case record.Equality, record.Match, record.Ore, record.Ope, record.OreBlock: + return o, nil + } + return "", fmt.Errorf("unknown index %q", s) +} + func targetString(f vcvalue.Field) (string, error) { s, ok := f.Value.(string) if !ok { diff --git a/languages/golang/encrypt/ciphertext.go b/languages/golang/encrypt/ciphertext.go index 242d85d67..7d70475d8 100644 --- a/languages/golang/encrypt/ciphertext.go +++ b/languages/golang/encrypt/ciphertext.go @@ -50,8 +50,9 @@ func scanBytes(kind string, src any) ([]byte, error) { } // otherLeaf is a leaf of a kind the SDK does not store: the authenticated -// markers for an absent value, an empty sequence and an empty map, which a -// record field never produces. Decoding one is an error at the use site. +// markers for an empty sequence and an empty map, which a record field never +// produces. Decoding one is an error at the use site. An absent value is a +// sealed null, a Ciphertext like any other value. type otherLeaf struct { kind vcffi.LeafKind bytes []byte diff --git a/languages/golang/encrypt/client.go b/languages/golang/encrypt/client.go index 4f89bd4ff..042a7d09c 100644 --- a/languages/golang/encrypt/client.go +++ b/languages/golang/encrypt/client.go @@ -8,7 +8,7 @@ import ( "net/http" "runtime" "strconv" - "sync" + "time" "github.com/cipherstash/stack/languages/golang/internal/guest" "github.com/cipherstash/vitaminc/bindings/go/vcffi" @@ -17,8 +17,19 @@ import ( // Client is one wasm instance holding one ZeroKMS client: its key, its // default keyset, and the keysets it has loaded since. It is safe for -// concurrent use; calls are serialised internally, because a wasm instance -// is single-threaded. Close it when done, as with any resource. +// concurrent use; the guest runs one call's step at a time, because a wasm +// instance is single-threaded, but never waits on the network inside one: +// a ZeroKMS request is performed between steps, with the client free for +// other callers (see step.go). Close it when done, as with any resource. +// +// A call whose context ends fails with the context's error, and only that +// call: one request's cancellation or deadline never closes the client for +// the others sharing it. A call waiting its turn leaves the queue when its +// context ends; a call waiting on ZeroKMS has its request cancelled, and +// the error matches both the context's error and [ErrTransport]. The +// client closes itself, and every later call is [ErrState], only when the +// guest's state is unknown: it trapped, or it went on running long after +// its caller gave up, which is a bug. // // Its key material lives in the guest's linear memory, which this package // supplies: reserved once so it never moves, locked in RAM and excluded @@ -29,13 +40,23 @@ import ( // where the protection lives. [Client.MemoryLocked] reports whether the // lock was granted. type Client struct { - mu sync.Mutex + mu guest.Lock inst *instance transport *transport - // closed refuses further calls: either Close ran, or an interrupted - // call took the module down under us (see Client.call). released is - // the runtime's own state, tracked apart from it because those two - // things come apart: an interrupted call closes the module — and so + // stuck is how long a call may run on after its context ends before + // the guest is treated as stuck and closed. guest.StuckAfter; tests + // shorten it. + stuck time.Duration + // paused bounds the operations that may be paused in the guest at once + // to its cap (maxPaused): an operation that can reach ZeroKMS takes a + // slot before its first step and gives it back when it ends, so a + // caller past the cap waits here for one rather than being refused by + // the guest. + paused chan struct{} + // closed refuses further calls: either Close ran, or a stuck or + // trapped call took the module down under us (see Client.call). + // released is the runtime's own state, tracked apart from it because + // those two things come apart: a stuck call closes the module — and so // the client — while the runtime and the host modules beside it are // still allocated. Close is what frees those, so it must do its work // even on a client that is already closed. @@ -175,7 +196,7 @@ func NewClient(ctx context.Context, opts ...ClientOption) (_ *Client, err error) c.releaseCredentials = resolved.Close c.credentialsLockErr = resolved.MemoryLockError owned = true - out, err := c.call(ctx, func(inst *instance) ([]byte, error) { + out, err := c.callKMS(ctx, func(ctx context.Context, inst *instance) ([]byte, error) { return inst.call(ctx, inst.cipherInit, buf(encoded)) }) if err != nil { @@ -217,7 +238,7 @@ func consumeUnresolved(creds Credentials) { // instance, not the client: a cleanup whose argument reaches its object // keeps that object alive forever. func newClient(inst *instance, t *transport) *Client { - c := &Client{inst: inst, transport: t} + c := &Client{inst: inst, transport: t, stuck: guest.StuckAfter, paused: make(chan struct{}, maxPaused)} c.cleanup = runtime.AddCleanup(c, func(inst *instance) { _ = inst.release() }, inst) return c } @@ -323,8 +344,8 @@ func encodeConfig(cfg initConfig) ([]byte, error) { func (c *Client) Close() error { c.mu.Lock() defer c.mu.Unlock() - // Idempotency turns on the runtime, not on the client: a client an - // interrupted call already closed has never released its runtime. + // Idempotency turns on the runtime, not on the client: a client a + // stuck call already closed has never released its runtime. if c.released { return nil } @@ -368,7 +389,7 @@ func (c *Client) resolveKeyset(ctx context.Context, sel KeysetSelector) (KeysetI if err != nil { return KeysetID{}, err } - out, err := c.call(ctx, func(inst *instance) ([]byte, error) { + out, err := c.callKMS(ctx, func(ctx context.Context, inst *instance) ([]byte, error) { return inst.call(ctx, inst.keyset, buf(encoded)) }) if err != nil { @@ -382,35 +403,203 @@ func (c *Client) resolveKeyset(ctx context.Context, sel KeysetSelector) (KeysetI return id, nil } -// call runs f on the instance under the client's lock. +// maxPaused is the most operations the guest holds paused at once: MAX_OPEN +// in stack-guest-abi's step module. One more is refused there, so the +// client never starts more than this many operations that can pause. +const maxPaused = 64 + +// call runs an operation that never reaches ZeroKMS (a plan check, the +// catalog, a verify): the step loop with no token to fetch. +func (c *Client) call(ctx context.Context, f stepFunc) ([]byte, error) { + return c.drive(ctx, false, f) +} + +// callKMS runs an operation that can reach ZeroKMS: the step loop, with a +// token fetched before its first step. +func (c *Client) callKMS(ctx context.Context, f stepFunc) ([]byte, error) { + return c.drive(ctx, true, f) +} + +// drive runs an operation to its end, a step at a time (see step.go): its +// first step is f, and each later one resumes it with the responses to the +// requests the step before asked for. The client's lock is held for each +// step and released while the requests run, so callers share the client +// between steps rather than queue behind a round trip. // -// A call interrupted by its context (the runtime closes the module on a -// deadline or cancellation, see newInstance) leaves the instance closed: -// its key material is gone with its memory and no further call can run. -// The client is then closed, so later calls are ErrState rather than a -// runtime error, and Close releases the runtime without a shutdown call. -// A guest that trapped is closed the same way, by this method: the guest -// builds with panic-as-abort, so a trap is an abort mid-export, after -// which its state is unknown and its keys are better wiped than reused. -func (c *Client) call(ctx context.Context, f func(*instance) ([]byte, error)) ([]byte, error) { +// When kms is set, the operation first takes one of the guest's paused +// slots (see Client.paused), and the token is fetched once, before its +// first step and outside the lock; every step of the operation hands it +// over, so the guest's token_get never waits. The token source is asked +// once per such operation, including one the guest answers from its cache +// without asking for the token (a query term on a loaded keyset, a cached +// keyset lookup): which operations reach ZeroKMS is known only once the +// guest has run. A source that fails is reported only if the guest asks. +// +// The caller's context governs everything the host waits on: the slot, the +// token fetch, the wait for the lock and the requests, which run under it. It +// never reaches the guest, which runs each step under guest.Detach (see +// step). An operation whose caller gives up between steps is aborted, +// which drops it in the guest with everything it held, and the call fails +// with the context's error; one abandoned while it waited on ZeroKMS +// matches ErrTransport too. +func (c *Client) drive(ctx context.Context, kms bool, f stepFunc) ([]byte, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + token, tokenErr := "", error(nil) + if kms { + select { + case c.paused <- struct{}{}: + defer func() { <-c.paused }() + case <-ctx.Done(): + return nil, ctx.Err() + } + } + if kms && c.transport != nil { + token, tokenErr = c.transport.fetchToken(ctx) + // A caller that gave up waiting on its token never starts the + // operation. + if err := ctx.Err(); err != nil { + return nil, err + } + } + step, resumed := f, false + var handle uint64 + var responses []byte + for { + out, io, err := c.step(ctx, token, tokenErr, step) + if resumed { + // The resume step wipes its responses when it runs; when it + // never ran (its caller gave up waiting for the client, or the + // client closed meanwhile) they are wiped here. + wipe(responses) + } + if errors.Is(err, errLockWait) { + // The step never ran: a paused operation is still the + // guest's to drop. + if resumed { + c.abort(handle) + } + return nil, ctx.Err() + } + if io == nil { + return settle(ctx, out, err) + } + // A caller that gave up while the guest ran sends no request. + if err := ctx.Err(); err != nil { + io.wipe() + c.abort(io.handle) + return nil, fmt.Errorf("%w: %w", err, ErrTransport) + } + responses, err = c.transport.performStep(ctx, io) + if err != nil { + c.abort(io.handle) + return nil, err + } + // The requests ran under the caller's context, so one it cancelled + // came back a transport failure: the guest is not resumed with it. + if err := ctx.Err(); err != nil { + wipe(responses) + c.abort(io.handle) + return nil, fmt.Errorf("%w: %w", err, ErrTransport) + } + h, pending := io.handle, responses + handle, resumed = h, true + step = func(ctx context.Context, inst *instance) ([]byte, error) { + defer wipe(pending) + return inst.call(ctx, inst.resume, scalar(h), buf(pending)) + } + } +} + +// settle is how an operation ends for its caller. A call that failed after +// its context ended failed because of it, or as good as: the caller asks +// errors.Is(err, context.Canceled) or DeadlineExceeded, and the cause +// stays matchable too. A call that succeeded after its context ended fails +// as well: work the guest does without the network (a query term, a batch +// under a cached key) runs on through a cancellation, and the caller was +// promised the context's error. The output nobody will read is wiped. +func settle(ctx context.Context, out []byte, err error) ([]byte, error) { + ctxErr := ctx.Err() + if err != nil { + if ctxErr != nil && !errors.Is(err, ctxErr) { + err = fmt.Errorf("%w: %w", ctxErr, err) + } + return nil, err + } + if ctxErr != nil { + wipe(out) + return nil, ctxErr + } + return out, nil +} + +// abort drops a paused operation in the guest, for a caller that gave up +// on it between steps. It waits for the lock however long that takes, and +// runs under no deadline: the operation's plaintext and keys are wiped +// only when it is dropped. +func (c *Client) abort(handle uint64) { c.mu.Lock() defer c.mu.Unlock() + if c.closed || c.inst.module.IsClosed() { + return + } + ctx := context.Background() + _, _ = c.inst.call(ctx, c.inst.abort, scalar(handle)) +} + +// errLockWait marks a step that never ran because its caller's context +// ended while it waited for the client. +var errLockWait = errors.New("encrypt: the caller's context ended waiting for the client") + +// step runs one step of an operation on the instance under the client's +// lock: the operation's output when it finishes, or its next step when it +// pauses, taken from the guest before the lock is released (a host reads a +// step before it runs any other). +// +// The caller's context bounds the wait for the lock, and is refused if it +// has already ended, so a step whose caller timed out in the queue never +// reaches the guest. The guest then runs under guest.Detach, never the +// caller's context: the runtime closes a module whose call context ends, +// and one caller giving up must not close the client for the others. +// +// The client is closed, so later calls are ErrState rather than a runtime +// error, when the guest's state is unknown. A guest that trapped is one: +// it builds with panic-as-abort, so a trap is an abort mid-export, after +// which its keys are better wiped than reused. A guest still running +// c.stuck after its caller's context ended is the other: Detach's context +// ends, the runtime closes the module (WithCloseOnContextDone, see +// newInstance), and its key material goes with its memory. +func (c *Client) step(ctx context.Context, token string, tokenErr error, f stepFunc) ([]byte, *ioStep, error) { + if err := c.mu.Acquire(ctx); err != nil { + return nil, nil, fmt.Errorf("%w: %w", errLockWait, err) + } + defer c.mu.Unlock() if c.closed || c.inst.module.IsClosed() { c.closed = true - return nil, ErrState + return nil, nil, ErrState } growth := c.inst.mem.GrowthRefusal() if c.transport != nil { - c.transport.tokenErr = nil - } - out, err := f(c.inst) + c.transport.held, c.transport.fetchErr, c.transport.tokenErr = token, tokenErr, nil + defer func() { c.transport.held, c.transport.fetchErr = "", nil }() + } + run, done := guest.Detach(ctx, c.stuck) + out, err := f(run, c.inst) + var io *ioStep + if errors.Is(err, guest.ErrNeedsIO) { + // A step that cannot be read cannot be resumed; the guest drops + // the operation at shutdown, and refuses new pauses past its cap. + io, err = c.inst.takeIO(run) + } + done() switch { case c.inst.module.IsClosed(): c.closed = true if err == nil { err = ErrState } - err = fmt.Errorf("%w: interrupted call closed the client", err) + err = fmt.Errorf("%w: the guest ran on after its caller's context ended, and the client is closed", err) case errors.Is(err, errGuestTrap): // No guest code is running (f has returned), so the close is // immediate: the module's memory is wiped and freed here. @@ -439,7 +628,7 @@ func (c *Client) call(ctx context.Context, f func(*instance) ([]byte, error)) ([ } } if err != nil { - return nil, err + return nil, nil, err } - return out, nil + return out, io, nil } diff --git a/languages/golang/encrypt/context_field_test.go b/languages/golang/encrypt/context_field_test.go index 7952df4e7..303d71ae4 100644 --- a/languages/golang/encrypt/context_field_test.go +++ b/languages/golang/encrypt/context_field_test.go @@ -109,19 +109,19 @@ func TestContextFieldQueriesNameTheContext(t *testing.T) { t.Fatal(err) } // A query through a cipher with no context has nothing to derive under. - if _, err := testusers.NoteFields.Text.Equality(ctx, cipher, "hello"); !errors.Is(err, encrypt.ErrEncoding) || !strings.Contains(err.Error(), "Cipher.Context") { + if _, err := testusers.NoteFields.Text.Eq(ctx, cipher, "hello"); !errors.Is(err, encrypt.ErrEncoding) || !strings.Contains(err.Error(), "Cipher.Context") { t.Fatalf("a query with no context: %v, want ErrEncoding naming Cipher.Context", err) } // Under the tenant it singles out that tenant's row. acme := cipher.Context("tenants/acme") - probe, err := testusers.NoteFields.Text.Equality(ctx, acme, "hello") + probe, err := testusers.NoteFields.Text.Eq(ctx, acme, "hello") if err != nil { t.Fatal(err) } if !probe.Equal(encrypted[0].Text.Equality) || probe.Equal(encrypted[1].Text.Equality) || probe.Equal(encrypted[2].Text.Equality) { t.Error("the probe under tenants/acme does not single out acme's hello") } - globex, err := testusers.NoteFields.Text.Equality(ctx, cipher.Context("tenants/globex"), "hello") + globex, err := testusers.NoteFields.Text.Eq(ctx, cipher.Context("tenants/globex"), "hello") if err != nil || !globex.Equal(encrypted[1].Text.Equality) { t.Fatalf("the probe under tenants/globex: %v", err) } @@ -223,7 +223,7 @@ func TestGeneratedCodeOpensTheRustContextFieldFixture(t *testing.T) { t.Fatalf("through the globex cipher: %v, want ErrContextMismatch", err) } // The term Go derives under the tenant is the bytes Rust stored. - eq, err := testusers.NoteFields.Text.Equality(ctx, cipher.Context(want.Tenant), want.Text) + eq, err := testusers.NoteFields.Text.Eq(ctx, cipher.Context(want.Tenant), want.Text) if err != nil || !eq.Equal(stored.Text.Equality) { t.Errorf("text equality: %v, equal=%v", err, eq.Equal(stored.Text.Equality)) } diff --git a/languages/golang/encrypt/diagnostic_test.go b/languages/golang/encrypt/diagnostic_test.go index 811496d44..c632bde94 100644 --- a/languages/golang/encrypt/diagnostic_test.go +++ b/languages/golang/encrypt/diagnostic_test.go @@ -36,7 +36,7 @@ func wantCode(t *testing.T, err error, code string) { func TestGuestRefusalsCarryTheirDiagnostic(t *testing.T) { ctx := context.Background() c := rawInstance(t) - _, err := c.call(ctx, func(inst *instance) ([]byte, error) { + _, err := c.call(ctx, func(ctx context.Context, inst *instance) ([]byte, error) { return inst.call(ctx, inst.planCheck, buf([]byte{0xff})) }) if !errors.Is(err, ErrEncoding) { @@ -48,7 +48,7 @@ func TestGuestRefusalsCarryTheirDiagnostic(t *testing.T) { if err != nil { t.Fatal(err) } - _, err = c.call(ctx, func(inst *instance) ([]byte, error) { + _, err = c.call(ctx, func(ctx context.Context, inst *instance) ([]byte, error) { return inst.call(ctx, inst.keyset, buf(selector)) }) if !errors.Is(err, ErrState) { @@ -65,7 +65,7 @@ func TestAGuestWithoutLastErrorFailsWithTheBareSentinel(t *testing.T) { ctx := context.Background() c := rawInstance(t) c.inst.exports.LastError = nil - _, err := c.call(ctx, func(inst *instance) ([]byte, error) { + _, err := c.call(ctx, func(ctx context.Context, inst *instance) ([]byte, error) { return inst.call(ctx, inst.planCheck, buf([]byte{0xff})) }) if err != ErrEncoding { diff --git a/languages/golang/encrypt/doc.go b/languages/golang/encrypt/doc.go index 7b114f0e3..7cf110511 100644 --- a/languages/golang/encrypt/doc.go +++ b/languages/golang/encrypt/doc.go @@ -15,10 +15,28 @@ // type User struct { // _ struct{} `stash:"context=users"` // ID int64 `stash:"id,passthrough"` -// Email string `stash:"email,encrypt,index=equality;match"` -// Age uint32 `stash:"age,encrypt,index=equality;ore"` +// Email string `stash:"email,encrypt_into=TextEq"` +// Name string `stash:"name,encrypt_into=TextMatch"` +// Age int32 `stash:"age,encrypt_into=Integer"` // } // +// encrypt_into names the EQL type the field is stored as. The Go field type +// must match it: TextEq, TextMatch and Text take a string, Smallint and +// SmallintEq an int16, Integer and IntegerEq an int32, Bigint and BigintEq +// an int64, Boolean a bool, Real and RealEq a float32, Double and DoubleEq a +// float64, Date and DateEq a [Date], Timestamp and TimestampEq a time.Time, +// and Numeric and NumericEq a [Decimal]. An integer EQL type +// also takes any other built-in Go integer type, sealed at the type's width: +// one every value of which fits it (an int16 into Integer) always seals, and +// a wider one (an int64 into Integer) is range checked, a value that does not +// fit refused with [ErrEncoding]. The *Eq types answer Eq; the *Ord and +// *OrdOpe types (IntegerOrd, SmallintOrd, BigintOrd, RealOrd, DoubleOrd, +// TextOrd and their OrdOpe twins) answer Lt, Lte, Gt and Gte, and Eq too on +// text; Smallint, Integer, Bigint, Boolean, Real, Double, Text, Date, +// Timestamp and Numeric are stored and read only. NumericEq compares +// decimals with their scale normalised (1.5 = 1.50), and TimestampEq +// compares instants to the microsecond, truncated, never rounded. +// // 3. Run the generator. It writes user_stash.go beside the struct: // // go generate ./... @@ -35,15 +53,24 @@ // cipher := client.Keyset(encrypt.KeysetName("tenant-42")) // encrypted, err := users.Encrypt(ctx, cipher, people) // []users.EncryptedUser // opened, err := users.Decrypt(ctx, cipher, encrypted) // []users.User -// term, err := users.Fields.Email.Equality(ctx, cipher, "bob@example.com") +// term, err := users.Fields.Email.Eq(ctx, cipher, "bob@example.com") // // Encrypt and Decrypt send one ZeroKMS request for each 500 sealed values in -// the batch, plus one the first time a keyset is used. +// the batch, plus one the first time a keyset is used. A query derives its +// term locally, with no request of its own; only the first use of a keyset, +// by a query or anything else, loads it from ZeroKMS. See Queries below. +// +// 6. Store the encrypted type. Each encrypt_into field is one EQL v4 column +// (an eql.TextEq goes in a public.eql_v4_text_eq column), implementing +// driver.Valuer and sql.Scanner, so a database library binds and scans it +// like any other value. Install EQL v4 into PostgreSQL first, with the +// eql-v4-install.sql attached to the release. A query casts its parameter to +// the column's query type: +// +// SELECT id, email, name, age FROM users WHERE email = $1::eql_v4.query_text_eq // -// 6. Store the encrypted type. Each sealed field is one or more columns of -// bytes ([Ciphertext] and the term types), each implementing driver.Valuer -// and sql.Scanner, so a database library binds and scans each one as bytes; -// map each one to its own column. +// A field tagged encrypt or index= stores its ciphertext ([Ciphertext]) and +// each term as separate byte columns instead; see Terms below. // // 7. Run the generator again after each change to the struct or to a tag. A // change to the fields of the struct stops the build until you do. @@ -89,11 +116,20 @@ // plan the Rust chain and the derive run. A program never builds or names a // plan, and no call takes a context or an index choice of its own. // +// A sealed field outside an opaque struct is one value: a string, []byte, +// bool, an integer of any width (int8 to int64, uint8 to uint64, int, uint, +// or [Int128] and [Uint128]), a float32 or float64, a [Decimal], a [Date] or a +// time.Time (a timestamp), or a type defined over one of the built-in types. +// Each integer seals at its own width. Equality applies to every one but a +// bool; ORE and OPE to the integers, text, bytes and floats, and not yet to a +// decimal, date or timestamp, whose order terms wait for new cllw-ore and +// ore-rs releases (#1236). +// // Passthrough fields never cross the binding: the engine does nothing to a // passthrough value that a program could observe, and the FFI codec cannot -// carry every Go type a program stores beside a ciphertext (a time.Time, a -// driver.Valuer). An opaque struct crosses as one JSON document and is one -// column. +// carry every Go type a program stores beside a ciphertext (a +// driver.Valuer, a sql.NullTime). An opaque struct crosses as one JSON +// document and is one column. // // # Terms // @@ -104,18 +140,89 @@ // [Match], [Ore] and [Ope]; [JSON] is declared and refused until the engine // derives it. Every stored type implements driver.Valuer and sql.Scanner. // -// A match index needs text with at least one token: the engine derives no -// match term for an empty string, separator-only text, or text shorter than -// the n-gram length (3 characters), because an empty term would match every -// row. Encrypt then fails for the whole batch with [ErrTerm], naming the row, -// the field and the index, so an optional or short value does not belong -// under match. +// A match index stores text with no token (an empty string, separator-only +// text, or text shorter than the n-gram length of 3 characters) as the empty +// filter: a valid value that no query matches, so an optional or short value +// can live under match. The empty filter tells anyone who can read the +// column that the value is one of those. A match *query* of such text is +// refused with [ErrTerm], because an empty filter as a query would match every +// row. +// +// # Queries +// +// A generated Fields entry has one method for each operator the engine +// answers on the field, named for it: Eq, Lt, Lte, Gt, Gte, Between and +// Matches. A field has no method for an operator it does not answer, so +// such a query does not compile; the generator asks the engine which +// operators each field answers, and holds no copy of the rules. Each method +// takes the field's own Go type. +// +// Eq returns an [EqualityTerm] (or, for a field with encrypt_into, the EQL +// type's query value), an ordering an [OreTerm] or [OpeTerm], and Between a +// [Range] of two. Their answers are exact. The orderings are answered on +// integers, on floats, which order as PostgreSQL's float8 does (-0.0 ties +// 0.0, NaN sorts last), and on text, which orders by its folded form (case +// and accents folded, under Unicode 16), so Café and cafe tie and alice +// orders before Zoe. Matches returns a [MatchQuery], +// whose answer can hold rows that do not match: size the fetch with +// [MatchQuery.TargetLimit], then decrypt the rows and keep those +// [MatchQuery.Verify] reports true, one call into the engine for the page. +// On a field with encrypt_into=TextMatch, Matches returns an +// [EQLMatchQuery], the type's query value with the same two methods. +// Every query type implements driver.Valuer, so it binds as a parameter +// through database/sql or pgx. With separate byte columns, as the example/ +// program declares its fields (Email `stash:"email,encrypt,index=equality;match"`, +// Age `stash:"age,encrypt,index=equality;ore"`), not the encrypt_into User +// above: +// +// q, err := users.Fields.Email.Matches(ctx, cipher, "alice") +// limit, err := q.TargetLimit(ctx, 10, encrypt.Stats{Rows: rowCount}) +// rows, err := db.QueryContext(ctx, `SELECT id, email, email_eq, email_match, age, age_eq, age_ore +// FROM users WHERE match_positions(email_match) @> match_positions($1) +// ORDER BY id LIMIT $2`, q, limit.FetchOr(50)) +// // scan into []users.EncryptedUser, then +// people, err := users.Decrypt(ctx, cipher, encrypted) +// emails := make([]string, len(people)) +// for i, p := range people { +// emails[i] = p.Email +// } +// verified, err := q.Verify(ctx, emails) // one bool per row +// +// match_positions is a SQL function of your own that reads a match term's +// little-endian uint16 positions into an int[]; the package README has one. +// A MatchQuery keeps the text it was derived from, because Verify cuts its +// tokens again in the engine. Go cannot wipe a string, so the text stays in +// memory until the garbage collector reclaims it; a MatchQuery never prints +// it. +// +// # Cancellation +// +// Every call takes a [context.Context], and a call whose context is +// cancelled or passes its deadline fails with that context's error. Only +// that call fails: one Client is shared by every request in a service, and +// one request giving up never closes it for the others. A call waiting +// for the client, which runs one call at a time, leaves the queue when its +// context ends and never starts. A call waiting on ZeroKMS has that request +// cancelled; its error matches both the context's error and +// [ErrTransport]. A call waiting on its auth strategy for a token returns +// too, with the context's error. If the strategy is refreshing a device +// session, the refresh finishes in the background, for up to 10 seconds: +// the service rotates the refresh token when it answers, and dropping that +// answer would leave the strategy unable to refresh. +// +// A Client closes itself, and every later call returns [ErrState], only +// when the guest's state is unknown: the guest trapped, or a call ran on +// for 30 seconds after its context ended, which means a bug, not a caller +// that gave up. A service that sees ErrState from a client it did not +// close builds a new one. // // # Transport and auth // -// The guest imports exactly two host functions: an HTTP send, served by any -// [net/http.RoundTripper], and a bearer-token fetch, served by the -// credentials' auth strategy. What crosses per ZeroKMS call is what would +// The guest imports exactly one host function, a bearer-token fetch, served +// by the credentials' auth strategy. It never performs a request itself: an +// operation that needs ZeroKMS pauses and hands the request to the client, +// which sends it with any [net/http.RoundTripper] and resumes the operation +// with the response. What crosses per ZeroKMS call is what would // cross TLS anyway; derived key material never leaves the guest. Under // [AutoCredentials] and [OIDCFederation] the same RoundTripper also carries // the authentication requests to CTS, so one scoped to the ZeroKMS host alone @@ -143,7 +250,8 @@ // // The generator and the compiler find a mistake in a declaration, so no call // returns an error for one. A call returns an error for a key, for the -// network, or for stored data: [ErrForeignKeyset] when a *Cipher is given a +// network, or for stored data, or because its context ended (see +// Cancellation): [ErrForeignKeyset] when a *Cipher is given a // record another keyset sealed, [ErrAuthentication] or [ErrForbidden] for a // ciphertext that does not open under its field's context, [ErrEncoding] for // a stored value that does not fit its declaration. Read the kind with diff --git a/languages/golang/encrypt/eql/eql.go b/languages/golang/encrypt/eql/eql.go index 24b3d7c17..d6efdcb1a 100644 --- a/languages/golang/encrypt/eql/eql.go +++ b/languages/golang/encrypt/eql/eql.go @@ -3,7 +3,7 @@ // the type's domain, and the query types that match them. // // A field with `stash:"email,encrypt_into=TextEq"` is an [TextEq] in the -// generated encrypted type, and its Fields entry's Query returns a +// generated encrypted type, and its Fields entry's Eq returns a // [TextEqQuery]. The guest builds every value: generated code stores what // the guest returns and never assembles an EQL value itself (ADR-0007, // amended 2026-10-06). Each type implements driver.Valuer and sql.Scanner @@ -18,9 +18,9 @@ // // The types and the [Types] table are generated from the EQL catalog by // eql-codegen (`mise run types:generate` in packages/eql); eql_gen.go is -// committed and drift-gated. The engine produces [TextEq] today; every -// other type is listed with the reason it cannot be produced yet, and -// stashgen refuses it. +// committed and drift-gated. [Types] says which types the engine produces +// today; every other type is listed with the reason it cannot be produced +// yet, and stashgen refuses it. package eql import ( diff --git a/languages/golang/encrypt/eql/eql_gen.go b/languages/golang/encrypt/eql/eql_gen.go index 911a82c97..b47e285eb 100644 --- a/languages/golang/encrypt/eql/eql_gen.go +++ b/languages/golang/encrypt/eql/eql_gen.go @@ -9,61 +9,61 @@ import "database/sql/driver" // engine produces the type today; stashgen refuses encrypt_into for one // it does not, with the Reason. var Types = []Type{ - {Name: "Integer", GoName: "Integer", Family: "integer", Suffix: "", Plaintext: "", SQLDomain: "public.eql_v3_integer", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, - {Name: "IntegerEq", GoName: "IntegerEq", Family: "integer", Suffix: "Eq", Plaintext: "", SQLDomain: "public.eql_v3_integer_eq", Indexes: []string{"eq"}, Query: "IntegerEqQuery", QuerySQLDomain: "eql_v3.query_integer_eq", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, - {Name: "IntegerOrdOre", GoName: "IntegerOrdOre", Family: "integer", Suffix: "OrdOre", Plaintext: "", SQLDomain: "public.eql_v3_integer_ord_ore", Indexes: []string{"ore"}, Query: "IntegerOrdOreQuery", QuerySQLDomain: "eql_v3.query_integer_ord_ore", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, - {Name: "IntegerOrd", GoName: "IntegerOrd", Family: "integer", Suffix: "Ord", Plaintext: "", SQLDomain: "public.eql_v3_integer_ord", Indexes: []string{"ope"}, Query: "IntegerOrdQuery", QuerySQLDomain: "eql_v3.query_integer_ord", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, - {Name: "IntegerOrdOpe", GoName: "IntegerOrdOpe", Family: "integer", Suffix: "OrdOpe", Plaintext: "", SQLDomain: "public.eql_v3_integer_ord_ope", Indexes: []string{"ope"}, Query: "IntegerOrdOpeQuery", QuerySQLDomain: "eql_v3.query_integer_ord_ope", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, - {Name: "Smallint", GoName: "Smallint", Family: "smallint", Suffix: "", Plaintext: "", SQLDomain: "public.eql_v3_smallint", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, - {Name: "SmallintEq", GoName: "SmallintEq", Family: "smallint", Suffix: "Eq", Plaintext: "", SQLDomain: "public.eql_v3_smallint_eq", Indexes: []string{"eq"}, Query: "SmallintEqQuery", QuerySQLDomain: "eql_v3.query_smallint_eq", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, - {Name: "SmallintOrdOre", GoName: "SmallintOrdOre", Family: "smallint", Suffix: "OrdOre", Plaintext: "", SQLDomain: "public.eql_v3_smallint_ord_ore", Indexes: []string{"ore"}, Query: "SmallintOrdOreQuery", QuerySQLDomain: "eql_v3.query_smallint_ord_ore", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, - {Name: "SmallintOrd", GoName: "SmallintOrd", Family: "smallint", Suffix: "Ord", Plaintext: "", SQLDomain: "public.eql_v3_smallint_ord", Indexes: []string{"ope"}, Query: "SmallintOrdQuery", QuerySQLDomain: "eql_v3.query_smallint_ord", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, - {Name: "SmallintOrdOpe", GoName: "SmallintOrdOpe", Family: "smallint", Suffix: "OrdOpe", Plaintext: "", SQLDomain: "public.eql_v3_smallint_ord_ope", Indexes: []string{"ope"}, Query: "SmallintOrdOpeQuery", QuerySQLDomain: "eql_v3.query_smallint_ord_ope", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, - {Name: "Bigint", GoName: "Bigint", Family: "bigint", Suffix: "", Plaintext: "", SQLDomain: "public.eql_v3_bigint", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, - {Name: "BigintEq", GoName: "BigintEq", Family: "bigint", Suffix: "Eq", Plaintext: "", SQLDomain: "public.eql_v3_bigint_eq", Indexes: []string{"eq"}, Query: "BigintEqQuery", QuerySQLDomain: "eql_v3.query_bigint_eq", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, - {Name: "BigintOrdOre", GoName: "BigintOrdOre", Family: "bigint", Suffix: "OrdOre", Plaintext: "", SQLDomain: "public.eql_v3_bigint_ord_ore", Indexes: []string{"ore"}, Query: "BigintOrdOreQuery", QuerySQLDomain: "eql_v3.query_bigint_ord_ore", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, - {Name: "BigintOrd", GoName: "BigintOrd", Family: "bigint", Suffix: "Ord", Plaintext: "", SQLDomain: "public.eql_v3_bigint_ord", Indexes: []string{"ope"}, Query: "BigintOrdQuery", QuerySQLDomain: "eql_v3.query_bigint_ord", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, - {Name: "BigintOrdOpe", GoName: "BigintOrdOpe", Family: "bigint", Suffix: "OrdOpe", Plaintext: "", SQLDomain: "public.eql_v3_bigint_ord_ope", Indexes: []string{"ope"}, Query: "BigintOrdOpeQuery", QuerySQLDomain: "eql_v3.query_bigint_ord_ope", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, - {Name: "Date", GoName: "Date", Family: "date", Suffix: "", Plaintext: "", SQLDomain: "public.eql_v3_date", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, - {Name: "DateEq", GoName: "DateEq", Family: "date", Suffix: "Eq", Plaintext: "", SQLDomain: "public.eql_v3_date_eq", Indexes: []string{"eq"}, Query: "DateEqQuery", QuerySQLDomain: "eql_v3.query_date_eq", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, - {Name: "DateOrdOre", GoName: "DateOrdOre", Family: "date", Suffix: "OrdOre", Plaintext: "", SQLDomain: "public.eql_v3_date_ord_ore", Indexes: []string{"ore"}, Query: "DateOrdOreQuery", QuerySQLDomain: "eql_v3.query_date_ord_ore", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, - {Name: "DateOrd", GoName: "DateOrd", Family: "date", Suffix: "Ord", Plaintext: "", SQLDomain: "public.eql_v3_date_ord", Indexes: []string{"ope"}, Query: "DateOrdQuery", QuerySQLDomain: "eql_v3.query_date_ord", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, - {Name: "DateOrdOpe", GoName: "DateOrdOpe", Family: "date", Suffix: "OrdOpe", Plaintext: "", SQLDomain: "public.eql_v3_date_ord_ope", Indexes: []string{"ope"}, Query: "DateOrdOpeQuery", QuerySQLDomain: "eql_v3.query_date_ord_ope", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, - {Name: "Timestamp", GoName: "Timestamp", Family: "timestamp", Suffix: "", Plaintext: "", SQLDomain: "public.eql_v3_timestamp", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, - {Name: "TimestampEq", GoName: "TimestampEq", Family: "timestamp", Suffix: "Eq", Plaintext: "", SQLDomain: "public.eql_v3_timestamp_eq", Indexes: []string{"eq"}, Query: "TimestampEqQuery", QuerySQLDomain: "eql_v3.query_timestamp_eq", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, - {Name: "TimestampOrdOre", GoName: "TimestampOrdOre", Family: "timestamp", Suffix: "OrdOre", Plaintext: "", SQLDomain: "public.eql_v3_timestamp_ord_ore", Indexes: []string{"ore"}, Query: "TimestampOrdOreQuery", QuerySQLDomain: "eql_v3.query_timestamp_ord_ore", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, - {Name: "TimestampOrd", GoName: "TimestampOrd", Family: "timestamp", Suffix: "Ord", Plaintext: "", SQLDomain: "public.eql_v3_timestamp_ord", Indexes: []string{"ope"}, Query: "TimestampOrdQuery", QuerySQLDomain: "eql_v3.query_timestamp_ord", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, - {Name: "TimestampOrdOpe", GoName: "TimestampOrdOpe", Family: "timestamp", Suffix: "OrdOpe", Plaintext: "", SQLDomain: "public.eql_v3_timestamp_ord_ope", Indexes: []string{"ope"}, Query: "TimestampOrdOpeQuery", QuerySQLDomain: "eql_v3.query_timestamp_ord_ope", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, - {Name: "Numeric", GoName: "Numeric", Family: "numeric", Suffix: "", Plaintext: "", SQLDomain: "public.eql_v3_numeric", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, - {Name: "NumericEq", GoName: "NumericEq", Family: "numeric", Suffix: "Eq", Plaintext: "", SQLDomain: "public.eql_v3_numeric_eq", Indexes: []string{"eq"}, Query: "NumericEqQuery", QuerySQLDomain: "eql_v3.query_numeric_eq", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, - {Name: "NumericOrdOre", GoName: "NumericOrdOre", Family: "numeric", Suffix: "OrdOre", Plaintext: "", SQLDomain: "public.eql_v3_numeric_ord_ore", Indexes: []string{"ore"}, Query: "NumericOrdOreQuery", QuerySQLDomain: "eql_v3.query_numeric_ord_ore", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, - {Name: "NumericOrd", GoName: "NumericOrd", Family: "numeric", Suffix: "Ord", Plaintext: "", SQLDomain: "public.eql_v3_numeric_ord", Indexes: []string{"ope"}, Query: "NumericOrdQuery", QuerySQLDomain: "eql_v3.query_numeric_ord", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, - {Name: "NumericOrdOpe", GoName: "NumericOrdOpe", Family: "numeric", Suffix: "OrdOpe", Plaintext: "", SQLDomain: "public.eql_v3_numeric_ord_ope", Indexes: []string{"ope"}, Query: "NumericOrdOpeQuery", QuerySQLDomain: "eql_v3.query_numeric_ord_ope", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, - {Name: "Text", GoName: "Text", Family: "text", Suffix: "", Plaintext: "string", SQLDomain: "public.eql_v3_text", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: false, Reason: "the storage-only text domain follows once TextEq is proven end to end in PostgreSQL"}, - {Name: "TextEq", GoName: "TextEq", Family: "text", Suffix: "Eq", Plaintext: "string", SQLDomain: "public.eql_v3_text_eq", Indexes: []string{"eq"}, Query: "TextEqQuery", QuerySQLDomain: "eql_v3.query_text_eq", Producible: true, Reason: ""}, - {Name: "TextMatch", GoName: "TextMatch", Family: "text", Suffix: "Match", Plaintext: "string", SQLDomain: "public.eql_v3_text_match", Indexes: []string{"match"}, Query: "TextMatchQuery", QuerySQLDomain: "eql_v3.query_text_match", Producible: false, Reason: "the engine derives match and OPE terms, but no EQL type is built from them yet"}, - {Name: "TextOrdOre", GoName: "TextOrdOre", Family: "text", Suffix: "OrdOre", Plaintext: "string", SQLDomain: "public.eql_v3_text_ord_ore", Indexes: []string{"eq", "ore"}, Query: "TextOrdOreQuery", QuerySQLDomain: "eql_v3.query_text_ord_ore", Producible: false, Reason: "EQL stores a block ORE term and the engine derives a CLLW ORE term; the two are different algorithms"}, - {Name: "TextOrd", GoName: "TextOrd", Family: "text", Suffix: "Ord", Plaintext: "string", SQLDomain: "public.eql_v3_text_ord", Indexes: []string{"eq", "ope"}, Query: "TextOrdQuery", QuerySQLDomain: "eql_v3.query_text_ord", Producible: false, Reason: "the engine derives match and OPE terms, but no EQL type is built from them yet"}, - {Name: "TextOrdOpe", GoName: "TextOrdOpe", Family: "text", Suffix: "OrdOpe", Plaintext: "string", SQLDomain: "public.eql_v3_text_ord_ope", Indexes: []string{"eq", "ope"}, Query: "TextOrdOpeQuery", QuerySQLDomain: "eql_v3.query_text_ord_ope", Producible: false, Reason: "the engine derives match and OPE terms, but no EQL type is built from them yet"}, - {Name: "TextSearchOre", GoName: "TextSearchOre", Family: "text", Suffix: "SearchOre", Plaintext: "string", SQLDomain: "public.eql_v3_text_search_ore", Indexes: []string{"eq", "ore", "match"}, Query: "TextSearchOreQuery", QuerySQLDomain: "eql_v3.query_text_search_ore", Producible: false, Reason: "EQL stores a block ORE term and the engine derives a CLLW ORE term; the two are different algorithms"}, - {Name: "TextSearch", GoName: "TextSearch", Family: "text", Suffix: "Search", Plaintext: "string", SQLDomain: "public.eql_v3_text_search", Indexes: []string{"eq", "ope", "match"}, Query: "TextSearchQuery", QuerySQLDomain: "eql_v3.query_text_search", Producible: false, Reason: "the engine derives match and OPE terms, but no EQL type is built from them yet"}, - {Name: "Boolean", GoName: "Boolean", Family: "boolean", Suffix: "", Plaintext: "", SQLDomain: "public.eql_v3_boolean", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, - {Name: "Real", GoName: "Real", Family: "real", Suffix: "", Plaintext: "", SQLDomain: "public.eql_v3_real", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, - {Name: "RealEq", GoName: "RealEq", Family: "real", Suffix: "Eq", Plaintext: "", SQLDomain: "public.eql_v3_real_eq", Indexes: []string{"eq"}, Query: "RealEqQuery", QuerySQLDomain: "eql_v3.query_real_eq", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, - {Name: "RealOrdOre", GoName: "RealOrdOre", Family: "real", Suffix: "OrdOre", Plaintext: "", SQLDomain: "public.eql_v3_real_ord_ore", Indexes: []string{"ore"}, Query: "RealOrdOreQuery", QuerySQLDomain: "eql_v3.query_real_ord_ore", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, - {Name: "RealOrd", GoName: "RealOrd", Family: "real", Suffix: "Ord", Plaintext: "", SQLDomain: "public.eql_v3_real_ord", Indexes: []string{"ope"}, Query: "RealOrdQuery", QuerySQLDomain: "eql_v3.query_real_ord", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, - {Name: "RealOrdOpe", GoName: "RealOrdOpe", Family: "real", Suffix: "OrdOpe", Plaintext: "", SQLDomain: "public.eql_v3_real_ord_ope", Indexes: []string{"ope"}, Query: "RealOrdOpeQuery", QuerySQLDomain: "eql_v3.query_real_ord_ope", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, - {Name: "Double", GoName: "Double", Family: "double", Suffix: "", Plaintext: "", SQLDomain: "public.eql_v3_double", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, - {Name: "DoubleEq", GoName: "DoubleEq", Family: "double", Suffix: "Eq", Plaintext: "", SQLDomain: "public.eql_v3_double_eq", Indexes: []string{"eq"}, Query: "DoubleEqQuery", QuerySQLDomain: "eql_v3.query_double_eq", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, - {Name: "DoubleOrdOre", GoName: "DoubleOrdOre", Family: "double", Suffix: "OrdOre", Plaintext: "", SQLDomain: "public.eql_v3_double_ord_ore", Indexes: []string{"ore"}, Query: "DoubleOrdOreQuery", QuerySQLDomain: "eql_v3.query_double_ord_ore", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, - {Name: "DoubleOrd", GoName: "DoubleOrd", Family: "double", Suffix: "Ord", Plaintext: "", SQLDomain: "public.eql_v3_double_ord", Indexes: []string{"ope"}, Query: "DoubleOrdQuery", QuerySQLDomain: "eql_v3.query_double_ord", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, - {Name: "DoubleOrdOpe", GoName: "DoubleOrdOpe", Family: "double", Suffix: "OrdOpe", Plaintext: "", SQLDomain: "public.eql_v3_double_ord_ope", Indexes: []string{"ope"}, Query: "DoubleOrdOpeQuery", QuerySQLDomain: "eql_v3.query_double_ord_ope", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, - {Name: "SteVecDocument", GoName: "SteVecDocument", Family: "json", Suffix: "Search", Plaintext: "", SQLDomain: "public.eql_v3_json_search", Indexes: []string{"json"}, Query: "SteVecQuery", QuerySQLDomain: "eql_v3.query_json", Producible: false, Reason: "the JSON index is a new operation in the engine"}, - {Name: "Json", GoName: "JSON", Family: "json", Suffix: "", Plaintext: "", SQLDomain: "public.eql_v3_json", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: false, Reason: "how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is"}, -} - -// Integer is the EQL type stored in public.eql_v3_integer: the integer family with no index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. + {Name: "Integer", GoName: "Integer", Family: "integer", Suffix: "", Plaintext: "int32", SQLDomain: "public.eql_v4_integer", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: true, Reason: ""}, + {Name: "IntegerEq", GoName: "IntegerEq", Family: "integer", Suffix: "Eq", Plaintext: "int32", SQLDomain: "public.eql_v4_integer_eq", Indexes: []string{"eq"}, Query: "IntegerEqQuery", QuerySQLDomain: "eql_v4.query_integer_eq", Producible: true, Reason: ""}, + {Name: "IntegerOrdOre", GoName: "IntegerOrdOre", Family: "integer", Suffix: "OrdOre", Plaintext: "int32", SQLDomain: "public.eql_v4_integer_ord_ore", Indexes: []string{"ore_block"}, Query: "IntegerOrdOreQuery", QuerySQLDomain: "eql_v4.query_integer_ord_ore", Producible: true, Reason: ""}, + {Name: "IntegerOrd", GoName: "IntegerOrd", Family: "integer", Suffix: "Ord", Plaintext: "int32", SQLDomain: "public.eql_v4_integer_ord", Indexes: []string{"ope"}, Query: "IntegerOrdQuery", QuerySQLDomain: "eql_v4.query_integer_ord", Producible: true, Reason: ""}, + {Name: "IntegerOrdOpe", GoName: "IntegerOrdOpe", Family: "integer", Suffix: "OrdOpe", Plaintext: "int32", SQLDomain: "public.eql_v4_integer_ord_ope", Indexes: []string{"ope"}, Query: "IntegerOrdOpeQuery", QuerySQLDomain: "eql_v4.query_integer_ord_ope", Producible: true, Reason: ""}, + {Name: "Smallint", GoName: "Smallint", Family: "smallint", Suffix: "", Plaintext: "int16", SQLDomain: "public.eql_v4_smallint", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: true, Reason: ""}, + {Name: "SmallintEq", GoName: "SmallintEq", Family: "smallint", Suffix: "Eq", Plaintext: "int16", SQLDomain: "public.eql_v4_smallint_eq", Indexes: []string{"eq"}, Query: "SmallintEqQuery", QuerySQLDomain: "eql_v4.query_smallint_eq", Producible: true, Reason: ""}, + {Name: "SmallintOrdOre", GoName: "SmallintOrdOre", Family: "smallint", Suffix: "OrdOre", Plaintext: "int16", SQLDomain: "public.eql_v4_smallint_ord_ore", Indexes: []string{"ore_block"}, Query: "SmallintOrdOreQuery", QuerySQLDomain: "eql_v4.query_smallint_ord_ore", Producible: true, Reason: ""}, + {Name: "SmallintOrd", GoName: "SmallintOrd", Family: "smallint", Suffix: "Ord", Plaintext: "int16", SQLDomain: "public.eql_v4_smallint_ord", Indexes: []string{"ope"}, Query: "SmallintOrdQuery", QuerySQLDomain: "eql_v4.query_smallint_ord", Producible: true, Reason: ""}, + {Name: "SmallintOrdOpe", GoName: "SmallintOrdOpe", Family: "smallint", Suffix: "OrdOpe", Plaintext: "int16", SQLDomain: "public.eql_v4_smallint_ord_ope", Indexes: []string{"ope"}, Query: "SmallintOrdOpeQuery", QuerySQLDomain: "eql_v4.query_smallint_ord_ope", Producible: true, Reason: ""}, + {Name: "Bigint", GoName: "Bigint", Family: "bigint", Suffix: "", Plaintext: "int64", SQLDomain: "public.eql_v4_bigint", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: true, Reason: ""}, + {Name: "BigintEq", GoName: "BigintEq", Family: "bigint", Suffix: "Eq", Plaintext: "int64", SQLDomain: "public.eql_v4_bigint_eq", Indexes: []string{"eq"}, Query: "BigintEqQuery", QuerySQLDomain: "eql_v4.query_bigint_eq", Producible: true, Reason: ""}, + {Name: "BigintOrdOre", GoName: "BigintOrdOre", Family: "bigint", Suffix: "OrdOre", Plaintext: "int64", SQLDomain: "public.eql_v4_bigint_ord_ore", Indexes: []string{"ore_block"}, Query: "BigintOrdOreQuery", QuerySQLDomain: "eql_v4.query_bigint_ord_ore", Producible: true, Reason: ""}, + {Name: "BigintOrd", GoName: "BigintOrd", Family: "bigint", Suffix: "Ord", Plaintext: "int64", SQLDomain: "public.eql_v4_bigint_ord", Indexes: []string{"ope"}, Query: "BigintOrdQuery", QuerySQLDomain: "eql_v4.query_bigint_ord", Producible: true, Reason: ""}, + {Name: "BigintOrdOpe", GoName: "BigintOrdOpe", Family: "bigint", Suffix: "OrdOpe", Plaintext: "int64", SQLDomain: "public.eql_v4_bigint_ord_ope", Indexes: []string{"ope"}, Query: "BigintOrdOpeQuery", QuerySQLDomain: "eql_v4.query_bigint_ord_ope", Producible: true, Reason: ""}, + {Name: "Date", GoName: "Date", Family: "date", Suffix: "", Plaintext: "date", SQLDomain: "public.eql_v4_date", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: true, Reason: ""}, + {Name: "DateEq", GoName: "DateEq", Family: "date", Suffix: "Eq", Plaintext: "date", SQLDomain: "public.eql_v4_date_eq", Indexes: []string{"eq"}, Query: "DateEqQuery", QuerySQLDomain: "eql_v4.query_date_eq", Producible: true, Reason: ""}, + {Name: "DateOrdOre", GoName: "DateOrdOre", Family: "date", Suffix: "OrdOre", Plaintext: "date", SQLDomain: "public.eql_v4_date_ord_ore", Indexes: []string{"ore_block"}, Query: "DateOrdOreQuery", QuerySQLDomain: "eql_v4.query_date_ord_ore", Producible: false, Reason: "the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236)"}, + {Name: "DateOrd", GoName: "DateOrd", Family: "date", Suffix: "Ord", Plaintext: "date", SQLDomain: "public.eql_v4_date_ord", Indexes: []string{"ope"}, Query: "DateOrdQuery", QuerySQLDomain: "eql_v4.query_date_ord", Producible: false, Reason: "the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236)"}, + {Name: "DateOrdOpe", GoName: "DateOrdOpe", Family: "date", Suffix: "OrdOpe", Plaintext: "date", SQLDomain: "public.eql_v4_date_ord_ope", Indexes: []string{"ope"}, Query: "DateOrdOpeQuery", QuerySQLDomain: "eql_v4.query_date_ord_ope", Producible: false, Reason: "the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236)"}, + {Name: "Timestamp", GoName: "Timestamp", Family: "timestamp", Suffix: "", Plaintext: "timestamp", SQLDomain: "public.eql_v4_timestamp", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: true, Reason: ""}, + {Name: "TimestampEq", GoName: "TimestampEq", Family: "timestamp", Suffix: "Eq", Plaintext: "timestamp", SQLDomain: "public.eql_v4_timestamp_eq", Indexes: []string{"eq"}, Query: "TimestampEqQuery", QuerySQLDomain: "eql_v4.query_timestamp_eq", Producible: true, Reason: ""}, + {Name: "TimestampOrdOre", GoName: "TimestampOrdOre", Family: "timestamp", Suffix: "OrdOre", Plaintext: "timestamp", SQLDomain: "public.eql_v4_timestamp_ord_ore", Indexes: []string{"ore_block"}, Query: "TimestampOrdOreQuery", QuerySQLDomain: "eql_v4.query_timestamp_ord_ore", Producible: false, Reason: "the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236)"}, + {Name: "TimestampOrd", GoName: "TimestampOrd", Family: "timestamp", Suffix: "Ord", Plaintext: "timestamp", SQLDomain: "public.eql_v4_timestamp_ord", Indexes: []string{"ope"}, Query: "TimestampOrdQuery", QuerySQLDomain: "eql_v4.query_timestamp_ord", Producible: false, Reason: "the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236)"}, + {Name: "TimestampOrdOpe", GoName: "TimestampOrdOpe", Family: "timestamp", Suffix: "OrdOpe", Plaintext: "timestamp", SQLDomain: "public.eql_v4_timestamp_ord_ope", Indexes: []string{"ope"}, Query: "TimestampOrdOpeQuery", QuerySQLDomain: "eql_v4.query_timestamp_ord_ope", Producible: false, Reason: "the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236)"}, + {Name: "Numeric", GoName: "Numeric", Family: "numeric", Suffix: "", Plaintext: "decimal", SQLDomain: "public.eql_v4_numeric", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: true, Reason: ""}, + {Name: "NumericEq", GoName: "NumericEq", Family: "numeric", Suffix: "Eq", Plaintext: "decimal", SQLDomain: "public.eql_v4_numeric_eq", Indexes: []string{"eq"}, Query: "NumericEqQuery", QuerySQLDomain: "eql_v4.query_numeric_eq", Producible: true, Reason: ""}, + {Name: "NumericOrdOre", GoName: "NumericOrdOre", Family: "numeric", Suffix: "OrdOre", Plaintext: "decimal", SQLDomain: "public.eql_v4_numeric_ord_ore", Indexes: []string{"ore_block"}, Query: "NumericOrdOreQuery", QuerySQLDomain: "eql_v4.query_numeric_ord_ore", Producible: false, Reason: "the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236)"}, + {Name: "NumericOrd", GoName: "NumericOrd", Family: "numeric", Suffix: "Ord", Plaintext: "decimal", SQLDomain: "public.eql_v4_numeric_ord", Indexes: []string{"ope"}, Query: "NumericOrdQuery", QuerySQLDomain: "eql_v4.query_numeric_ord", Producible: false, Reason: "the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236)"}, + {Name: "NumericOrdOpe", GoName: "NumericOrdOpe", Family: "numeric", Suffix: "OrdOpe", Plaintext: "decimal", SQLDomain: "public.eql_v4_numeric_ord_ope", Indexes: []string{"ope"}, Query: "NumericOrdOpeQuery", QuerySQLDomain: "eql_v4.query_numeric_ord_ope", Producible: false, Reason: "the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236)"}, + {Name: "Text", GoName: "Text", Family: "text", Suffix: "", Plaintext: "string", SQLDomain: "public.eql_v4_text", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: true, Reason: ""}, + {Name: "TextEq", GoName: "TextEq", Family: "text", Suffix: "Eq", Plaintext: "string", SQLDomain: "public.eql_v4_text_eq", Indexes: []string{"eq"}, Query: "TextEqQuery", QuerySQLDomain: "eql_v4.query_text_eq", Producible: true, Reason: ""}, + {Name: "TextMatch", GoName: "TextMatch", Family: "text", Suffix: "Match", Plaintext: "string", SQLDomain: "public.eql_v4_text_match", Indexes: []string{"match"}, Query: "TextMatchQuery", QuerySQLDomain: "eql_v4.query_text_match", Producible: true, Reason: ""}, + {Name: "TextOrdOre", GoName: "TextOrdOre", Family: "text", Suffix: "OrdOre", Plaintext: "string", SQLDomain: "public.eql_v4_text_ord_ore", Indexes: []string{"eq", "ore_block"}, Query: "TextOrdOreQuery", QuerySQLDomain: "eql_v4.query_text_ord_ore", Producible: true, Reason: ""}, + {Name: "TextOrd", GoName: "TextOrd", Family: "text", Suffix: "Ord", Plaintext: "string", SQLDomain: "public.eql_v4_text_ord", Indexes: []string{"eq", "ope"}, Query: "TextOrdQuery", QuerySQLDomain: "eql_v4.query_text_ord", Producible: true, Reason: ""}, + {Name: "TextOrdOpe", GoName: "TextOrdOpe", Family: "text", Suffix: "OrdOpe", Plaintext: "string", SQLDomain: "public.eql_v4_text_ord_ope", Indexes: []string{"eq", "ope"}, Query: "TextOrdOpeQuery", QuerySQLDomain: "eql_v4.query_text_ord_ope", Producible: true, Reason: ""}, + {Name: "TextSearchOre", GoName: "TextSearchOre", Family: "text", Suffix: "SearchOre", Plaintext: "string", SQLDomain: "public.eql_v4_text_search_ore", Indexes: []string{"eq", "ore_block", "match"}, Query: "TextSearchOreQuery", QuerySQLDomain: "eql_v4.query_text_search_ore", Producible: true, Reason: ""}, + {Name: "TextSearch", GoName: "TextSearch", Family: "text", Suffix: "Search", Plaintext: "string", SQLDomain: "public.eql_v4_text_search", Indexes: []string{"eq", "ope", "match"}, Query: "TextSearchQuery", QuerySQLDomain: "eql_v4.query_text_search", Producible: true, Reason: ""}, + {Name: "Boolean", GoName: "Boolean", Family: "boolean", Suffix: "", Plaintext: "bool", SQLDomain: "public.eql_v4_boolean", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: true, Reason: ""}, + {Name: "Real", GoName: "Real", Family: "real", Suffix: "", Plaintext: "float32", SQLDomain: "public.eql_v4_real", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: true, Reason: ""}, + {Name: "RealEq", GoName: "RealEq", Family: "real", Suffix: "Eq", Plaintext: "float32", SQLDomain: "public.eql_v4_real_eq", Indexes: []string{"eq"}, Query: "RealEqQuery", QuerySQLDomain: "eql_v4.query_real_eq", Producible: true, Reason: ""}, + {Name: "RealOrdOre", GoName: "RealOrdOre", Family: "real", Suffix: "OrdOre", Plaintext: "float32", SQLDomain: "public.eql_v4_real_ord_ore", Indexes: []string{"ore_block"}, Query: "RealOrdOreQuery", QuerySQLDomain: "eql_v4.query_real_ord_ore", Producible: true, Reason: ""}, + {Name: "RealOrd", GoName: "RealOrd", Family: "real", Suffix: "Ord", Plaintext: "float32", SQLDomain: "public.eql_v4_real_ord", Indexes: []string{"ope"}, Query: "RealOrdQuery", QuerySQLDomain: "eql_v4.query_real_ord", Producible: true, Reason: ""}, + {Name: "RealOrdOpe", GoName: "RealOrdOpe", Family: "real", Suffix: "OrdOpe", Plaintext: "float32", SQLDomain: "public.eql_v4_real_ord_ope", Indexes: []string{"ope"}, Query: "RealOrdOpeQuery", QuerySQLDomain: "eql_v4.query_real_ord_ope", Producible: true, Reason: ""}, + {Name: "Double", GoName: "Double", Family: "double", Suffix: "", Plaintext: "float64", SQLDomain: "public.eql_v4_double", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: true, Reason: ""}, + {Name: "DoubleEq", GoName: "DoubleEq", Family: "double", Suffix: "Eq", Plaintext: "float64", SQLDomain: "public.eql_v4_double_eq", Indexes: []string{"eq"}, Query: "DoubleEqQuery", QuerySQLDomain: "eql_v4.query_double_eq", Producible: true, Reason: ""}, + {Name: "DoubleOrdOre", GoName: "DoubleOrdOre", Family: "double", Suffix: "OrdOre", Plaintext: "float64", SQLDomain: "public.eql_v4_double_ord_ore", Indexes: []string{"ore_block"}, Query: "DoubleOrdOreQuery", QuerySQLDomain: "eql_v4.query_double_ord_ore", Producible: true, Reason: ""}, + {Name: "DoubleOrd", GoName: "DoubleOrd", Family: "double", Suffix: "Ord", Plaintext: "float64", SQLDomain: "public.eql_v4_double_ord", Indexes: []string{"ope"}, Query: "DoubleOrdQuery", QuerySQLDomain: "eql_v4.query_double_ord", Producible: true, Reason: ""}, + {Name: "DoubleOrdOpe", GoName: "DoubleOrdOpe", Family: "double", Suffix: "OrdOpe", Plaintext: "float64", SQLDomain: "public.eql_v4_double_ord_ope", Indexes: []string{"ope"}, Query: "DoubleOrdOpeQuery", QuerySQLDomain: "eql_v4.query_double_ord_ope", Producible: true, Reason: ""}, + {Name: "SteVecDocument", GoName: "SteVecDocument", Family: "json", Suffix: "Search", Plaintext: "", SQLDomain: "public.eql_v4_json_search", Indexes: []string{"json"}, Query: "SteVecQuery", QuerySQLDomain: "eql_v4.query_json", Producible: false, Reason: "the JSON index is a new operation in the engine (#1060)"}, + {Name: "Json", GoName: "JSON", Family: "json", Suffix: "", Plaintext: "", SQLDomain: "public.eql_v4_json", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: false, Reason: "a JSON value seals as a tree of leaves and EQL's `c` holds one; how a document becomes one sealed leaf is not decided"}, +} + +// Integer is the EQL type stored in public.eql_v4_integer: the integer family with no index. +// The engine produces it from an int32. type Integer []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -71,7 +71,7 @@ func (v Integer) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_integer column. +// Scan reads a public.eql_v4_integer column. func (v *Integer) Scan(src any) error { return scan((*[]byte)(v), src, "Integer") } @@ -86,8 +86,8 @@ func (v *Integer) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// IntegerEq is the EQL type stored in public.eql_v3_integer_eq: the integer family with the eq index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. +// IntegerEq is the EQL type stored in public.eql_v4_integer_eq: the integer family with the eq index. +// The engine produces it from an int32. type IntegerEq []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -95,7 +95,7 @@ func (v IntegerEq) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_integer_eq column. +// Scan reads a public.eql_v4_integer_eq column. func (v *IntegerEq) Scan(src any) error { return scan((*[]byte)(v), src, "IntegerEq") } @@ -110,7 +110,7 @@ func (v *IntegerEq) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// IntegerEqQuery is the query value for IntegerEq: the operand eql_v3.query_integer_eq takes. +// IntegerEqQuery is the query value for IntegerEq: the operand eql_v4.query_integer_eq takes. type IntegerEqQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -118,7 +118,7 @@ func (v IntegerEqQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v3.query_integer_eq column. +// Scan reads a eql_v4.query_integer_eq column. func (v *IntegerEqQuery) Scan(src any) error { return scan((*[]byte)(v), src, "IntegerEqQuery") } @@ -133,8 +133,8 @@ func (v *IntegerEqQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// IntegerOrdOre is the EQL type stored in public.eql_v3_integer_ord_ore: the integer family with the ore index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. +// IntegerOrdOre is the EQL type stored in public.eql_v4_integer_ord_ore: the integer family with the ore_block index. +// The engine produces it from an int32. type IntegerOrdOre []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -142,7 +142,7 @@ func (v IntegerOrdOre) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_integer_ord_ore column. +// Scan reads a public.eql_v4_integer_ord_ore column. func (v *IntegerOrdOre) Scan(src any) error { return scan((*[]byte)(v), src, "IntegerOrdOre") } @@ -157,7 +157,7 @@ func (v *IntegerOrdOre) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// IntegerOrdOreQuery is the query value for IntegerOrdOre: the operand eql_v3.query_integer_ord_ore takes. +// IntegerOrdOreQuery is the query value for IntegerOrdOre: the operand eql_v4.query_integer_ord_ore takes. type IntegerOrdOreQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -165,7 +165,7 @@ func (v IntegerOrdOreQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v3.query_integer_ord_ore column. +// Scan reads a eql_v4.query_integer_ord_ore column. func (v *IntegerOrdOreQuery) Scan(src any) error { return scan((*[]byte)(v), src, "IntegerOrdOreQuery") } @@ -180,8 +180,8 @@ func (v *IntegerOrdOreQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// IntegerOrd is the EQL type stored in public.eql_v3_integer_ord: the integer family with the ope index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. +// IntegerOrd is the EQL type stored in public.eql_v4_integer_ord: the integer family with the ope index. +// The engine produces it from an int32. type IntegerOrd []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -189,7 +189,7 @@ func (v IntegerOrd) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_integer_ord column. +// Scan reads a public.eql_v4_integer_ord column. func (v *IntegerOrd) Scan(src any) error { return scan((*[]byte)(v), src, "IntegerOrd") } @@ -204,7 +204,7 @@ func (v *IntegerOrd) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// IntegerOrdQuery is the query value for IntegerOrd: the operand eql_v3.query_integer_ord takes. +// IntegerOrdQuery is the query value for IntegerOrd: the operand eql_v4.query_integer_ord takes. type IntegerOrdQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -212,7 +212,7 @@ func (v IntegerOrdQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v3.query_integer_ord column. +// Scan reads a eql_v4.query_integer_ord column. func (v *IntegerOrdQuery) Scan(src any) error { return scan((*[]byte)(v), src, "IntegerOrdQuery") } @@ -227,8 +227,8 @@ func (v *IntegerOrdQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// IntegerOrdOpe is the EQL type stored in public.eql_v3_integer_ord_ope: the integer family with the ope index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. +// IntegerOrdOpe is the EQL type stored in public.eql_v4_integer_ord_ope: the integer family with the ope index. +// The engine produces it from an int32. type IntegerOrdOpe []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -236,7 +236,7 @@ func (v IntegerOrdOpe) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_integer_ord_ope column. +// Scan reads a public.eql_v4_integer_ord_ope column. func (v *IntegerOrdOpe) Scan(src any) error { return scan((*[]byte)(v), src, "IntegerOrdOpe") } @@ -251,7 +251,7 @@ func (v *IntegerOrdOpe) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// IntegerOrdOpeQuery is the query value for IntegerOrdOpe: the operand eql_v3.query_integer_ord_ope takes. +// IntegerOrdOpeQuery is the query value for IntegerOrdOpe: the operand eql_v4.query_integer_ord_ope takes. type IntegerOrdOpeQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -259,7 +259,7 @@ func (v IntegerOrdOpeQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v3.query_integer_ord_ope column. +// Scan reads a eql_v4.query_integer_ord_ope column. func (v *IntegerOrdOpeQuery) Scan(src any) error { return scan((*[]byte)(v), src, "IntegerOrdOpeQuery") } @@ -274,8 +274,8 @@ func (v *IntegerOrdOpeQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// Smallint is the EQL type stored in public.eql_v3_smallint: the smallint family with no index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. +// Smallint is the EQL type stored in public.eql_v4_smallint: the smallint family with no index. +// The engine produces it from an int16. type Smallint []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -283,7 +283,7 @@ func (v Smallint) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_smallint column. +// Scan reads a public.eql_v4_smallint column. func (v *Smallint) Scan(src any) error { return scan((*[]byte)(v), src, "Smallint") } @@ -298,8 +298,8 @@ func (v *Smallint) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// SmallintEq is the EQL type stored in public.eql_v3_smallint_eq: the smallint family with the eq index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. +// SmallintEq is the EQL type stored in public.eql_v4_smallint_eq: the smallint family with the eq index. +// The engine produces it from an int16. type SmallintEq []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -307,7 +307,7 @@ func (v SmallintEq) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_smallint_eq column. +// Scan reads a public.eql_v4_smallint_eq column. func (v *SmallintEq) Scan(src any) error { return scan((*[]byte)(v), src, "SmallintEq") } @@ -322,7 +322,7 @@ func (v *SmallintEq) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// SmallintEqQuery is the query value for SmallintEq: the operand eql_v3.query_smallint_eq takes. +// SmallintEqQuery is the query value for SmallintEq: the operand eql_v4.query_smallint_eq takes. type SmallintEqQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -330,7 +330,7 @@ func (v SmallintEqQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v3.query_smallint_eq column. +// Scan reads a eql_v4.query_smallint_eq column. func (v *SmallintEqQuery) Scan(src any) error { return scan((*[]byte)(v), src, "SmallintEqQuery") } @@ -345,8 +345,8 @@ func (v *SmallintEqQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// SmallintOrdOre is the EQL type stored in public.eql_v3_smallint_ord_ore: the smallint family with the ore index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. +// SmallintOrdOre is the EQL type stored in public.eql_v4_smallint_ord_ore: the smallint family with the ore_block index. +// The engine produces it from an int16. type SmallintOrdOre []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -354,7 +354,7 @@ func (v SmallintOrdOre) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_smallint_ord_ore column. +// Scan reads a public.eql_v4_smallint_ord_ore column. func (v *SmallintOrdOre) Scan(src any) error { return scan((*[]byte)(v), src, "SmallintOrdOre") } @@ -369,7 +369,7 @@ func (v *SmallintOrdOre) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// SmallintOrdOreQuery is the query value for SmallintOrdOre: the operand eql_v3.query_smallint_ord_ore takes. +// SmallintOrdOreQuery is the query value for SmallintOrdOre: the operand eql_v4.query_smallint_ord_ore takes. type SmallintOrdOreQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -377,7 +377,7 @@ func (v SmallintOrdOreQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v3.query_smallint_ord_ore column. +// Scan reads a eql_v4.query_smallint_ord_ore column. func (v *SmallintOrdOreQuery) Scan(src any) error { return scan((*[]byte)(v), src, "SmallintOrdOreQuery") } @@ -392,8 +392,8 @@ func (v *SmallintOrdOreQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// SmallintOrd is the EQL type stored in public.eql_v3_smallint_ord: the smallint family with the ope index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. +// SmallintOrd is the EQL type stored in public.eql_v4_smallint_ord: the smallint family with the ope index. +// The engine produces it from an int16. type SmallintOrd []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -401,7 +401,7 @@ func (v SmallintOrd) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_smallint_ord column. +// Scan reads a public.eql_v4_smallint_ord column. func (v *SmallintOrd) Scan(src any) error { return scan((*[]byte)(v), src, "SmallintOrd") } @@ -416,7 +416,7 @@ func (v *SmallintOrd) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// SmallintOrdQuery is the query value for SmallintOrd: the operand eql_v3.query_smallint_ord takes. +// SmallintOrdQuery is the query value for SmallintOrd: the operand eql_v4.query_smallint_ord takes. type SmallintOrdQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -424,7 +424,7 @@ func (v SmallintOrdQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v3.query_smallint_ord column. +// Scan reads a eql_v4.query_smallint_ord column. func (v *SmallintOrdQuery) Scan(src any) error { return scan((*[]byte)(v), src, "SmallintOrdQuery") } @@ -439,8 +439,8 @@ func (v *SmallintOrdQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// SmallintOrdOpe is the EQL type stored in public.eql_v3_smallint_ord_ope: the smallint family with the ope index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. +// SmallintOrdOpe is the EQL type stored in public.eql_v4_smallint_ord_ope: the smallint family with the ope index. +// The engine produces it from an int16. type SmallintOrdOpe []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -448,7 +448,7 @@ func (v SmallintOrdOpe) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_smallint_ord_ope column. +// Scan reads a public.eql_v4_smallint_ord_ope column. func (v *SmallintOrdOpe) Scan(src any) error { return scan((*[]byte)(v), src, "SmallintOrdOpe") } @@ -463,7 +463,7 @@ func (v *SmallintOrdOpe) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// SmallintOrdOpeQuery is the query value for SmallintOrdOpe: the operand eql_v3.query_smallint_ord_ope takes. +// SmallintOrdOpeQuery is the query value for SmallintOrdOpe: the operand eql_v4.query_smallint_ord_ope takes. type SmallintOrdOpeQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -471,7 +471,7 @@ func (v SmallintOrdOpeQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v3.query_smallint_ord_ope column. +// Scan reads a eql_v4.query_smallint_ord_ope column. func (v *SmallintOrdOpeQuery) Scan(src any) error { return scan((*[]byte)(v), src, "SmallintOrdOpeQuery") } @@ -486,8 +486,8 @@ func (v *SmallintOrdOpeQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// Bigint is the EQL type stored in public.eql_v3_bigint: the bigint family with no index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. +// Bigint is the EQL type stored in public.eql_v4_bigint: the bigint family with no index. +// The engine produces it from an int64. type Bigint []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -495,7 +495,7 @@ func (v Bigint) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_bigint column. +// Scan reads a public.eql_v4_bigint column. func (v *Bigint) Scan(src any) error { return scan((*[]byte)(v), src, "Bigint") } @@ -510,8 +510,8 @@ func (v *Bigint) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// BigintEq is the EQL type stored in public.eql_v3_bigint_eq: the bigint family with the eq index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. +// BigintEq is the EQL type stored in public.eql_v4_bigint_eq: the bigint family with the eq index. +// The engine produces it from an int64. type BigintEq []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -519,7 +519,7 @@ func (v BigintEq) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_bigint_eq column. +// Scan reads a public.eql_v4_bigint_eq column. func (v *BigintEq) Scan(src any) error { return scan((*[]byte)(v), src, "BigintEq") } @@ -534,7 +534,7 @@ func (v *BigintEq) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// BigintEqQuery is the query value for BigintEq: the operand eql_v3.query_bigint_eq takes. +// BigintEqQuery is the query value for BigintEq: the operand eql_v4.query_bigint_eq takes. type BigintEqQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -542,7 +542,7 @@ func (v BigintEqQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v3.query_bigint_eq column. +// Scan reads a eql_v4.query_bigint_eq column. func (v *BigintEqQuery) Scan(src any) error { return scan((*[]byte)(v), src, "BigintEqQuery") } @@ -557,8 +557,8 @@ func (v *BigintEqQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// BigintOrdOre is the EQL type stored in public.eql_v3_bigint_ord_ore: the bigint family with the ore index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. +// BigintOrdOre is the EQL type stored in public.eql_v4_bigint_ord_ore: the bigint family with the ore_block index. +// The engine produces it from an int64. type BigintOrdOre []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -566,7 +566,7 @@ func (v BigintOrdOre) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_bigint_ord_ore column. +// Scan reads a public.eql_v4_bigint_ord_ore column. func (v *BigintOrdOre) Scan(src any) error { return scan((*[]byte)(v), src, "BigintOrdOre") } @@ -581,7 +581,7 @@ func (v *BigintOrdOre) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// BigintOrdOreQuery is the query value for BigintOrdOre: the operand eql_v3.query_bigint_ord_ore takes. +// BigintOrdOreQuery is the query value for BigintOrdOre: the operand eql_v4.query_bigint_ord_ore takes. type BigintOrdOreQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -589,7 +589,7 @@ func (v BigintOrdOreQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v3.query_bigint_ord_ore column. +// Scan reads a eql_v4.query_bigint_ord_ore column. func (v *BigintOrdOreQuery) Scan(src any) error { return scan((*[]byte)(v), src, "BigintOrdOreQuery") } @@ -604,8 +604,8 @@ func (v *BigintOrdOreQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// BigintOrd is the EQL type stored in public.eql_v3_bigint_ord: the bigint family with the ope index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. +// BigintOrd is the EQL type stored in public.eql_v4_bigint_ord: the bigint family with the ope index. +// The engine produces it from an int64. type BigintOrd []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -613,7 +613,7 @@ func (v BigintOrd) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_bigint_ord column. +// Scan reads a public.eql_v4_bigint_ord column. func (v *BigintOrd) Scan(src any) error { return scan((*[]byte)(v), src, "BigintOrd") } @@ -628,7 +628,7 @@ func (v *BigintOrd) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// BigintOrdQuery is the query value for BigintOrd: the operand eql_v3.query_bigint_ord takes. +// BigintOrdQuery is the query value for BigintOrd: the operand eql_v4.query_bigint_ord takes. type BigintOrdQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -636,7 +636,7 @@ func (v BigintOrdQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v3.query_bigint_ord column. +// Scan reads a eql_v4.query_bigint_ord column. func (v *BigintOrdQuery) Scan(src any) error { return scan((*[]byte)(v), src, "BigintOrdQuery") } @@ -651,8 +651,8 @@ func (v *BigintOrdQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// BigintOrdOpe is the EQL type stored in public.eql_v3_bigint_ord_ope: the bigint family with the ope index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. +// BigintOrdOpe is the EQL type stored in public.eql_v4_bigint_ord_ope: the bigint family with the ope index. +// The engine produces it from an int64. type BigintOrdOpe []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -660,7 +660,7 @@ func (v BigintOrdOpe) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_bigint_ord_ope column. +// Scan reads a public.eql_v4_bigint_ord_ope column. func (v *BigintOrdOpe) Scan(src any) error { return scan((*[]byte)(v), src, "BigintOrdOpe") } @@ -675,7 +675,7 @@ func (v *BigintOrdOpe) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// BigintOrdOpeQuery is the query value for BigintOrdOpe: the operand eql_v3.query_bigint_ord_ope takes. +// BigintOrdOpeQuery is the query value for BigintOrdOpe: the operand eql_v4.query_bigint_ord_ope takes. type BigintOrdOpeQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -683,7 +683,7 @@ func (v BigintOrdOpeQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v3.query_bigint_ord_ope column. +// Scan reads a eql_v4.query_bigint_ord_ope column. func (v *BigintOrdOpeQuery) Scan(src any) error { return scan((*[]byte)(v), src, "BigintOrdOpeQuery") } @@ -698,8 +698,8 @@ func (v *BigintOrdOpeQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// Date is the EQL type stored in public.eql_v3_date: the date family with no index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. +// Date is the EQL type stored in public.eql_v4_date: the date family with no index. +// The engine produces it from a date. type Date []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -707,7 +707,7 @@ func (v Date) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_date column. +// Scan reads a public.eql_v4_date column. func (v *Date) Scan(src any) error { return scan((*[]byte)(v), src, "Date") } @@ -722,8 +722,8 @@ func (v *Date) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// DateEq is the EQL type stored in public.eql_v3_date_eq: the date family with the eq index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. +// DateEq is the EQL type stored in public.eql_v4_date_eq: the date family with the eq index. +// The engine produces it from a date. type DateEq []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -731,7 +731,7 @@ func (v DateEq) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_date_eq column. +// Scan reads a public.eql_v4_date_eq column. func (v *DateEq) Scan(src any) error { return scan((*[]byte)(v), src, "DateEq") } @@ -746,7 +746,7 @@ func (v *DateEq) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// DateEqQuery is the query value for DateEq: the operand eql_v3.query_date_eq takes. +// DateEqQuery is the query value for DateEq: the operand eql_v4.query_date_eq takes. type DateEqQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -754,7 +754,7 @@ func (v DateEqQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v3.query_date_eq column. +// Scan reads a eql_v4.query_date_eq column. func (v *DateEqQuery) Scan(src any) error { return scan((*[]byte)(v), src, "DateEqQuery") } @@ -769,8 +769,8 @@ func (v *DateEqQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// DateOrdOre is the EQL type stored in public.eql_v3_date_ord_ore: the date family with the ore index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. +// DateOrdOre is the EQL type stored in public.eql_v4_date_ord_ore: the date family with the ore_block index. +// The engine cannot produce it yet: the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236). type DateOrdOre []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -778,7 +778,7 @@ func (v DateOrdOre) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_date_ord_ore column. +// Scan reads a public.eql_v4_date_ord_ore column. func (v *DateOrdOre) Scan(src any) error { return scan((*[]byte)(v), src, "DateOrdOre") } @@ -793,7 +793,7 @@ func (v *DateOrdOre) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// DateOrdOreQuery is the query value for DateOrdOre: the operand eql_v3.query_date_ord_ore takes. +// DateOrdOreQuery is the query value for DateOrdOre: the operand eql_v4.query_date_ord_ore takes. type DateOrdOreQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -801,7 +801,7 @@ func (v DateOrdOreQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v3.query_date_ord_ore column. +// Scan reads a eql_v4.query_date_ord_ore column. func (v *DateOrdOreQuery) Scan(src any) error { return scan((*[]byte)(v), src, "DateOrdOreQuery") } @@ -816,8 +816,8 @@ func (v *DateOrdOreQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// DateOrd is the EQL type stored in public.eql_v3_date_ord: the date family with the ope index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. +// DateOrd is the EQL type stored in public.eql_v4_date_ord: the date family with the ope index. +// The engine cannot produce it yet: the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236). type DateOrd []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -825,7 +825,7 @@ func (v DateOrd) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_date_ord column. +// Scan reads a public.eql_v4_date_ord column. func (v *DateOrd) Scan(src any) error { return scan((*[]byte)(v), src, "DateOrd") } @@ -840,7 +840,7 @@ func (v *DateOrd) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// DateOrdQuery is the query value for DateOrd: the operand eql_v3.query_date_ord takes. +// DateOrdQuery is the query value for DateOrd: the operand eql_v4.query_date_ord takes. type DateOrdQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -848,7 +848,7 @@ func (v DateOrdQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v3.query_date_ord column. +// Scan reads a eql_v4.query_date_ord column. func (v *DateOrdQuery) Scan(src any) error { return scan((*[]byte)(v), src, "DateOrdQuery") } @@ -863,8 +863,8 @@ func (v *DateOrdQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// DateOrdOpe is the EQL type stored in public.eql_v3_date_ord_ope: the date family with the ope index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. +// DateOrdOpe is the EQL type stored in public.eql_v4_date_ord_ope: the date family with the ope index. +// The engine cannot produce it yet: the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236). type DateOrdOpe []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -872,7 +872,7 @@ func (v DateOrdOpe) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_date_ord_ope column. +// Scan reads a public.eql_v4_date_ord_ope column. func (v *DateOrdOpe) Scan(src any) error { return scan((*[]byte)(v), src, "DateOrdOpe") } @@ -887,7 +887,7 @@ func (v *DateOrdOpe) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// DateOrdOpeQuery is the query value for DateOrdOpe: the operand eql_v3.query_date_ord_ope takes. +// DateOrdOpeQuery is the query value for DateOrdOpe: the operand eql_v4.query_date_ord_ope takes. type DateOrdOpeQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -895,7 +895,7 @@ func (v DateOrdOpeQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v3.query_date_ord_ope column. +// Scan reads a eql_v4.query_date_ord_ope column. func (v *DateOrdOpeQuery) Scan(src any) error { return scan((*[]byte)(v), src, "DateOrdOpeQuery") } @@ -910,8 +910,8 @@ func (v *DateOrdOpeQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// Timestamp is the EQL type stored in public.eql_v3_timestamp: the timestamp family with no index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. +// Timestamp is the EQL type stored in public.eql_v4_timestamp: the timestamp family with no index. +// The engine produces it from a timestamp. type Timestamp []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -919,7 +919,7 @@ func (v Timestamp) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_timestamp column. +// Scan reads a public.eql_v4_timestamp column. func (v *Timestamp) Scan(src any) error { return scan((*[]byte)(v), src, "Timestamp") } @@ -934,8 +934,8 @@ func (v *Timestamp) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// TimestampEq is the EQL type stored in public.eql_v3_timestamp_eq: the timestamp family with the eq index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. +// TimestampEq is the EQL type stored in public.eql_v4_timestamp_eq: the timestamp family with the eq index. +// The engine produces it from a timestamp. type TimestampEq []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -943,7 +943,7 @@ func (v TimestampEq) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_timestamp_eq column. +// Scan reads a public.eql_v4_timestamp_eq column. func (v *TimestampEq) Scan(src any) error { return scan((*[]byte)(v), src, "TimestampEq") } @@ -958,7 +958,7 @@ func (v *TimestampEq) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// TimestampEqQuery is the query value for TimestampEq: the operand eql_v3.query_timestamp_eq takes. +// TimestampEqQuery is the query value for TimestampEq: the operand eql_v4.query_timestamp_eq takes. type TimestampEqQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -966,7 +966,7 @@ func (v TimestampEqQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v3.query_timestamp_eq column. +// Scan reads a eql_v4.query_timestamp_eq column. func (v *TimestampEqQuery) Scan(src any) error { return scan((*[]byte)(v), src, "TimestampEqQuery") } @@ -981,8 +981,8 @@ func (v *TimestampEqQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// TimestampOrdOre is the EQL type stored in public.eql_v3_timestamp_ord_ore: the timestamp family with the ore index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. +// TimestampOrdOre is the EQL type stored in public.eql_v4_timestamp_ord_ore: the timestamp family with the ore_block index. +// The engine cannot produce it yet: the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236). type TimestampOrdOre []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -990,7 +990,7 @@ func (v TimestampOrdOre) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_timestamp_ord_ore column. +// Scan reads a public.eql_v4_timestamp_ord_ore column. func (v *TimestampOrdOre) Scan(src any) error { return scan((*[]byte)(v), src, "TimestampOrdOre") } @@ -1005,7 +1005,7 @@ func (v *TimestampOrdOre) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// TimestampOrdOreQuery is the query value for TimestampOrdOre: the operand eql_v3.query_timestamp_ord_ore takes. +// TimestampOrdOreQuery is the query value for TimestampOrdOre: the operand eql_v4.query_timestamp_ord_ore takes. type TimestampOrdOreQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1013,7 +1013,7 @@ func (v TimestampOrdOreQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v3.query_timestamp_ord_ore column. +// Scan reads a eql_v4.query_timestamp_ord_ore column. func (v *TimestampOrdOreQuery) Scan(src any) error { return scan((*[]byte)(v), src, "TimestampOrdOreQuery") } @@ -1028,8 +1028,8 @@ func (v *TimestampOrdOreQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// TimestampOrd is the EQL type stored in public.eql_v3_timestamp_ord: the timestamp family with the ope index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. +// TimestampOrd is the EQL type stored in public.eql_v4_timestamp_ord: the timestamp family with the ope index. +// The engine cannot produce it yet: the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236). type TimestampOrd []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1037,7 +1037,7 @@ func (v TimestampOrd) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_timestamp_ord column. +// Scan reads a public.eql_v4_timestamp_ord column. func (v *TimestampOrd) Scan(src any) error { return scan((*[]byte)(v), src, "TimestampOrd") } @@ -1052,7 +1052,7 @@ func (v *TimestampOrd) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// TimestampOrdQuery is the query value for TimestampOrd: the operand eql_v3.query_timestamp_ord takes. +// TimestampOrdQuery is the query value for TimestampOrd: the operand eql_v4.query_timestamp_ord takes. type TimestampOrdQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1060,7 +1060,7 @@ func (v TimestampOrdQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v3.query_timestamp_ord column. +// Scan reads a eql_v4.query_timestamp_ord column. func (v *TimestampOrdQuery) Scan(src any) error { return scan((*[]byte)(v), src, "TimestampOrdQuery") } @@ -1075,8 +1075,8 @@ func (v *TimestampOrdQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// TimestampOrdOpe is the EQL type stored in public.eql_v3_timestamp_ord_ope: the timestamp family with the ope index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. +// TimestampOrdOpe is the EQL type stored in public.eql_v4_timestamp_ord_ope: the timestamp family with the ope index. +// The engine cannot produce it yet: the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236). type TimestampOrdOpe []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1084,7 +1084,7 @@ func (v TimestampOrdOpe) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_timestamp_ord_ope column. +// Scan reads a public.eql_v4_timestamp_ord_ope column. func (v *TimestampOrdOpe) Scan(src any) error { return scan((*[]byte)(v), src, "TimestampOrdOpe") } @@ -1099,7 +1099,7 @@ func (v *TimestampOrdOpe) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// TimestampOrdOpeQuery is the query value for TimestampOrdOpe: the operand eql_v3.query_timestamp_ord_ope takes. +// TimestampOrdOpeQuery is the query value for TimestampOrdOpe: the operand eql_v4.query_timestamp_ord_ope takes. type TimestampOrdOpeQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1107,7 +1107,7 @@ func (v TimestampOrdOpeQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v3.query_timestamp_ord_ope column. +// Scan reads a eql_v4.query_timestamp_ord_ope column. func (v *TimestampOrdOpeQuery) Scan(src any) error { return scan((*[]byte)(v), src, "TimestampOrdOpeQuery") } @@ -1122,8 +1122,8 @@ func (v *TimestampOrdOpeQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// Numeric is the EQL type stored in public.eql_v3_numeric: the numeric family with no index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. +// Numeric is the EQL type stored in public.eql_v4_numeric: the numeric family with no index. +// The engine produces it from a decimal. type Numeric []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1131,7 +1131,7 @@ func (v Numeric) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_numeric column. +// Scan reads a public.eql_v4_numeric column. func (v *Numeric) Scan(src any) error { return scan((*[]byte)(v), src, "Numeric") } @@ -1146,8 +1146,8 @@ func (v *Numeric) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// NumericEq is the EQL type stored in public.eql_v3_numeric_eq: the numeric family with the eq index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. +// NumericEq is the EQL type stored in public.eql_v4_numeric_eq: the numeric family with the eq index. +// The engine produces it from a decimal. type NumericEq []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1155,7 +1155,7 @@ func (v NumericEq) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_numeric_eq column. +// Scan reads a public.eql_v4_numeric_eq column. func (v *NumericEq) Scan(src any) error { return scan((*[]byte)(v), src, "NumericEq") } @@ -1170,7 +1170,7 @@ func (v *NumericEq) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// NumericEqQuery is the query value for NumericEq: the operand eql_v3.query_numeric_eq takes. +// NumericEqQuery is the query value for NumericEq: the operand eql_v4.query_numeric_eq takes. type NumericEqQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1178,7 +1178,7 @@ func (v NumericEqQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v3.query_numeric_eq column. +// Scan reads a eql_v4.query_numeric_eq column. func (v *NumericEqQuery) Scan(src any) error { return scan((*[]byte)(v), src, "NumericEqQuery") } @@ -1193,8 +1193,8 @@ func (v *NumericEqQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// NumericOrdOre is the EQL type stored in public.eql_v3_numeric_ord_ore: the numeric family with the ore index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. +// NumericOrdOre is the EQL type stored in public.eql_v4_numeric_ord_ore: the numeric family with the ore_block index. +// The engine cannot produce it yet: the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236). type NumericOrdOre []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1202,7 +1202,7 @@ func (v NumericOrdOre) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_numeric_ord_ore column. +// Scan reads a public.eql_v4_numeric_ord_ore column. func (v *NumericOrdOre) Scan(src any) error { return scan((*[]byte)(v), src, "NumericOrdOre") } @@ -1217,7 +1217,7 @@ func (v *NumericOrdOre) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// NumericOrdOreQuery is the query value for NumericOrdOre: the operand eql_v3.query_numeric_ord_ore takes. +// NumericOrdOreQuery is the query value for NumericOrdOre: the operand eql_v4.query_numeric_ord_ore takes. type NumericOrdOreQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1225,7 +1225,7 @@ func (v NumericOrdOreQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v3.query_numeric_ord_ore column. +// Scan reads a eql_v4.query_numeric_ord_ore column. func (v *NumericOrdOreQuery) Scan(src any) error { return scan((*[]byte)(v), src, "NumericOrdOreQuery") } @@ -1240,8 +1240,8 @@ func (v *NumericOrdOreQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// NumericOrd is the EQL type stored in public.eql_v3_numeric_ord: the numeric family with the ope index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. +// NumericOrd is the EQL type stored in public.eql_v4_numeric_ord: the numeric family with the ope index. +// The engine cannot produce it yet: the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236). type NumericOrd []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1249,7 +1249,7 @@ func (v NumericOrd) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_numeric_ord column. +// Scan reads a public.eql_v4_numeric_ord column. func (v *NumericOrd) Scan(src any) error { return scan((*[]byte)(v), src, "NumericOrd") } @@ -1264,7 +1264,7 @@ func (v *NumericOrd) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// NumericOrdQuery is the query value for NumericOrd: the operand eql_v3.query_numeric_ord takes. +// NumericOrdQuery is the query value for NumericOrd: the operand eql_v4.query_numeric_ord takes. type NumericOrdQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1272,7 +1272,7 @@ func (v NumericOrdQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v3.query_numeric_ord column. +// Scan reads a eql_v4.query_numeric_ord column. func (v *NumericOrdQuery) Scan(src any) error { return scan((*[]byte)(v), src, "NumericOrdQuery") } @@ -1287,8 +1287,8 @@ func (v *NumericOrdQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// NumericOrdOpe is the EQL type stored in public.eql_v3_numeric_ord_ope: the numeric family with the ope index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. +// NumericOrdOpe is the EQL type stored in public.eql_v4_numeric_ord_ope: the numeric family with the ope index. +// The engine cannot produce it yet: the engine derives no order term for a date, timestamp or decimal yet: how they are ordered is not decided (#1236). type NumericOrdOpe []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1296,7 +1296,7 @@ func (v NumericOrdOpe) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_numeric_ord_ope column. +// Scan reads a public.eql_v4_numeric_ord_ope column. func (v *NumericOrdOpe) Scan(src any) error { return scan((*[]byte)(v), src, "NumericOrdOpe") } @@ -1311,7 +1311,7 @@ func (v *NumericOrdOpe) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// NumericOrdOpeQuery is the query value for NumericOrdOpe: the operand eql_v3.query_numeric_ord_ope takes. +// NumericOrdOpeQuery is the query value for NumericOrdOpe: the operand eql_v4.query_numeric_ord_ope takes. type NumericOrdOpeQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1319,7 +1319,7 @@ func (v NumericOrdOpeQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v3.query_numeric_ord_ope column. +// Scan reads a eql_v4.query_numeric_ord_ope column. func (v *NumericOrdOpeQuery) Scan(src any) error { return scan((*[]byte)(v), src, "NumericOrdOpeQuery") } @@ -1334,8 +1334,8 @@ func (v *NumericOrdOpeQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// Text is the EQL type stored in public.eql_v3_text: the text family with no index. -// The engine cannot produce it yet: the storage-only text domain follows once TextEq is proven end to end in PostgreSQL. +// Text is the EQL type stored in public.eql_v4_text: the text family with no index. +// The engine produces it from a string. type Text []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1343,7 +1343,7 @@ func (v Text) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_text column. +// Scan reads a public.eql_v4_text column. func (v *Text) Scan(src any) error { return scan((*[]byte)(v), src, "Text") } @@ -1358,7 +1358,7 @@ func (v *Text) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// TextEq is the EQL type stored in public.eql_v3_text_eq: the text family with the eq index. +// TextEq is the EQL type stored in public.eql_v4_text_eq: the text family with the eq index. // The engine produces it from a string. type TextEq []byte @@ -1367,7 +1367,7 @@ func (v TextEq) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_text_eq column. +// Scan reads a public.eql_v4_text_eq column. func (v *TextEq) Scan(src any) error { return scan((*[]byte)(v), src, "TextEq") } @@ -1382,7 +1382,7 @@ func (v *TextEq) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// TextEqQuery is the query value for TextEq: the operand eql_v3.query_text_eq takes. +// TextEqQuery is the query value for TextEq: the operand eql_v4.query_text_eq takes. type TextEqQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1390,7 +1390,7 @@ func (v TextEqQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v3.query_text_eq column. +// Scan reads a eql_v4.query_text_eq column. func (v *TextEqQuery) Scan(src any) error { return scan((*[]byte)(v), src, "TextEqQuery") } @@ -1405,8 +1405,8 @@ func (v *TextEqQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// TextMatch is the EQL type stored in public.eql_v3_text_match: the text family with the match index. -// The engine cannot produce it yet: the engine derives match and OPE terms, but no EQL type is built from them yet. +// TextMatch is the EQL type stored in public.eql_v4_text_match: the text family with the match index. +// The engine produces it from a string. type TextMatch []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1414,7 +1414,7 @@ func (v TextMatch) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_text_match column. +// Scan reads a public.eql_v4_text_match column. func (v *TextMatch) Scan(src any) error { return scan((*[]byte)(v), src, "TextMatch") } @@ -1429,7 +1429,7 @@ func (v *TextMatch) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// TextMatchQuery is the query value for TextMatch: the operand eql_v3.query_text_match takes. +// TextMatchQuery is the query value for TextMatch: the operand eql_v4.query_text_match takes. type TextMatchQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1437,7 +1437,7 @@ func (v TextMatchQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v3.query_text_match column. +// Scan reads a eql_v4.query_text_match column. func (v *TextMatchQuery) Scan(src any) error { return scan((*[]byte)(v), src, "TextMatchQuery") } @@ -1452,8 +1452,8 @@ func (v *TextMatchQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// TextOrdOre is the EQL type stored in public.eql_v3_text_ord_ore: the text family with the eq and ore indexes. -// The engine cannot produce it yet: EQL stores a block ORE term and the engine derives a CLLW ORE term; the two are different algorithms. +// TextOrdOre is the EQL type stored in public.eql_v4_text_ord_ore: the text family with the eq and ore_block indexes. +// The engine produces it from a string. type TextOrdOre []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1461,7 +1461,7 @@ func (v TextOrdOre) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_text_ord_ore column. +// Scan reads a public.eql_v4_text_ord_ore column. func (v *TextOrdOre) Scan(src any) error { return scan((*[]byte)(v), src, "TextOrdOre") } @@ -1476,7 +1476,7 @@ func (v *TextOrdOre) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// TextOrdOreQuery is the query value for TextOrdOre: the operand eql_v3.query_text_ord_ore takes. +// TextOrdOreQuery is the query value for TextOrdOre: the operand eql_v4.query_text_ord_ore takes. type TextOrdOreQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1484,7 +1484,7 @@ func (v TextOrdOreQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v3.query_text_ord_ore column. +// Scan reads a eql_v4.query_text_ord_ore column. func (v *TextOrdOreQuery) Scan(src any) error { return scan((*[]byte)(v), src, "TextOrdOreQuery") } @@ -1499,8 +1499,8 @@ func (v *TextOrdOreQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// TextOrd is the EQL type stored in public.eql_v3_text_ord: the text family with the eq and ope indexes. -// The engine cannot produce it yet: the engine derives match and OPE terms, but no EQL type is built from them yet. +// TextOrd is the EQL type stored in public.eql_v4_text_ord: the text family with the eq and ope indexes. +// The engine produces it from a string. type TextOrd []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1508,7 +1508,7 @@ func (v TextOrd) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_text_ord column. +// Scan reads a public.eql_v4_text_ord column. func (v *TextOrd) Scan(src any) error { return scan((*[]byte)(v), src, "TextOrd") } @@ -1523,7 +1523,7 @@ func (v *TextOrd) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// TextOrdQuery is the query value for TextOrd: the operand eql_v3.query_text_ord takes. +// TextOrdQuery is the query value for TextOrd: the operand eql_v4.query_text_ord takes. type TextOrdQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1531,7 +1531,7 @@ func (v TextOrdQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v3.query_text_ord column. +// Scan reads a eql_v4.query_text_ord column. func (v *TextOrdQuery) Scan(src any) error { return scan((*[]byte)(v), src, "TextOrdQuery") } @@ -1546,8 +1546,8 @@ func (v *TextOrdQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// TextOrdOpe is the EQL type stored in public.eql_v3_text_ord_ope: the text family with the eq and ope indexes. -// The engine cannot produce it yet: the engine derives match and OPE terms, but no EQL type is built from them yet. +// TextOrdOpe is the EQL type stored in public.eql_v4_text_ord_ope: the text family with the eq and ope indexes. +// The engine produces it from a string. type TextOrdOpe []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1555,7 +1555,7 @@ func (v TextOrdOpe) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_text_ord_ope column. +// Scan reads a public.eql_v4_text_ord_ope column. func (v *TextOrdOpe) Scan(src any) error { return scan((*[]byte)(v), src, "TextOrdOpe") } @@ -1570,7 +1570,7 @@ func (v *TextOrdOpe) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// TextOrdOpeQuery is the query value for TextOrdOpe: the operand eql_v3.query_text_ord_ope takes. +// TextOrdOpeQuery is the query value for TextOrdOpe: the operand eql_v4.query_text_ord_ope takes. type TextOrdOpeQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1578,7 +1578,7 @@ func (v TextOrdOpeQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v3.query_text_ord_ope column. +// Scan reads a eql_v4.query_text_ord_ope column. func (v *TextOrdOpeQuery) Scan(src any) error { return scan((*[]byte)(v), src, "TextOrdOpeQuery") } @@ -1593,8 +1593,8 @@ func (v *TextOrdOpeQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// TextSearchOre is the EQL type stored in public.eql_v3_text_search_ore: the text family with the eq and ore and match indexes. -// The engine cannot produce it yet: EQL stores a block ORE term and the engine derives a CLLW ORE term; the two are different algorithms. +// TextSearchOre is the EQL type stored in public.eql_v4_text_search_ore: the text family with the eq and ore_block and match indexes. +// The engine produces it from a string. type TextSearchOre []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1602,7 +1602,7 @@ func (v TextSearchOre) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_text_search_ore column. +// Scan reads a public.eql_v4_text_search_ore column. func (v *TextSearchOre) Scan(src any) error { return scan((*[]byte)(v), src, "TextSearchOre") } @@ -1617,7 +1617,7 @@ func (v *TextSearchOre) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// TextSearchOreQuery is the query value for TextSearchOre: the operand eql_v3.query_text_search_ore takes. +// TextSearchOreQuery is the query value for TextSearchOre: the operand eql_v4.query_text_search_ore takes. type TextSearchOreQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1625,7 +1625,7 @@ func (v TextSearchOreQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v3.query_text_search_ore column. +// Scan reads a eql_v4.query_text_search_ore column. func (v *TextSearchOreQuery) Scan(src any) error { return scan((*[]byte)(v), src, "TextSearchOreQuery") } @@ -1640,8 +1640,8 @@ func (v *TextSearchOreQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// TextSearch is the EQL type stored in public.eql_v3_text_search: the text family with the eq and ope and match indexes. -// The engine cannot produce it yet: the engine derives match and OPE terms, but no EQL type is built from them yet. +// TextSearch is the EQL type stored in public.eql_v4_text_search: the text family with the eq and ope and match indexes. +// The engine produces it from a string. type TextSearch []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1649,7 +1649,7 @@ func (v TextSearch) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_text_search column. +// Scan reads a public.eql_v4_text_search column. func (v *TextSearch) Scan(src any) error { return scan((*[]byte)(v), src, "TextSearch") } @@ -1664,7 +1664,7 @@ func (v *TextSearch) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// TextSearchQuery is the query value for TextSearch: the operand eql_v3.query_text_search takes. +// TextSearchQuery is the query value for TextSearch: the operand eql_v4.query_text_search takes. type TextSearchQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1672,7 +1672,7 @@ func (v TextSearchQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v3.query_text_search column. +// Scan reads a eql_v4.query_text_search column. func (v *TextSearchQuery) Scan(src any) error { return scan((*[]byte)(v), src, "TextSearchQuery") } @@ -1687,8 +1687,8 @@ func (v *TextSearchQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// Boolean is the EQL type stored in public.eql_v3_boolean: the boolean family with no index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. +// Boolean is the EQL type stored in public.eql_v4_boolean: the boolean family with no index. +// The engine produces it from a bool. type Boolean []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1696,7 +1696,7 @@ func (v Boolean) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_boolean column. +// Scan reads a public.eql_v4_boolean column. func (v *Boolean) Scan(src any) error { return scan((*[]byte)(v), src, "Boolean") } @@ -1711,8 +1711,8 @@ func (v *Boolean) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// Real is the EQL type stored in public.eql_v3_real: the real family with no index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. +// Real is the EQL type stored in public.eql_v4_real: the real family with no index. +// The engine produces it from a float32. type Real []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1720,7 +1720,7 @@ func (v Real) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_real column. +// Scan reads a public.eql_v4_real column. func (v *Real) Scan(src any) error { return scan((*[]byte)(v), src, "Real") } @@ -1735,8 +1735,8 @@ func (v *Real) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// RealEq is the EQL type stored in public.eql_v3_real_eq: the real family with the eq index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. +// RealEq is the EQL type stored in public.eql_v4_real_eq: the real family with the eq index. +// The engine produces it from a float32. type RealEq []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1744,7 +1744,7 @@ func (v RealEq) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_real_eq column. +// Scan reads a public.eql_v4_real_eq column. func (v *RealEq) Scan(src any) error { return scan((*[]byte)(v), src, "RealEq") } @@ -1759,7 +1759,7 @@ func (v *RealEq) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// RealEqQuery is the query value for RealEq: the operand eql_v3.query_real_eq takes. +// RealEqQuery is the query value for RealEq: the operand eql_v4.query_real_eq takes. type RealEqQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1767,7 +1767,7 @@ func (v RealEqQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v3.query_real_eq column. +// Scan reads a eql_v4.query_real_eq column. func (v *RealEqQuery) Scan(src any) error { return scan((*[]byte)(v), src, "RealEqQuery") } @@ -1782,8 +1782,8 @@ func (v *RealEqQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// RealOrdOre is the EQL type stored in public.eql_v3_real_ord_ore: the real family with the ore index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. +// RealOrdOre is the EQL type stored in public.eql_v4_real_ord_ore: the real family with the ore_block index. +// The engine produces it from a float32. type RealOrdOre []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1791,7 +1791,7 @@ func (v RealOrdOre) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_real_ord_ore column. +// Scan reads a public.eql_v4_real_ord_ore column. func (v *RealOrdOre) Scan(src any) error { return scan((*[]byte)(v), src, "RealOrdOre") } @@ -1806,7 +1806,7 @@ func (v *RealOrdOre) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// RealOrdOreQuery is the query value for RealOrdOre: the operand eql_v3.query_real_ord_ore takes. +// RealOrdOreQuery is the query value for RealOrdOre: the operand eql_v4.query_real_ord_ore takes. type RealOrdOreQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1814,7 +1814,7 @@ func (v RealOrdOreQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v3.query_real_ord_ore column. +// Scan reads a eql_v4.query_real_ord_ore column. func (v *RealOrdOreQuery) Scan(src any) error { return scan((*[]byte)(v), src, "RealOrdOreQuery") } @@ -1829,8 +1829,8 @@ func (v *RealOrdOreQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// RealOrd is the EQL type stored in public.eql_v3_real_ord: the real family with the ope index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. +// RealOrd is the EQL type stored in public.eql_v4_real_ord: the real family with the ope index. +// The engine produces it from a float32. type RealOrd []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1838,7 +1838,7 @@ func (v RealOrd) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_real_ord column. +// Scan reads a public.eql_v4_real_ord column. func (v *RealOrd) Scan(src any) error { return scan((*[]byte)(v), src, "RealOrd") } @@ -1853,7 +1853,7 @@ func (v *RealOrd) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// RealOrdQuery is the query value for RealOrd: the operand eql_v3.query_real_ord takes. +// RealOrdQuery is the query value for RealOrd: the operand eql_v4.query_real_ord takes. type RealOrdQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1861,7 +1861,7 @@ func (v RealOrdQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v3.query_real_ord column. +// Scan reads a eql_v4.query_real_ord column. func (v *RealOrdQuery) Scan(src any) error { return scan((*[]byte)(v), src, "RealOrdQuery") } @@ -1876,8 +1876,8 @@ func (v *RealOrdQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// RealOrdOpe is the EQL type stored in public.eql_v3_real_ord_ope: the real family with the ope index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. +// RealOrdOpe is the EQL type stored in public.eql_v4_real_ord_ope: the real family with the ope index. +// The engine produces it from a float32. type RealOrdOpe []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1885,7 +1885,7 @@ func (v RealOrdOpe) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_real_ord_ope column. +// Scan reads a public.eql_v4_real_ord_ope column. func (v *RealOrdOpe) Scan(src any) error { return scan((*[]byte)(v), src, "RealOrdOpe") } @@ -1900,7 +1900,7 @@ func (v *RealOrdOpe) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// RealOrdOpeQuery is the query value for RealOrdOpe: the operand eql_v3.query_real_ord_ope takes. +// RealOrdOpeQuery is the query value for RealOrdOpe: the operand eql_v4.query_real_ord_ope takes. type RealOrdOpeQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1908,7 +1908,7 @@ func (v RealOrdOpeQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v3.query_real_ord_ope column. +// Scan reads a eql_v4.query_real_ord_ope column. func (v *RealOrdOpeQuery) Scan(src any) error { return scan((*[]byte)(v), src, "RealOrdOpeQuery") } @@ -1923,8 +1923,8 @@ func (v *RealOrdOpeQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// Double is the EQL type stored in public.eql_v3_double: the double family with no index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. +// Double is the EQL type stored in public.eql_v4_double: the double family with no index. +// The engine produces it from a float64. type Double []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1932,7 +1932,7 @@ func (v Double) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_double column. +// Scan reads a public.eql_v4_double column. func (v *Double) Scan(src any) error { return scan((*[]byte)(v), src, "Double") } @@ -1947,8 +1947,8 @@ func (v *Double) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// DoubleEq is the EQL type stored in public.eql_v3_double_eq: the double family with the eq index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. +// DoubleEq is the EQL type stored in public.eql_v4_double_eq: the double family with the eq index. +// The engine produces it from a float64. type DoubleEq []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1956,7 +1956,7 @@ func (v DoubleEq) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_double_eq column. +// Scan reads a public.eql_v4_double_eq column. func (v *DoubleEq) Scan(src any) error { return scan((*[]byte)(v), src, "DoubleEq") } @@ -1971,7 +1971,7 @@ func (v *DoubleEq) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// DoubleEqQuery is the query value for DoubleEq: the operand eql_v3.query_double_eq takes. +// DoubleEqQuery is the query value for DoubleEq: the operand eql_v4.query_double_eq takes. type DoubleEqQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -1979,7 +1979,7 @@ func (v DoubleEqQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v3.query_double_eq column. +// Scan reads a eql_v4.query_double_eq column. func (v *DoubleEqQuery) Scan(src any) error { return scan((*[]byte)(v), src, "DoubleEqQuery") } @@ -1994,8 +1994,8 @@ func (v *DoubleEqQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// DoubleOrdOre is the EQL type stored in public.eql_v3_double_ord_ore: the double family with the ore index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. +// DoubleOrdOre is the EQL type stored in public.eql_v4_double_ord_ore: the double family with the ore_block index. +// The engine produces it from a float64. type DoubleOrdOre []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -2003,7 +2003,7 @@ func (v DoubleOrdOre) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_double_ord_ore column. +// Scan reads a public.eql_v4_double_ord_ore column. func (v *DoubleOrdOre) Scan(src any) error { return scan((*[]byte)(v), src, "DoubleOrdOre") } @@ -2018,7 +2018,7 @@ func (v *DoubleOrdOre) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// DoubleOrdOreQuery is the query value for DoubleOrdOre: the operand eql_v3.query_double_ord_ore takes. +// DoubleOrdOreQuery is the query value for DoubleOrdOre: the operand eql_v4.query_double_ord_ore takes. type DoubleOrdOreQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -2026,7 +2026,7 @@ func (v DoubleOrdOreQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v3.query_double_ord_ore column. +// Scan reads a eql_v4.query_double_ord_ore column. func (v *DoubleOrdOreQuery) Scan(src any) error { return scan((*[]byte)(v), src, "DoubleOrdOreQuery") } @@ -2041,8 +2041,8 @@ func (v *DoubleOrdOreQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// DoubleOrd is the EQL type stored in public.eql_v3_double_ord: the double family with the ope index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. +// DoubleOrd is the EQL type stored in public.eql_v4_double_ord: the double family with the ope index. +// The engine produces it from a float64. type DoubleOrd []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -2050,7 +2050,7 @@ func (v DoubleOrd) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_double_ord column. +// Scan reads a public.eql_v4_double_ord column. func (v *DoubleOrd) Scan(src any) error { return scan((*[]byte)(v), src, "DoubleOrd") } @@ -2065,7 +2065,7 @@ func (v *DoubleOrd) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// DoubleOrdQuery is the query value for DoubleOrd: the operand eql_v3.query_double_ord takes. +// DoubleOrdQuery is the query value for DoubleOrd: the operand eql_v4.query_double_ord takes. type DoubleOrdQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -2073,7 +2073,7 @@ func (v DoubleOrdQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v3.query_double_ord column. +// Scan reads a eql_v4.query_double_ord column. func (v *DoubleOrdQuery) Scan(src any) error { return scan((*[]byte)(v), src, "DoubleOrdQuery") } @@ -2088,8 +2088,8 @@ func (v *DoubleOrdQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// DoubleOrdOpe is the EQL type stored in public.eql_v3_double_ord_ope: the double family with the ope index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. +// DoubleOrdOpe is the EQL type stored in public.eql_v4_double_ord_ope: the double family with the ope index. +// The engine produces it from a float64. type DoubleOrdOpe []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -2097,7 +2097,7 @@ func (v DoubleOrdOpe) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_double_ord_ope column. +// Scan reads a public.eql_v4_double_ord_ope column. func (v *DoubleOrdOpe) Scan(src any) error { return scan((*[]byte)(v), src, "DoubleOrdOpe") } @@ -2112,7 +2112,7 @@ func (v *DoubleOrdOpe) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// DoubleOrdOpeQuery is the query value for DoubleOrdOpe: the operand eql_v3.query_double_ord_ope takes. +// DoubleOrdOpeQuery is the query value for DoubleOrdOpe: the operand eql_v4.query_double_ord_ope takes. type DoubleOrdOpeQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -2120,7 +2120,7 @@ func (v DoubleOrdOpeQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v3.query_double_ord_ope column. +// Scan reads a eql_v4.query_double_ord_ope column. func (v *DoubleOrdOpeQuery) Scan(src any) error { return scan((*[]byte)(v), src, "DoubleOrdOpeQuery") } @@ -2135,8 +2135,8 @@ func (v *DoubleOrdOpeQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// SteVecDocument is the EQL type stored in public.eql_v3_json_search: the json family with the json index. -// The engine cannot produce it yet: the JSON index is a new operation in the engine. +// SteVecDocument is the EQL type stored in public.eql_v4_json_search: the json family with the json index. +// The engine cannot produce it yet: the JSON index is a new operation in the engine (#1060). type SteVecDocument []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -2144,7 +2144,7 @@ func (v SteVecDocument) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_json_search column. +// Scan reads a public.eql_v4_json_search column. func (v *SteVecDocument) Scan(src any) error { return scan((*[]byte)(v), src, "SteVecDocument") } @@ -2159,7 +2159,7 @@ func (v *SteVecDocument) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// SteVecQuery is the query value for SteVecDocument: the operand eql_v3.query_json takes. +// SteVecQuery is the query value for SteVecDocument: the operand eql_v4.query_json takes. type SteVecQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -2167,7 +2167,7 @@ func (v SteVecQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v3.query_json column. +// Scan reads a eql_v4.query_json column. func (v *SteVecQuery) Scan(src any) error { return scan((*[]byte)(v), src, "SteVecQuery") } @@ -2182,8 +2182,8 @@ func (v *SteVecQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// JSON is the EQL type stored in public.eql_v3_json: the json family with no index. -// The engine cannot produce it yet: how this family encodes a plaintext for the stack-encrypt producer profile is not specified; only the text family's encoding is. +// JSON is the EQL type stored in public.eql_v4_json: the json family with no index. +// The engine cannot produce it yet: a JSON value seals as a tree of leaves and EQL's `c` holds one; how a document becomes one sealed leaf is not decided. type JSON []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -2191,7 +2191,7 @@ func (v JSON) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v3_json column. +// Scan reads a public.eql_v4_json column. func (v *JSON) Scan(src any) error { return scan((*[]byte)(v), src, "JSON") } @@ -2205,3 +2205,435 @@ func (v JSON) MarshalJSON() ([]byte, error) { func (v *JSON) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } + +// IntegerRangeOreQuery is a range query value over the ore_block index of the integer family: both bounds of one BETWEEN, +// included, as the one operand eql_v4.query_integer_range_ore takes in eql_v4.between. It ranges over IntegerOrdOre. +type IntegerRangeOreQuery []byte + +// Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. +func (v IntegerRangeOreQuery) Value() (driver.Value, error) { + return value(v) +} + +// Scan reads a eql_v4.query_integer_range_ore column. +func (v *IntegerRangeOreQuery) Scan(src any) error { + return scan((*[]byte)(v), src, "IntegerRangeOreQuery") +} + +// MarshalJSON is the EQL value as the JSON it is, or null when empty. +func (v IntegerRangeOreQuery) MarshalJSON() ([]byte, error) { + return marshal(v, "IntegerRangeOreQuery") +} + +// UnmarshalJSON reads the EQL value from JSON, as it is. +func (v *IntegerRangeOreQuery) UnmarshalJSON(b []byte) error { + return unmarshal((*[]byte)(v), b) +} + +// IntegerRangeOpeQuery is a range query value over the ope index of the integer family: both bounds of one BETWEEN, +// included, as the one operand eql_v4.query_integer_range_ope takes in eql_v4.between. It ranges over IntegerOrd, IntegerOrdOpe. +type IntegerRangeOpeQuery []byte + +// Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. +func (v IntegerRangeOpeQuery) Value() (driver.Value, error) { + return value(v) +} + +// Scan reads a eql_v4.query_integer_range_ope column. +func (v *IntegerRangeOpeQuery) Scan(src any) error { + return scan((*[]byte)(v), src, "IntegerRangeOpeQuery") +} + +// MarshalJSON is the EQL value as the JSON it is, or null when empty. +func (v IntegerRangeOpeQuery) MarshalJSON() ([]byte, error) { + return marshal(v, "IntegerRangeOpeQuery") +} + +// UnmarshalJSON reads the EQL value from JSON, as it is. +func (v *IntegerRangeOpeQuery) UnmarshalJSON(b []byte) error { + return unmarshal((*[]byte)(v), b) +} + +// SmallintRangeOreQuery is a range query value over the ore_block index of the smallint family: both bounds of one BETWEEN, +// included, as the one operand eql_v4.query_smallint_range_ore takes in eql_v4.between. It ranges over SmallintOrdOre. +type SmallintRangeOreQuery []byte + +// Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. +func (v SmallintRangeOreQuery) Value() (driver.Value, error) { + return value(v) +} + +// Scan reads a eql_v4.query_smallint_range_ore column. +func (v *SmallintRangeOreQuery) Scan(src any) error { + return scan((*[]byte)(v), src, "SmallintRangeOreQuery") +} + +// MarshalJSON is the EQL value as the JSON it is, or null when empty. +func (v SmallintRangeOreQuery) MarshalJSON() ([]byte, error) { + return marshal(v, "SmallintRangeOreQuery") +} + +// UnmarshalJSON reads the EQL value from JSON, as it is. +func (v *SmallintRangeOreQuery) UnmarshalJSON(b []byte) error { + return unmarshal((*[]byte)(v), b) +} + +// SmallintRangeOpeQuery is a range query value over the ope index of the smallint family: both bounds of one BETWEEN, +// included, as the one operand eql_v4.query_smallint_range_ope takes in eql_v4.between. It ranges over SmallintOrd, SmallintOrdOpe. +type SmallintRangeOpeQuery []byte + +// Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. +func (v SmallintRangeOpeQuery) Value() (driver.Value, error) { + return value(v) +} + +// Scan reads a eql_v4.query_smallint_range_ope column. +func (v *SmallintRangeOpeQuery) Scan(src any) error { + return scan((*[]byte)(v), src, "SmallintRangeOpeQuery") +} + +// MarshalJSON is the EQL value as the JSON it is, or null when empty. +func (v SmallintRangeOpeQuery) MarshalJSON() ([]byte, error) { + return marshal(v, "SmallintRangeOpeQuery") +} + +// UnmarshalJSON reads the EQL value from JSON, as it is. +func (v *SmallintRangeOpeQuery) UnmarshalJSON(b []byte) error { + return unmarshal((*[]byte)(v), b) +} + +// BigintRangeOreQuery is a range query value over the ore_block index of the bigint family: both bounds of one BETWEEN, +// included, as the one operand eql_v4.query_bigint_range_ore takes in eql_v4.between. It ranges over BigintOrdOre. +type BigintRangeOreQuery []byte + +// Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. +func (v BigintRangeOreQuery) Value() (driver.Value, error) { + return value(v) +} + +// Scan reads a eql_v4.query_bigint_range_ore column. +func (v *BigintRangeOreQuery) Scan(src any) error { + return scan((*[]byte)(v), src, "BigintRangeOreQuery") +} + +// MarshalJSON is the EQL value as the JSON it is, or null when empty. +func (v BigintRangeOreQuery) MarshalJSON() ([]byte, error) { + return marshal(v, "BigintRangeOreQuery") +} + +// UnmarshalJSON reads the EQL value from JSON, as it is. +func (v *BigintRangeOreQuery) UnmarshalJSON(b []byte) error { + return unmarshal((*[]byte)(v), b) +} + +// BigintRangeOpeQuery is a range query value over the ope index of the bigint family: both bounds of one BETWEEN, +// included, as the one operand eql_v4.query_bigint_range_ope takes in eql_v4.between. It ranges over BigintOrd, BigintOrdOpe. +type BigintRangeOpeQuery []byte + +// Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. +func (v BigintRangeOpeQuery) Value() (driver.Value, error) { + return value(v) +} + +// Scan reads a eql_v4.query_bigint_range_ope column. +func (v *BigintRangeOpeQuery) Scan(src any) error { + return scan((*[]byte)(v), src, "BigintRangeOpeQuery") +} + +// MarshalJSON is the EQL value as the JSON it is, or null when empty. +func (v BigintRangeOpeQuery) MarshalJSON() ([]byte, error) { + return marshal(v, "BigintRangeOpeQuery") +} + +// UnmarshalJSON reads the EQL value from JSON, as it is. +func (v *BigintRangeOpeQuery) UnmarshalJSON(b []byte) error { + return unmarshal((*[]byte)(v), b) +} + +// DateRangeOreQuery is a range query value over the ore_block index of the date family: both bounds of one BETWEEN, +// included, as the one operand eql_v4.query_date_range_ore takes in eql_v4.between. It ranges over DateOrdOre. +type DateRangeOreQuery []byte + +// Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. +func (v DateRangeOreQuery) Value() (driver.Value, error) { + return value(v) +} + +// Scan reads a eql_v4.query_date_range_ore column. +func (v *DateRangeOreQuery) Scan(src any) error { + return scan((*[]byte)(v), src, "DateRangeOreQuery") +} + +// MarshalJSON is the EQL value as the JSON it is, or null when empty. +func (v DateRangeOreQuery) MarshalJSON() ([]byte, error) { + return marshal(v, "DateRangeOreQuery") +} + +// UnmarshalJSON reads the EQL value from JSON, as it is. +func (v *DateRangeOreQuery) UnmarshalJSON(b []byte) error { + return unmarshal((*[]byte)(v), b) +} + +// DateRangeOpeQuery is a range query value over the ope index of the date family: both bounds of one BETWEEN, +// included, as the one operand eql_v4.query_date_range_ope takes in eql_v4.between. It ranges over DateOrd, DateOrdOpe. +type DateRangeOpeQuery []byte + +// Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. +func (v DateRangeOpeQuery) Value() (driver.Value, error) { + return value(v) +} + +// Scan reads a eql_v4.query_date_range_ope column. +func (v *DateRangeOpeQuery) Scan(src any) error { + return scan((*[]byte)(v), src, "DateRangeOpeQuery") +} + +// MarshalJSON is the EQL value as the JSON it is, or null when empty. +func (v DateRangeOpeQuery) MarshalJSON() ([]byte, error) { + return marshal(v, "DateRangeOpeQuery") +} + +// UnmarshalJSON reads the EQL value from JSON, as it is. +func (v *DateRangeOpeQuery) UnmarshalJSON(b []byte) error { + return unmarshal((*[]byte)(v), b) +} + +// TimestampRangeOreQuery is a range query value over the ore_block index of the timestamp family: both bounds of one BETWEEN, +// included, as the one operand eql_v4.query_timestamp_range_ore takes in eql_v4.between. It ranges over TimestampOrdOre. +type TimestampRangeOreQuery []byte + +// Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. +func (v TimestampRangeOreQuery) Value() (driver.Value, error) { + return value(v) +} + +// Scan reads a eql_v4.query_timestamp_range_ore column. +func (v *TimestampRangeOreQuery) Scan(src any) error { + return scan((*[]byte)(v), src, "TimestampRangeOreQuery") +} + +// MarshalJSON is the EQL value as the JSON it is, or null when empty. +func (v TimestampRangeOreQuery) MarshalJSON() ([]byte, error) { + return marshal(v, "TimestampRangeOreQuery") +} + +// UnmarshalJSON reads the EQL value from JSON, as it is. +func (v *TimestampRangeOreQuery) UnmarshalJSON(b []byte) error { + return unmarshal((*[]byte)(v), b) +} + +// TimestampRangeOpeQuery is a range query value over the ope index of the timestamp family: both bounds of one BETWEEN, +// included, as the one operand eql_v4.query_timestamp_range_ope takes in eql_v4.between. It ranges over TimestampOrd, TimestampOrdOpe. +type TimestampRangeOpeQuery []byte + +// Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. +func (v TimestampRangeOpeQuery) Value() (driver.Value, error) { + return value(v) +} + +// Scan reads a eql_v4.query_timestamp_range_ope column. +func (v *TimestampRangeOpeQuery) Scan(src any) error { + return scan((*[]byte)(v), src, "TimestampRangeOpeQuery") +} + +// MarshalJSON is the EQL value as the JSON it is, or null when empty. +func (v TimestampRangeOpeQuery) MarshalJSON() ([]byte, error) { + return marshal(v, "TimestampRangeOpeQuery") +} + +// UnmarshalJSON reads the EQL value from JSON, as it is. +func (v *TimestampRangeOpeQuery) UnmarshalJSON(b []byte) error { + return unmarshal((*[]byte)(v), b) +} + +// NumericRangeOreQuery is a range query value over the ore_block index of the numeric family: both bounds of one BETWEEN, +// included, as the one operand eql_v4.query_numeric_range_ore takes in eql_v4.between. It ranges over NumericOrdOre. +type NumericRangeOreQuery []byte + +// Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. +func (v NumericRangeOreQuery) Value() (driver.Value, error) { + return value(v) +} + +// Scan reads a eql_v4.query_numeric_range_ore column. +func (v *NumericRangeOreQuery) Scan(src any) error { + return scan((*[]byte)(v), src, "NumericRangeOreQuery") +} + +// MarshalJSON is the EQL value as the JSON it is, or null when empty. +func (v NumericRangeOreQuery) MarshalJSON() ([]byte, error) { + return marshal(v, "NumericRangeOreQuery") +} + +// UnmarshalJSON reads the EQL value from JSON, as it is. +func (v *NumericRangeOreQuery) UnmarshalJSON(b []byte) error { + return unmarshal((*[]byte)(v), b) +} + +// NumericRangeOpeQuery is a range query value over the ope index of the numeric family: both bounds of one BETWEEN, +// included, as the one operand eql_v4.query_numeric_range_ope takes in eql_v4.between. It ranges over NumericOrd, NumericOrdOpe. +type NumericRangeOpeQuery []byte + +// Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. +func (v NumericRangeOpeQuery) Value() (driver.Value, error) { + return value(v) +} + +// Scan reads a eql_v4.query_numeric_range_ope column. +func (v *NumericRangeOpeQuery) Scan(src any) error { + return scan((*[]byte)(v), src, "NumericRangeOpeQuery") +} + +// MarshalJSON is the EQL value as the JSON it is, or null when empty. +func (v NumericRangeOpeQuery) MarshalJSON() ([]byte, error) { + return marshal(v, "NumericRangeOpeQuery") +} + +// UnmarshalJSON reads the EQL value from JSON, as it is. +func (v *NumericRangeOpeQuery) UnmarshalJSON(b []byte) error { + return unmarshal((*[]byte)(v), b) +} + +// TextRangeOreQuery is a range query value over the ore_block index of the text family: both bounds of one BETWEEN, +// included, as the one operand eql_v4.query_text_range_ore takes in eql_v4.between. It ranges over TextOrdOre, TextSearchOre. +type TextRangeOreQuery []byte + +// Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. +func (v TextRangeOreQuery) Value() (driver.Value, error) { + return value(v) +} + +// Scan reads a eql_v4.query_text_range_ore column. +func (v *TextRangeOreQuery) Scan(src any) error { + return scan((*[]byte)(v), src, "TextRangeOreQuery") +} + +// MarshalJSON is the EQL value as the JSON it is, or null when empty. +func (v TextRangeOreQuery) MarshalJSON() ([]byte, error) { + return marshal(v, "TextRangeOreQuery") +} + +// UnmarshalJSON reads the EQL value from JSON, as it is. +func (v *TextRangeOreQuery) UnmarshalJSON(b []byte) error { + return unmarshal((*[]byte)(v), b) +} + +// TextRangeOpeQuery is a range query value over the ope index of the text family: both bounds of one BETWEEN, +// included, as the one operand eql_v4.query_text_range_ope takes in eql_v4.between. It ranges over TextOrd, TextOrdOpe, TextSearch. +type TextRangeOpeQuery []byte + +// Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. +func (v TextRangeOpeQuery) Value() (driver.Value, error) { + return value(v) +} + +// Scan reads a eql_v4.query_text_range_ope column. +func (v *TextRangeOpeQuery) Scan(src any) error { + return scan((*[]byte)(v), src, "TextRangeOpeQuery") +} + +// MarshalJSON is the EQL value as the JSON it is, or null when empty. +func (v TextRangeOpeQuery) MarshalJSON() ([]byte, error) { + return marshal(v, "TextRangeOpeQuery") +} + +// UnmarshalJSON reads the EQL value from JSON, as it is. +func (v *TextRangeOpeQuery) UnmarshalJSON(b []byte) error { + return unmarshal((*[]byte)(v), b) +} + +// RealRangeOreQuery is a range query value over the ore_block index of the real family: both bounds of one BETWEEN, +// included, as the one operand eql_v4.query_real_range_ore takes in eql_v4.between. It ranges over RealOrdOre. +type RealRangeOreQuery []byte + +// Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. +func (v RealRangeOreQuery) Value() (driver.Value, error) { + return value(v) +} + +// Scan reads a eql_v4.query_real_range_ore column. +func (v *RealRangeOreQuery) Scan(src any) error { + return scan((*[]byte)(v), src, "RealRangeOreQuery") +} + +// MarshalJSON is the EQL value as the JSON it is, or null when empty. +func (v RealRangeOreQuery) MarshalJSON() ([]byte, error) { + return marshal(v, "RealRangeOreQuery") +} + +// UnmarshalJSON reads the EQL value from JSON, as it is. +func (v *RealRangeOreQuery) UnmarshalJSON(b []byte) error { + return unmarshal((*[]byte)(v), b) +} + +// RealRangeOpeQuery is a range query value over the ope index of the real family: both bounds of one BETWEEN, +// included, as the one operand eql_v4.query_real_range_ope takes in eql_v4.between. It ranges over RealOrd, RealOrdOpe. +type RealRangeOpeQuery []byte + +// Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. +func (v RealRangeOpeQuery) Value() (driver.Value, error) { + return value(v) +} + +// Scan reads a eql_v4.query_real_range_ope column. +func (v *RealRangeOpeQuery) Scan(src any) error { + return scan((*[]byte)(v), src, "RealRangeOpeQuery") +} + +// MarshalJSON is the EQL value as the JSON it is, or null when empty. +func (v RealRangeOpeQuery) MarshalJSON() ([]byte, error) { + return marshal(v, "RealRangeOpeQuery") +} + +// UnmarshalJSON reads the EQL value from JSON, as it is. +func (v *RealRangeOpeQuery) UnmarshalJSON(b []byte) error { + return unmarshal((*[]byte)(v), b) +} + +// DoubleRangeOreQuery is a range query value over the ore_block index of the double family: both bounds of one BETWEEN, +// included, as the one operand eql_v4.query_double_range_ore takes in eql_v4.between. It ranges over DoubleOrdOre. +type DoubleRangeOreQuery []byte + +// Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. +func (v DoubleRangeOreQuery) Value() (driver.Value, error) { + return value(v) +} + +// Scan reads a eql_v4.query_double_range_ore column. +func (v *DoubleRangeOreQuery) Scan(src any) error { + return scan((*[]byte)(v), src, "DoubleRangeOreQuery") +} + +// MarshalJSON is the EQL value as the JSON it is, or null when empty. +func (v DoubleRangeOreQuery) MarshalJSON() ([]byte, error) { + return marshal(v, "DoubleRangeOreQuery") +} + +// UnmarshalJSON reads the EQL value from JSON, as it is. +func (v *DoubleRangeOreQuery) UnmarshalJSON(b []byte) error { + return unmarshal((*[]byte)(v), b) +} + +// DoubleRangeOpeQuery is a range query value over the ope index of the double family: both bounds of one BETWEEN, +// included, as the one operand eql_v4.query_double_range_ope takes in eql_v4.between. It ranges over DoubleOrd, DoubleOrdOpe. +type DoubleRangeOpeQuery []byte + +// Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. +func (v DoubleRangeOpeQuery) Value() (driver.Value, error) { + return value(v) +} + +// Scan reads a eql_v4.query_double_range_ope column. +func (v *DoubleRangeOpeQuery) Scan(src any) error { + return scan((*[]byte)(v), src, "DoubleRangeOpeQuery") +} + +// MarshalJSON is the EQL value as the JSON it is, or null when empty. +func (v DoubleRangeOpeQuery) MarshalJSON() ([]byte, error) { + return marshal(v, "DoubleRangeOpeQuery") +} + +// UnmarshalJSON reads the EQL value from JSON, as it is. +func (v *DoubleRangeOpeQuery) UnmarshalJSON(b []byte) error { + return unmarshal((*[]byte)(v), b) +} diff --git a/languages/golang/encrypt/eql/eql_test.go b/languages/golang/encrypt/eql/eql_test.go index d951dc36f..94067930a 100644 --- a/languages/golang/encrypt/eql/eql_test.go +++ b/languages/golang/encrypt/eql/eql_test.go @@ -7,7 +7,7 @@ import ( "github.com/cipherstash/stack/languages/golang/encrypt/eql" ) -func TestTypesTableNamesTextEqAsTheOneProducibleType(t *testing.T) { +func TestTypesTableNamesTheProducibleTypes(t *testing.T) { var producible []string for _, typ := range eql.Types { if typ.Producible { @@ -19,11 +19,31 @@ func TestTypesTableNamesTextEqAsTheOneProducibleType(t *testing.T) { t.Errorf("%s: an unproducible type has a reason", typ.Name) } } - if len(producible) != 1 || producible[0] != "TextEq" { - t.Fatalf("producible = %v, want [TextEq]", producible) + want := []string{ + "Integer", "IntegerEq", "IntegerOrdOre", "IntegerOrd", "IntegerOrdOpe", + "Smallint", "SmallintEq", "SmallintOrdOre", "SmallintOrd", "SmallintOrdOpe", + "Bigint", "BigintEq", "BigintOrdOre", "BigintOrd", "BigintOrdOpe", + "Date", "DateEq", "Timestamp", "TimestampEq", "Numeric", "NumericEq", + "Text", "TextEq", "TextMatch", "TextOrdOre", "TextOrd", "TextOrdOpe", "TextSearchOre", "TextSearch", + "Boolean", + "Real", "RealEq", "RealOrdOre", "RealOrd", "RealOrdOpe", + "Double", "DoubleEq", "DoubleOrdOre", "DoubleOrd", "DoubleOrdOpe", + } + if len(producible) != len(want) { + t.Fatalf("producible = %v, want %v", producible, want) + } + for i := range want { + if producible[i] != want[i] { + t.Fatalf("producible = %v, want %v", producible, want) + } + } + for name, plaintext := range map[string]string{"Integer": "int32", "IntegerEq": "int32", "IntegerOrdOre": "int32", "TextOrdOre": "string", "IntegerOrd": "int32", "Smallint": "int16", "SmallintEq": "int16", "SmallintOrdOre": "int16", "SmallintOrdOpe": "int16", "Bigint": "int64", "BigintEq": "int64", "BigintOrdOpe": "int64", "Boolean": "bool", "Real": "float32", "RealEq": "float32", "RealOrd": "float32", "Double": "float64", "DoubleEq": "float64", "DoubleOrdOpe": "float64", "TextOrd": "string", "Date": "date", "DateEq": "date", "Timestamp": "timestamp", "TimestampEq": "timestamp", "Numeric": "decimal", "NumericEq": "decimal"} { + if typ, ok := eql.Lookup(name); !ok || typ.Plaintext != plaintext { + t.Errorf("%s: Plaintext = %q, want %q", name, typ.Plaintext, plaintext) + } } textEq, ok := eql.Lookup("TextEq") - if !ok || textEq.GoName != "TextEq" || textEq.Plaintext != "string" || textEq.Query != "TextEqQuery" || textEq.SQLDomain != "public.eql_v3_text_eq" || len(textEq.Indexes) != 1 || textEq.Indexes[0] != "eq" { + if !ok || textEq.GoName != "TextEq" || textEq.Plaintext != "string" || textEq.Query != "TextEqQuery" || textEq.SQLDomain != "public.eql_v4_text_eq" || len(textEq.Indexes) != 1 || textEq.Indexes[0] != "eq" { t.Fatalf("TextEq = %+v", textEq) } if j, ok := eql.Lookup("Json"); !ok || j.GoName != "JSON" { @@ -35,7 +55,7 @@ func TestTypesTableNamesTextEqAsTheOneProducibleType(t *testing.T) { } func TestValuesRoundTripThroughTheDriverAndJSON(t *testing.T) { - doc := []byte(`{"v":3,"i":{"t":"users","c":"email"},"c":"stack-encrypt:1:AA==","hm":"00"}`) + doc := []byte(`{"v":4,"i":{"t":"users","c":"email"},"c":"stack-encrypt:1:AA==","hm":"00"}`) v := eql.TextEq(doc) value, err := v.Value() if err != nil || value != string(doc) { diff --git a/languages/golang/encrypt/eql_test.go b/languages/golang/encrypt/eql_test.go index 833724c85..de73e01bd 100644 --- a/languages/golang/encrypt/eql_test.go +++ b/languages/golang/encrypt/eql_test.go @@ -41,8 +41,8 @@ var contacts = []testusers.Contact{ {ID: 3, Email: "bob@example.com"}, } -// eqlEnvelope is the EQL v3 envelope of a TextEq value, as PostgreSQL's -// public.eql_v3_text_eq domain checks it. +// eqlEnvelope is the EQL v4 envelope of a TextEq value, as PostgreSQL's +// public.eql_v4_text_eq domain checks it. type eqlEnvelope struct { V int `json:"v"` I struct{ T, C string } @@ -91,8 +91,8 @@ func TestTextEqFieldSealsToTheEQLEnvelope(t *testing.T) { t.Errorf("row %d: passthrough id %d, want %d", i, e.ID, contacts[i].ID) } env := decodeEnvelope(t, e.Email) - if env.V != 3 { - t.Errorf("row %d: v = %d, want 3", i, env.V) + if env.V != 4 { + t.Errorf("row %d: v = %d, want 4", i, env.V) } if env.I.T != "users" || env.I.C != "email" { t.Errorf("row %d: i = %+v, want users/email", i, env.I) @@ -162,7 +162,7 @@ func TestTextEqQueryMatchesTheStoredValueAndTheRustFixture(t *testing.T) { t.Fatalf("the fixture's column changed: %+v", fixture) } - query, err := testusers.ContactFields.Email.Query(ctx, cipher, fixture.Plaintext) + query, err := testusers.ContactFields.Email.Eq(ctx, cipher, fixture.Plaintext) if err != nil { t.Fatal(err) } @@ -184,7 +184,7 @@ func TestTextEqQueryMatchesTheStoredValueAndTheRustFixture(t *testing.T) { if env := decodeEnvelope(t, stored); env.HM != probe.HM { t.Fatalf("the query's hm %s does not match the stored %s", probe.HM, env.HM) } - if other, _ := testusers.ContactFields.Email.Query(ctx, cipher, "Bob@example.com"); string(other) == fixture.Query { + if other, _ := testusers.ContactFields.Email.Eq(ctx, cipher, "Bob@example.com"); string(other) == fixture.Query { t.Fatal("no case folding: a different plaintext has a different term") } } @@ -217,7 +217,7 @@ func TestATamperedEQLValueDoesNotOpen(t *testing.T) { for name, value := range map[string]eql.TextEq{ "an empty object": eql.TextEq(`{}`), "not JSON": eql.TextEq(`not json`), - "a query value": eql.TextEq(`{"v":3,"i":{"t":"users","c":"email"},"hm":"00"}`), + "a query value": eql.TextEq(`{"v":4,"i":{"t":"users","c":"email"},"hm":"00"}`), } { junk[0].Email = value if _, err := testusers.DecryptContact(ctx, cipher, junk); !errors.Is(err, encrypt.ErrEncoding) { @@ -315,7 +315,7 @@ func TestTextEqIsRefusedByTheBuildWithoutEQLTypes(t *testing.T) { if err := eqlChecker.Check(ctx, plan); err != nil { t.Fatalf("the eql build refuses TextEq: %v", err) } - for _, name := range []string{"TextOrdOre", "Nope"} { + for _, name := range []string{"DateOrdOre", "Nope"} { refused := &record.Plan{Context: []string{"users"}, Fields: []record.Field{{Name: "email", Kind: record.String, Target: name}}} if err := eqlChecker.Check(ctx, refused); !errors.Is(err, encrypt.ErrEncoding) { t.Errorf("%s: Check = %v, want ErrEncoding", name, err) diff --git a/languages/golang/encrypt/eql_v4_ledger_test.go b/languages/golang/encrypt/eql_v4_ledger_test.go new file mode 100644 index 000000000..8e63c31a2 --- /dev/null +++ b/languages/golang/encrypt/eql_v4_ledger_test.go @@ -0,0 +1,358 @@ +package encrypt_test + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "testing" + "time" + + "github.com/cipherstash/stack/languages/golang/encrypt" + "github.com/cipherstash/stack/languages/golang/encrypt/internal/testusers" +) + +func mustDecimal(t *testing.T, s string) encrypt.Decimal { + t.Helper() + d, err := encrypt.ParseDecimal(s) + if err != nil { + t.Fatal(err) + } + return d +} + +func mustInstant(t *testing.T, s string) time.Time { + t.Helper() + v, err := time.Parse(time.RFC3339Nano, s) + if err != nil { + t.Fatal(err) + } + return v +} + +func storedLedger(e testusers.EncryptedLedger) map[string][]byte { + return map[string][]byte{"opened": e.Opened, "due": e.Due, "created": e.Created, "posted": e.Posted, "total": e.Total, "amount": e.Amount} +} + +// sameLedger compares two ledgers as values: instants with Equal, since an +// opened time.Time is in UTC and the original may not be. +func sameLedger(a, b testusers.Ledger) bool { + return a.ID == b.ID && a.Opened == b.Opened && a.Due == b.Due && + a.Created.Equal(b.Created) && a.Posted.Equal(b.Posted) && + a.Total == b.Total && a.Amount == b.Amount +} + +// The date, timestamp and numeric EQL v4 types, sealed through generated +// code over the deterministic guest build with the EQL types: an +// encrypt.Date, a time.Time and an encrypt.Decimal reach each family's +// storage and equality types, open back, and their Eq queries are the ones +// eql-bindings' own dispatch derives (v4_ledger_queries.json). +func TestV4DateTimestampNumericFieldsSealAndOpenBack(t *testing.T) { + c := deterministicEQLClient(t) + ctx := context.Background() + cipher := c.DefaultKeyset() + + east := time.FixedZone("UTC+10", 10*60*60) + ledgers := []testusers.Ledger{ + { + ID: 1, Opened: encrypt.Date{Year: 2024, Month: time.February, Day: 29}, Due: encrypt.Date{Year: 1, Month: time.January, Day: 1}, + Created: mustInstant(t, "1969-12-31T23:59:59.999999999Z"), Posted: time.Date(2024, time.March, 1, 9, 0, 0, 1, east), + Total: mustDecimal(t, "-79228162514264337593543950335"), Amount: mustDecimal(t, "1.50"), + }, + { + ID: 2, Opened: encrypt.Date{Year: -262143, Month: time.January, Day: 1}, Due: encrypt.Date{Year: 262142, Month: time.December, Day: 31}, + Created: mustInstant(t, "2024-02-29T12:34:56.123456789Z"), Posted: time.Unix(0, 0), + Total: mustDecimal(t, "0.0000000000000000000000000001"), Amount: mustDecimal(t, "-0.00"), + }, + } + encrypted, err := testusers.EncryptLedger(ctx, cipher, ledgers) + if err != nil { + t.Fatal(err) + } + for i, e := range encrypted { + for column, value := range storedLedger(e) { + want := "c,hm,i,v" + if column == "opened" || column == "created" || column == "total" { + want = "c,i,v" + } + if got := keysOf(t, column, value); got != want { + t.Errorf("row %d %s: keys %s, want %s", i, column, got, want) + } + } + } + decrypted, err := testusers.DecryptLedger(ctx, cipher, encrypted) + if err != nil { + t.Fatal(err) + } + for i := range ledgers { + if !sameLedger(decrypted[i], ledgers[i]) { + t.Errorf("row %d: %+v, want %+v", i, decrypted[i], ledgers[i]) + } + // An instant opens in UTC, with its nanoseconds. + if decrypted[i].Posted.Location() != time.UTC || decrypted[i].Posted.Nanosecond() != ledgers[i].Posted.Nanosecond() { + t.Errorf("row %d: posted opens as %v", i, decrypted[i].Posted) + } + } + // A decimal keeps its scale; a negative zero opens as zero. + if got := decrypted[0].Amount.String(); got != "1.50" { + t.Errorf("amount opens as %s, want 1.50", got) + } + if got := decrypted[1].Amount.String(); got != "0.00" { + t.Errorf("amount opens as %s, want 0.00", got) + } + + // Equality follows the baseline: a decimal's scale is normalised, and + // an instant keeps the microsecond. + amount := func(s string) []byte { + q, err := testusers.LedgerFields.Amount.Eq(ctx, cipher, mustDecimal(t, s)) + if err != nil { + t.Fatal(err) + } + return q + } + posted := func(v time.Time) []byte { + q, err := testusers.LedgerFields.Posted.Eq(ctx, cipher, v) + if err != nil { + t.Fatal(err) + } + return q + } + if !bytes.Equal(amount("1.5"), amount("1.50")) || bytes.Equal(amount("1.5"), amount("1.51")) { + t.Error("Amount.Eq does not normalise the scale") + } + early := posted(mustInstant(t, "2024-02-29T12:00:00.123456001Z")) + if !bytes.Equal(early, posted(mustInstant(t, "2024-02-29T12:00:00.123456999Z"))) || bytes.Equal(early, posted(mustInstant(t, "2024-02-29T12:00:00.123457Z"))) { + t.Error("Posted.Eq does not compare to the microsecond") + } + // The instant is compared, not the wall clock: the same instant in two + // locations is one term. + if !bytes.Equal(early, posted(time.Date(2024, time.February, 29, 22, 0, 0, 123456001, east))) { + t.Error("Posted.Eq differs for the same instant in another location") + } + + // What the engine cannot store is refused on every path, naming the + // field: a zero Date, a date past the range, an instant past it, and + // a decimal scale past 28. + refusals := map[string]error{} + _, refusals["EncryptLedger zero date"] = testusers.EncryptLedger(ctx, cipher, []testusers.Ledger{{ID: 3, Opened: ledgers[0].Opened, Created: ledgers[0].Created, Posted: ledgers[0].Posted}}) + _, refusals["Due.Encrypt zero"] = testusers.LedgerFields.Due.Encrypt(ctx, cipher, encrypt.Date{}) + _, refusals["Due.Eq past the range"] = testusers.LedgerFields.Due.Eq(ctx, cipher, encrypt.Date{Year: 262143, Month: time.January, Day: 1}) + _, refusals["Posted.Eq past the range"] = testusers.LedgerFields.Posted.Eq(ctx, cipher, time.Date(262143, time.January, 1, 0, 0, 0, 0, time.UTC)) + _, refusals["Amount.Eq scale 29"] = testusers.LedgerFields.Amount.Eq(ctx, cipher, encrypt.Decimal{Scale: 29}) + for path, err := range refusals { + if !errors.Is(err, encrypt.ErrEncoding) { + t.Errorf("%s: %v, want ErrEncoding", path, err) + } + } + + raw, err := os.ReadFile(fixturePath("v4_ledger_queries.json")) + if err != nil { + t.Fatal(err) + } + var fixture struct { + Due, Posted, Amount struct { + Plaintext string + Query string + } + } + if err := json.Unmarshal(raw, &fixture); err != nil { + t.Fatal(err) + } + queries := map[string]func(string) ([]byte, error){ + "due": func(s string) ([]byte, error) { + v, err := time.Parse(time.DateOnly, s) + if err != nil { + return nil, err + } + return testusers.LedgerFields.Due.Eq(ctx, cipher, encrypt.DateOf(v)) + }, + "posted": func(s string) ([]byte, error) { + return testusers.LedgerFields.Posted.Eq(ctx, cipher, mustInstant(t, s)) + }, + "amount": func(s string) ([]byte, error) { + return testusers.LedgerFields.Amount.Eq(ctx, cipher, mustDecimal(t, s)) + }, + } + wants := map[string]struct { + Plaintext string + Query string + }{"due": fixture.Due, "posted": fixture.Posted, "amount": fixture.Amount} + for column, derive := range queries { + want := wants[column] + got, err := derive(want.Plaintext) + if err != nil { + t.Fatal(err) + } + if want.Query == "" || string(got) != want.Query { + t.Errorf("%s.Eq = %s\nwant %s", column, got, want.Query) + } + } +} + +// ledgerRows are the rows v4_ledger_rows.json holds: the range's edges, a +// day, instant and number each of the fixture's queries finds, a value in +// the same microsecond and an amount at another scale, which the queries +// find too, and rows they do not. +func ledgerRows(t *testing.T) []testusers.Ledger { + return []testusers.Ledger{ + { + ID: 1, Opened: encrypt.Date{Year: -262143, Month: time.January, Day: 1}, Due: encrypt.Date{Year: 2024, Month: time.February, Day: 29}, + Created: mustInstant(t, "1969-12-31T23:59:59.999999999Z"), Posted: mustInstant(t, "2024-02-29T12:34:56.123456Z"), + Total: mustDecimal(t, "-79228162514264337593543950335"), Amount: mustDecimal(t, "1.50"), + }, + { + ID: 2, Opened: encrypt.Date{Year: 262142, Month: time.December, Day: 31}, Due: encrypt.Date{Year: 2024, Month: time.March, Day: 1}, + Created: mustInstant(t, "2024-02-29T12:34:56.123456789Z"), Posted: mustInstant(t, "2024-02-29T12:34:56.123456999Z"), + Total: mustDecimal(t, "0.0000000000000000000000000001"), Amount: mustDecimal(t, "1.5"), + }, + { + ID: 3, Opened: encrypt.Date{Year: 1970, Month: time.January, Day: 1}, Due: encrypt.Date{Year: 2024, Month: time.February, Day: 29}, + Created: mustInstant(t, "1970-01-01T00:00:00Z"), Posted: mustInstant(t, "2024-02-29T12:34:56.123457Z"), + Total: mustDecimal(t, "1234.5600"), Amount: mustDecimal(t, "1.51"), + }, + } +} + +type ledgerRowsFixture struct { + Comment string `json:"_comment"` + Seed string `json:"seed"` + Table string `json:"table"` + Rows []ledgerRow `json:"rows"` +} + +type ledgerRow struct { + ID int64 `json:"id"` + Plaintext ledgerPlain `json:"plaintext"` + Stored map[string]json.RawMessage `json:"stored"` +} + +// ledgerPlain is a row's plaintext as text: dates in ISO 8601, instants in +// RFC 3339 with nanoseconds, decimals as fixed-point text with their scale. +type ledgerPlain struct { + Opened string `json:"opened"` + Due string `json:"due"` + Created string `json:"created"` + Posted string `json:"posted"` + Total string `json:"total"` + Amount string `json:"amount"` +} + +// isoDate is a Date as ISO 8601, with the signed year chrono reads for +// years outside 0000 to 9999. +func isoDate(d encrypt.Date) string { + switch { + case d.Year < 0: + return fmt.Sprintf("-%04d-%02d-%02d", -d.Year, d.Month, d.Day) + case d.Year > 9999: + return fmt.Sprintf("+%04d-%02d-%02d", d.Year, d.Month, d.Day) + } + return fmt.Sprintf("%04d-%02d-%02d", d.Year, d.Month, d.Day) +} + +func plainLedger(l testusers.Ledger) ledgerPlain { + return ledgerPlain{ + Opened: isoDate(l.Opened), Due: isoDate(l.Due), + Created: l.Created.UTC().Format(time.RFC3339Nano), Posted: l.Posted.UTC().Format(time.RFC3339Nano), + Total: l.Total.String(), Amount: l.Amount.String(), + } +} + +// TestV4LedgerRowsMatchTheCommittedFixture checks the committed +// v4_ledger_rows.json against what the generated code seals today, as +// TestV4RowsMatchTheCommittedFixture does for v4_rows.json: everything but +// each `c` must match, and the committed `c` values must open. The Rust +// suite opens the same rows and stores them in PostgreSQL. EQL_UPDATE_FIXTURES +// rewrites the file when something other than the ciphertexts moved. +func TestV4LedgerRowsMatchTheCommittedFixture(t *testing.T) { + c := deterministicEQLClient(t) + ctx := context.Background() + rows := ledgerRows(t) + encrypted, err := testusers.EncryptLedger(ctx, c.DefaultKeyset(), rows) + if err != nil { + t.Fatal(err) + } + fixture := ledgerRowsFixture{ + Comment: "Ledger rows sealed by the Go SDK's generated code over the deterministic EQL guest with the seed below; the Rust suite opens and queries them. Regenerate with EQL_UPDATE_FIXTURES=1 go test ./encrypt -run TestV4LedgerRows, do not edit.", + Seed: string(testSeed[:]), + Table: "profiles", + } + for i, e := range encrypted { + row := ledgerRow{ID: rows[i].ID, Plaintext: plainLedger(rows[i]), Stored: map[string]json.RawMessage{}} + for column, value := range storedLedger(e) { + row.Stored[column] = json.RawMessage(value) + } + fixture.Rows = append(fixture.Rows, row) + } + rendered, err := json.MarshalIndent(fixture, "", " ") + if err != nil { + t.Fatal(err) + } + rendered = append(rendered, '\n') + path := fixturePath("v4_ledger_rows.json") + var committed ledgerRowsFixture + committedBytes, err := os.ReadFile(path) + if err == nil { + err = json.Unmarshal(committedBytes, &committed) + } + same := err == nil && bytes.Equal(ledgerWithoutCiphertexts(t, committed), ledgerWithoutCiphertexts(t, fixture)) + if os.Getenv("EQL_UPDATE_FIXTURES") != "" && !same { + if err := os.WriteFile(path, rendered, 0o644); err != nil { + t.Fatal(err) + } + t.Logf("%s: rewritten", path) + committed, same, err = fixture, true, nil + } + if err != nil { + t.Fatalf("%v: run with EQL_UPDATE_FIXTURES=1 to write it", err) + } + if !same { + t.Errorf("%s differs from what the generated code seals today beyond the ciphertexts; regenerate it with EQL_UPDATE_FIXTURES=1", path) + } + + sealed := make([]testusers.EncryptedLedger, len(committed.Rows)) + for i, row := range committed.Rows { + s := row.Stored + sealed[i] = testusers.EncryptedLedger{ + ID: row.ID, Opened: []byte(s["opened"]), Due: []byte(s["due"]), Created: []byte(s["created"]), + Posted: []byte(s["posted"]), Total: []byte(s["total"]), Amount: []byte(s["amount"]), + } + } + opened, err := testusers.DecryptLedger(ctx, c.DefaultKeyset(), sealed) + if err != nil { + t.Fatal(err) + } + for i, row := range committed.Rows { + if got := plainLedger(opened[i]); got != row.Plaintext { + t.Errorf("row %d opens to %+v, want %+v", row.ID, got, row.Plaintext) + } + } +} + +func ledgerWithoutCiphertexts(t *testing.T, f ledgerRowsFixture) []byte { + t.Helper() + out := f + out.Rows = nil + for _, row := range f.Rows { + stripped := ledgerRow{ID: row.ID, Plaintext: row.Plaintext, Stored: map[string]json.RawMessage{}} + for column, value := range row.Stored { + var fields map[string]json.RawMessage + if err := json.Unmarshal(value, &fields); err != nil { + t.Fatal(err) + } + delete(fields, "c") + b, err := json.Marshal(fields) + if err != nil { + t.Fatal(err) + } + stripped.Stored[column] = b + } + out.Rows = append(out.Rows, stripped) + } + b, err := json.Marshal(out) + if err != nil { + t.Fatal(err) + } + return b +} diff --git a/languages/golang/encrypt/eql_v4_order_test.go b/languages/golang/encrypt/eql_v4_order_test.go new file mode 100644 index 000000000..2a4619997 --- /dev/null +++ b/languages/golang/encrypt/eql_v4_order_test.go @@ -0,0 +1,386 @@ +package encrypt_test + +import ( + "bytes" + "context" + "encoding/json" + "math" + "os" + "sort" + "strings" + "testing" + + "github.com/cipherstash/stack/languages/golang/encrypt/internal/testusers" +) + +// The OPE-ordered EQL v4 types (IntegerOrd, BigintOrd, RealOrd, DoubleOrd, +// TextOrd and their OrdOpe twins), sealed through generated code over the +// deterministic guest build with the EQL types. PostgreSQL sees these +// values through the Rust suite: TestV4OrderedRowsMatchTheCommittedFixture +// commits them to v4_ordered_rows.json, and +// packages/eql/tests/encryption/tests/v4_order.rs inserts that file into +// eql_v4 columns and compares it. + +// rankings are ordered by every field's order key: each row's value is +// below the next row's in every column, except the pairs rankingTies +// records. +var rankings = []testusers.Ranking{ + {ID: 1, Level: -5, Floor: -5, Points: math.MinInt64, Total: -1 << 40, Speed: float32(math.Inf(-1)), Pace: -2.5, Score: -1.5, Mean: math.Inf(-1), Title: "alice", Handle: "al"}, + {ID: 2, Level: 0, Floor: 0, Points: 0, Total: 0, Speed: 0, Pace: 0, Score: 0, Mean: 0, Title: "Cafe", Handle: "bob"}, + {ID: 3, Level: 0, Floor: 0, Points: 0, Total: 0, Speed: float32(math.Copysign(0, -1)), Pace: float32(math.Copysign(0, -1)), Score: math.Copysign(0, -1), Mean: math.Copysign(0, -1), Title: "café", Handle: "BOB"}, + {ID: 4, Level: 7, Floor: 70000, Points: 5_000_000_001, Total: 1 << 40, Speed: 2.5, Pace: 1.5, Score: 9.75, Mean: 1e300, Title: "Mango", Handle: "carol"}, + {ID: 5, Level: math.MaxInt32, Floor: math.MaxInt32, Points: math.MaxInt64, Total: math.MaxInt64, Speed: float32(math.Inf(1)), Pace: float32(math.Inf(1)), Score: math.Inf(1), Mean: math.Inf(1), Title: "Zoe", Handle: "zed"}, +} + +// rankingTies are the adjacent pairs of rankings, by ID, that tie in every +// order key: equal integers, -0.0 against 0.0, and text that differs only +// in case and accents. They are written down rather than derived because +// the text ties need Unicode case folding, which the engine does and this +// module has no dependency for. +var rankingTies = map[[2]int64]bool{{2, 3}: true} + +// storedRanking is each sealed field of an encrypted ranking by column. +func storedRanking(e testusers.EncryptedRanking) map[string][]byte { + return map[string][]byte{ + "level": e.Level, + "floor": e.Floor, + "points": e.Points, + "total": e.Total, + "speed": e.Speed, + "pace": e.Pace, + "score": e.Score, + "mean": e.Mean, + "title": e.Title, + "handle": e.Handle, + } +} + +// orderTerm is a stored value's or query's `op`, as its hex. Lower-case +// hex sorts as the bytes it encodes, which is how EQL compares `op`. +func orderTerm(t *testing.T, value []byte) string { + t.Helper() + var v struct { + OP string `json:"op"` + } + if err := json.Unmarshal(value, &v); err != nil || v.OP == "" { + t.Fatalf("no op in %s: %v", value, err) + } + return v.OP +} + +func TestV4OrderedFieldsSealToTheirEQLTypesAndOpenBack(t *testing.T) { + c := deterministicEQLClient(t) + ctx := context.Background() + cipher := c.DefaultKeyset() + + encrypted, err := testusers.EncryptRanking(ctx, cipher, rankings) + if err != nil { + t.Fatal(err) + } + for i, e := range encrypted { + for column, value := range storedRanking(e) { + want := "c,i,op,v" + if column == "title" || column == "handle" { + want = "c,hm,i,op,v" + } + if got := rankingKeys(t, column, value); got != want { + t.Errorf("row %d %s: keys %s, want %s", i, column, got, want) + } + } + } + + // The stored order terms sort as the rows do: each row below the next, + // except the pairs rankingTies records. A recorded tie that is not an + // adjacent pair of rankings would be checked as nothing, so each one + // must be found, its numbers must be equal in Go too, and there must be + // at least one. + if len(rankingTies) == 0 { + t.Fatal("rankingTies records no tie: the equal-order-key case is untested") + } + found := map[[2]int64]bool{} + for i := 0; i+1 < len(rankings); i++ { + a, b := rankings[i], rankings[i+1] + pair := [2]int64{a.ID, b.ID} + if !rankingTies[pair] { + continue + } + found[pair] = true + if a.Level != b.Level || a.Floor != b.Floor || a.Points != b.Points || a.Total != b.Total || + a.Speed != b.Speed || a.Pace != b.Pace || a.Score != b.Score || a.Mean != b.Mean { + t.Errorf("rows %d and %d are recorded as a tie but their numbers differ", a.ID, b.ID) + } + } + for pair := range rankingTies { + if !found[pair] { + t.Errorf("rankingTies records rows %d and %d, which are not adjacent in rankings", pair[0], pair[1]) + } + } + for column := range storedRanking(encrypted[0]) { + for i := 0; i+1 < len(encrypted); i++ { + a := orderTerm(t, storedRanking(encrypted[i])[column]) + b := orderTerm(t, storedRanking(encrypted[i+1])[column]) + tie := rankingTies[[2]int64{rankings[i].ID, rankings[i+1].ID}] + switch { + case tie && a != b: + t.Errorf("%s: rows %d and %d tie in the order key but store different terms", column, rankings[i].ID, rankings[i+1].ID) + case !tie && a >= b: + t.Errorf("%s: row %d's term does not sort below row %d's", column, rankings[i].ID, rankings[i+1].ID) + } + } + } + + decrypted, err := testusers.DecryptRanking(ctx, cipher, encrypted) + if err != nil { + t.Fatal(err) + } + for i := range rankings { + got, want := decrypted[i], rankings[i] + // The ciphertext keeps -0.0's sign; == would not tell. + if math.Signbit(float64(got.Speed)) != math.Signbit(float64(want.Speed)) || math.Signbit(got.Mean) != math.Signbit(want.Mean) { + t.Errorf("row %d: a zero lost its sign", want.ID) + } + if got != want { + t.Errorf("row %d: %+v, want %+v", i, got, want) + } + } +} + +// rankingKeys is the sorted top-level keys of an EQL value stored under +// rankings, checking its envelope as keysOf does for profiles. +func rankingKeys(t *testing.T, column string, value []byte) string { + t.Helper() + var raw map[string]json.RawMessage + if err := json.Unmarshal(value, &raw); err != nil { + t.Fatalf("the EQL value is not JSON: %v: %s", err, value) + } + var i struct{ T, C string } + if err := json.Unmarshal(raw["i"], &i); err != nil || i.T != "rankings" || i.C != column { + t.Errorf("i = %s, want {\"t\":\"rankings\",\"c\":%q}", raw["i"], column) + } + if string(raw["v"]) != "4" { + t.Errorf("v = %s, want 4", raw["v"]) + } + keys := make([]string, 0, len(raw)) + for k := range raw { + keys = append(keys, k) + } + sort.Strings(keys) + return strings.Join(keys, ",") +} + +func TestV4OrderedQueriesMatchTheRustFixture(t *testing.T) { + c := deterministicEQLClient(t) + ctx := context.Background() + cipher := c.DefaultKeyset() + + raw, err := os.ReadFile(fixturePath("v4_ordered_queries.json")) + if err != nil { + t.Fatal(err) + } + type entry[T any] struct { + Plaintext T + Query string + } + var fixture struct { + Table string + Level entry[int32] + Floor entry[int32] + Points entry[int64] + Total entry[int64] + Speed entry[float32] + Pace entry[float32] + Score entry[float64] + Mean entry[float64] + Title entry[string] + Handle entry[string] + } + if err := json.Unmarshal(raw, &fixture); err != nil { + t.Fatal(err) + } + if fixture.Table != "rankings" { + t.Fatalf("the fixture's table changed: %q", fixture.Table) + } + f := testusers.RankingFields + // Byte for byte what eql-bindings' own dispatch derived under the same + // index key. Every ordering method of a field returns one operand: the + // operator is SQL's. + cases := []struct { + field string + lt func() ([]byte, error) + gte func() ([]byte, error) + want string + }{ + {"Level", func() ([]byte, error) { return f.Level.Lt(ctx, cipher, fixture.Level.Plaintext) }, func() ([]byte, error) { return f.Level.Gte(ctx, cipher, fixture.Level.Plaintext) }, fixture.Level.Query}, + {"Floor", func() ([]byte, error) { return f.Floor.Lt(ctx, cipher, fixture.Floor.Plaintext) }, func() ([]byte, error) { return f.Floor.Gte(ctx, cipher, fixture.Floor.Plaintext) }, fixture.Floor.Query}, + {"Points", func() ([]byte, error) { return f.Points.Lt(ctx, cipher, fixture.Points.Plaintext) }, func() ([]byte, error) { return f.Points.Gte(ctx, cipher, fixture.Points.Plaintext) }, fixture.Points.Query}, + {"Total", func() ([]byte, error) { return f.Total.Lt(ctx, cipher, fixture.Total.Plaintext) }, func() ([]byte, error) { return f.Total.Gte(ctx, cipher, fixture.Total.Plaintext) }, fixture.Total.Query}, + {"Speed", func() ([]byte, error) { return f.Speed.Lt(ctx, cipher, fixture.Speed.Plaintext) }, func() ([]byte, error) { return f.Speed.Gte(ctx, cipher, fixture.Speed.Plaintext) }, fixture.Speed.Query}, + {"Pace", func() ([]byte, error) { return f.Pace.Lt(ctx, cipher, fixture.Pace.Plaintext) }, func() ([]byte, error) { return f.Pace.Gte(ctx, cipher, fixture.Pace.Plaintext) }, fixture.Pace.Query}, + {"Score", func() ([]byte, error) { return f.Score.Lt(ctx, cipher, fixture.Score.Plaintext) }, func() ([]byte, error) { return f.Score.Gte(ctx, cipher, fixture.Score.Plaintext) }, fixture.Score.Query}, + {"Mean", func() ([]byte, error) { return f.Mean.Lt(ctx, cipher, fixture.Mean.Plaintext) }, func() ([]byte, error) { return f.Mean.Gte(ctx, cipher, fixture.Mean.Plaintext) }, fixture.Mean.Query}, + {"Title", func() ([]byte, error) { return f.Title.Lt(ctx, cipher, fixture.Title.Plaintext) }, func() ([]byte, error) { return f.Title.Eq(ctx, cipher, fixture.Title.Plaintext) }, fixture.Title.Query}, + {"Handle", func() ([]byte, error) { return f.Handle.Lt(ctx, cipher, fixture.Handle.Plaintext) }, func() ([]byte, error) { return f.Handle.Eq(ctx, cipher, fixture.Handle.Plaintext) }, fixture.Handle.Query}, + } + for _, tc := range cases { + for _, method := range []func() ([]byte, error){tc.lt, tc.gte} { + got, err := method() + if err != nil { + t.Fatal(err) + } + if tc.want == "" || string(got) != tc.want { + t.Errorf("%s = %s\nwant %s", tc.field, got, tc.want) + } + } + } + + // Text with a code point Unicode 16 does not assign has no fold, so no + // order term. + if q, err := f.Title.Lt(ctx, cipher, "a͸b"); err == nil { + t.Errorf("Title.Lt over unassigned text = %s, want a refusal", q) + } +} + +// orderedRowsFixture is v4_ordered_rows.json: the rankings above as the +// generated code seals them under the deterministic guest and testSeed. +type orderedRowsFixture struct { + Comment string `json:"_comment"` + Seed string `json:"seed"` + Table string `json:"table"` + Rows []orderedRow `json:"rows"` +} + +type orderedRow struct { + ID int64 `json:"id"` + Plaintext orderedPlain `json:"plaintext"` + Stored map[string]json.RawMessage `json:"stored"` +} + +// orderedPlain is a ranking's plaintext as the fixture records it. JSON has +// no infinity, so the fixture's rows hold finite values only; the infinite +// and -0.0 cases are covered by TestV4OrderedFieldsSealToTheirEQLTypesAndOpenBack +// and the Rust suite. +type orderedPlain struct { + Level int32 `json:"level"` + Floor int32 `json:"floor"` + Points int64 `json:"points"` + Total int64 `json:"total"` + Speed float32 `json:"speed"` + Pace float32 `json:"pace"` + Score float64 `json:"score"` + Mean float64 `json:"mean"` + Title string `json:"title"` + Handle string `json:"handle"` +} + +func orderedPlainOf(r testusers.Ranking) orderedPlain { + return orderedPlain{r.Level, r.Floor, r.Points, r.Total, r.Speed, r.Pace, r.Score, r.Mean, r.Title, r.Handle} +} + +// fixtureRankings are the rows v4_ordered_rows.json holds: finite values +// on both sides of the fixture's query plaintexts in every column. +var fixtureRankings = []testusers.Ranking{ + {ID: 1, Level: -20, Floor: -30, Points: -5, Total: -9, Speed: -1.25, Pace: -3.5, Score: -2.5, Mean: -0.25, Title: "apple", Handle: "Alice"}, + {ID: 2, Level: 10, Floor: -3, Points: 5_000_000_000, Total: -1, Speed: 2.5, Pace: -0.5, Score: 9.75, Mean: 0, Title: "mango", Handle: "Carol"}, + {ID: 3, Level: 99, Floor: 4, Points: 6_000_000_000, Total: 12, Speed: 8, Pace: 0.75, Score: 100, Mean: 3.5, Title: "Zucchini", Handle: "dave"}, +} + +// TestV4OrderedRowsMatchTheCommittedFixture checks the committed +// v4_ordered_rows.json against what the generated code seals today, as +// TestV4RowsMatchTheCommittedFixture does for v4_rows.json: everything but +// each `c` must match, and the committed `c` values must open. +// EQL_UPDATE_FIXTURES rewrites the file when something other than the +// ciphertexts moved. +func TestV4OrderedRowsMatchTheCommittedFixture(t *testing.T) { + c := deterministicEQLClient(t) + ctx := context.Background() + encrypted, err := testusers.EncryptRanking(ctx, c.DefaultKeyset(), fixtureRankings) + if err != nil { + t.Fatal(err) + } + fixture := orderedRowsFixture{ + Comment: "Ranking rows sealed by the Go SDK's generated code over the deterministic EQL guest with the seed below; the Rust suite opens and compares them. Regenerate with EQL_UPDATE_FIXTURES=1 go test ./encrypt -run TestV4OrderedRows, do not edit.", + Seed: string(testSeed[:]), + Table: "rankings", + } + for i, e := range encrypted { + row := orderedRow{ID: fixtureRankings[i].ID, Plaintext: orderedPlainOf(fixtureRankings[i]), Stored: map[string]json.RawMessage{}} + for column, value := range storedRanking(e) { + row.Stored[column] = json.RawMessage(value) + } + fixture.Rows = append(fixture.Rows, row) + } + rendered, err := json.MarshalIndent(fixture, "", " ") + if err != nil { + t.Fatal(err) + } + rendered = append(rendered, '\n') + path := fixturePath("v4_ordered_rows.json") + var committed orderedRowsFixture + committedBytes, err := os.ReadFile(path) + if err == nil { + err = json.Unmarshal(committedBytes, &committed) + } + same := err == nil && bytes.Equal(orderedWithoutCiphertexts(t, committed), orderedWithoutCiphertexts(t, fixture)) + if os.Getenv("EQL_UPDATE_FIXTURES") != "" && !same { + if err := os.WriteFile(path, rendered, 0o644); err != nil { + t.Fatal(err) + } + t.Logf("%s: rewritten", path) + committed, same, err = fixture, true, nil + } + if err != nil { + t.Fatalf("%v: run with EQL_UPDATE_FIXTURES=1 to write it", err) + } + if !same { + t.Errorf("%s differs from what the generated code seals today beyond the ciphertexts; regenerate it with EQL_UPDATE_FIXTURES=1", path) + } + + // The committed ciphertexts still open, to the committed plaintexts. + sealed := make([]testusers.EncryptedRanking, len(committed.Rows)) + for i, row := range committed.Rows { + s := row.Stored + sealed[i] = testusers.EncryptedRanking{ + ID: row.ID, Level: []byte(s["level"]), Floor: []byte(s["floor"]), Points: []byte(s["points"]), Total: []byte(s["total"]), + Speed: []byte(s["speed"]), Pace: []byte(s["pace"]), Score: []byte(s["score"]), Mean: []byte(s["mean"]), + Title: []byte(s["title"]), Handle: []byte(s["handle"]), + } + } + opened, err := testusers.DecryptRanking(ctx, c.DefaultKeyset(), sealed) + if err != nil { + t.Fatal(err) + } + for i, row := range committed.Rows { + if got := orderedPlainOf(opened[i]); got != row.Plaintext { + t.Errorf("row %d opens to %+v, want %+v", row.ID, got, row.Plaintext) + } + } +} + +// orderedWithoutCiphertexts renders a fixture with every stored value's `c` +// removed: the part of the file that is the same on every run. +func orderedWithoutCiphertexts(t *testing.T, f orderedRowsFixture) []byte { + t.Helper() + out := f + out.Rows = nil + for _, row := range f.Rows { + stripped := orderedRow{ID: row.ID, Plaintext: row.Plaintext, Stored: map[string]json.RawMessage{}} + for column, value := range row.Stored { + var fields map[string]json.RawMessage + if err := json.Unmarshal(value, &fields); err != nil { + t.Fatal(err) + } + delete(fields, "c") + b, err := json.Marshal(fields) + if err != nil { + t.Fatal(err) + } + stripped.Stored[column] = b + } + out.Rows = append(out.Rows, stripped) + } + b, err := json.Marshal(out) + if err != nil { + t.Fatal(err) + } + return b +} diff --git a/languages/golang/encrypt/eql_v4_smallint_test.go b/languages/golang/encrypt/eql_v4_smallint_test.go new file mode 100644 index 000000000..59dff0686 --- /dev/null +++ b/languages/golang/encrypt/eql_v4_smallint_test.go @@ -0,0 +1,232 @@ +package encrypt_test + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "math" + "os" + "strings" + "testing" + + "github.com/cipherstash/stack/languages/golang/encrypt" + "github.com/cipherstash/stack/languages/golang/encrypt/internal/testusers" +) + +// The smallint EQL v4 types, sealed through generated code over the +// deterministic guest build with the EQL types: an int16 field reaches +// Smallint and SmallintEq, opens back, and its Eq query is the one +// eql-bindings' own dispatch derives (v4_queries.json, key tally). +func TestV4SmallintFieldsSealAndOpenBack(t *testing.T) { + c := deterministicEQLClient(t) + ctx := context.Background() + cipher := c.DefaultKeyset() + + tallies := []testusers.Tally{ + {ID: 1, Small: math.MinInt16, Count: -12, Wide: 7}, + {ID: 2, Small: math.MaxInt16, Count: 0, Wide: math.MinInt16}, + } + encrypted, err := testusers.EncryptTally(ctx, cipher, tallies) + if err != nil { + t.Fatal(err) + } + for i, e := range encrypted { + for column, value := range map[string][]byte{"small": e.Small, "tally": e.Count, "wide": e.Wide} { + want := "c,hm,i,v" + if column == "small" { + want = "c,i,v" + } + if got := keysOf(t, column, value); got != want { + t.Errorf("row %d %s: keys %s, want %s", i, column, got, want) + } + } + } + decrypted, err := testusers.DecryptTally(ctx, cipher, encrypted) + if err != nil { + t.Fatal(err) + } + for i := range tallies { + if decrypted[i] != tallies[i] { + t.Errorf("row %d: %+v, want %+v", i, decrypted[i], tallies[i]) + } + } + + // An int32 field in a smallint type seals at 16 bits: a value that does + // not fit is refused, not truncated, on every path, and the error names + // the field. Both bounds, as widened_test.go pins them for IntegerEq. + for _, v := range []int32{math.MaxInt16 + 1, math.MinInt16 - 1} { + refusals := map[string]error{} + _, refusals["EncryptTally"] = testusers.EncryptTally(ctx, cipher, []testusers.Tally{{ID: 3, Wide: v}}) + _, refusals["Wide.Encrypt"] = testusers.TallyFields.Wide.Encrypt(ctx, cipher, v) + _, refusals["Wide.Eq"] = testusers.TallyFields.Wide.Eq(ctx, cipher, v) + for path, err := range refusals { + if !errors.Is(err, encrypt.ErrEncoding) { + t.Errorf("%s(%d): %v, want ErrEncoding", path, v, err) + } else if !strings.Contains(err.Error(), "wide") { + t.Errorf("%s(%d): %v, want the field named", path, v, err) + } + } + } + // The bounds themselves fit. + for _, v := range []int32{math.MaxInt16, math.MinInt16} { + if _, err := testusers.TallyFields.Wide.Eq(ctx, cipher, v); err != nil { + t.Errorf("Wide.Eq(%d): %v", v, err) + } + } + + raw, err := os.ReadFile(fixturePath("v4_queries.json")) + if err != nil { + t.Fatal(err) + } + var fixture struct { + Tally struct { + Plaintext int16 + Query string + } + } + if err := json.Unmarshal(raw, &fixture); err != nil { + t.Fatal(err) + } + q, err := testusers.TallyFields.Count.Eq(ctx, cipher, fixture.Tally.Plaintext) + if err != nil { + t.Fatal(err) + } + if fixture.Tally.Query == "" || string(q) != fixture.Tally.Query { + t.Errorf("Count.Eq = %s\nwant %s", q, fixture.Tally.Query) + } +} + +// tallyRows are the rows v4_smallint_rows.json holds: both extremes of the +// width, and an int32 field at the extremes too. +var tallyRows = []testusers.Tally{ + {ID: 1, Small: math.MinInt16, Count: -12, Wide: math.MinInt16}, + {ID: 2, Small: math.MaxInt16, Count: 7, Wide: math.MaxInt16}, + {ID: 3, Small: 0, Count: -12, Wide: -1}, +} + +type smallintRowsFixture struct { + Comment string `json:"_comment"` + Seed string `json:"seed"` + Table string `json:"table"` + Rows []smallintRow `json:"rows"` +} + +type smallintRow struct { + ID int64 `json:"id"` + Plaintext smallintPlain `json:"plaintext"` + Stored map[string]json.RawMessage `json:"stored"` +} + +type smallintPlain struct { + Small int16 `json:"small"` + Tally int16 `json:"tally"` + Wide int32 `json:"wide"` +} + +func storedTally(e testusers.EncryptedTally) map[string][]byte { + return map[string][]byte{"small": e.Small, "tally": e.Count, "wide": e.Wide} +} + +// TestV4SmallintRowsMatchTheCommittedFixture checks the committed +// v4_smallint_rows.json against what the generated code seals today, as +// TestV4RowsMatchTheCommittedFixture does for v4_rows.json: everything but +// each `c` must match, and the committed `c` values must open. +// EQL_UPDATE_FIXTURES rewrites the file when something other than the +// ciphertexts moved, so a change that moves only `c` leaves the committed +// file as it was. EQL_V4_SMALLINT_ROWS_OUT names a file to write today's +// rows to, ciphertexts and all: CI hands it to the Rust suite +// (EQL_V4_SMALLINT_ROWS), so Rust opens the smallint ciphertexts the Go SDK +// seals now, not only the committed ones. +func TestV4SmallintRowsMatchTheCommittedFixture(t *testing.T) { + c := deterministicEQLClient(t) + ctx := context.Background() + encrypted, err := testusers.EncryptTally(ctx, c.DefaultKeyset(), tallyRows) + if err != nil { + t.Fatal(err) + } + fixture := smallintRowsFixture{ + Comment: "Tally rows sealed by the Go SDK's generated code over the deterministic EQL guest with the seed below; the Rust suite opens and queries them. Regenerate with EQL_UPDATE_FIXTURES=1 go test ./encrypt -run TestV4SmallintRows, do not edit.", + Seed: string(testSeed[:]), + Table: "profiles", + } + for i, e := range encrypted { + r := tallyRows[i] + row := smallintRow{ID: r.ID, Plaintext: smallintPlain{r.Small, r.Count, r.Wide}, Stored: map[string]json.RawMessage{}} + for column, value := range storedTally(e) { + row.Stored[column] = json.RawMessage(value) + } + fixture.Rows = append(fixture.Rows, row) + } + rendered, err := json.MarshalIndent(fixture, "", " ") + if err != nil { + t.Fatal(err) + } + rendered = append(rendered, '\n') + if out := os.Getenv("EQL_V4_SMALLINT_ROWS_OUT"); out != "" { + if err := os.WriteFile(out, rendered, 0o644); err != nil { + t.Fatal(err) + } + } + path := fixturePath("v4_smallint_rows.json") + var committed smallintRowsFixture + committedBytes, err := os.ReadFile(path) + if err == nil { + err = json.Unmarshal(committedBytes, &committed) + } + same := err == nil && bytes.Equal(smallintWithoutCiphertexts(t, committed), smallintWithoutCiphertexts(t, fixture)) + if os.Getenv("EQL_UPDATE_FIXTURES") != "" && !same { + if err := os.WriteFile(path, rendered, 0o644); err != nil { + t.Fatal(err) + } + t.Logf("%s: rewritten", path) + committed, same, err = fixture, true, nil + } + if err != nil { + t.Fatalf("%v: run with EQL_UPDATE_FIXTURES=1 to write it", err) + } + if !same { + t.Errorf("%s differs from what the generated code seals today beyond the ciphertexts; regenerate it with EQL_UPDATE_FIXTURES=1", path) + } + + sealed := make([]testusers.EncryptedTally, len(committed.Rows)) + for i, row := range committed.Rows { + sealed[i] = testusers.EncryptedTally{ID: row.ID, Small: []byte(row.Stored["small"]), Count: []byte(row.Stored["tally"]), Wide: []byte(row.Stored["wide"])} + } + opened, err := testusers.DecryptTally(ctx, c.DefaultKeyset(), sealed) + if err != nil { + t.Fatal(err) + } + for i, row := range committed.Rows { + if got := (smallintPlain{opened[i].Small, opened[i].Count, opened[i].Wide}); got != row.Plaintext { + t.Errorf("row %d opens to %+v, want %+v", row.ID, got, row.Plaintext) + } + } +} + +func smallintWithoutCiphertexts(t *testing.T, f smallintRowsFixture) []byte { + t.Helper() + out := f + out.Rows = nil + for _, row := range f.Rows { + stripped := smallintRow{ID: row.ID, Plaintext: row.Plaintext, Stored: map[string]json.RawMessage{}} + for column, value := range row.Stored { + var fields map[string]json.RawMessage + if err := json.Unmarshal(value, &fields); err != nil { + t.Fatal(err) + } + delete(fields, "c") + b, err := json.Marshal(fields) + if err != nil { + t.Fatal(err) + } + stripped.Stored[column] = b + } + out.Rows = append(out.Rows, stripped) + } + b, err := json.Marshal(out) + if err != nil { + t.Fatal(err) + } + return b +} diff --git a/languages/golang/encrypt/eql_v4_test.go b/languages/golang/encrypt/eql_v4_test.go new file mode 100644 index 000000000..c6c457a23 --- /dev/null +++ b/languages/golang/encrypt/eql_v4_test.go @@ -0,0 +1,546 @@ +package encrypt_test + +import ( + "bytes" + "context" + "encoding/json" + "os" + "path/filepath" + "slices" + "sort" + "strings" + "testing" + + "github.com/cipherstash/stack/languages/golang/encrypt" + "github.com/cipherstash/stack/languages/golang/encrypt/internal/testusers" +) + +// The EQL v4 types the engine produces besides TextEq, sealed through +// generated code over the deterministic guest build with the EQL types: +// each Go field type reaches its v4 type's own plan, and the values open +// back to what went in. PostgreSQL sees these values through the Rust +// suite: TestV4RowsMatchTheCommittedFixture commits them to +// v4_rows.json, and packages/eql/tests/encryption inserts that file into +// eql_v4 columns and queries it. + +var profiles = []testusers.Profile{ + {ID: 1, Visits: 3, Balance: 1234567890123, Active: true, Ratio: 0.5, Score: 2.25, Note: "a private note", Name: "alice", Age: 42, Rank: 7, Weight: 1.5, Price: 9.75, Email: "alice@example.com"}, + {ID: 2, Visits: 0, Balance: -1, Active: false, Ratio: -1.5, Score: 0, Note: "", Name: "bob smith", Age: 42, Rank: -7, Weight: -1.5, Price: 0, Email: "bob@example.com"}, +} + +func fixturePath(name string) string { + return filepath.Join("..", "..", "..", "packages", "eql", "tests", "encryption", "fixtures", name) +} + +// keysOf is the sorted top-level keys of an EQL value. It also checks the +// envelope: version 4, and the table and column the value was stored under. +func keysOf(t *testing.T, column string, value []byte) string { + t.Helper() + var raw map[string]json.RawMessage + if err := json.Unmarshal(value, &raw); err != nil { + t.Fatalf("the EQL value is not JSON: %v: %s", err, value) + } + if string(raw["v"]) != "4" { + t.Errorf("v = %s, want 4: %s", raw["v"], value) + } + var i struct{ T, C string } + if err := json.Unmarshal(raw["i"], &i); err != nil || i.T != "profiles" || i.C != column { + t.Errorf("i = %s, want {\"t\":\"profiles\",\"c\":%q}", raw["i"], column) + } + keys := make([]string, 0, len(raw)) + for k := range raw { + keys = append(keys, k) + } + sort.Strings(keys) + return strings.Join(keys, ",") +} + +// stored is each sealed field of an encrypted profile by column. +func stored(e testusers.EncryptedProfile) map[string][]byte { + return map[string][]byte{ + "visits": e.Visits, + "balance": e.Balance, + "active": e.Active, + "ratio": e.Ratio, + "score": e.Score, + "note": e.Note, + "name": e.Name, + "age": e.Age, + "rank": e.Rank, + "weight": e.Weight, + "price": e.Price, + "email": e.Email, + } +} + +func TestV4FieldsSealToTheirEQLTypesAndOpenBack(t *testing.T) { + c := deterministicEQLClient(t) + ctx := context.Background() + cipher := c.DefaultKeyset() + + encrypted, err := testusers.EncryptProfile(ctx, cipher, profiles) + if err != nil { + t.Fatal(err) + } + want := map[string]string{ + "visits": "c,i,v", + "balance": "c,i,v", + "active": "c,i,v", + "ratio": "c,i,v", + "score": "c,i,v", + "note": "c,i,v", + "name": "bf,c,i,v", + "age": "c,hm,i,v", + "rank": "c,hm,i,v", + "weight": "c,hm,i,v", + "price": "c,hm,i,v", + "email": "c,hm,i,v", + } + for i, e := range encrypted { + for column, value := range stored(e) { + if got := keysOf(t, column, value); got != want[column] { + t.Errorf("row %d %s: keys %s, want %s", i, column, got, want[column]) + } + } + } + + decrypted, err := testusers.DecryptProfile(ctx, cipher, encrypted) + if err != nil { + t.Fatal(err) + } + for i := range profiles { + if decrypted[i] != profiles[i] { + t.Errorf("row %d: %+v, want %+v", i, decrypted[i], profiles[i]) + } + } +} + +// bloom is the set of filter positions in a TextMatch value or query. +func bloom(t *testing.T, value []byte) map[int]bool { + t.Helper() + var v struct { + BF []int `json:"bf"` + } + if err := json.Unmarshal(value, &v); err != nil { + t.Fatal(err) + } + set := make(map[int]bool, len(v.BF)) + for _, p := range v.BF { + set[p] = true + } + return set +} + +func subset(a, b map[int]bool) bool { + for p := range a { + if !b[p] { + return false + } + } + return true +} + +func TestV4QueriesMatchTheRustFixture(t *testing.T) { + c := deterministicEQLClient(t) + ctx := context.Background() + cipher := c.DefaultKeyset() + + raw, err := os.ReadFile(fixturePath("v4_queries.json")) + if err != nil { + t.Fatal(err) + } + var fixture struct { + Table string + Name struct { + Plaintext string + Query string + } + Age struct { + Plaintext int32 + Query string + } + Rank struct { + Plaintext int64 + Query string + } + Weight struct { + Plaintext float32 + Query string + } + Price struct { + Plaintext float64 + Query string + } + Email struct { + Plaintext string + Query string + } + } + if err := json.Unmarshal(raw, &fixture); err != nil { + t.Fatal(err) + } + if fixture.Table != "profiles" { + t.Fatalf("the fixture's table changed: %q", fixture.Table) + } + + // Byte for byte what eql-bindings' own dispatch derived under the same + // index key: the Go field runs its v4 type's plan and no other. + name, err := testusers.ProfileFields.Name.Matches(ctx, cipher, fixture.Name.Plaintext) + if err != nil { + t.Fatal(err) + } + if string(name.Query()) != fixture.Name.Query { + t.Errorf("Name.Matches = %s\nwant %s", name.Query(), fixture.Name.Query) + } + equality := []struct { + field string + query func() ([]byte, error) + want string + }{ + {"Age", func() ([]byte, error) { + return testusers.ProfileFields.Age.Eq(ctx, cipher, fixture.Age.Plaintext) + }, fixture.Age.Query}, + {"Rank", func() ([]byte, error) { + return testusers.ProfileFields.Rank.Eq(ctx, cipher, fixture.Rank.Plaintext) + }, fixture.Rank.Query}, + {"Weight", func() ([]byte, error) { + return testusers.ProfileFields.Weight.Eq(ctx, cipher, fixture.Weight.Plaintext) + }, fixture.Weight.Query}, + {"Price", func() ([]byte, error) { + return testusers.ProfileFields.Price.Eq(ctx, cipher, fixture.Price.Plaintext) + }, fixture.Price.Query}, + {"Email", func() ([]byte, error) { + return testusers.ProfileFields.Email.Eq(ctx, cipher, fixture.Email.Plaintext) + }, fixture.Email.Query}, + } + for _, eq := range equality { + got, err := eq.query() + if err != nil { + t.Fatal(err) + } + if eq.want == "" || string(got) != eq.want { + t.Errorf("%s.Eq = %s\nwant %s", eq.field, got, eq.want) + } + } + + // And a stored value holds every filter bit of a query for text it + // contains, which is what `@@` tests, and not those of text it lacks. + // The first is a property of the filter. The second is not: a Bloom + // filter can cover an absent needle's bits by chance, and "zzz" and + // "bob" are clear of it under testSeed and the pinned configuration + // only, so a change to either can flip them with no bug behind it. + encrypted, err := testusers.EncryptProfile(ctx, cipher, profiles[:1]) + if err != nil { + t.Fatal(err) + } + all := bloom(t, encrypted[0].Name) + for needle, contained := range map[string]bool{"alice": true, "LIC": true, "zzz": false, "bob": false} { + q, err := testusers.ProfileFields.Name.Matches(ctx, cipher, needle) + if err != nil { + t.Fatal(err) + } + if got := subset(bloom(t, q.Query()), all); got != contained { + t.Errorf("%q in %q: %v, want %v", needle, profiles[0].Name, got, contained) + } + // Verify decides on the decrypted text, under TextMatch's own + // tokenizer, whatever the filter says. Verify takes any decrypted + // values, so the page also holds the case the tokenizer folds and + // text holding every 3-gram of the needle out of order. + names := []string{profiles[0].Name, profiles[1].Name, strings.ToUpper(needle), scrambled(needle)} + verified, err := q.Verify(ctx, names) + if err != nil { + t.Fatal(err) + } + for i, name := range names { + if want := holdsTrigrams(name, needle); verified[i] != want { + t.Errorf("Verify(%q) over %q = %v, want %v", needle, name, verified[i], want) + } + } + } + + // The fetch is sized from the twin's bits: with no measured histogram a + // data plan states no prior, so no finite fetch is promised. + q, err := testusers.ProfileFields.Name.Matches(ctx, cipher, "alice") + if err != nil { + t.Fatal(err) + } + limit, err := q.TargetLimit(ctx, 10, encrypt.Stats{Rows: 1000}) + if err != nil { + t.Fatal(err) + } + if rows, bounded := limit.Rows(); bounded { + t.Errorf("TargetLimit with no histogram = %d, want unbounded", rows) + } + limit, err = q.TargetLimit(ctx, 10, encrypt.Stats{Rows: 1000, Histogram: []encrypt.Bucket{{Bits: 60, Rows: 1000}}}) + if err != nil { + t.Fatal(err) + } + if rows, bounded := limit.Rows(); !bounded || rows < 10 { + t.Errorf("TargetLimit with a histogram = %d (bounded %v), want at least 10", rows, bounded) + } + + // A needle too short for a 3-gram has no filter bits, and an empty + // filter would be a subset of every row's: the engine refuses it. + for _, short := range []string{"", "al"} { + if q, err := testusers.ProfileFields.Name.Matches(ctx, cipher, short); err == nil { + t.Errorf("Name.Matches(%q) = %s, want a refusal", short, q) + } + } +} + +// rowsFixture is v4_rows.json: the profiles above as the generated code seals +// them under the deterministic guest and testSeed. The Rust suite opens them +// with stack-kms's DeterministicSource over the same seed, and inserts and +// queries them in PostgreSQL. +type rowsFixture struct { + Comment string `json:"_comment"` + Seed string `json:"seed"` + Table string `json:"table"` + Rows []fixtureRow `json:"rows"` +} + +type fixtureRow struct { + ID int64 `json:"id"` + Plaintext fixturePlain `json:"plaintext"` + Stored map[string]json.RawMessage `json:"stored"` +} + +type fixturePlain struct { + Visits int32 `json:"visits"` + Balance int64 `json:"balance"` + Active bool `json:"active"` + Ratio float32 `json:"ratio"` + Score float64 `json:"score"` + Note string `json:"note"` + Name string `json:"name"` + Age int32 `json:"age"` + Rank int64 `json:"rank"` + Weight float32 `json:"weight"` + Price float64 `json:"price"` + Email string `json:"email"` +} + +// TestV4RowsMatchTheCommittedFixture checks the committed v4_rows.json +// against what the generated code seals today. The deterministic guest +// derives every data key from the seed, so the envelopes and the terms +// (`bf`) are the same every run; the AEAD nonce inside each `c` is random, +// so `c` is not. The test therefore compares everything but `c`, and opens +// the committed `c` values with today's code. +// +// EQL_UPDATE_FIXTURES rewrites the file only when something other than the +// ciphertexts moved, so a regeneration that changes nothing leaves no diff. +// EQL_V4_ROWS_OUT names a file to write today's rows to, ciphertexts and +// all: CI hands it to the Rust suite (EQL_V4_ROWS), so the ciphertexts the +// Go SDK seals now are opened by Rust, not only the committed ones. +func TestV4RowsMatchTheCommittedFixture(t *testing.T) { + c := deterministicEQLClient(t) + encrypted, err := testusers.EncryptProfile(context.Background(), c.DefaultKeyset(), profiles) + if err != nil { + t.Fatal(err) + } + fixture := rowsFixture{ + Comment: "Profile rows sealed by the Go SDK's generated code over the deterministic EQL guest with the seed below; the Rust suite opens and queries them. Regenerate with EQL_UPDATE_FIXTURES=1 go test ./encrypt -run TestV4Rows, do not edit.", + Seed: string(testSeed[:]), + Table: "profiles", + } + for i, e := range encrypted { + p := profiles[i] + row := fixtureRow{ + ID: p.ID, + Plaintext: plainOf(p), + Stored: map[string]json.RawMessage{}, + } + for column, value := range stored(e) { + row.Stored[column] = json.RawMessage(value) + } + fixture.Rows = append(fixture.Rows, row) + } + rendered, err := json.MarshalIndent(fixture, "", " ") + if err != nil { + t.Fatal(err) + } + rendered = append(rendered, '\n') + if out := os.Getenv("EQL_V4_ROWS_OUT"); out != "" { + if err := os.WriteFile(out, rendered, 0o644); err != nil { + t.Fatal(err) + } + } + path := fixturePath("v4_rows.json") + var committed rowsFixture + committedBytes, err := os.ReadFile(path) + if err == nil { + err = json.Unmarshal(committedBytes, &committed) + } + same := err == nil && bytes.Equal(withoutCiphertexts(t, committed), withoutCiphertexts(t, fixture)) + if os.Getenv("EQL_UPDATE_FIXTURES") != "" { + if same { + t.Logf("%s: unchanged beyond the ciphertexts; left as it is", path) + } else { + if err := os.WriteFile(path, rendered, 0o644); err != nil { + t.Fatal(err) + } + t.Logf("%s: rewritten", path) + committed, same = fixture, true + } + } + if err != nil && !same { + t.Fatalf("%v: run with EQL_UPDATE_FIXTURES=1 to write it", err) + } + if !same { + t.Errorf("%s differs from what the generated code seals today beyond the ciphertexts; regenerate it with EQL_UPDATE_FIXTURES=1", path) + } + + // The committed ciphertexts still open, to the committed plaintexts. + opensToItsPlaintexts(t, c, committed.Rows) +} + +// TestFrozenV4RowsStillOpen holds stored_v4_rows_v1.json, rows the Go SDK +// stored with the next build. Unlike v4_rows.json it is never regenerated: +// nothing writes it. Sealing its plaintexts today must give the same rows +// apart from each `c` (the envelopes and the terms), and its committed `c` +// values must still open to its plaintexts. A failure here means a change +// broke rows already stored; see the README in that directory before +// touching the file. +func TestFrozenV4RowsStillOpen(t *testing.T) { + raw, err := os.ReadFile(fixturePath("stored_v4_rows_v1.json")) + if err != nil { + t.Fatal(err) + } + var frozen rowsFixture + if err := json.Unmarshal(raw, &frozen); err != nil { + t.Fatal(err) + } + if frozen.Seed != string(testSeed[:]) || frozen.Table != "profiles" || len(frozen.Rows) == 0 { + t.Fatalf("the frozen fixture changed shape: seed %q, table %q, %d rows", frozen.Seed, frozen.Table, len(frozen.Rows)) + } + c := deterministicEQLClient(t) + in := make([]testusers.Profile, len(frozen.Rows)) + for i, row := range frozen.Rows { + in[i] = profileOf(row.ID, row.Plaintext) + } + encrypted, err := testusers.EncryptProfile(context.Background(), c.DefaultKeyset(), in) + if err != nil { + t.Fatal(err) + } + today := rowsFixture{Comment: frozen.Comment, Seed: frozen.Seed, Table: frozen.Table} + for i, e := range encrypted { + row := fixtureRow{ID: frozen.Rows[i].ID, Plaintext: frozen.Rows[i].Plaintext, Stored: map[string]json.RawMessage{}} + for column, value := range stored(e) { + row.Stored[column] = json.RawMessage(value) + } + today.Rows = append(today.Rows, row) + } + if !bytes.Equal(withoutCiphertexts(t, frozen), withoutCiphertexts(t, today)) { + t.Errorf("sealing the frozen rows' plaintexts today stores different envelopes or terms: a change broke stored rows. Do not regenerate stored_v4_rows_v1.json") + } + opensToItsPlaintexts(t, c, frozen.Rows) +} + +// opensToItsPlaintexts opens each row's committed `c` values with today's +// code and checks them against the row's plaintext. +func opensToItsPlaintexts(t *testing.T, c *encrypt.Client, rows []fixtureRow) { + t.Helper() + sealed := make([]testusers.EncryptedProfile, len(rows)) + for i, row := range rows { + sealed[i] = testusers.EncryptedProfile{ + ID: row.ID, + Visits: []byte(row.Stored["visits"]), + Balance: []byte(row.Stored["balance"]), + Active: []byte(row.Stored["active"]), + Ratio: []byte(row.Stored["ratio"]), + Score: []byte(row.Stored["score"]), + Note: []byte(row.Stored["note"]), + Name: []byte(row.Stored["name"]), + Age: []byte(row.Stored["age"]), + Rank: []byte(row.Stored["rank"]), + Weight: []byte(row.Stored["weight"]), + Price: []byte(row.Stored["price"]), + Email: []byte(row.Stored["email"]), + } + } + opened, err := testusers.DecryptProfile(context.Background(), c.DefaultKeyset(), sealed) + if err != nil { + t.Fatal(err) + } + for i, row := range rows { + if got := plainOf(opened[i]); got != row.Plaintext { + t.Errorf("row %d opens to %+v, want %+v", row.ID, got, row.Plaintext) + } + } +} + +// profileOf is the profile a fixture row records. +func profileOf(id int64, p fixturePlain) testusers.Profile { + return testusers.Profile{ + ID: id, Visits: p.Visits, Balance: p.Balance, Active: p.Active, Ratio: p.Ratio, Score: p.Score, + Note: p.Note, Name: p.Name, Age: p.Age, Rank: p.Rank, Weight: p.Weight, Price: p.Price, Email: p.Email, + } +} + +// plainOf is a profile's plaintext as the fixture records it. +func plainOf(p testusers.Profile) fixturePlain { + return fixturePlain{p.Visits, p.Balance, p.Active, p.Ratio, p.Score, p.Note, p.Name, p.Age, p.Rank, p.Weight, p.Price, p.Email} +} + +// withoutCiphertexts renders a fixture with every stored value's `c` +// removed: the part of the file that is the same on every run. +func withoutCiphertexts(t *testing.T, f rowsFixture) []byte { + t.Helper() + out := f + out.Rows = nil + for _, row := range f.Rows { + stripped := fixtureRow{ID: row.ID, Plaintext: row.Plaintext, Stored: map[string]json.RawMessage{}} + for column, value := range row.Stored { + var fields map[string]json.RawMessage + if err := json.Unmarshal(value, &fields); err != nil { + t.Fatal(err) + } + delete(fields, "c") + b, err := json.Marshal(fields) + if err != nil { + t.Fatal(err) + } + stripped.Stored[column] = b + } + out.Rows = append(out.Rows, stripped) + } + b, err := json.Marshal(out) + if err != nil { + t.Fatal(err) + } + return b +} + +// holdsTrigrams is TextMatch's verify as a test oracle: the value holds +// every 3-gram of the query, both lower-cased (EqlMatch's options are 3-grams +// with downcasing), and the query cuts at least one. +func holdsTrigrams(value, query string) bool { + have := map[string]bool{} + for _, g := range trigrams(strings.ToLower(value)) { + have[g] = true + } + want := trigrams(strings.ToLower(query)) + for _, g := range want { + if !have[g] { + return false + } + } + return len(want) > 0 +} + +func trigrams(s string) []string { + r := []rune(s) + var out []string + for i := 0; i+3 <= len(r); i++ { + out = append(out, string(r[i:i+3])) + } + return out +} + +// scrambled holds every 3-gram of s, last first, with a space between them: +// no substring of it is s, and a token test still passes it. +func scrambled(s string) string { + g := trigrams(s) + slices.Reverse(g) + return strings.Join(g, " ") +} diff --git a/languages/golang/encrypt/eql_v4_text_search_test.go b/languages/golang/encrypt/eql_v4_text_search_test.go new file mode 100644 index 000000000..8f23f64d8 --- /dev/null +++ b/languages/golang/encrypt/eql_v4_text_search_test.go @@ -0,0 +1,159 @@ +package encrypt_test + +import ( + "context" + "encoding/json" + "errors" + "sort" + "strings" + "testing" + + "github.com/cipherstash/stack/languages/golang/encrypt" + "github.com/cipherstash/stack/languages/golang/encrypt/internal/testusers" +) + +// EQL v4's TextSearch, sealed and queried through generated code over the +// deterministic guest build with the EQL types. A stored value carries all +// three terms, short text with the empty filter; a query carries only the +// one its operator reads (ADR-0002, amended 2026-10-19), so Eq and Lt take a +// two-letter name and Matches refuses one. +// packages/eql/tests/encryption/tests/v4_text_search.rs holds the same rules +// against PostgreSQL. + +// keysAndTerms is an EQL value's sorted top-level keys, and the value of +// each key. +func keysAndTerms(t *testing.T, value []byte) (string, map[string]json.RawMessage) { + t.Helper() + var raw map[string]json.RawMessage + if err := json.Unmarshal(value, &raw); err != nil { + t.Fatalf("the EQL value is not JSON: %v", err) + } + if string(raw["v"]) != "4" { + t.Errorf("v = %s, want 4", raw["v"]) + } + keys := make([]string, 0, len(raw)) + for k := range raw { + keys = append(keys, k) + } + sort.Strings(keys) + return strings.Join(keys, ","), raw +} + +func TestV4TextSearchStoresEveryTermAndQueriesCarryOne(t *testing.T) { + c := deterministicEQLClient(t) + ctx := context.Background() + cipher := c.DefaultKeyset() + + encrypted, err := testusers.EncryptEntry(ctx, cipher, []testusers.Entry{{ID: 1, Name: "Alice"}}) + if err != nil { + t.Fatal(err) + } + keys, stored := keysAndTerms(t, encrypted[0].Name) + if keys != "bf,c,hm,i,op,v" { + t.Errorf("stored keys %s, want bf,c,hm,i,op,v", keys) + } + decrypted, err := testusers.DecryptEntry(ctx, cipher, encrypted) + if err != nil { + t.Fatal(err) + } + if decrypted[0].Name != "Alice" { + t.Errorf("opened %q", decrypted[0].Name) + } + + f := testusers.EntryFields.Name + // Each method's query carries its operator's term alone, the stored one. + for _, tc := range []struct { + method string + query func(string) ([]byte, error) + term string + }{ + {"Eq", func(v string) ([]byte, error) { return f.Eq(ctx, cipher, v) }, "hm"}, + {"Lt", func(v string) ([]byte, error) { return f.Lt(ctx, cipher, v) }, "op"}, + {"Gte", func(v string) ([]byte, error) { return f.Gte(ctx, cipher, v) }, "op"}, + } { + q, err := tc.query("Alice") + if err != nil { + t.Fatalf("%s: %v", tc.method, err) + } + keys, terms := keysAndTerms(t, q) + if want := strings.Join(sortedWith("i", "v", tc.term), ","); keys != want { + t.Errorf("%s query keys %s, want %s", tc.method, keys, want) + } + if string(terms[tc.term]) != string(stored[tc.term]) { + t.Errorf("%s query's %s is not the stored one", tc.method, tc.term) + } + // Text too short for a 3-gram still has an equality and an order + // term. + short, err := tc.query("Al") + if err != nil { + t.Fatalf("%s on two-letter text: %v", tc.method, err) + } + if keys, _ := keysAndTerms(t, short); keys != strings.Join(sortedWith("i", "v", tc.term), ",") { + t.Errorf("%s query keys on two-letter text %s", tc.method, keys) + } + } + + m, err := f.Matches(ctx, cipher, "Alice") + if err != nil { + t.Fatal(err) + } + keys, terms := keysAndTerms(t, m.Query()) + if keys != "bf,i,v" { + t.Errorf("Matches query keys %s, want bf,i,v", keys) + } + if string(terms["bf"]) != string(stored["bf"]) { + t.Error("the match query's bf is not the stored filter") + } + if ok, err := m.Verify(ctx, []string{"alice", "Bob"}); err != nil || !ok[0] || ok[1] { + t.Errorf("Verify = %v, %v", ok, err) + } + + // A match query on text too short for a 3-gram is refused, never sent + // as an empty filter. + if q, err := f.Matches(ctx, cipher, "Al"); !errors.Is(err, encrypt.ErrTerm) { + t.Errorf("Matches on two-letter text = %v, %v, want ErrTerm", q, err) + } + // Storing such text, an empty string included, keeps every term, the + // match term as the empty filter (where TextMatch refuses the text): Eq + // and Lt find the row through hm and op, and no Matches query does. + short, err := testusers.EncryptEntry(ctx, cipher, []testusers.Entry{{ID: 2, Name: "Al"}, {ID: 3, Name: ""}}) + if err != nil { + t.Fatalf("storing short text in a TextSearch: %v", err) + } + for i, name := range []string{"Al", ""} { + keys, terms := keysAndTerms(t, short[i].Name) + if keys != "bf,c,hm,i,op,v" { + t.Errorf("stored keys for %q %s, want bf,c,hm,i,op,v", name, keys) + } + if string(terms["bf"]) != "[]" { + t.Errorf("stored filter for %q is %s, want []", name, terms["bf"]) + } + eq, err := f.Eq(ctx, cipher, name) + if err != nil { + t.Fatalf("Eq(%q): %v", name, err) + } + if _, q := keysAndTerms(t, eq); string(q["hm"]) != string(terms["hm"]) { + t.Errorf("Eq(%q)'s hm is not the stored one", name) + } + lt, err := f.Lt(ctx, cipher, name) + if err != nil { + t.Fatalf("Lt(%q): %v", name, err) + } + if _, q := keysAndTerms(t, lt); string(q["op"]) != string(terms["op"]) { + t.Errorf("Lt(%q)'s op is not the stored one", name) + } + } + opened, err := testusers.DecryptEntry(ctx, cipher, short) + if err != nil { + t.Fatal(err) + } + if opened[0].Name != "Al" || opened[1].Name != "" { + t.Errorf("opened %q and %q", opened[0].Name, opened[1].Name) + } +} + +// sortedWith is the given keys, sorted. +func sortedWith(keys ...string) []string { + sort.Strings(keys) + return keys +} diff --git a/languages/golang/encrypt/errors_test.go b/languages/golang/encrypt/errors_test.go index e7253d348..2ba3f1566 100644 --- a/languages/golang/encrypt/errors_test.go +++ b/languages/golang/encrypt/errors_test.go @@ -46,7 +46,7 @@ func TestEachKindCarriesItsDiagnostic(t *testing.T) { return err }, encrypt.ErrAuthentication, "stack_encrypt::aead"}, {"match text with no token", func() error { - _, err := testusers.Fields.Email.Match(ctx, cipher, "") + _, err := testusers.Fields.Email.Matches(ctx, cipher, "") return err }, encrypt.ErrTerm, "stack_encrypt::empty_term_text"}, {"a ciphertext that is not a sealed value", func() error { diff --git a/languages/golang/encrypt/example/README.md b/languages/golang/encrypt/example/README.md index 0a995caca..1e1e3c2c1 100644 --- a/languages/golang/encrypt/example/README.md +++ b/languages/golang/encrypt/example/README.md @@ -2,7 +2,7 @@ A runnable tour of the Go SDK against real ZeroKMS: declare a struct with `stash` tags, generate its encrypted type, seal a batch in one request, derive -a query term, and open the batch again. +a query term, open the batch again, and verify a matches query over it. ## Running it @@ -28,5 +28,5 @@ until they are built. - `model.go`: the struct and its `go:generate` line. - `user_stash.go`: what `stashgen` wrote from the tags. Regenerate it with `go generate ./...`; CI fails when that changes a committed file. -- `main.go`: `Encrypt`, `Fields.Email.Equality` and `Decrypt`, the only three - calls a program makes. +- `main.go`: `Encrypt`, `Fields.Email.Eq`, `Decrypt`, and `Fields.Email.Matches` + with `Verify` on the decrypted rows. diff --git a/languages/golang/encrypt/example/main.go b/languages/golang/encrypt/example/main.go index 2157699d7..ef451565f 100644 --- a/languages/golang/encrypt/example/main.go +++ b/languages/golang/encrypt/example/main.go @@ -47,7 +47,7 @@ func run(ctx context.Context) error { } // A query term for one field compares against the stored terms. - term, err := Fields.Email.Equality(ctx, cipher, "bob@example.com") + term, err := Fields.Email.Eq(ctx, cipher, "bob@example.com") if err != nil { return fmt.Errorf("query term: %w", err) } @@ -66,5 +66,23 @@ func run(ctx context.Context) error { for _, u := range back { fmt.Printf("decrypted id %d\n", u.ID) } + + // A matches query's answer can hold rows that do not match, so the + // rows it finds are decrypted and verified: one call for the page. + q, err := Fields.Email.Matches(ctx, cipher, "bob") + if err != nil { + return fmt.Errorf("matches: %w", err) + } + emails := make([]string, len(back)) + for i, u := range back { + emails[i] = u.Email + } + verified, err := q.Verify(ctx, emails) + if err != nil { + return fmt.Errorf("verify: %w", err) + } + for i, u := range back { + fmt.Printf("id %d holds bob: %v\n", u.ID, verified[i]) + } return nil } diff --git a/languages/golang/encrypt/example/user_stash.go b/languages/golang/encrypt/example/user_stash.go index 13cf2b6ef..4dc7ec03a 100644 --- a/languages/golang/encrypt/example/user_stash.go +++ b/languages/golang/encrypt/example/user_stash.go @@ -142,12 +142,15 @@ func (f EmailField) Encrypt(ctx context.Context, c *encrypt.Cipher, v string) (E return EncryptedUserEmail{Ciphertext: out.Ciphertext, Equality: out.Equality, Match: out.Match}, nil } -func (f EmailField) Equality(ctx context.Context, c *encrypt.Cipher, v string) (encrypt.EqualityTerm, error) { - return f.field.Equality(ctx, c, v) +func (f EmailField) Eq(ctx context.Context, c *encrypt.Cipher, v string) (encrypt.EqualityTerm, error) { + term, err := f.field.Term(ctx, c, "eq", v) + return encrypt.EqualityTerm(term), err } -func (f EmailField) Match(ctx context.Context, c *encrypt.Cipher, v string) (encrypt.MatchTerm, error) { - return f.field.Match(ctx, c, v) +// Matches derives the matches query for v. Its answer can hold rows that do not +// match: decrypt the rows it fetches and keep those Verify reports true. +func (f EmailField) Matches(ctx context.Context, c *encrypt.Cipher, v string) (encrypt.MatchQuery[string], error) { + return gensupport.Matches(ctx, f.field, c, v) } type AgeField struct { @@ -162,10 +165,32 @@ func (f AgeField) Encrypt(ctx context.Context, c *encrypt.Cipher, v uint32) (Enc return EncryptedUserAge{Ciphertext: out.Ciphertext, Equality: out.Equality, Ore: out.Ore}, nil } -func (f AgeField) Equality(ctx context.Context, c *encrypt.Cipher, v uint32) (encrypt.EqualityTerm, error) { - return f.field.Equality(ctx, c, v) +func (f AgeField) Eq(ctx context.Context, c *encrypt.Cipher, v uint32) (encrypt.EqualityTerm, error) { + term, err := f.field.Term(ctx, c, "eq", v) + return encrypt.EqualityTerm(term), err } -func (f AgeField) Ore(ctx context.Context, c *encrypt.Cipher, v uint32) (encrypt.OreTerm, error) { - return f.field.Ore(ctx, c, v) +func (f AgeField) Lt(ctx context.Context, c *encrypt.Cipher, v uint32) (encrypt.OreTerm, error) { + term, err := f.field.Term(ctx, c, "lt", v) + return encrypt.OreTerm(term), err +} + +func (f AgeField) Lte(ctx context.Context, c *encrypt.Cipher, v uint32) (encrypt.OreTerm, error) { + term, err := f.field.Term(ctx, c, "lte", v) + return encrypt.OreTerm(term), err +} + +func (f AgeField) Gt(ctx context.Context, c *encrypt.Cipher, v uint32) (encrypt.OreTerm, error) { + term, err := f.field.Term(ctx, c, "gt", v) + return encrypt.OreTerm(term), err +} + +func (f AgeField) Gte(ctx context.Context, c *encrypt.Cipher, v uint32) (encrypt.OreTerm, error) { + term, err := f.field.Term(ctx, c, "gte", v) + return encrypt.OreTerm(term), err +} + +func (f AgeField) Between(ctx context.Context, c *encrypt.Cipher, lo, hi uint32) (encrypt.Range[encrypt.OreTerm], error) { + low, high, err := f.field.Range(ctx, c, "between", lo, hi) + return encrypt.Range[encrypt.OreTerm]{Low: low, High: high}, err } diff --git a/languages/golang/encrypt/export_test.go b/languages/golang/encrypt/export_test.go index 8fd779c95..753d1a196 100644 --- a/languages/golang/encrypt/export_test.go +++ b/languages/golang/encrypt/export_test.go @@ -118,7 +118,7 @@ func deterministicClient(ctx context.Context, wasm []byte, seed [32]byte) (*Clie _ = c.Close() return nil, err } - out, err := c.call(ctx, func(inst *instance) ([]byte, error) { + out, err := c.call(ctx, func(ctx context.Context, inst *instance) ([]byte, error) { return inst.call(ctx, inst.cipherInit, buf(encoded)) }) if err != nil { diff --git a/languages/golang/encrypt/fixture_test.go b/languages/golang/encrypt/fixture_test.go index e992a9679..5f92793a7 100644 --- a/languages/golang/encrypt/fixture_test.go +++ b/languages/golang/encrypt/fixture_test.go @@ -122,21 +122,21 @@ func TestGeneratedCodeOpensTheRustRecordFixture(t *testing.T) { } } // The terms Go derives are the bytes Rust stored. - eq, err := testusers.Fields.Age.Equality(ctx, cipher, f.Plaintext.Age) + eq, err := testusers.Fields.Age.Eq(ctx, cipher, f.Plaintext.Age) if err != nil || !eq.Equal(stored.Age.Equality) { t.Errorf("age equality: %v, equal=%v", err, eq.Equal(stored.Age.Equality)) } - ore, err := testusers.Fields.Age.Ore(ctx, cipher, f.Plaintext.Age) + ore, err := testusers.Fields.Age.Lt(ctx, cipher, f.Plaintext.Age) if err != nil || !bytes.Equal(ore, stored.Age.Ore) { t.Errorf("age ore: %v, equal=%v", err, bytes.Equal(ore, stored.Age.Ore)) } - emailEq, err := testusers.Fields.Email.Equality(ctx, cipher, f.Plaintext.Email) + emailEq, err := testusers.Fields.Email.Eq(ctx, cipher, f.Plaintext.Email) if err != nil || !emailEq.Equal(stored.Email.Equality) { t.Errorf("email equality: %v, equal=%v", err, emailEq.Equal(stored.Email.Equality)) } - match, err := testusers.Fields.Email.Match(ctx, cipher, f.Plaintext.Email) - if err != nil || !bytes.Equal(match, stored.Email.Match) { - t.Errorf("email match: %v, equal=%v", err, bytes.Equal(match, stored.Email.Match)) + match, err := testusers.Fields.Email.Matches(ctx, cipher, f.Plaintext.Email) + if err != nil || !bytes.Equal(match.Term(), stored.Email.Match) { + t.Errorf("email match: %v, equal=%v", err, bytes.Equal(match.Term(), stored.Email.Match)) } // A fresh Go record of the same plaintext derives the same terms. again, err := testusers.Encrypt(ctx, cipher, []testusers.User{{ID: int64(f.Plaintext.ID), Age: f.Plaintext.Age, Email: f.Plaintext.Email, Notes: f.Plaintext.Notes}}) diff --git a/languages/golang/encrypt/generated_paths_test.go b/languages/golang/encrypt/generated_paths_test.go index efd787fbc..03011a841 100644 --- a/languages/golang/encrypt/generated_paths_test.go +++ b/languages/golang/encrypt/generated_paths_test.go @@ -117,19 +117,19 @@ func TestGeneratedCodeFromForAndFromAPolicyRunsAgainstTheGuest(t *testing.T) { } // A term derived through each path's Fields is the tag path's. - probe, err := testusers.Fields.Email.Equality(ctx, cipher, "bob@example.com") + probe, err := testusers.Fields.Email.Eq(ctx, cipher, "bob@example.com") if err != nil { t.Fatal(err) } - forProbe, err := testusers.MemberFields.Email.Equality(ctx, cipher, "bob@example.com") + forProbe, err := testusers.MemberFields.Email.Eq(ctx, cipher, "bob@example.com") if err != nil || !forProbe.Equal(probe) { t.Fatalf("the -for path's probe: %v, equal=%v", err, forProbe.Equal(probe)) } - policyProbe, err := testpolicy.Fields.Email.Equality(ctx, cipher, "bob@example.com") + policyProbe, err := testpolicy.Fields.Email.Eq(ctx, cipher, "bob@example.com") if err != nil || !policyProbe.Equal(probe) { t.Fatalf("the policy path's probe: %v, equal=%v", err, policyProbe.Equal(probe)) } - ore, err := testpolicy.Fields.Age.Ore(ctx, cipher, 29) + ore, err := testpolicy.Fields.Age.Lt(ctx, cipher, 29) if err != nil || !bytes.Equal(ore, tagged[1].Age.Ore) { t.Fatalf("the policy path's ORE probe: %v", err) } diff --git a/languages/golang/encrypt/gensupport/codec.go b/languages/golang/encrypt/gensupport/codec.go index 77e23cb60..fa94c2a16 100644 --- a/languages/golang/encrypt/gensupport/codec.go +++ b/languages/golang/encrypt/gensupport/codec.go @@ -203,7 +203,11 @@ func (c *Codec[P, E]) split(vals Values) (record.Source, map[string]any, error) return nil, nil, fmt.Errorf("the generated Source gave no field %q", f.name) } if f.crosses() { - row[f.name] = v + converted, err := asKind(f.kind, v) + if err != nil { + return nil, nil, fmt.Errorf("field %q: %w", f.name, err) + } + row[f.name] = converted } else { keep[f.name] = v } diff --git a/languages/golang/encrypt/gensupport/convert.go b/languages/golang/encrypt/gensupport/convert.go index 22162f8ea..678c527f0 100644 --- a/languages/golang/encrypt/gensupport/convert.go +++ b/languages/golang/encrypt/gensupport/convert.go @@ -13,10 +13,14 @@ import ( // convert writes an opened sealed value into out, a pointer to the Go type // the struct declares. The engine returns a value at the field's declared -// kind — int32, int64, uint32, uint64, float32, float64, string, []byte, -// bool, or a vcvalue.Object for a composite — and the struct's type is in -// the same family, narrower at most. A value outside the target's range, or -// of another family, is an error. +// kind — an integer of its own width (int8 to uint64, or vcvalue.Int128 and +// vcvalue.Uint128), float32, float64, string, []byte, bool, time.Time for a +// timestamp, vcvalue.Date, vcvalue.Decimal, or a vcvalue.Object for a +// composite — and the struct's type is that type, or in the same family and +// narrower at most (an EQL type's int32 opening into the int16 field it was +// widened from). A value outside the target's range, or of another family, +// is an error. A time, date, decimal or 128-bit integer is read only into +// its own type, which Get matches before it calls this. // // An error names types and positions, never the value: the value is // decrypted plaintext, and an error is what a program logs. @@ -300,6 +304,14 @@ func outOfRange(want any) error { func setInt[T ~int | ~int8 | ~int16 | ~int32 | ~int64](v any, out *T, lo, hi int64) error { var n int64 switch i := v.(type) { + case int8: + n = int64(i) + case int16: + n = int64(i) + case uint8: + n = int64(i) + case uint16: + n = int64(i) case int32: n = int64(i) case int64: @@ -326,6 +338,20 @@ func setInt[T ~int | ~int8 | ~int16 | ~int32 | ~int64](v any, out *T, lo, hi int func setUint[T ~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64](v any, out *T, hi uint64) error { var n uint64 switch i := v.(type) { + case uint8: + n = uint64(i) + case uint16: + n = uint64(i) + case int8: + if i < 0 { + return outOfRange(*out) + } + n = uint64(i) + case int16: + if i < 0 { + return outOfRange(*out) + } + n = uint64(i) case uint32: n = uint64(i) case uint64: diff --git a/languages/golang/encrypt/gensupport/declaration.go b/languages/golang/encrypt/gensupport/declaration.go index 339044827..57d3e8c31 100644 --- a/languages/golang/encrypt/gensupport/declaration.go +++ b/languages/golang/encrypt/gensupport/declaration.go @@ -19,20 +19,35 @@ import ( // the engine refuses a plan whose indexed field has no type. type Kind string -// The kinds. int8, int16 and int32 are Int32; int and int64 are Int64; -// uint8, uint16 and uint32 are Uint32; uint and uint64 are Uint64; []byte is -// Bytes. A type defined over one of these has its underlying kind. +// The kinds. Each Go integer type has the kind of its own width: int8 is +// Int8, int16 Int16, int32 Int32, int and int64 Int64, uint8 Uint8, uint16 +// Uint16, uint32 Uint32, uint and uint64 Uint64, and encrypt.Int128 and +// encrypt.Uint128 the 128-bit kinds. time.Time is Timestamp, encrypt.Date +// Date and encrypt.Decimal Decimal. []byte is Bytes. A type defined over a +// basic type has its underlying kind. A field stored as an EQL type takes +// the type's kind instead, when every value of its Go type fits it (an int16 +// into Integer's int32), and its value is converted to that kind before it +// crosses. const ( - Untyped Kind = "" - Bool Kind = "bool" - Int32 Kind = "int32" - Int64 Kind = "int64" - Uint32 Kind = "uint32" - Uint64 Kind = "uint64" - Float32 Kind = "float32" - Float64 Kind = "float64" - String Kind = "string" - Bytes Kind = "bytes" + Untyped Kind = "" + Bool Kind = "bool" + Int8 Kind = "int8" + Int16 Kind = "int16" + Int32 Kind = "int32" + Int64 Kind = "int64" + Int128 Kind = "int128" + Uint8 Kind = "uint8" + Uint16 Kind = "uint16" + Uint32 Kind = "uint32" + Uint64 Kind = "uint64" + Uint128 Kind = "uint128" + Float32 Kind = "float32" + Float64 Kind = "float64" + Decimal Kind = "decimal" + Date Kind = "date" + Timestamp Kind = "timestamp" + String Kind = "string" + Bytes Kind = "bytes" ) // OpaqueField is the one field of an opaque declaration: the whole struct, diff --git a/languages/golang/encrypt/gensupport/field.go b/languages/golang/encrypt/gensupport/field.go index 8006b0a01..f4e8ebb32 100644 --- a/languages/golang/encrypt/gensupport/field.go +++ b/languages/golang/encrypt/gensupport/field.go @@ -10,7 +10,7 @@ import ( // Field is one sealed field's entry in a generated Fields value: it encrypts // one value of the field, for an update of one column, and derives the -// terms the field declares, for a query. T is the field's Go type. +// query term of each operator the field answers. T is the field's Go type. type Field[T any] struct { name string plan *record.Plan @@ -36,6 +36,14 @@ func NewField[T any](d Declaration, name string) Field[T] { return f } +// kind is the field's declared kind, as its plan holds it. +func (f Field[T]) kind() Kind { + if pf := f.plan.Field(f.name); pf != nil { + return Kind(pf.Kind) + } + return Untyped +} + // Encrypt seals one value of the field: its ciphertext and every term it // declares, in one request. func (f Field[T]) Encrypt(ctx context.Context, c *encrypt.Cipher, v T) (Output, error) { @@ -45,7 +53,11 @@ func (f Field[T]) Encrypt(ctx context.Context, c *encrypt.Cipher, v T) (Output, if c == nil { return Output{}, fmt.Errorf("gensupport: %s: Encrypt needs a cipher", f.name) } - row := record.Source{f.name: v} + value, err := asKind(f.kind(), v) + if err != nil { + return Output{}, fmt.Errorf("gensupport: %s: %w", f.name, err) + } + row := record.Source{f.name: value} if f.plan.ContextField != "" { // One value of one field has no record to take its context from: // the cipher names it, as it does for a query. @@ -65,58 +77,84 @@ func (f Field[T]) Encrypt(ctx context.Context, c *encrypt.Cipher, v T) (Output, return outputOf(sealed[0][f.name]), nil } -// Query derives the EQL query value for one value of an encrypt_into field: -// the operand that matches stored values of the field, in Output.EQL. The -// engine runs the EQL type's own query plan, with no data key. -func (f Field[T]) Query(ctx context.Context, c *encrypt.Cipher, v T) (Output, error) { - if f.err != nil { - return Output{}, f.err - } - if c == nil { - return Output{}, fmt.Errorf("gensupport: %s: Query needs a cipher", f.name) - } - out, err := c.Query(ctx, f.plan, f.name, v) +// Term derives the term of an exact operator the field answers, for one +// value: an index term for a field declared with indexes, the query value's +// JSON for a field that names an EQL type. The generated method for the +// operator converts it to its query type. The engine picks the index that +// answers the operator ([encrypt.Cipher.QueryTerm]). +func (f Field[T]) Term(ctx context.Context, c *encrypt.Cipher, operator string, v T) ([]byte, error) { + terms, err := f.query(ctx, c, operator, v, 1) if err != nil { - return Output{}, err + return nil, err } - return Output{EQL: out}, nil -} - -// Equality derives the field's equality term for one value. -func (f Field[T]) Equality(ctx context.Context, c *encrypt.Cipher, v T) (encrypt.EqualityTerm, error) { - return f.term(ctx, c, record.Equality, v) -} - -// Match derives the field's match term for one value. -func (f Field[T]) Match(ctx context.Context, c *encrypt.Cipher, v T) (encrypt.MatchTerm, error) { - return f.term(ctx, c, record.Match, v) -} - -// Ore derives the field's ORE term for one value. -func (f Field[T]) Ore(ctx context.Context, c *encrypt.Cipher, v T) (encrypt.OreTerm, error) { - return f.term(ctx, c, record.Ore, v) + return terms[0], nil } -// Ope derives the field's OPE term for one value. -func (f Field[T]) Ope(ctx context.Context, c *encrypt.Cipher, v T) (encrypt.OpeTerm, error) { - return f.term(ctx, c, record.Ope, v) +// Range derives the two terms of a range operator the field answers, such +// as between: the low bound's term, then the high bound's. +func (f Field[T]) Range(ctx context.Context, c *encrypt.Cipher, operator string, lo, hi T) ([]byte, []byte, error) { + terms, err := f.query(ctx, c, operator, []any{lo, hi}, 2) + if err != nil { + return nil, nil, err + } + return terms[0], terms[1], nil } -// JSON derives the field's json index term. The engine does not derive it -// yet, so it fails. -func (f Field[T]) JSON(context.Context, *encrypt.Cipher, T) (encrypt.JSONTerm, error) { +// query derives an exact operator's terms, and refuses an answer of another +// shape than the generated method's: a probabilistic term has a query type +// of its own (Matches). Each operand is converted to the field's kind as a +// written value is ([asKind]), so the query finds what a write stored. +func (f Field[T]) query(ctx context.Context, c *encrypt.Cipher, operator string, input any, parts int) ([][]byte, error) { if f.err != nil { return nil, f.err } - return nil, fmt.Errorf("gensupport: %s: the engine does not derive the json index yet", f.name) + if c == nil { + return nil, fmt.Errorf("%w: gensupport: %s: a query needs a cipher", encrypt.ErrEncoding, f.name) + } + pair, isPair := input.([]any) + values := []any{input} + if isPair { + values = pair + } + converted := make([]any, len(values)) + for i, v := range values { + value, err := asKind(f.kind(), v) + if err != nil { + return nil, fmt.Errorf("gensupport: %s: %w", f.name, err) + } + converted[i] = value + } + operand := converted[0] + if isPair { + operand = converted + } + term, err := c.QueryTerm(ctx, f.plan, f.name, operator, operand) + if err != nil { + return nil, err + } + if term.Fidelity != record.Exact || len(term.Terms) != parts { + return nil, fmt.Errorf("gensupport: %s: %s came back as a %s term of %d parts, not an exact one of %d", f.name, operator, term.Fidelity, len(term.Terms), parts) + } + return term.Terms, nil } -func (f Field[T]) term(ctx context.Context, c *encrypt.Cipher, output record.Output, v T) ([]byte, error) { +// Matches derives the matches query for one value of a field declared with +// a match index: the term, and what Verify and TargetLimit need. A function, +// not a method, because only a string field answers matches. A nil cipher +// is refused by [encrypt.NewMatchQuery], with [encrypt.ErrEncoding]. +func Matches[T ~string](ctx context.Context, f Field[T], c *encrypt.Cipher, v T) (encrypt.MatchQuery[T], error) { if f.err != nil { - return nil, f.err + return encrypt.MatchQuery[T]{}, f.err } - if c == nil { - return nil, fmt.Errorf("gensupport: %s: a term needs a cipher", f.name) + return encrypt.NewMatchQuery(ctx, c, f.plan, f.name, v) +} + +// EQLMatches derives the matches query for one value of a field that names +// an EQL type with a match index: the type's query value Q, and what Verify +// and TargetLimit need. +func EQLMatches[Q ~[]byte, T ~string](ctx context.Context, f Field[T], c *encrypt.Cipher, v T) (encrypt.EQLMatchQuery[Q, T], error) { + if f.err != nil { + return encrypt.EQLMatchQuery[Q, T]{}, f.err } - return c.Derive(ctx, f.plan, f.name, output, v) + return encrypt.NewEQLMatchQuery[Q](ctx, c, f.plan, f.name, v) } diff --git a/languages/golang/encrypt/gensupport/gensupport_internal_test.go b/languages/golang/encrypt/gensupport/gensupport_internal_test.go index 72a40f043..b39b57f65 100644 --- a/languages/golang/encrypt/gensupport/gensupport_internal_test.go +++ b/languages/golang/encrypt/gensupport/gensupport_internal_test.go @@ -12,6 +12,7 @@ import ( "github.com/cipherstash/stack/languages/golang/encrypt" "github.com/cipherstash/stack/languages/golang/internal/record" + "github.com/cipherstash/vitaminc/bindings/go/vcffi" "github.com/cipherstash/vitaminc/bindings/go/vcvalue" ) @@ -394,3 +395,68 @@ func check(err error, ok bool) error { } return nil } + +// A value sealed at an EQL integer type's kind is converted to it on the +// host: any Go integer of a value that fits, from a narrower type or a wider +// one, and a value that does not fit is refused with ErrEncoding, naming the +// kind and not the value (ADR-0002). +func TestAsKindConvertsAnIntegerThatFitsAndRefusesOneThatDoesNot(t *testing.T) { + for _, c := range []struct { + kind Kind + in any + want any + }{ + {Int32, int8(-7), int32(-7)}, + {Int32, uint16(65535), int32(65535)}, + {Int32, int64(math.MaxInt32), int32(math.MaxInt32)}, + {Int32, int(math.MinInt32), int32(math.MinInt32)}, + {Int64, uint32(math.MaxUint32), int64(math.MaxUint32)}, + {Int64, uint64(math.MaxInt64), int64(math.MaxInt64)}, + {Int8, int8(3), int8(3)}, + {String, "s", "s"}, + } { + got, err := asKind(c.kind, c.in) + if err != nil || got != c.want { + t.Errorf("asKind(%s, %T) = %v (%T), %v; want %v (%T)", c.kind, c.in, got, got, err, c.want, c.want) + } + } + for _, c := range []struct { + kind Kind + in any + }{ + {Int32, int64(math.MaxInt32) + 1}, + {Int32, int64(math.MinInt32) - 1}, + {Int32, uint32(math.MaxUint32)}, + {Int64, uint64(math.MaxInt64) + 1}, + {Uint8, int8(-1)}, + } { + _, err := asKind(c.kind, c.in) + if !errors.Is(err, encrypt.ErrEncoding) { + t.Errorf("asKind(%s, %T out of range) = %v, want ErrEncoding", c.kind, c.in, err) + } + if err != nil && strings.Contains(err.Error(), fmt.Sprint(c.in)) { + t.Errorf("asKind's error names the value: %v", err) + } + } +} + +// encodedInt is an integer type that writes itself, as an int32 of its value +// plus 100. +type encodedInt int16 + +func (v encodedInt) EncryptValue(enc vcffi.Encoder) error { + enc.Int32(int32(v) + 100) + return nil +} + +// A value whose type writes itself crosses as it is, whatever the kind: a +// conversion to a built-in integer would drop its EncryptValue, so a write +// would store one value and a query, which keeps it, ask for another. +func TestAsKindLeavesAValueThatWritesItself(t *testing.T) { + for _, kind := range []Kind{Int16, Int32, Int64} { + got, err := asKind(kind, encodedInt(7)) + if err != nil || got != encodedInt(7) { + t.Errorf("asKind(%s, encodedInt(7)) = %v (%T), %v; want encodedInt(7)", kind, got, got, err) + } + } +} diff --git a/languages/golang/encrypt/gensupport/widen.go b/languages/golang/encrypt/gensupport/widen.go new file mode 100644 index 000000000..02e13c735 --- /dev/null +++ b/languages/golang/encrypt/gensupport/widen.go @@ -0,0 +1,139 @@ +package gensupport + +import ( + "fmt" + "math" + "reflect" + "time" + + "github.com/cipherstash/stack/languages/golang/encrypt" + "github.com/cipherstash/vitaminc/bindings/go/vcffi" + "github.com/cipherstash/vitaminc/bindings/go/vcvalue" +) + +// asKind is a sealed field's value as the Go type of its declared kind, the +// width the engine seals it at. A field's kind is its Go type's own unless the +// field is stored as an EQL type of another integer kind (an int16 stored as +// Integer is sealed as an int32, and so is an int64, ADR-0002): then the value +// is converted here, on the host, because nothing below the binding converts. +// Only an integer changes; every other value, and an integer already of its +// kind's width, crosses as it is. A value outside the kind's range is refused +// with encrypt.ErrEncoding: a Go type wider than the kind is accepted by +// stashgen and range checked here, value by value. +// +// A value whose type writes itself (vcffi.Encryptable) crosses as it is too: +// converting it to a built-in integer would drop its EncryptValue, and the +// engine holds what it writes to the field's kind. A write and a query of one +// field both convert here ([Field.Encrypt], [Field.Term]), so they agree. +// +// The error names the kind and not the value, which is plaintext. +func asKind(kind Kind, v any) (any, error) { + if err := checkValue(v); err != nil { + return nil, err + } + if !isInteger(kind) { + return v, nil + } + switch v.(type) { + case vcvalue.Int128, vcvalue.Uint128, vcffi.Encryptable: + return v, nil + } + rv := reflect.ValueOf(v) + if !rv.IsValid() { + return v, nil + } + var signed *int64 + var unsigned *uint64 + switch rv.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + i := rv.Int() + signed = &i + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + u := rv.Uint() + unsigned = &u + default: + return v, nil + } + out, ok := integerAs(kind, signed, unsigned) + if !ok { + return nil, fmt.Errorf("%w: a value does not fit the field's kind %s", encrypt.ErrEncoding, kind) + } + return out, nil +} + +// checkValue refuses a date, decimal or timestamp the codec cannot write, before +// it reaches the codec, so the caller can name the field: the codec's own +// refusal names neither. A zero encrypt.Date (year 0, month 0, day 0) is not +// a date and is refused here. The error is vitaminc's, which names the type +// and not the value. +func checkValue(v any) error { + var err error + switch v := v.(type) { + case vcvalue.Date: + err = v.Validate() + case vcvalue.Decimal: + err = v.Validate() + case time.Time: + err = vcvalue.Timestamp{Seconds: v.Unix(), Nanoseconds: uint32(v.Nanosecond())}.Validate() //nolint:gosec // Nanosecond is in [0, 1e9) + } + if err != nil { + return fmt.Errorf("%w: %w", encrypt.ErrEncoding, err) + } + return nil +} + +func isInteger(k Kind) bool { + switch k { + case Int8, Int16, Int32, Int64, Int128, Uint8, Uint16, Uint32, Uint64, Uint128: + return true + } + return false +} + +// integerAs is the integer, given as exactly one of a signed or an unsigned +// value, as the Go type of kind, if it fits. A 128-bit kind has no arm: its +// only Go types are encrypt.Int128 and Uint128, which asKind passes through, +// and stashgen refuses a built-in integer field stored as a 128-bit kind, so a +// built-in integer never reaches one here; if one did, it is refused as not +// fitting. +func integerAs(kind Kind, signed *int64, unsigned *uint64) (any, bool) { + inSigned := func(lo, hi int64) (int64, bool) { + if signed != nil { + return *signed, *signed >= lo && *signed <= hi + } + return int64(*unsigned), *unsigned <= uint64(hi) //nolint:gosec // range-checked + } + inUnsigned := func(hi uint64) (uint64, bool) { + if unsigned != nil { + return *unsigned, *unsigned <= hi + } + return uint64(*signed), *signed >= 0 && uint64(*signed) <= hi //nolint:gosec // range-checked + } + switch kind { + case Int8: + i, ok := inSigned(math.MinInt8, math.MaxInt8) + return int8(i), ok //nolint:gosec // range-checked: ok is false when it does not fit + case Int16: + i, ok := inSigned(math.MinInt16, math.MaxInt16) + return int16(i), ok //nolint:gosec // range-checked: ok is false when it does not fit + case Int32: + i, ok := inSigned(math.MinInt32, math.MaxInt32) + return int32(i), ok //nolint:gosec // range-checked: ok is false when it does not fit + case Int64: + i, ok := inSigned(math.MinInt64, math.MaxInt64) + return i, ok + case Uint8: + u, ok := inUnsigned(math.MaxUint8) + return uint8(u), ok //nolint:gosec // range-checked: ok is false when it does not fit + case Uint16: + u, ok := inUnsigned(math.MaxUint16) + return uint16(u), ok //nolint:gosec // range-checked: ok is false when it does not fit + case Uint32: + u, ok := inUnsigned(math.MaxUint32) + return uint32(u), ok //nolint:gosec // range-checked: ok is false when it does not fit + case Uint64: + u, ok := inUnsigned(math.MaxUint64) + return u, ok + } + return nil, false +} diff --git a/languages/golang/encrypt/guest.go b/languages/golang/encrypt/guest.go index c5bbecbd4..04d01730c 100644 --- a/languages/golang/encrypt/guest.go +++ b/languages/golang/encrypt/guest.go @@ -82,11 +82,15 @@ type instance struct { // on it. mem *guest.Allocator - exports guest.Exports - cipherInit, shutdown, keyset api.Function - term, query api.Function - encryptRecord, decryptRecord api.Function - planCheck, targets api.Function + exports guest.Exports + cipherInit, shutdown, keyset api.Function + term api.Function + queryTerm, verify, limit api.Function + encryptRecord, decryptRecord api.Function + planCheck, targets, operators api.Function + // The step loop (step.go): take a paused export's step, resume it with + // the responses, or abort it. + io, resume, abort api.Function } // guestModuleConfig is the module configuration every guest instance runs @@ -112,10 +116,13 @@ func guestModuleConfig() wazero.ModuleConfig { // strict policy, memory that cannot be locked fails instantiation with // ErrMemoryLock. func newInstance(ctx context.Context, wasm []byte, t *transport, policy guest.LockPolicy) (*instance, error) { - // WithCloseOnContextDone lets a caller's deadline or cancellation - // interrupt an in-flight guest call — which otherwise holds the Client's - // lock against every other user. An interrupted call closes the module, - // so the Client is done afterwards; the alternative is a wedged process. + // WithCloseOnContextDone closes the module when a call's context ends. + // A guest call never runs under the caller's context: Client.call runs + // it under guest.Detach, whose context ends only when the guest has run + // on long after its caller gave up. Closing the module then frees the + // Client's lock, which a stuck call would otherwise hold against every + // other user, at the cost of the Client: the alternative is a wedged + // process. config := wazero.NewRuntimeConfig(). WithCompilationCache(compilationCache()). WithCloseOnContextDone(true) @@ -171,11 +178,17 @@ func newInstance(ctx context.Context, wasm []byte, t *transport, policy guest.Lo "se_shutdown": &inst.shutdown, "se_keyset": &inst.keyset, "se_term": &inst.term, - "se_query": &inst.query, + "se_query_term": &inst.queryTerm, + "se_verify": &inst.verify, + "se_target_limit": &inst.limit, "se_encrypt_record": &inst.encryptRecord, "se_decrypt_record": &inst.decryptRecord, "se_plan_check": &inst.planCheck, "se_targets": &inst.targets, + "se_operators": &inst.operators, + "se_io": &inst.io, + "se_resume": &inst.resume, + "se_abort": &inst.abort, } for name, slot := range exports { if *slot = module.ExportedFunction(name); *slot == nil { @@ -192,8 +205,8 @@ func newInstance(ctx context.Context, wasm []byte, t *transport, policy guest.Lo // release runs the guest's shutdown — the client key and every loaded // index key wiped inside the instance — and closes the runtime, which // frees the linear memory through the allocator's wipe. It is what Close -// does, and what the cleanup on an unreachable Client does. A module an -// interrupted call or a trap already closed cannot run se_shutdown; the +// does, and what the cleanup on an unreachable Client does. A module a +// stuck call or a trap already closed cannot run se_shutdown; the // runtime close still wipes and frees its memory, so nothing is left // behind either way. func (inst *instance) release() error { diff --git a/languages/golang/encrypt/guest/Cargo.lock b/languages/golang/encrypt/guest/Cargo.lock index 88e3723f7..f9bc1224b 100644 --- a/languages/golang/encrypt/guest/Cargo.lock +++ b/languages/golang/encrypt/guest/Cargo.lock @@ -21,6 +21,7 @@ dependencies = [ "cfg-if", "cipher", "cpufeatures 0.2.17", + "zeroize", ] [[package]] @@ -218,6 +219,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "block-modes" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e2211b0817f061502a8dd9f11a37e879e79763e3c698d2418cf824d8cb2f21e" + [[package]] name = "bumpalo" version = "3.20.3" @@ -230,6 +237,12 @@ version = "1.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.12.1" @@ -338,15 +351,17 @@ dependencies = [ [[package]] name = "cllw-ore" -version = "0.5.0" +version = "0.5.2-rc.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "476f300d37a5029d3d9dd57145d4db50a23f40ae9b4d1374c44543978b906191" +checksum = "24eb4a774d070a6c2b52175b98273cd7193c61e1bda39c1a98b50b7a2d91375a" dependencies = [ "blake3", "hex", "subtle", "thiserror 1.0.69", "unicode-normalization", + "vitaminc-ore", + "vitaminc-protected 0.6.1", "zeroize", ] @@ -484,7 +499,7 @@ dependencies = [ "url", "utoipa", "uuid", - "vitaminc", + "vitaminc 0.5.1", ] [[package]] @@ -709,8 +724,7 @@ dependencies = [ "stack-encrypt", "thiserror 2.0.20", "ts-rs", - "vitaminc-aead-value", - "vitaminc-prf", + "vitaminc-aead 0.6.1", "zeroize", ] @@ -1343,6 +1357,20 @@ dependencies = [ "memchr", ] +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + [[package]] name = "num-bigint" version = "0.4.8" @@ -1353,6 +1381,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.2" @@ -1368,6 +1405,27 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1399,6 +1457,35 @@ dependencies = [ "libc", ] +[[package]] +name = "orderable-bytes" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7d30b72ffca73d9936fe5fc80a9c9520c73f5b4f64839a810211903d5ace3c" + +[[package]] +name = "ore-rs" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e1d6cb8c1249643266bd60269547c93d3cd263d5385f1262902bbb295ff9a5e" +dependencies = [ + "aes", + "block-modes", + "byteorder", + "hex", + "num", + "orderable-bytes", + "rand 0.8.8", + "rand_chacha 0.3.1", + "subtle-ng", + "thiserror 1.0.69", + "vitaminc-hmac", + "vitaminc-ore", + "vitaminc-prf", + "vitaminc-protected 0.6.1", + "zeroize", +] + [[package]] name = "parking_lot" version = "0.12.5" @@ -1486,6 +1573,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro-error-attr2" version = "2.0.0" @@ -2055,8 +2151,8 @@ dependencies = [ "tracing", "url", "uuid", - "vitaminc", - "vitaminc-protected", + "vitaminc 0.6.1", + "vitaminc-protected 0.6.1", "web-time", "zeroize", "zerokms-protocol", @@ -2064,29 +2160,32 @@ dependencies = [ [[package]] name = "stack-encrypt" -version = "0.2.0" +version = "0.3.0" dependencies = [ "base64ct", "cllw-ore", "miette", + "ore-rs", "serde", "serde_json", "stack-encrypt-derive", "stack-kms", "thiserror 1.0.69", "uuid", - "vitaminc-aead", - "vitaminc-aead-value", - "vitaminc-encrypt", + "vitaminc-aead 0.6.1", + "vitaminc-aead-frame", + "vitaminc-encrypt 0.6.1", "vitaminc-hmac", + "vitaminc-normalize", + "vitaminc-ore", "vitaminc-prf", - "vitaminc-protected", + "vitaminc-protected 0.6.1", "zeroize", ] [[package]] name = "stack-encrypt-derive" -version = "0.2.0" +version = "0.3.0" dependencies = [ "proc-macro2", "quote", @@ -2109,8 +2208,9 @@ dependencies = [ "stack-guest-abi", "stack-kms", "uuid", - "vitaminc-aead-value", - "vitaminc-protected", + "vitaminc-aead 0.6.1", + "vitaminc-aead-frame", + "vitaminc-protected 0.6.1", "zeroize", "zerokms-protocol", ] @@ -2122,8 +2222,9 @@ dependencies = [ "miette", "serde_json", "thiserror 1.0.69", - "vitaminc-aead-value", - "vitaminc-protected", + "vitaminc-aead 0.6.1", + "vitaminc-aead-frame", + "vitaminc-protected 0.6.1", "zeroize", ] @@ -2149,8 +2250,8 @@ dependencies = [ "tracing", "url", "uuid", - "vitaminc", - "vitaminc-protected", + "vitaminc 0.6.1", + "vitaminc-protected 0.6.1", "zeroize", "zerokms-protocol", ] @@ -2186,6 +2287,12 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "subtle-ng" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "734676eb262c623cec13c3155096e08d1f8f29adce39ba17948b18dad1e54142" + [[package]] name = "syn" version = "2.0.119" @@ -2357,6 +2464,36 @@ dependencies = [ "syn 3.0.4", ] +[[package]] +name = "toml_datetime" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b86d767906c6c42421dcba507eb9d203e779497710a47782a224bb871653053" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.17+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3641d5bbb5349a79e1020a242d251efbc546ad8048d133958323ce9c40a9c9c" +dependencies = [ + "indexmap", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.1.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baa693a8032d7e1cada7d0041e96126df243179ff061456783ac7f12bda4744c" +dependencies = [ + "winnow", +] + [[package]] name = "tracing" version = "0.1.44" @@ -2424,6 +2561,12 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" +[[package]] +name = "unicode-general-category" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b993bddc193ae5bd0d623b49ec06ac3e9312875fdae725a975c51db1cc1677f" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -2573,12 +2716,26 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35042fb7a81d72dc43c53fec835e61027f18b4d01cc0f7e12d6c6a6ffbf3bfb9" dependencies = [ - "vitaminc-aead", - "vitaminc-context", - "vitaminc-encrypt", - "vitaminc-protected", - "vitaminc-random", - "vitaminc-traits", + "vitaminc-aead 0.5.1", + "vitaminc-context 0.5.1", + "vitaminc-encrypt 0.5.1", + "vitaminc-protected 0.5.1", + "vitaminc-random 0.5.1", + "vitaminc-traits 0.5.1", +] + +[[package]] +name = "vitaminc" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac5c15b6c25efcb9a6111f7108ab8101947a7bf7e9af3ff6d6a73358a1ace0da" +dependencies = [ + "vitaminc-aead 0.6.1", + "vitaminc-context 0.6.1", + "vitaminc-encrypt 0.6.1", + "vitaminc-protected 0.6.1", + "vitaminc-random 0.6.1", + "vitaminc-traits 0.6.1", ] [[package]] @@ -2589,10 +2746,27 @@ checksum = "62cacd4dae485e98cfe5407e919187a8468dd933ea4234be5115b7d13a30f3c6" dependencies = [ "bytes", "serde", - "vitaminc-aead-derive", - "vitaminc-context", - "vitaminc-protected", - "vitaminc-random", + "vitaminc-aead-derive 0.5.1", + "vitaminc-context 0.5.1", + "vitaminc-protected 0.5.1", + "vitaminc-random 0.5.1", + "zeroize", +] + +[[package]] +name = "vitaminc-aead" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71b8c5c4ced92545dce5fd7cf04581d08210def174cc81475cf227fa2506c5b7" +dependencies = [ + "bytes", + "futures-util", + "serde", + "thiserror 2.0.20", + "vitaminc-aead-derive 0.6.1", + "vitaminc-context 0.6.1", + "vitaminc-protected 0.6.1", + "vitaminc-random 0.6.1", "zeroize", ] @@ -2608,13 +2782,24 @@ dependencies = [ ] [[package]] -name = "vitaminc-aead-value" -version = "0.5.1" +name = "vitaminc-aead-derive" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d78cdadbc6374c3bd7c19fb4d99abca693b96953b18fef8c608efbe092cf3ea" +checksum = "4c546497a972d0f0498f2f8d5f537c514bad82df8662951c63d0c80023808900" dependencies = [ - "vitaminc-aead", - "vitaminc-protected", + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "vitaminc-aead-frame" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e35992a269bcaa5b201f1938e54bfbf167bfa6fe975d83cbdbdb54c2a363c5ef" +dependencies = [ + "serde", + "vitaminc-aead 0.6.1", "zeroize", ] @@ -2625,7 +2810,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7df52501fe33cdb3ee9e155b5242fac184c037d147e58c96e5143ed011252a8" dependencies = [ "mutants", - "vitaminc-protected", + "vitaminc-protected 0.5.1", +] + +[[package]] +name = "vitaminc-context" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4c4bc2bd9811253eb427657565a45bd4826e2379e93ee1c4d00c808a769882" +dependencies = [ + "mutants", + "vitaminc-protected 0.6.1", ] [[package]] @@ -2636,35 +2831,79 @@ checksum = "9fbd0e1748055e381e02033e08af385de879c4dcf32e0f6739ac0c0cf0c6f13d" dependencies = [ "aes-gcm", "aws-lc-rs", - "vitaminc-aead", - "vitaminc-protected", - "vitaminc-random", + "vitaminc-aead 0.5.1", + "vitaminc-protected 0.5.1", + "vitaminc-random 0.5.1", + "zeroize", +] + +[[package]] +name = "vitaminc-encrypt" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0923694602855b33a296d9af473e282e8096c1cfe1b30b8695d3b6651717fc5" +dependencies = [ + "aes-gcm", + "aws-lc-rs", + "vitaminc-aead 0.6.1", + "vitaminc-protected 0.6.1", + "vitaminc-random 0.6.1", "zeroize", ] [[package]] name = "vitaminc-hmac" -version = "0.5.1" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15fe558a5e92d4f2b3f811c5b67d863d9255da5f66d08a71d4f0dd3d0011c563" +checksum = "bd361f626e95cbe770b9d012e4a86c43cfefe4aefb3d5b3ed9339a83085bdc53" dependencies = [ "hmac", "sha2 0.11.0", "vitaminc-prf", - "vitaminc-protected", + "vitaminc-protected 0.6.1", "zeroize", ] +[[package]] +name = "vitaminc-normalize" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "214b2a9d085924c3bc3c91f3ba74238e8848bf54990e5f48f170c71fbacea07a" +dependencies = [ + "thiserror 2.0.20", + "unicode-general-category", + "unicode-normalization", + "vitaminc-aead 0.6.1", + "vitaminc-protected 0.6.1", +] + +[[package]] +name = "vitaminc-ore" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed88c54b5f06f2177ecbfb7666812adf645406f6fb555f4e2f0fd7fa218e71c4" +dependencies = [ + "thiserror 2.0.20", + "unicode-general-category", + "unicode-normalization", + "vitaminc-aead 0.6.1", + "vitaminc-context 0.6.1", + "vitaminc-normalize", + "vitaminc-protected 0.6.1", +] + [[package]] name = "vitaminc-prf" -version = "0.5.1" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712d31d46d76b38c6b62deb35f516fc5636a35f90f39c0213d0ca9a052b4400b" +checksum = "c2715cca25006b4656223bc932ab4fd7f70d387aa4ec63a4e2ef33aa8cd2b3a5" dependencies = [ "mutants", "thiserror 2.0.20", - "vitaminc-context", - "vitaminc-protected", + "vitaminc-aead 0.6.1", + "vitaminc-context 0.6.1", + "vitaminc-normalize", + "vitaminc-protected 0.6.1", ] [[package]] @@ -2680,7 +2919,24 @@ dependencies = [ "serde_bytes", "subtle", "thiserror 2.0.20", - "vitaminc-protected-derive", + "vitaminc-protected-derive 0.5.1", + "zeroize", +] + +[[package]] +name = "vitaminc-protected" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1bb76dd473f5aa89b844f92018ebe3041deb8e5390e7061cca4283d959ff855" +dependencies = [ + "bitvec", + "digest 0.11.3", + "libc", + "serde", + "serde_bytes", + "subtle", + "thiserror 2.0.20", + "vitaminc-protected-derive 0.6.1", "zeroize", ] @@ -2695,6 +2951,18 @@ dependencies = [ "syn 3.0.4", ] +[[package]] +name = "vitaminc-protected-derive" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7478ed160c019059746e75fc613563dcdcd5266d155a34601d29377a1c4afc3" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 3.0.4", +] + [[package]] name = "vitaminc-random" version = "0.5.1" @@ -2705,8 +2973,23 @@ dependencies = [ "getrandom 0.4.3", "rand 0.10.2", "thiserror 2.0.20", - "vitaminc-protected", - "vitaminc-random-derives", + "vitaminc-protected 0.5.1", + "vitaminc-random-derives 0.5.1", + "zeroize", +] + +[[package]] +name = "vitaminc-random" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b477af7c01c3d9ef2647e46952ad3df7f246b3f806b1d4009ee83beecec613a" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand 0.10.2", + "thiserror 2.0.20", + "vitaminc-protected 0.6.1", + "vitaminc-random-derives 0.6.1", "zeroize", ] @@ -2721,6 +3004,17 @@ dependencies = [ "syn 3.0.4", ] +[[package]] +name = "vitaminc-random-derives" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab5a0de8da428c6789dc067a578a0462e411099721d616736cf29e1622230c35" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + [[package]] name = "vitaminc-traits" version = "0.5.1" @@ -2732,8 +3026,24 @@ dependencies = [ "rmp-serde", "serde", "thiserror 2.0.20", - "vitaminc-protected", - "vitaminc-random", + "vitaminc-protected 0.5.1", + "vitaminc-random 0.5.1", + "zeroize", +] + +[[package]] +name = "vitaminc-traits" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a900f0edcacd49c8557ab54722175869fec66926054a6fbdc53a05069db3c922" +dependencies = [ + "anyhow", + "bytes", + "rmp-serde", + "serde", + "thiserror 2.0.20", + "vitaminc-protected 0.6.1", + "vitaminc-random 0.6.1", "zeroize", ] @@ -2979,6 +3289,15 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] + [[package]] name = "wit-bindgen" version = "0.57.1" diff --git a/languages/golang/encrypt/guest/Cargo.toml b/languages/golang/encrypt/guest/Cargo.toml index ef9814ecf..e39ab37b0 100644 --- a/languages/golang/encrypt/guest/Cargo.toml +++ b/languages/golang/encrypt/guest/Cargo.toml @@ -47,27 +47,27 @@ eql = ["dep:eql-bindings"] # HTTP comes from the host (see `wasm:wasi-check` in the suite root). stack-auth = { path = "../../../../packages/stack-auth", default-features = false } # `dynamic`: the runtime value model this guest speaks — contexts, index -# terms and record plans over `FfiValue`, shared with every other binding. +# terms and record plans read from frames, shared with every other binding. stack-encrypt = { path = "../../../../packages/stack-encrypt", default-features = false, features = ["dynamic"] } # The guest ABI every guest under bindings/go shares: the allocator and # buffer registry (and with them the `se_alloc`/`se_dealloc` exports), the -# packed-result helpers, the status table and the `transport_send` import. +# packed-result helpers, the status table and the step loop. # This crate defines only the exports that are its own. stack-guest-abi = { path = "../../../../packages/stack-guest-abi" } stack-kms = { path = "../../../../packages/stack-kms", default-features = false } # The EQL types, for the `eql` build only. By path, like every other # eql-bindings dependency in the repository: the SQL that stores a payload # and the Rust that emits it must agree (scripts/lint-no-eql-registry-pins.mjs). -eql-bindings = { path = "../../../../packages/eql/crates/eql-bindings", features = ["stack-encrypt"], optional = true } +eql-bindings = { path = "../../../../packages/eql/crates/eql-bindings", features = ["targets"], optional = true } zerokms-protocol = "=0.12.31" -# The FFI codec + `FfiValue` tree, shared with vitaminc's own guest — one -# codec, not a fork. Same vitaminc version stack-encrypt builds against -# (see the comment in packages/stack-encrypt/Cargo.toml). A different -# version here fails at the `FfiValue: Decrypt` bound, since the aead crate -# would be duplicated. -vitaminc-aead-value = "0.5.1" -vitaminc-protected = "0.5.1" +# The transport frame and ciphertext codec, shared with vitaminc's own +# guest — one codec, not a fork. Same vitaminc version stack-encrypt builds +# against (see the comment in packages/stack-encrypt/Cargo.toml). A +# different version here fails at the `ValueFrame: Decrypt` bound, since the +# aead crate would be duplicated. +vitaminc-aead-frame = { version = "0.6.1", features = ["serde"] } +vitaminc-protected = "0.6.1" futures = { version = "0.3", default-features = false, features = ["executor"] } serde = "1" @@ -76,6 +76,8 @@ uuid = "1" zeroize = "1" [dev-dependencies] +# The test tree (`tests/support`) walks values through vitaminc's sinks. +vitaminc-aead = "0.6.1" stack-kms = { path = "../../../../packages/stack-kms", default-features = false, features = ["test-support"] } # For minting a valid client-key fixture in the config tests. recipher = "=0.3.1" diff --git a/languages/golang/encrypt/guest/src/abi.rs b/languages/golang/encrypt/guest/src/abi.rs index 5e5d1c675..2d6e47e52 100644 --- a/languages/golang/encrypt/guest/src/abi.rs +++ b/languages/golang/encrypt/guest/src/abi.rs @@ -6,25 +6,42 @@ //! //! - **Every export handed plaintext wipes that buffer in place before it //! returns**, rather than leaving it for `se_dealloc`: [`se_cipher_init`] -//! (the config carries the client key), and [`se_term`], [`se_query`] and -//! [`se_encrypt_record`] (their value/source buffers). The host's plaintext therefore lives no longer +//! (the config carries the client key), and [`se_term`], +//! [`se_query_term`], [`se_verify`] and [`se_encrypt_record`] (their +//! value, source and input buffers, and [`se_verify`]'s page of decrypted +//! values). The host's plaintext therefore lives no longer //! than the call, instead of until the host gets round to releasing it. //! **A host must not read a plaintext input buffer back after the call, or //! pass the same buffer to two calls** — it will be zeros. Option, context, //! AAD and plan buffers are not secret and are left untouched. //! - Output buffers from the decrypt exports contain plaintext; the host must //! copy them out and immediately `se_dealloc` (which zeroizes). +//! - **The guest never waits on the network: it steps.** An export that can +//! reach ZeroKMS ([`se_cipher_init`], [`se_keyset`], [`se_term`], +//! [`se_encrypt_record`], [`se_decrypt_record`], [`se_query_term`]) runs +//! its work as a future, polled once. Done, it returns its output as +//! always. Waiting on a request, it returns `STATUS_NEEDS_IO` instead: the +//! host takes the paused operation's handle and requests with [`se_io`], +//! performs them, and hands the responses to [`se_resume`], which returns +//! the same three ways; or it gives up with [`se_abort`] +//! (`stack_guest_abi::step`, and [`crate::io`] for the wire form). Several +//! operations may be paused at once, each under its own handle. A paused +//! operation owns its inputs — the plaintext among them, wiped when it +//! ends or is aborted — so a host's input buffers are free once the +//! export returns, paused or not. A host reads a step's requests (and a +//! failure's `se_last_error`) before it steps another operation. //! - **One instance is one client.** [`se_cipher_init`] runs once per //! instance: it builds the //! `StackCipher>` (one -//! `load-keyset` round trip through the host transport for the default +//! `load-keyset` round trip, through the step loop, for the default //! keyset) and returns that keyset's id. There is no cipher handle: the //! keysets a client uses are selected per call through the options object //! ([`crate::options`]), loaded on first use through the cipher's own //! cache. Nothing crosses the boundary that the host could allocate, //! alias or free. -//! - [`se_shutdown`] is the one lifetime call: it drops the cipher (client -//! key and every loaded index key wiped by `ZeroizeOnDrop`) and wipes +//! - [`se_shutdown`] is the one lifetime call: it drops every paused +//! operation, then the cipher (client +//! key and every loaded index key wiped by `ZeroizeOnDrop`), and wipes //! every buffer the registry still holds. It exists because closing a //! wasm instance frees linear memory without running Rust destructors — //! without it, key material would sit in freed host memory. After it, a @@ -35,10 +52,10 @@ //! [`se_shutdown`] is a no-op, and a *malformed* call is //! `STATUS_ENCODING` in any state, because validation runs first (see //! below). -//! - During an entry-point call the host's imported functions may re-enter -//! the guest **only** through `se_alloc` (to place the transport response -//! / token); calling any other export from inside a host import is -//! undefined behaviour of the embedding, not of this module. +//! - During an entry-point call the host's imported function may re-enter +//! the guest **only** through `se_alloc` (to place the token); calling any +//! other export from inside a host import is undefined behaviour of the +//! embedding, not of this module. //! - **Every export runs through `stack_guest_abi::abi::export`**, so a //! failure leaves its full error for `se_last_error`: the code, message, //! help and fields of the `stack_encrypt`, `stack_kms` or `stack_auth` @@ -52,8 +69,9 @@ //! fields, so their contexts must be non-empty (`STATUS_ENCODING` //! otherwise): each is a [`stack_encrypt::NonEmpty`] from the moment it is //! parsed, and the sealing and opening sides bind that one value. -//! [`se_plan_check`] and [`se_targets`] answer the Go generator's questions -//! and need no cipher. +//! [`se_plan_check`], [`se_targets`] and [`se_operators`] answer the Go +//! generator's questions and need no cipher; nor do [`se_verify`] and [`se_target_limit`], a +//! probabilistic query's two questions, which use no key. //! //! Every export decodes and validates *all* of its inputs — the operation //! payload, the plan or context, the term kind, the value against the @@ -66,24 +84,25 @@ //! inside the call, after every check. //! //! Wasm modules are single-threaded; the host must serialize calls into one -//! instance. +//! instance. Between calls it may interleave steps of different operations. use std::cell::{Cell, RefCell}; use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::rc::Rc; -use futures::executor::block_on; -use stack_encrypt::{KeysetCipher, StackCipher}; +use stack_encrypt::StackCipher; use stack_guest_abi::abi::{export, input, take_plaintext, wipe_input}; use stack_guest_abi::buffers; +use stack_guest_abi::call; use stack_guest_abi::last_error::{self, malformed, out_of_order}; -use vitaminc_aead_value::transport as codec; -use vitaminc_aead_value::FfiValue; +use stack_guest_abi::status::STATUS_NEEDS_IO; +use stack_guest_abi::step::{self, Operation, Step}; +use crate::io; use crate::ops; use crate::options::{parse_options, parse_selector, scope_for, KeysetSelector, Side}; #[cfg(not(feature = "deterministic-kms"))] use crate::status::STATUS_INTERNAL; -use stack_encrypt::dynamic::Scope; #[cfg(not(feature = "deterministic-kms"))] use stack_encrypt::ErrorPayload; @@ -101,63 +120,51 @@ type GuestCipher = StackCipher; thread_local! { // Wasm is single-threaded, so a thread-local `RefCell` is a plain owner - // of the cipher — no `Send`/`Sync` bounds required. - static CIPHER: RefCell> = const { RefCell::new(None) }; + // of the cipher — no `Send`/`Sync` bounds required. Paused operations + // share it through the `Rc`, so their handles are dropped before it at + // shutdown, and the last owner wipes it. + static CIPHER: RefCell>> = const { RefCell::new(None) }; /// Set by `se_shutdown`: after it, `se_cipher_init` is refused too, so /// an instance the host has torn down cannot be quietly revived with /// stale buffers around. static SHUT_DOWN: Cell = const { Cell::new(false) }; + /// Set while an `se_cipher_init` is paused, so a second one is refused + /// before the first has installed its cipher. + static INITIALISING: Cell = const { Cell::new(false) }; } -/// Decode one codec-encoded input. -fn decode(bytes: &[u8]) -> Result { - codec::decode_value(&mut codec::Reader::new(bytes)) - .map_err(|_| malformed("an input is not a value in the transport codec")) -} - -/// Run `f` with the instance's cipher, or report `STATUS_STATE` when there -/// is none (never initialised, or shut down). -fn with_cipher(f: impl FnOnce(&GuestCipher) -> Result) -> Result { - CIPHER.with(|c| { - let c = c.borrow(); - let cipher = c.as_ref().ok_or_else(|| { - out_of_order( - "there is no cipher: call se_cipher_init first, and none after se_shutdown", - ) - })?; - f(cipher) +/// The instance's cipher, or `STATUS_STATE` when there is none (never +/// initialised, still initialising, or shut down). +fn cipher() -> Result, u32> { + CIPHER.with(|c| c.borrow().clone()).ok_or_else(|| { + out_of_order("there is no cipher: call se_cipher_init first, and none after se_shutdown") }) } -/// Run `f` with the keyset the mint-side options in `opts` select, -/// resolving it through the cipher (a first use is one `load-keyset` round -/// trip). The options are decoded and validated before the cipher is -/// consulted. -fn with_keyset( - opts: &[u8], - f: impl FnOnce(&KeysetCipher<'_, GuestKms>) -> Result, -) -> Result { - let options = parse_options(decode(opts)?, Side::Mint)?; - with_cipher(|cipher| { - let keyset = block_on(options.keyset.resolve(cipher))?; - f(&keyset) - }) +/// Turn what came of an operation's step into an export's result: the +/// output when done, `STATUS_NEEDS_IO` with the step registered for +/// [`se_io`] when paused, the status when failed. +fn deliver(step: Result) -> Result, u32> { + match step? { + Step::Done(out) => Ok(out), + Step::NeedsIo { handle, requests } => { + let encoded = io::encode_step(handle, &requests); + drop(requests); + let encoded = encoded.inspect_err(|_| { + let _ = step::abort(handle); + })?; + step::set_pending(encoded); + Err(STATUS_NEEDS_IO) + } + } } -/// Run `f` with the scope the open-side options in `opts` select: the -/// client for `{"any"}`, one keyset's cipher otherwise. -fn with_scope( - opts: &[u8], - f: impl FnOnce(Scope<'_, GuestKms>) -> Result, -) -> Result { - let options = parse_options(decode(opts)?, Side::Open)?; - with_cipher(|cipher| { - let scope = block_on(scope_for(cipher, &options.keyset))?; - f(scope) - }) +/// [`deliver`] an operation's first step. +fn run(work: Operation) -> Result, u32> { + deliver(step::start(work)) } -/// Initialise the instance's cipher from an FFI-codec-encoded config object +/// Initialise the instance's cipher from a config frame /// (see [`crate::config`]). Performs one `load-keyset` round trip through /// the host transport for the default keyset, and returns that keyset's id /// (16 raw UUID bytes) as the output buffer. The raw config buffer — which @@ -180,33 +187,43 @@ pub unsafe extern "C" fn se_cipher_init(cfg_ptr: *mut u8, cfg_len: u32) -> u64 { export(|| { // The pointer/length pair is validated first and on its own: a pair // that fails here returns before anything touches the range, which - // is `wipe_input`'s precondition. Only a validated buffer is decoded - // and, whatever the decode outcome, wiped. + // is `wipe_input`'s precondition. Only a validated buffer is parsed + // and, whatever the outcome, wiped. // SAFETY: host-owned ranges the export was handed; the borrows end // before it returns and before any wipe of an overlapping range. let bytes = unsafe { input(cfg_ptr, cfg_len)? }; - let decoded = decode(bytes); - // The borrow of the raw buffer ends with `decoded` owned; wipe the - // buffer now — it holds the client-key hex — before parsing (and - // before the init round trip), whatever the decode outcome. + // The parser borrows the buffer's strings and owns only what it + // builds from them, so the borrow ends here; wipe the buffer now — + // it holds the client key — before the init round trip, whatever + // the parse outcome. + let config = parse_init(bytes); unsafe { wipe_input(cfg_ptr, cfg_len) }; - cipher_init(decoded?) + cipher_init(config?) }) } #[cfg(not(feature = "deterministic-kms"))] -fn cipher_init(decoded: FfiValue) -> Result, u32> { +use crate::config::{parse_config as parse_init, CipherConfig as InitConfig}; +#[cfg(feature = "deterministic-kms")] +use crate::deterministic::Seed as InitConfig; +#[cfg(feature = "deterministic-kms")] +fn parse_init(bytes: &[u8]) -> Result { + InitConfig::parse(bytes) +} + +#[cfg(not(feature = "deterministic-kms"))] +fn cipher_init(config: InitConfig) -> Result, u32> { use crate::host::{HostTokenStrategy, WasiHostConnection}; use crate::status::STATUS_KMS_TRANSPORT; use stack_kms::{ClientOpts, StackKms}; - let config = crate::config::parse_config(decoded).map_err(|e| malformed(e.describe()))?; - if SHUT_DOWN.with(Cell::get) || CIPHER.with(|c| c.borrow().is_some()) { + if init_refused() { return Err(init_out_of_order()); } - // One request at a time: the host import is synchronous, so concurrency - // would only interleave nothing; keep the executor honest about it. + // One request at a time: a step hands the host one request, and the + // first version of the step loop keeps it that way. Raising it later + // changes no ABI: a step's request list can hold several. // // `max_keys_per_req` stays at the client default (500). That is what // bounds "one ZeroKMS call": a batch is assembled once, then @@ -236,7 +253,16 @@ fn cipher_init(decoded: FfiValue) -> Result, u32> { if let Some(size) = config.keyset_cache_size { builder = builder.keyset_cache_size(size); } - install(block_on(builder.init()).map_err(|e| crate::status::fail_error(&e))?) + let initialising = Initialising::begin(); + run(Box::pin(async move { + let _initialising = initialising; + install( + builder + .init() + .await + .map_err(|e| crate::status::fail_error(&e))?, + ) + })) } /// The `deterministic-kms` build's init: the config buffer is the @@ -244,35 +270,63 @@ fn cipher_init(decoded: FfiValue) -> Result, u32> { /// endpoint, no token, no network. A test artefact, never shipped; see /// [`crate::deterministic`]. #[cfg(feature = "deterministic-kms")] -fn cipher_init(decoded: FfiValue) -> Result, u32> { - use vitaminc_protected::Controlled; - - let not_a_seed = || malformed("the deterministic build's config is a 32-byte seed"); - let FfiValue::Bytes(seed) = decoded else { - return Err(not_a_seed()); - }; - let seed: [u8; 32] = seed - .risky_ref() - .as_slice() - .try_into() - .map_err(|_| not_a_seed())?; - if SHUT_DOWN.with(Cell::get) || CIPHER.with(|c| c.borrow().is_some()) { +fn cipher_init(seed: InitConfig) -> Result, u32> { + if init_refused() { return Err(init_out_of_order()); } - let builder = StackCipher::builder().kms(crate::deterministic::DeterministicSource::new(seed)); - install(block_on(builder.init()).map_err(|e| crate::status::fail_error(&e))?) + let builder = StackCipher::builder().kms(seed.source()); + let initialising = Initialising::begin(); + run(Box::pin(async move { + let _initialising = initialising; + install( + builder + .init() + .await + .map_err(|e| crate::status::fail_error(&e))?, + ) + })) +} + +/// Whether an `se_cipher_init` must be refused: after shutdown, with a +/// cipher installed, or with another init paused. +fn init_refused() -> bool { + SHUT_DOWN.with(Cell::get) + || INITIALISING.with(Cell::get) + || CIPHER.with(|c| c.borrow().is_some()) +} + +/// Marks an init in flight from the moment it starts until its work ends: +/// done, failed or aborted. +struct Initialising; + +impl Initialising { + fn begin() -> Self { + INITIALISING.with(|i| i.set(true)); + Initialising + } +} + +impl Drop for Initialising { + fn drop(&mut self) { + INITIALISING.with(|i| i.set(false)); + } } -/// `se_cipher_init` called a second time, or after `se_shutdown`. +/// `se_cipher_init` called a second time, while one is paused, or after +/// `se_shutdown`. fn init_out_of_order() -> u32 { - out_of_order("the cipher is already initialised, or the instance is shut down") + out_of_order("the cipher is already initialised or initialising, or the instance is shut down") } /// Install the built cipher as the instance's and return its default /// keyset's id. fn install(cipher: GuestCipher) -> Result, u32> { + // A shutdown that overtook a paused init installs nothing. + if SHUT_DOWN.with(Cell::get) { + return Err(init_out_of_order()); + } let default = cipher.default_keyset().keyset_id().as_bytes().to_vec(); - CIPHER.with(|c| *c.borrow_mut() = Some(cipher)); + CIPHER.with(|c| *c.borrow_mut() = Some(Rc::new(cipher))); Ok(default) } @@ -285,16 +339,15 @@ fn install(cipher: GuestCipher) -> Result, u32> { /// is `STATUS_STATE`, [`se_cipher_init`] included; a malformed one is /// `STATUS_ENCODING` first, as in any other state. /// -/// The index keys' wipe holds because each `HmacSha256Prf` clone a -/// derivation takes is created and dropped inside one `block_on`'d call, -/// and the guest is single-threaded, so no clone is alive when the host -/// calls this. That is the precondition, not a property of the drop: -/// anything that later parks a PRF clone beyond an ABI call turns this -/// wipe into a no-op for that key. +/// Every paused operation is dropped first: each holds the cipher, and may +/// hold a PRF clone a derivation took, plaintext, or keys it was handed. +/// Dropping them before the cipher is what makes the cipher's drop the last +/// owner's, and so its wipe real. #[no_mangle] pub extern "C" fn se_shutdown() { let _ = catch_unwind(AssertUnwindSafe(|| { SHUT_DOWN.with(|s| s.set(true)); + step::drop_all(); CIPHER.with(|c| { let _ = c.borrow_mut().take(); }); @@ -317,14 +370,15 @@ pub unsafe extern "C" fn se_keyset(sel_ptr: *const u8, sel_len: u32) -> u64 { export(|| { // SAFETY: host-owned ranges the export was handed; the borrows end // before it returns and before any wipe of an overlapping range. - let selector = parse_selector(decode(unsafe { input(sel_ptr, sel_len)? })?)?; + let selector = parse_selector(unsafe { input(sel_ptr, sel_len)? })?; if selector == KeysetSelector::Any { return Err(malformed(r#"{"any"} is not a keyset"#)); } - with_cipher(|cipher| { - let keyset = block_on(selector.resolve(cipher))?; + let cipher = cipher()?; + run(Box::pin(async move { + let keyset = selector.resolve(&cipher).await?; Ok(keyset.keyset_id().as_bytes().to_vec()) - }) + })) }) } @@ -336,7 +390,7 @@ pub unsafe extern "C" fn se_keyset(sel_ptr: *const u8, sel_len: u32) -> u64 { /// /// The context is one part — a string, bytes, or an `i32`/`i64`/`u32`/`u64` /// — or an array of parts, which may nest as deep as the transport codec -/// allows (`vitaminc_aead_value::transport::MAX_DEPTH` levels, counted from +/// allows (`vitaminc_aead_frame::MAX_DEPTH` levels, counted from /// the root of the encoded value; deeper is refused as `STATUS_ENCODING` /// before the context is parsed). [`stack_encrypt::dynamic::context`] is the one home of /// that grammar and of which Rust context each shape spells. @@ -362,15 +416,17 @@ pub unsafe extern "C" fn se_term( ) -> u64 { export(|| { let value = unsafe { take_plaintext(val_ptr, val_len)? }; - let value = value.as_slice(); // SAFETY: host-owned ranges the export was handed; the borrows end // before it returns and before any wipe of an overlapping range. - let context = unsafe { input(ctx_ptr, ctx_len)? }; + let context = unsafe { input(ctx_ptr, ctx_len)? }.to_vec(); let opts = unsafe { input(opt_ptr, opt_len)? }; - ops::validate::term(value, context, kind)?; - with_keyset(opts, |keyset| { - block_on(ops::term(keyset, value, context, kind)) - }) + ops::validate::term(&value, &context, kind)?; + let options = parse_options(opts, Side::Mint)?; + let cipher = cipher()?; + run(Box::pin(async move { + let keyset = options.keyset.resolve(&cipher).await?; + ops::term(&keyset, &value, &context, kind).await + })) }) } @@ -395,15 +451,17 @@ pub unsafe extern "C" fn se_encrypt_record( ) -> u64 { export(|| { let source = unsafe { take_plaintext(src_ptr, src_len)? }; - let source = source.as_slice(); // SAFETY: host-owned ranges the export was handed; the borrows end // before it returns and before any wipe of an overlapping range. - let plan = unsafe { input(plan_ptr, plan_len)? }; + let plan = unsafe { input(plan_ptr, plan_len)? }.to_vec(); let opts = unsafe { input(opt_ptr, opt_len)? }; - ops::validate::record(source, plan)?; - with_keyset(opts, |keyset| { - block_on(ops::encrypt_record(keyset, source, plan)) - }) + ops::validate::record(&source, &plan)?; + let options = parse_options(opts, Side::Mint)?; + let cipher = cipher()?; + run(Box::pin(async move { + let keyset = options.keyset.resolve(&cipher).await?; + ops::encrypt_record(&keyset, &source, &plan).await + })) }) } @@ -435,14 +493,16 @@ pub unsafe extern "C" fn se_decrypt_record( export(|| { // SAFETY: host-owned ranges the export was handed; the borrows end // before it returns and before any wipe of an overlapping range. - let record = unsafe { input(rec_ptr, rec_len)? }; - let plan = unsafe { input(plan_ptr, plan_len)? }; + let record = unsafe { input(rec_ptr, rec_len)? }.to_vec(); + let plan = unsafe { input(plan_ptr, plan_len)? }.to_vec(); let opts = unsafe { input(opt_ptr, opt_len)? }; - let expected = parse_options(decode(opts)?, Side::Open)?.context; - ops::validate::record_tree(record, plan, expected.as_ref())?; - with_scope(opts, |scope| { - block_on(ops::decrypt_record(scope, record, plan, expected)) - }) + let options = parse_options(opts, Side::Open)?; + ops::validate::record_tree(&record, &plan, options.context.as_ref())?; + let cipher = cipher()?; + run(Box::pin(async move { + let scope = scope_for(&cipher, &options.keyset).await?; + ops::decrypt_record(scope, &record, &plan, options.context).await + })) }) } @@ -465,39 +525,125 @@ pub unsafe extern "C" fn se_plan_check(plan_ptr: *const u8, plan_len: u32) -> u6 }) } -/// Derive the EQL query value of one target field under the keyset `opts` -/// selects: a codec-encoded plaintext, the codec-encoded plan and the -/// field's name (UTF-8 bytes) in, the query value's JSON bytes out -/// ([`ops::query`]). The plan is parsed against this build's resolver, so -/// a build without EQL types refuses it as `STATUS_ENCODING` before the -/// cipher is consulted, as it refuses the same plan at [`se_plan_check`]. +/// Derive the query term for one plan field, an operator and its input +/// under the keyset `opts` selects ([`ops::query_term`]): a codec-encoded +/// input (one value, or for `between` a list of two), the codec-encoded +/// plan, and the field's and the operator's names (UTF-8 bytes) in; the +/// term's codec-encoded object out. An operator the field does not answer +/// is `STATUS_ENCODING` naming the field, as a mistyped value is, before +/// the cipher is consulted; the refusals the typed path makes before a term +/// exists (match text that cuts no token or does not normalize) are +/// `STATUS_TERM`, also before the cipher. A range the wrong way round is +/// not refused: it matches no row, as SQL's `BETWEEN` does. The input +/// buffer is wiped in place. /// /// # Safety /// /// As for [`se_term`]. #[no_mangle] -pub unsafe extern "C" fn se_query( +pub unsafe extern "C" fn se_query_term( val_ptr: *mut u8, val_len: u32, plan_ptr: *const u8, plan_len: u32, field_ptr: *const u8, field_len: u32, + op_ptr: *const u8, + op_len: u32, opt_ptr: *const u8, opt_len: u32, ) -> u64 { export(|| { let value = unsafe { take_plaintext(val_ptr, val_len)? }; - let value = value.as_slice(); // SAFETY: host-owned ranges the export was handed; the borrows end // before it returns and before any wipe of an overlapping range. + let plan = unsafe { input(plan_ptr, plan_len)? }.to_vec(); + let field = unsafe { input(field_ptr, field_len)? }.to_vec(); + let operator = unsafe { input(op_ptr, op_len)? }.to_vec(); + let opts = unsafe { input(opt_ptr, opt_len)? }; + ops::validate::query_term(&value, &plan, &field, &operator)?; + let options = parse_options(opts, Side::Mint)?; + let cipher = cipher()?; + run(Box::pin(async move { + let keyset = options.keyset.resolve(&cipher).await?; + ops::query_term(&keyset, &value, &plan, &field, &operator).await + })) + }) +} + +/// Verify a page of decrypted values against a probabilistic query +/// ([`ops::verify`]): the query's codec-encoded input and a codec-encoded +/// list of values, the codec-encoded plan, and the field's and the +/// operator's names in; a codec-encoded list of booleans out, one per value. +/// Both plaintext buffers, the input and the values, are wiped in place. An +/// exact operator has nothing to verify and is `STATUS_ENCODING`. Needs no +/// cipher: it works in every instance state, as [`se_plan_check`] does. +/// +/// # Safety +/// +/// As for [`se_term`]. +#[no_mangle] +pub unsafe extern "C" fn se_verify( + in_ptr: *mut u8, + in_len: u32, + vals_ptr: *mut u8, + vals_len: u32, + plan_ptr: *const u8, + plan_len: u32, + field_ptr: *const u8, + field_len: u32, + op_ptr: *const u8, + op_len: u32, +) -> u64 { + export(|| { + // Both buffers are taken, and so wiped, before either result is + // acted on: a bad query pointer must not leave the page of + // decrypted values unwiped behind an early return. + let query = unsafe { take_plaintext(in_ptr, in_len) }; + let values = unsafe { take_plaintext(vals_ptr, vals_len) }; + let (query, values) = (query?, values?); + // SAFETY: host-owned ranges the export was handed; the borrows end + // before it returns. let plan = unsafe { input(plan_ptr, plan_len)? }; let field = unsafe { input(field_ptr, field_len)? }; - let opts = unsafe { input(opt_ptr, opt_len)? }; - ops::validate::query(value, plan, field)?; - with_keyset(opts, |keyset| { - block_on(ops::query(keyset, value, plan, field)) - }) + let operator = unsafe { input(op_ptr, op_len)? }; + ops::verify(query.as_slice(), values.as_slice(), plan, field, operator) + }) +} + +/// How many rows a probabilistic query should fetch for `want` of them to +/// verify ([`ops::target_limit`]): the term's bytes as [`se_query_term`] +/// returned them, the codec-encoded statistics, the codec-encoded plan, +/// and the field's and the operator's names in, with `want`; the +/// codec-encoded limit out, an unsigned integer or null. An exact operator +/// has no fetch to size and is `STATUS_ENCODING`. Needs no cipher. +/// +/// # Safety +/// +/// As for [`se_term`]. +#[no_mangle] +pub unsafe extern "C" fn se_target_limit( + term_ptr: *const u8, + term_len: u32, + stats_ptr: *const u8, + stats_len: u32, + plan_ptr: *const u8, + plan_len: u32, + field_ptr: *const u8, + field_len: u32, + op_ptr: *const u8, + op_len: u32, + want: u64, +) -> u64 { + export(|| { + // SAFETY: host-owned ranges the export was handed; the borrows end + // before it returns. + let term = unsafe { input(term_ptr, term_len)? }; + let stats = unsafe { input(stats_ptr, stats_len)? }; + let plan = unsafe { input(plan_ptr, plan_len)? }; + let field = unsafe { input(field_ptr, field_len)? }; + let operator = unsafe { input(op_ptr, op_len)? }; + ops::target_limit(term, stats, plan, field, operator, want) }) } @@ -508,3 +654,74 @@ pub unsafe extern "C" fn se_query( pub extern "C" fn se_targets() -> u64 { export(ops::targets) } + +/// The operators one plan field answers ([`ops::operators`]): the +/// codec-encoded plan and the field's name (UTF-8 bytes) in, a codec-encoded +/// list of operator objects out. A field the plan does not have is +/// `STATUS_ENCODING`. Needs no cipher: it works in every instance state, as +/// [`se_plan_check`] does. +/// +/// # Safety +/// +/// As for [`se_term`]. +#[no_mangle] +pub unsafe extern "C" fn se_operators( + plan_ptr: *const u8, + plan_len: u32, + field_ptr: *const u8, + field_len: u32, +) -> u64 { + export(|| { + // SAFETY: host-owned ranges the export was handed; the borrows end + // before they return. + let plan = unsafe { input(plan_ptr, plan_len)? }; + let field = unsafe { input(field_ptr, field_len)? }; + ops::operators(plan, field) + }) +} + +/// The step a paused export left: its handle and the requests it waits on, +/// as a packed buffer ([`crate::io`]), handed over once — the slot is +/// emptied, so a second call returns zero, as does a call with no step +/// waiting. The host calls it after `STATUS_NEEDS_IO`, before it runs any +/// other export, copies the buffer out and releases it with `se_dealloc` +/// (which wipes it: it carries the bearer token). +#[no_mangle] +pub extern "C" fn se_io() -> u64 { + match step::take_pending() { + Some((ptr, len)) => call::pack_buffer(ptr as usize as u32, len as u32), + None => 0, + } +} + +/// Hand a paused operation the responses to the requests its last step +/// asked for ([`crate::io`] has the frame), and run its next step. Returns +/// as the export that started it does: the output when done, +/// `STATUS_NEEDS_IO` when paused again (under the same handle), or the +/// status of its failure. The responses buffer can carry wrapped keys and +/// is wiped in place. A handle that names no paused operation is +/// `STATUS_STATE`; responses that do not decode, or are not one per +/// request, are `STATUS_ENCODING`, and the operation is dropped. +/// +/// # Safety +/// +/// As for [`se_term`]. +#[no_mangle] +pub unsafe extern "C" fn se_resume(handle: u64, resp_ptr: *mut u8, resp_len: u32) -> u64 { + export(|| { + let bytes = unsafe { take_plaintext(resp_ptr, resp_len) }; + let responses = bytes.and_then(|bytes| io::decode_responses(&bytes)); + let responses = responses.inspect_err(|_| { + let _ = step::abort(handle); + })?; + deliver(step::resume(handle, responses)) + }) +} + +/// Give up on a paused operation: drop it, wiping everything it held. An +/// empty output; `STATUS_STATE` for a handle that names no paused +/// operation (one that finished, failed or was aborted). +#[no_mangle] +pub extern "C" fn se_abort(handle: u64) -> u64 { + export(|| step::abort(handle).map(|()| Vec::new())) +} diff --git a/languages/golang/encrypt/guest/src/config.rs b/languages/golang/encrypt/guest/src/config.rs index 800461444..a08f576ce 100644 --- a/languages/golang/encrypt/guest/src/config.rs +++ b/languages/golang/encrypt/guest/src/config.rs @@ -1,8 +1,8 @@ //! Parsing of the `se_cipher_init` configuration. //! -//! The config crosses the boundary as one FFI-codec-encoded -//! [`FfiValue::Object`] — the same codec every other entry point uses, so -//! there is no second config format. Recognised keys (all string values): +//! The config crosses the boundary as one frame object, read by serde +//! (`vitaminc_aead_frame::serde`), the same format as every other control +//! message. Recognised keys (all string values): //! //! | key | required | meaning | //! |---------------|----------|---------| @@ -17,60 +17,26 @@ //! Every other keyset is selected per call (see [`crate::options`]). Unknown //! keys are rejected — a typo'd optional key must not silently fall back to a //! default — so a host still sending `keyset` is told so rather than quietly -//! encrypting somewhere else. +//! encrypting somewhere else. A key given twice is refused by the frame. //! -//! The parsed [`FfiValue`] holds the client-key hex inside -//! `Protected`, which wipes on drop; the raw config *buffer* is wiped by the -//! ABI layer immediately after decoding (see [`crate::abi`]). +//! Every string is borrowed from the config buffer, so the only copy of the +//! client key this parser makes is the [`ClientKey`] it builds. The ABI +//! layer wipes the buffer as soon as [`parse_config`] returns (see +//! [`crate::abi`]). +//! +//! A refusal is [`STATUS_ENCODING`](crate::status::STATUS_ENCODING), with a +//! message that names the key at fault and never a value, because the +//! config holds the client key. An unrecognised key is not named either: +//! if a value is pasted where a key should be, the unrecognised key *is* +//! that value. use std::num::NonZeroUsize; +use serde::{Deserialize, Deserializer}; +use stack_guest_abi::last_error::malformed; use stack_kms::{ClientKey, ZeroKmsEndpoint}; use uuid::Uuid; -use vitaminc_aead_value::FfiValue; -use zeroize::Zeroizing; - -/// A parse failure, carrying which key was at fault. Maps to -/// `STATUS_ENCODING` at the ABI, and [`describe`](Self::describe) is the -/// message the host reads through `se_last_error`: the name of the key at -/// fault crosses the boundary, its value never does. -#[derive(Debug, PartialEq, Eq)] -pub enum ConfigError { - /// The config was not an object of string values. - NotAnObject, - /// A required key was absent. - Missing(&'static str), - /// A key held something other than a string. - NotAString(&'static str), - /// A key's value failed its own validation (bad UUID, bad hex, bad URL). - Invalid(&'static str), - /// A key appeared twice. The codec rejects duplicate object keys before - /// this parser runs, but `parse_config` is `pub` and takes any - /// [`FfiValue`] — last-write-wins on, say, `client_key` must never be - /// silent. - Duplicate(&'static str), - /// A key this version does not recognise. - UnknownKey(String), -} - -impl ConfigError { - /// Says what was wrong with the config, for the host to read. - /// - /// The message names the key at fault but never its value, because the - /// config holds the client key. An unrecognised key is not named - /// either: if a value is pasted where a key should be, the unrecognised - /// key *is* that value. - pub fn describe(&self) -> String { - match self { - Self::NotAnObject => "the config is not an object of string values".to_owned(), - Self::Missing(key) => format!("the config has no {key}"), - Self::NotAString(key) => format!("the config's {key} is not a string"), - Self::Invalid(key) => format!("the config's {key} is not valid"), - Self::Duplicate(key) => format!("the config gives {key} twice"), - Self::UnknownKey(_) => "the config has a key this version does not know".to_owned(), - } - } -} +use vitaminc_aead_frame::serde::from_slice; /// Everything `se_cipher_init` needs to build the cipher. pub struct CipherConfig { @@ -79,93 +45,93 @@ pub struct CipherConfig { pub keyset_cache_size: Option, } -/// Parse a decoded config value. Consumes it so the client-key material has -/// one owner; the `Protected` payloads are wiped when the strings drop here. -pub fn parse_config(value: FfiValue) -> Result { - let FfiValue::Object(entries) = value else { - return Err(ConfigError::NotAnObject); - }; +/// Parse the config frame a host sent. See the [module docs](self). +pub fn parse_config(bytes: &[u8]) -> Result { + read(bytes).map_err(malformed) +} - // Every slot is a `Zeroizing`, not just the key one: the - // client-key slot *must* wipe on every exit path (any of the `?`s below - // can fire while it holds a full encoding of the client root key), and - // making one slot special invites the next edit to add an early return - // above the wipe. Uniform is cheaper than remembering. - let mut client_id: Option> = None; - let mut client_key_encoded: Option> = None; - let mut url: Option> = None; - let mut cache_size: Option> = None; +/// [`parse_config`], with the refusal's message. +fn read(bytes: &[u8]) -> Result { + let wire: Wire<'_> = from_slice(bytes).map_err(|e| format!("the config: {e}"))?; + // `from_encoded_v1` is lenient by design: hex in either case *or* the + // base64 `secretkey.json` holds, matching every native loader. + let client_key = ClientKey::from_encoded_v1(wire.client_id.0, wire.client_key) + .map_err(|_| "the config: client_key: a client key that is not valid".to_owned())?; + Ok(CipherConfig { + client_key, + endpoint: wire.zerokms_url.map(|url| url.0), + keyset_cache_size: wire.keyset_cache_size.map(|size| size.0), + }) +} - for (key, value) in entries { - let slot = match key.as_str() { - "client_id" => &mut client_id, - "client_key" => &mut client_key_encoded, - "zerokms_url" => &mut url, - "keyset_cache_size" => &mut cache_size, - _ => return Err(ConfigError::UnknownKey(key)), - }; - // The codec already rejects duplicate object keys, so on the ABI - // path the slot is always vacant — but this function accepts any - // `FfiValue`, so enforce it rather than assume it. - if slot.is_some() { - return Err(ConfigError::Duplicate(name_of(&key))); - } - let FfiValue::String(s) = value else { - return Err(ConfigError::NotAString(name_of(&key))); - }; - let text = Zeroizing::new( - std::str::from_utf8(s.risky_ref()) - .map_err(|_| ConfigError::NotAString(name_of(&key)))? - .to_string(), - ); - *slot = Some(text); - } +/// The config as a host writes it. Each field validates as it reads, +/// through `TryFrom`. The frame's serde never repeats a `TryFrom` error's +/// text (it could carry the value), so a refusal names the key and says +/// only that its value was not accepted; each error here is still a fixed +/// string, for the reader. +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Wire<'a> { + client_id: ClientId, + client_key: &'a str, + #[serde(default, deserialize_with = "present")] + zerokms_url: Option, + #[serde(default, deserialize_with = "present")] + keyset_cache_size: Option, +} - let client_id = client_id.ok_or(ConfigError::Missing("client_id"))?; - let client_id = Uuid::parse_str(&client_id).map_err(|_| ConfigError::Invalid("client_id"))?; +/// Absent is `None`. Present, a value must be a string: a null is refused. +fn present<'de, D: Deserializer<'de>, T: Deserialize<'de>>(d: D) -> Result, D::Error> { + T::deserialize(d).map(Some) +} - // Lenient by design: `from_encoded_v1` takes hex in either case *or* the - // base64 `secretkey.json` holds, matching every native loader. The - // `Zeroizing` slot wipes the encoded copy however this returns — the - // decoded `FfiValue`'s own copy was consumed above, and the raw input - // buffer is the ABI layer's to wipe. - let client_key_encoded = client_key_encoded.ok_or(ConfigError::Missing("client_key"))?; - let client_key = ClientKey::from_encoded_v1(client_id, &client_key_encoded) - .map_err(|_| ConfigError::Invalid("client_key"))?; +/// A client id: a UUID, as text. +#[derive(Deserialize)] +#[serde(try_from = "&str")] +struct ClientId(Uuid); - let endpoint = url - .map(|u| u.parse().map_err(|_| ConfigError::Invalid("zerokms_url"))) - .transpose()?; +impl TryFrom<&str> for ClientId { + type Error = &'static str; - let keyset_cache_size = cache_size - .map(|n| { - n.parse::() - .map_err(|_| ConfigError::Invalid("keyset_cache_size")) - }) - .transpose()?; + fn try_from(text: &str) -> Result { + Uuid::parse_str(text) + .map(ClientId) + .map_err(|_| "a client id that is not a UUID") + } +} - Ok(CipherConfig { - client_key, - endpoint, - keyset_cache_size, - }) +/// The ZeroKMS endpoint to pin. +#[derive(Deserialize)] +#[serde(try_from = "&str")] +struct Url(ZeroKmsEndpoint); + +impl TryFrom<&str> for Url { + type Error = &'static str; + + fn try_from(text: &str) -> Result { + text.parse().map(Url).map_err(|_| "a URL that is not valid") + } } -/// Intern the key name for error reporting (`&'static str` keeps -/// [`ConfigError`] cheap; unknown keys carry the owned string instead). -fn name_of(key: &str) -> &'static str { - match key { - "client_id" => "client_id", - "client_key" => "client_key", - "zerokms_url" => "zerokms_url", - "keyset_cache_size" => "keyset_cache_size", - _ => "unknown", +/// A positive integer, as a decimal string. +#[derive(Deserialize)] +#[serde(try_from = "&str")] +struct CacheSize(NonZeroUsize); + +impl TryFrom<&str> for CacheSize { + type Error = &'static str; + + fn try_from(text: &str) -> Result { + text.parse() + .map(CacheSize) + .map_err(|_| "a size that is not a positive decimal integer") } } #[cfg(test)] mod tests { use super::*; + use crate::test_frame::{frame, obj, s, V}; /// A valid v1 client key for fixtures, minted through the real key type /// so the hex exercises the actual `from_hex_v1` decoder. @@ -178,22 +144,27 @@ mod tests { (id, key.to_hex_v1().expect("encode fixture key")) } - fn obj(entries: Vec<(&str, &str)>) -> FfiValue { - FfiValue::Object( - entries - .into_iter() - .map(|(k, v)| (k.to_string(), FfiValue::String(v.into()))) - .collect(), - ) + fn config(entries: Vec<(&'static str, V)>) -> Result { + read(&frame(&obj(entries))) + } + + /// The config's refusal, checked to name neither the client key nor any + /// value or unknown key the test sent. + fn refused(v: &V, secrets: &[&str]) -> String { + let message = read(&frame(v)).err().expect("refused"); + for secret in secrets { + assert!(!message.contains(secret), "{message:?} echoes {secret:?}"); + } + message } #[test] fn parses_a_minimal_config() { let (id, hex) = client_key_hex(); - let cfg = parse_config(obj(vec![ - ("client_id", &id.to_string()), - ("client_key", &hex), - ])) + let cfg = config(vec![ + ("client_id", s(&id.to_string())), + ("client_key", s(&hex)), + ]) .expect("minimal config parses"); assert_eq!(cfg.client_key.key_id, id); assert!(cfg.endpoint.is_none()); @@ -203,28 +174,27 @@ mod tests { #[test] fn parses_the_keyset_cache_size() { let (id, hex) = client_key_hex(); - let id_s = id.to_string(); - let cfg = parse_config(obj(vec![ - ("client_id", &id_s), - ("client_key", &hex), - ("keyset_cache_size", "16"), - ])) - .expect("config parses"); + let with = |size: V| { + obj(vec![ + ("client_id", s(&id.to_string())), + ("client_key", s(&hex)), + ("keyset_cache_size", size), + ]) + }; + let cfg = read(&frame(&with(s("16")))).expect("config parses"); assert_eq!(cfg.keyset_cache_size, NonZeroUsize::new(16)); for bad in ["0", "-1", "sixteen", ""] { - assert!( - matches!( - parse_config(obj(vec![ - ("client_id", &id_s), - ("client_key", &hex), - ("keyset_cache_size", bad), - ])), - Err(ConfigError::Invalid("keyset_cache_size")) - ), + assert_eq!( + refused(&with(s(bad)), &[&hex]), + "the config: keyset_cache_size: a value its type does not accept", "{bad:?} must be refused" ); } + // A decimal string, not an integer, and not null. + for bad in [V::U32(16), V::Null] { + assert!(refused(&with(bad), &[&hex]).starts_with("the config: keyset_cache_size: ")); + } } /// A keyset is not a config key: the default is the server's and a @@ -235,25 +205,27 @@ mod tests { fn a_keyset_key_is_rejected_like_any_other_unknown_key() { let (id, hex) = client_key_hex(); for key in ["keyset", "keyset_id"] { - assert!(matches!( - parse_config(obj(vec![ - ("client_id", &id.to_string()), - ("client_key", &hex), - (key, "users"), - ])), - Err(ConfigError::UnknownKey(_)) - )); + let message = refused( + &obj(vec![ + ("client_id", s(&id.to_string())), + ("client_key", s(&hex)), + (key, s("users")), + ]), + // "keyset" alone is inside the expected `keyset_cache_size`. + &[&hex, "users", "keyset_id"], + ); + assert!(message.contains("an unknown key"), "{message}"); } } #[test] fn parses_the_zerokms_url() { let (id, hex) = client_key_hex(); - let cfg = parse_config(obj(vec![ - ("client_id", &id.to_string()), - ("client_key", &hex), - ("zerokms_url", "https://zerokms.example.com"), - ])) + let cfg = config(vec![ + ("client_id", s(&id.to_string())), + ("client_key", s(&hex)), + ("zerokms_url", s("https://zerokms.example.com")), + ]) .expect("config parses"); assert!(cfg.endpoint.is_some()); } @@ -266,7 +238,6 @@ mod tests { use base64ct::Encoding; let (id, hex) = client_key_hex(); - let id_s = id.to_string(); let bytes = base16ct::lower::decode_vec(&hex).expect("fixture hex"); let base64 = base64ct::Base64::encode_string(&bytes); @@ -275,8 +246,11 @@ mod tests { ("uppercase hex", hex.to_uppercase()), ("base64", base64), ] { - let cfg = parse_config(obj(vec![("client_id", &id_s), ("client_key", &encoded)])) - .unwrap_or_else(|e| panic!("{label} must parse, got {e:?}")); + let cfg = config(vec![ + ("client_id", s(&id.to_string())), + ("client_key", s(&encoded)), + ]) + .unwrap_or_else(|e| panic!("{label} must parse, got {e}")); assert_eq!(cfg.client_key.key_id, id, "{label}"); assert_eq!( cfg.client_key.to_hex_v1().expect("re-encode"), @@ -286,57 +260,96 @@ mod tests { } } + /// Every refusal names the key at fault and never a value: not the + /// client key, not a bad id or URL, not an unknown key. #[test] - fn rejects_bad_configs() { + fn rejects_bad_configs_without_echoing_a_value() { let (id, hex) = client_key_hex(); let id_s = id.to_string(); + let secrets = [ + hex.as_str(), + "nope", + "zerokms_urk", + "typo.example", + "deadbeef", + ]; + let cases = [ + (V::Null, "the config: expected "), + ( + V::Arr(vec![s(&id_s), s(&hex)]), + "the config: expected struct Wire, found an array", + ), + ( + obj(vec![ + ("client_id", s(&id_s)), + ("client_key", V::Bytes(hex.clone().into_bytes())), + ]), + "the config: client_key: expected a borrowed string, found bytes", + ), + ( + obj(vec![("client_id", s(&id_s))]), + "the config: missing field `client_key`", + ), + ( + obj(vec![("client_id", s("nope")), ("client_key", s(&hex))]), + "the config: client_id: a value its type does not accept", + ), + ( + obj(vec![("client_id", s(&id_s)), ("client_key", V::U64(1))]), + "the config: client_key: ", + ), + ( + // Valid hex, not a keyset. + obj(vec![("client_id", s(&id_s)), ("client_key", s("deadbeef"))]), + "the config: client_key: a client key that is not valid", + ), + ( + obj(vec![ + ("client_id", s(&id_s)), + ("client_key", s(&hex)), + ("zerokms_urk", s("https://typo.example.com")), + ]), + "the config: an unknown key, ", + ), + ( + obj(vec![ + ("client_id", s(&id_s)), + ("client_key", s(&hex)), + ("zerokms_url", s("not a url")), + ]), + "the config: zerokms_url: a value its type does not accept", + ), + ( + obj(vec![ + ("client_id", s(&id_s)), + ("client_key", s(&hex)), + ("zerokms_url", V::Null), + ]), + "the config: zerokms_url: ", + ), + ]; + for (config, expected) in &cases { + let message = refused(config, &secrets); + assert!( + message.starts_with(expected), + "{message:?} is not {expected:?}" + ); + } + } - assert!(matches!( - parse_config(FfiValue::Null), - Err(ConfigError::NotAnObject) - )); - assert!(matches!( - parse_config(obj(vec![("client_id", &id_s)])), - Err(ConfigError::Missing("client_key")) - )); - assert!(matches!( - parse_config(obj(vec![("client_id", "nope"), ("client_key", &hex)])), - Err(ConfigError::Invalid("client_id")) - )); - assert!(matches!( - parse_config(obj(vec![ - ("client_id", &id_s), - ("client_key", "deadbeef"), // valid hex, not a keyset - ])), - Err(ConfigError::Invalid("client_key")) - )); - assert!(matches!( - parse_config(obj(vec![ - ("client_id", &id_s), - ("client_key", &hex), - ("zerokms_urk", "https://typo.example.com"), - ])), - Err(ConfigError::UnknownKey(_)) - )); - // The codec refuses duplicate keys on the ABI path, but this - // function is `pub` over any `FfiValue`: a repeated `client_key` - // must be an error, never a silent last-write-wins on root key - // material. - assert!(matches!( - parse_config(obj(vec![ - ("client_id", &id_s), - ("client_key", &hex), - ("client_key", &hex), - ])), - Err(ConfigError::Duplicate("client_key")) - )); - assert!(matches!( - parse_config(obj(vec![ - ("client_id", &id_s), - ("client_key", &hex), - ("zerokms_url", "not a url"), - ])), - Err(ConfigError::Invalid("zerokms_url")) - )); + /// A repeated `client_key` is never a silent last-write-wins on root key + /// material: the frame refuses it. + #[test] + fn a_repeated_key_is_refused() { + let (id, hex) = client_key_hex(); + let message = refused( + &obj(vec![ + ("client_id", s(&id.to_string())), + ("client_key", s(&hex)), + ("client_key", s(&hex)), + ]), + &[&hex], + ); + assert!(message.starts_with("the config: "), "{message}"); } } diff --git a/languages/golang/encrypt/guest/src/deterministic.rs b/languages/golang/encrypt/guest/src/deterministic.rs index e58a26e74..88a789d51 100644 --- a/languages/golang/encrypt/guest/src/deterministic.rs +++ b/languages/golang/encrypt/guest/src/deterministic.rs @@ -11,3 +11,59 @@ //! binary carries it; the tests read it from disk. pub use stack_kms::DeterministicSource; + +use serde::Deserialize; +use stack_guest_abi::last_error::malformed; +use vitaminc_aead_frame::serde::from_slice; +use zeroize::Zeroize; + +/// The `deterministic-kms` build's whole config: the source's 32-byte seed, +/// a frame holding bytes and nothing else. Wiped when it drops. +#[derive(Deserialize)] +#[serde(try_from = "&[u8]")] +pub struct Seed([u8; 32]); + +impl TryFrom<&[u8]> for Seed { + type Error = &'static str; + + fn try_from(bytes: &[u8]) -> Result { + bytes + .try_into() + .map(Seed) + .map_err(|_| "a seed that is not 32 bytes") + } +} + +impl Drop for Seed { + fn drop(&mut self) { + self.0.zeroize(); + } +} + +impl Seed { + /// Read the seed frame the host sent in place of a config. + pub fn parse(bytes: &[u8]) -> Result { + from_slice(bytes) + .map_err(|_| malformed("the deterministic build's config is a 32-byte seed")) + } + + /// The key source this seed drives. + pub fn source(&self) -> DeterministicSource { + DeterministicSource::new(self.0) + } +} + +#[cfg(test)] +mod tests { + use super::Seed; + use crate::test_frame::{frame, V}; + use vitaminc_aead_frame::serde::from_slice; + + /// A seed is 32 bytes, and only bytes: 32 characters of text are refused. + #[test] + fn a_seed_is_32_bytes_never_text() { + assert!(from_slice::(&frame(&V::Bytes(vec![7; 32]))).is_ok()); + assert!(from_slice::(&frame(&V::Bytes(vec![7; 31]))).is_err()); + assert!(from_slice::(&frame(&V::Str("x".repeat(32)))).is_err()); + } +} diff --git a/languages/golang/encrypt/guest/src/host.rs b/languages/golang/encrypt/guest/src/host.rs index 7c7564f5f..605bf7d42 100644 --- a/languages/golang/encrypt/guest/src/host.rs +++ b/languages/golang/encrypt/guest/src/host.rs @@ -1,7 +1,16 @@ -//! The host imports and the types built over them: [`WasiHostConnection`] -//! (a [`ZeroKMSConnection`] whose transport is the host's HTTP client) and -//! [`HostTokenStrategy`] (an [`AuthStrategy`] that asks the host for the -//! bearer token). wasm32-only — everything here calls an imported function. +//! The guest's two connections to its host: [`WasiHostConnection`] (a +//! [`ZeroKMSConnection`] whose requests the host performs, through the step +//! loop) and [`HostTokenStrategy`] (an [`AuthStrategy`] that asks the host +//! for the bearer token). wasm32-only — the token is an imported function. +//! +//! # Requests +//! +//! The guest never waits on the network. A ZeroKMS request is recorded for +//! the step being run (`stack_guest_abi::step::request`) and the export +//! returns to the host with it, paused; the host performs it and resumes the +//! operation with the response ([`crate::io`] has the wire form). So a host +//! whose HTTP client cannot block — `fetch` on the edge — can run the guest, +//! and the guest is never running while a request is in flight. //! //! # Import contract (module `cipherstash_transport`) //! @@ -9,11 +18,9 @@ //! guest buffers with `se_alloc` and the guest reclaims them through the //! shared registry (`stack_guest_abi::buffers`). //! -//! - `transport_send(..)` — perform one HTTP request. The import and its -//! contract are `stack_guest_abi::transport`'s, shared with every guest; -//! this module only builds ZeroKMS requests over it. //! - `token_get(token_out) -> status` — hand over the current bearer token -//! (Phase-1 auth: minting and refresh stay on the host). `token_out` is a +//! (Phase-1 auth: minting and refresh stay on the host, which holds a +//! fresh token before each step, so this never waits). `token_out` is a //! `(ptr_out, len_out)` slot pair filled with an `se_alloc`'d buffer; //! status `0` is success, anything else a host-side failure. This //! guest's own: the credential guest supplies tokens rather than asking @@ -34,7 +41,7 @@ use std::sync::Mutex; use stack_auth::{AuthError, AuthStrategy, CustomError, SecretToken, ServiceToken}; use stack_guest_abi::buffers; use stack_guest_abi::headers::header_value; -use stack_guest_abi::transport; +use stack_guest_abi::step; use stack_kms::{BaseUrlUnresolved, ZeroKMSConnection, ZeroKMSConnectionInit, ZeroKmsEndpoint}; use zeroize::Zeroizing; use zerokms_protocol::{ViturRequest, ViturRequestError}; @@ -47,9 +54,9 @@ extern "C" { fn token_get(token_ptr_out: *mut u32, token_len_out: *mut u32) -> i32; } -/// A [`ZeroKMSConnection`] whose transport is the `transport_send` host -/// import. The host call is synchronous from the guest's perspective, so -/// `send` resolves immediately — `block_on` in the ABI layer never parks. +/// A [`ZeroKMSConnection`] whose requests the host performs: `send` records +/// its request for the step being run and resolves when the host resumes the +/// operation with the response (see the [module docs](self)). /// /// The endpoint is pinned at init when the cipher config named one; /// otherwise it is discovered by `StackKms` from the access token's @@ -126,11 +133,12 @@ impl ZeroKMSConnection for WasiHostConnection { let auth = Zeroizing::new(format!("Bearer {access_token}")); let headers = Zeroizing::new(request_headers(auth.as_str())); - // The shared import reclaims both response slots before judging - // either, and the body it hands back wipes on drop (it carries - // wrapped key material). - let response = transport::send("POST", url.as_str(), &headers, &body) - .map_err(|e| ViturRequestError::parse("Host response buffer failed validation", e))?; + // The request's headers and body wipe on drop, and so does the + // response's body: it carries wrapped key material. + let request = step::Request::new("POST", url.to_string(), headers, body); + let response = step::request(request) + .await + .map_err(|e| ViturRequestError::prepare("The host was not asked for the request", e))?; let content_type = header_value(&response.headers, "content-type"); map_response(response.status, content_type, &response.body) diff --git a/languages/golang/encrypt/guest/src/io.rs b/languages/golang/encrypt/guest/src/io.rs new file mode 100644 index 000000000..13618e8d2 --- /dev/null +++ b/languages/golang/encrypt/guest/src/io.rs @@ -0,0 +1,188 @@ +//! How a step of the step loop crosses the boundary +//! ([`stack_guest_abi::step`]): the requests a paused operation hands the +//! host, and the responses the host hands back. Both are transport frames, +//! read and written with serde. +//! +//! When an export pauses it returns `STATUS_NEEDS_IO` instead of an output, +//! and the host takes the step with `se_io`: +//! +//! ```text +//! { "handle": , +//! "requests": [ { "method": "POST", "url": , +//! "headers": , "body": }, ... ] } +//! ``` +//! +//! The host performs every request and calls `se_resume` with the handle and +//! their responses, in the same order: +//! +//! ```text +//! [ { "status": , "headers": , "body": }, ... ] +//! ``` +//! +//! `headers` is the `name: value` line format of +//! [`stack_guest_abi::headers`] both ways. A `status` is the HTTP status, or +//! negative for a request the host could not perform, with its error text as +//! the body: the contract `transport_send` had, field for field. A step +//! carries the bearer token and a request body that can hold key-material +//! contexts, and a response can carry wrapped keys, so the guest wipes its +//! copies, and the host wipes the buffers with `se_dealloc`. + +use serde::{Deserialize, Serialize, Serializer}; +use stack_guest_abi::last_error::malformed; +use stack_guest_abi::step::{Request, Response}; +use vitaminc_aead_frame::serde::{from_slice, to_vec}; +use zeroize::Zeroizing; + +use crate::status::STATUS_ENCODING; + +/// The step the host takes with `se_io`, as frame bytes. The bytes carry a +/// bearer token, so the caller registers them and the host's `se_dealloc` +/// wipes them; the staging buffer is wiped here. +pub fn encode_step(handle: u64, requests: &[Request]) -> Result, u32> { + #[derive(Serialize)] + struct Wire<'a> { + handle: u64, + requests: Vec>, + } + #[derive(Serialize)] + struct RequestWire<'a> { + method: &'a str, + url: &'a str, + headers: Bytes<'a>, + body: Bytes<'a>, + } + let wire = Wire { + handle, + requests: requests + .iter() + .map(|request| RequestWire { + method: request.method, + url: &request.url, + headers: Bytes(&request.headers), + body: Bytes(&request.body), + }) + .collect(), + }; + let frame = Zeroizing::new(to_vec(&wire).map_err(|_| STATUS_ENCODING)?); + // Sized exactly, as every output is: the registry would otherwise + // reallocate a buffer with spare capacity and leave the old one unwiped. + let mut out = Vec::new(); + out.try_reserve_exact(frame.len()) + .map_err(|_| stack_guest_abi::last_error::internal("no memory for a step's requests"))?; + out.extend_from_slice(&frame); + Ok(out) +} + +/// The responses the host handed `se_resume`, copied out of its buffer into +/// ones the guest wipes. +pub fn decode_responses(bytes: &[u8]) -> Result, u32> { + #[derive(Deserialize)] + #[serde(deny_unknown_fields)] + struct ResponseWire<'a> { + status: i32, + #[serde(borrow)] + headers: &'a [u8], + #[serde(borrow)] + body: &'a [u8], + } + let wire: Vec> = + from_slice(bytes).map_err(|e| malformed(format!("the responses: {e}")))?; + Ok(wire + .into_iter() + .map(|r| { + Response::new( + r.status, + r.headers.to_vec(), + Zeroizing::new(r.body.to_vec()), + ) + }) + .collect()) +} + +/// Bytes, written as a bytes leaf. +struct Bytes<'a>(&'a [u8]); + +impl Serialize for Bytes<'_> { + fn serialize(&self, s: S) -> Result { + s.serialize_bytes(self.0) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_frame::{frame, obj, V}; + + #[test] + fn a_step_is_its_handle_and_requests() { + let request = Request::new( + "POST", + "https://kms.example/keys".to_owned(), + Zeroizing::new(b"authorization: Bearer t\n".to_vec()), + Zeroizing::new(b"{\"k\":1}".to_vec()), + ); + let bytes = encode_step(7, &[request]).unwrap(); + let expected = frame(&obj(vec![ + ("handle", V::U64(7)), + ( + "requests", + V::Arr(vec![obj(vec![ + ("method", V::Str("POST".into())), + ("url", V::Str("https://kms.example/keys".into())), + ("headers", V::Bytes(b"authorization: Bearer t\n".to_vec())), + ("body", V::Bytes(b"{\"k\":1}".to_vec())), + ])]), + ), + ])); + assert_eq!(bytes, expected); + assert_eq!(bytes.capacity(), bytes.len()); + } + + #[test] + fn responses_read_in_order() { + let bytes = frame(&V::Arr(vec![ + obj(vec![ + ("status", V::I32(200)), + ( + "headers", + V::Bytes(b"content-type: application/json\n".to_vec()), + ), + ("body", V::Bytes(b"{}".to_vec())), + ]), + obj(vec![ + ("status", V::I32(-1)), + ("headers", V::Bytes(Vec::new())), + ("body", V::Bytes(b"dial tcp: refused".to_vec())), + ]), + ])); + let responses = decode_responses(&bytes).unwrap(); + assert_eq!(responses.len(), 2); + assert_eq!(responses[0].status, 200); + assert_eq!(&*responses[0].body, b"{}"); + assert_eq!(responses[1].status, -1); + assert_eq!(&*responses[1].body, b"dial tcp: refused"); + } + + #[test] + fn malformed_responses_are_refused_without_echoing_them() { + for bad in [ + V::Null, + obj(vec![("status", V::I32(200))]), + V::Arr(vec![obj(vec![ + ("status", V::I32(200)), + ("headers", V::Str("secret-looking".into())), + ("body", V::Bytes(Vec::new())), + ])]), + V::Arr(vec![obj(vec![ + ("status", V::I32(200)), + ("headers", V::Bytes(Vec::new())), + ("body", V::Bytes(Vec::new())), + ("extra", V::Bool(true)), + ])]), + ] { + assert_eq!(decode_responses(&frame(&bad)).err(), Some(STATUS_ENCODING)); + let error = stack_guest_abi::last_error::take().unwrap(); + assert!(!String::from_utf8_lossy(&error).contains("secret-looking")); + } + } +} diff --git a/languages/golang/encrypt/guest/src/lib.rs b/languages/golang/encrypt/guest/src/lib.rs index dfe7fbbdb..2d3c7acd3 100644 --- a/languages/golang/encrypt/guest/src/lib.rs +++ b/languages/golang/encrypt/guest/src/lib.rs @@ -45,9 +45,12 @@ //! `se_cipher_init`; derived data keys and index keys never leave. //! //! The exports are the record path (`se_encrypt_record`, `se_decrypt_record`), -//! per-field term derivation (`se_term`), a target field's EQL query value -//! (`se_query`), keyset resolution (`se_keyset`), the generator's two -//! questions (`se_plan_check`, `se_targets`) and the lifetime pair +//! the term of one index under a context (`se_term`, which the Go SDK asks +//! to name the field behind a record's term failure), a query named by +//! operator and a probabilistic query's two questions (`se_query_term`, +//! `se_verify`, `se_target_limit`; ADR-0007, amended 2026-10-16), keyset +//! resolution (`se_keyset`), the generator's three questions +//! (`se_plan_check`, `se_targets`, `se_operators`) and the lifetime pair //! (`se_cipher_init`, `se_shutdown`). There is no whole-value export: the //! Go SDK seals every value under a declaration (ADR-0007). //! @@ -73,13 +76,15 @@ //! - [`abi`], [`host`] (wasm32 only) — this guest's export surface and its //! token import. The conventions every guest shares — `se_alloc` / //! `se_dealloc`, the buffer registry, the packed result encoding, the -//! status table, the `transport_send` import — are `stack_guest_abi`'s; +//! status table, the step loop (`stack_guest_abi::step`) — are +//! `stack_guest_abi`'s, and [`io`] is how this guest's steps cross; //! [`abi`]'s module docs give this guest's contract on top of them. //! //! On wasm32 `vitaminc-encrypt` uses its pure-Rust (RustCrypto `aes-gcm`) //! backend; the trade-offs are documented there. Values cross the boundary -//! in the vitaminc FFI codec (`vitaminc_aead_value::transport` — an FFI -//! encoding, not a storage format); the leaves inside a ciphertext tree are +//! as vitaminc transport frames and ciphertext trees in vitaminc's +//! ciphertext codec (`vitaminc_aead_frame` — FFI encodings, not a storage +//! format); the leaves inside a ciphertext tree are //! the *frozen* `SealedValue` byte encoding from Phase 2, so a leaf lifted //! out of a tree is exactly what a database column holds. @@ -87,11 +92,17 @@ pub mod config; #[cfg(feature = "deterministic-kms")] pub mod deterministic; pub mod headers; +pub mod io; pub mod ops; pub mod options; pub mod response; pub mod status; +#[cfg(test)] +#[path = "../tests/support/mod.rs"] +mod support; pub mod targets; +#[cfg(test)] +mod test_frame; // The ABI's packed u64 results embed 32-bit pointers, its bounds checks // read the wasm linear-memory size, and `host` calls imported functions — diff --git a/languages/golang/encrypt/guest/src/ops.rs b/languages/golang/encrypt/guest/src/ops.rs index 9b0a8c9ac..c30b5430b 100644 --- a/languages/golang/encrypt/guest/src/ops.rs +++ b/languages/golang/encrypt/guest/src/ops.rs @@ -4,9 +4,10 @@ //! module wires them to the session table and the packed ABI; nothing in //! here knows about linear memory. //! -//! Values and ciphertext trees cross the boundary in the vitaminc FFI codec -//! (`vitaminc_aead_value::transport`) — the same codec the vitaminc guest -//! uses, so the Go side carries exactly one codec. A ciphertext tree's +//! Values cross the boundary as vitaminc transport frames, and ciphertext +//! trees in vitaminc's ciphertext codec (`vitaminc_aead_frame`) — the same +//! encodings the vitaminc guest uses, so the Go side carries exactly one +//! codec. A ciphertext tree's //! leaves are re-encoded through [`SealedValue::to_bytes`] / //! [`SealedValue::from_bytes`]: the codec sees an opaque byte-string leaf, //! and the bytes inside it are the frozen storage encoding a database column @@ -34,17 +35,24 @@ //! directions, buffers sized before a byte of plaintext is written, the //! ABI's numeric term kinds, and the mapping from a library error to a //! status code. - -use stack_encrypt::dynamic::{self, Scalar, Scope}; +//! +//! Plaintext is never decoded into a value tree. A term's value is parsed and +//! walked in place to the scalar it holds, the way vitaminc's own guest walks +//! a frame into encryption. A record's source, a query's input and a page of +//! values to verify are read as [`ValueFrame`]s: the frame is checked against +//! the engine's value model, and the engine walks it, splitting a source into +//! one frame per field as it goes. A record opens into one frame, which is +//! the output. Control data (a plan, a context, an options object, query +//! statistics) carries no secret and is read from its frame with serde. + +use stack_encrypt::dynamic::{self, Scope, ValueFrame}; use stack_encrypt::sem::MatchOptions; use stack_encrypt::target::IndexSpec; use stack_encrypt::ErrorPayload; -use stack_encrypt::{ - BoxedPassthrough, CipherText, KeysetCipher, Label, SealedValue, StackCipherText, -}; +use stack_encrypt::{CipherText, KeysetCipher, Label, Node, SealedValue, StackCipherText}; use stack_kms::DataKeySource; -use vitaminc_aead_value::{transport as codec, FfiValue}; -use vitaminc_protected::Controlled; +use vitaminc_aead_frame::{decode_ciphertext, encode_ciphertext, Frame}; +use zeroize::Zeroizing; use stack_guest_abi::last_error; @@ -63,7 +71,7 @@ pub const TERM_OPE: u32 = 4; /// A ciphertext tree whose leaves are the frozen [`SealedValue`] byte /// encoding — the shape that crosses the FFI codec. -type BytesTree = CipherText, BoxedPassthrough>; +type BytesTree = Node>; // ============================================================================= // Terms @@ -71,15 +79,15 @@ type BytesTree = CipherText, BoxedPassthrough>; /// Derive one index term: a codec-encoded scalar and a codec-encoded /// context in, the term's frozen byte encoding out (see `stack-encrypt`'s -/// `sem` module docs). Under the local HMAC backend the derivation is one +/// `sem` module docs). The scalar's frame is walked in place. Under the local HMAC backend the derivation is one /// PRF/CLLW computation with no ZeroKMS I/O; that is the backend's /// property, not this operation's contract — the term API is a `Pending` /// so a backend that derives terms at ZeroKMS settles the same way. /// /// The context is one part — a string, bytes, or an `i32`/`i64`/`u32`/`u64` /// — or an array of parts, nested as deep as the transport codec allows -/// ([`codec::MAX_DEPTH`] levels from the root of the encoded value; deeper -/// is [`STATUS_ENCODING`] before the context is parsed), exactly as a plan +/// ([`MAX_DEPTH`](vitaminc_aead_frame::MAX_DEPTH) levels from the root of the frame; deeper +/// is [`STATUS_ENCODING`] before the context is read), exactly as a plan /// field's; [`dynamic::context`] is the one home of that grammar. Shape is identity: /// `[x]` is a PAE-framed list and `x` is not, so a probe takes the context /// in the shape the field was sealed under — a plan field's context @@ -97,23 +105,33 @@ where { // The same proof every stack-encrypt leaf demands: an empty context is // `STATUS_ENCODING` here, before any derivation. - let context = dynamic::context(decode_value(context)?).map_err(|e| fail_dynamic(&e))?; - let (scalar, kind) = parse_term(decode_value(value)?, kind)?; - dynamic::term(cipher, scalar, &kind, context) + let context = parse_context(context)?; + let value = parse_frame(value)?; + let kind = parse_term(value, kind)?; + dynamic::term(cipher, value, &kind, context) .await .map_err(|e| fail_dynamic(&e)) } -/// The static half of a term: the kind is one of the ABI's table, the value -/// is a scalar, and the scheme defines the pair -/// ([`IndexSpec::supports`]). Shared by [`term`] and [`validate::term`] so -/// the ABI refuses exactly what the operation would, before any keyset is -/// resolved. +/// A term's context: one frame, read by [`dynamic::context`]. Bytes that +/// are not a frame are refused as every other input's are, before the +/// context's own grammar. +fn parse_context( + context: &[u8], +) -> Result>, u32> { + let _ = parse_frame(context)?; + dynamic::context(context).map_err(|e| fail_dynamic(&e)) +} + +/// The static half of a term: the kind is one of the ABI's table, and the +/// value is one scalar the index takes ([`IndexSpec::defines`]). Shared by +/// [`term`] and [`validate::term`] so the ABI refuses exactly what the +/// operation would, before any keyset is resolved. /// /// The ABI's kind codes carry no options, so [`TERM_MATCH`] is the match /// index under the default options, the same index a plan's bare `"match"` /// names. -fn parse_term(value: FfiValue, kind: u32) -> Result<(Scalar, IndexSpec), u32> { +fn parse_term(value: Frame<'_>, kind: u32) -> Result { let kind = match kind { TERM_EQUALITY => IndexSpec::Equality, TERM_MATCH => IndexSpec::Match(MatchOptions::default()), @@ -125,11 +143,10 @@ fn parse_term(value: FfiValue, kind: u32) -> Result<(Scalar, IndexSpec), u32> { )) } }; - let scalar = Scalar::of(&value, &kind).map_err(|e| fail_dynamic(&e))?; - if !kind.supports(&scalar) { + if !kind.defines(value) { return Err(fail_dynamic(&dynamic::Error::Term { field: None, kind })); } - Ok((scalar, kind)) + Ok(kind) } // ============================================================================= @@ -164,10 +181,11 @@ where K: DataKeySource + Sync + 'static, { let plan = parse_plan(plan)?; - let tree = dynamic::record::encrypt_with(cipher, decode_value(source)?, &plan, &resolver()) - .map_err(|e| fail_dynamic(&e))? - .await - .map_err(|e| fail_error(&e))?; + let tree = + dynamic::record::encrypt_with(cipher, checked_plaintext(source)?, &plan, &resolver()) + .map_err(|e| fail_dynamic(&e))? + .await + .map_err(|e| fail_error(&e))?; encode_tree(tree) } @@ -203,39 +221,148 @@ where .map_err(|e| fail_dynamic(&e))? .await .map_err(|e| fail_error(&e))?; - encode_value(value) + encode_frame(value) } -/// Derive the EQL query value of one target field: a codec-encoded -/// plaintext, the codec-encoded plan and the field's name (UTF-8) in, the -/// query value's JSON bytes out — the operand `eql_v3.query_` takes. -/// Runs the EQL type's own query plan through this build's resolver -/// ([`crate::targets`]); a build without EQL types refuses the plan before -/// this is reached. No data key is minted: under the local backend the -/// equality term is one PRF derivation. -pub async fn query( +// ============================================================================= +// Queries by operator +// ============================================================================= + +/// Derive the query term for one plan field, an operator and its input +/// (ADR-0009; ADR-0007, amended 2026-10-16): a codec-encoded input, the +/// codec-encoded plan, the field's name and the operator's name (both UTF-8) +/// in, the term's codec-encoded object out (the `Serialize` form of +/// [`dynamic::QueryTerm`]): +/// +/// ```text +/// { "operator": "matches", "fidelity": "probabilistic", "index": "match", +/// "domain": null, "term": } +/// ``` +/// +/// The input is one value for every operator but `between`, whose input is +/// a list of two, low then high. It is read as the field's declared kind +/// first, so a host's float `34.0` queries a `uint64` field. A field with +/// indexes derives the term of the index that answers the operator, as +/// bytes; `between` derives one per bound, as a list of two. A field that +/// names an EQL type runs the type's query twin through this build's +/// resolver, and `term` holds the twin's JSON bytes and `domain` its +/// PostgreSQL domain. No data key is minted. +pub async fn query_term( cipher: &KeysetCipher<'_, K>, - value: &[u8], + input: &[u8], plan: &[u8], field: &[u8], + operator: &[u8], ) -> Result, u32> where K: DataKeySource + Sync + 'static, { + let (plan, field, operator) = parse_query(plan, field, operator)?; + let term = dynamic::query_term( + cipher, + &plan, + field, + operator, + checked_plaintext(input)?, + &resolver(), + ) + .map_err(|e| fail_dynamic(&e))? + .await + .map_err(|e| fail_error(&e))?; + encode_answer(&term) +} + +/// Verify a page of decrypted values against a probabilistic query: the +/// query's codec-encoded input and a codec-encoded list of values, with the +/// plan, the field's name and the operator's name, in; a codec-encoded list +/// of booleans out, one per value in order ([`dynamic::verify`]). The query +/// is named as it was derived, and its tokens are cut again from the input: +/// they are plaintext, so no term carries them. A null value is `false`. +/// Needs no cipher and no key. +pub fn verify( + input: &[u8], + values: &[u8], + plan: &[u8], + field: &[u8], + operator: &[u8], +) -> Result, u32> { + let (plan, field, operator) = parse_query(plan, field, operator)?; + // Each value becomes a frame of its own (wiped on drop): vitaminc's + // `Frame` walks a sequence but does not hand out its elements' frames, + // and reading them out of the frame layout here would duplicate + // vitaminc's parser. + let values = decode_plaintext(values)? + .elements() + .ok_or_else(|| last_error::malformed("the values to verify are not a list"))?; + let verified = dynamic::verify( + &plan, + field, + operator, + checked_plaintext(input)?, + values, + &resolver(), + ) + .map_err(|e| fail_dynamic(&e))?; + encode_answer(&verified) +} + +/// How many rows a probabilistic query should fetch for `want` of them to +/// verify: the term's bytes as [`query_term`] returned them (not a frame), +/// the codec-encoded statistics +/// ([`Stats::from_frame`](stack_encrypt::op::Stats::from_frame)), the plan, +/// the field's name and the operator's name in; the codec-encoded limit out, +/// an unsigned 64-bit integer or null for no finite fetch +/// ([`Limit::rows`](stack_encrypt::op::Limit::rows)). Needs no cipher and +/// no key. +pub fn target_limit( + term: &[u8], + stats: &[u8], + plan: &[u8], + field: &[u8], + operator: &[u8], + want: u64, +) -> Result, u32> { + let (plan, field, operator) = parse_query(plan, field, operator)?; + // Statistics are read without a field to hand; the error names the + // query's, as every other refusal of a query does. + let stats = stack_encrypt::op::Stats::from_frame(stats) + .map_err(|e| fail_dynamic(&e.in_field(field)))?; + let limit = dynamic::target_limit(&plan, field, operator, term, want, &stats, &resolver()) + .map_err(|e| fail_dynamic(&e))?; + encode_answer(&limit.rows()) +} + +/// The plan, field name and operator a query names: the plan parsed against +/// this build's resolver, the field's name UTF-8, and the operator one the +/// engine knows ([`dynamic::OpSpec::parse`]); an unknown operator is +/// refused naming the field, as an operator the field does not answer is. +fn parse_query<'f>( + plan: &[u8], + field: &'f [u8], + operator: &[u8], +) -> Result<(dynamic::record::Plan, &'f str, dynamic::OpSpec), u32> { let plan = parse_plan(plan)?; let field = std::str::from_utf8(field) .map_err(|_| last_error::malformed("the field name is not UTF-8"))?; - dynamic::record::query(cipher, &plan, field, decode_value(value)?, &resolver()) - .map_err(|e| fail_dynamic(&e))? - .await - .map_err(|e| fail_error(&e)) + let operator = std::str::from_utf8(operator) + .ok() + .and_then(dynamic::OpSpec::parse) + .ok_or_else(|| { + fail_dynamic(&dynamic::Error::Source { + field: Some(field.to_owned()), + reason: dynamic::Reason::UnknownOperator, + }) + })?; + Ok((plan, field, operator)) } -/// A codec-encoded plan, parsed against this build's resolver: a target -/// name this build cannot run is refused here, the same way at every -/// export that takes a plan. +/// A plan's frame, parsed against this build's resolver: a target name this +/// build cannot run is refused here, the same way at every export that +/// takes a plan. Bytes that are not a frame are refused as every other +/// input's are, before the plan's own grammar. fn parse_plan(plan: &[u8]) -> Result { - dynamic::record::plan_with(decode_value(plan)?, &resolver()).map_err(|e| fail_dynamic(&e)) + let _ = parse_frame(plan)?; + dynamic::record::plan_with(plan, &resolver()).map_err(|e| fail_dynamic(&e)) } // ============================================================================= @@ -255,8 +382,8 @@ pub fn plan_check(plan: &[u8]) -> Result<(), u32> { /// The EQL types this build of the engine knows, as a codec-encoded /// `{"targets": [...]}`: one entry per type in -/// [`TargetDescriptor::to_value`](stack_encrypt::dynamic::TargetDescriptor::to_value)'s -/// wire form, producible or not, with the reason when not. The build +/// [`TargetDescriptor`](stack_encrypt::dynamic::TargetDescriptor)'s +/// `Serialize` form, producible or not, with the reason when not. The build /// without EQL types lists none, which a generator reads as "no /// `encrypt_into` type is available" and refuses the tag; the `eql` build /// lists the catalog, and a generator writes `encrypt_into` only for an @@ -278,18 +405,50 @@ pub fn plan_check(plan: &[u8]) -> Result<(), u32> { /// | `query_sql_domain` | string or null | the Postgres domain of the query value | /// | `producible` | bool | whether this build produces the type | /// | `reason` | string or null | why not, when `producible` is false | +/// | `operators` | list of objects | the operators the engine answers for the type, each `{"operator", "fidelity", "arity", "index"}` ([`TargetDescriptor`](stack_encrypt::dynamic::TargetDescriptor#wire-format)) | +/// +/// The first ten keys are eql-bindings' row; `operators` is the engine's, +/// computed from the row, so a generator asks the engine which operators a +/// type answers and how far each answer can be trusted. pub fn targets() -> Result, u32> { use stack_encrypt::dynamic::TargetResolver as _; - encode_value(FfiValue::Object(vec![( - "targets".to_string(), - FfiValue::Array( - resolver() - .targets() - .iter() - .map(|target| target.to_value()) - .collect(), - ), - )])) + #[derive(serde::Serialize)] + struct Targets { + targets: Vec, + } + encode_answer(&Targets { + targets: resolver().targets(), + }) +} + +/// The operators one field of a codec-encoded plan answers, as a +/// codec-encoded list: one object per operator, in the order the engine +/// lists them, each the `Serialize` form of a +/// [`dynamic::OperatorEntry`]: +/// +/// ```text +/// [ { "operator": "eq", "fidelity": "exact", "arity": "one", "index": "eq" }, ... ] +/// ``` +/// +/// Exactly the operators [`query_term`] derives a term for on that field +/// ([`dynamic::field_operators`]): the operators a field declared with +/// indexes answers, and those its EQL type answers for a field that names +/// one. The Go generator (`stashgen`) asks this for every sealed field it +/// writes and generates one query method per entry, so it holds no copy of +/// which index answers which operator. A field the plan does not have is +/// [`STATUS_ENCODING`]. Needs no cipher. +pub fn operators(plan: &[u8], field: &[u8]) -> Result, u32> { + let plan = parse_plan(plan)?; + let field = std::str::from_utf8(field) + .map_err(|_| last_error::malformed("the field name is not UTF-8"))?; + let operators = + dynamic::field_operators(&plan, field, &resolver()).map_err(|e| fail_dynamic(&e))?; + encode_answer( + &operators + .iter() + .map(|(operator, index)| dynamic::OperatorEntry::new(*operator, index)) + .collect::>(), + ) } // ============================================================================= @@ -308,13 +467,10 @@ pub mod validate { /// A term's inputs, as [`term`] takes them: the context decodes and is /// non-empty, the kind is one of [`TERM_EQUALITY`] .. [`TERM_OPE`], and - /// the value is a scalar the scheme defines that term for - /// ([`IndexSpec::supports`]). + /// the value is one scalar the index takes ([`IndexSpec::defines`]). pub fn term(value: &[u8], context: &[u8], kind: u32) -> Result<(), u32> { - dynamic::context(decode_value(context)?) - .map(drop) - .map_err(|e| fail_dynamic(&e))?; - parse_term(decode_value(value)?, kind).map(drop) + let _ = parse_context(context)?; + parse_term(parse_frame(value)?, kind).map(drop) } /// A record source against its plan, as [`encrypt_record`] takes them: @@ -323,38 +479,25 @@ pub mod validate { /// against its field's outputs). pub fn record(source: &[u8], plan: &[u8]) -> Result<(), u32> { let plan = parse_plan(plan)?; - dynamic::record::check_source(decode_value(source)?, &plan).map_err(|e| fail_dynamic(&e)) + dynamic::record::check_source(checked_plaintext(source)?, &plan) + .map_err(|e| fail_dynamic(&e)) } - /// A target query's inputs, as [`query`] takes them: the plan parses - /// against this build's resolver, the field is one of its target fields, - /// and the value is of the field's declared kind. - pub fn query(value: &[u8], plan: &[u8], field: &[u8]) -> Result<(), u32> { - let plan = parse_plan(plan)?; - let field = std::str::from_utf8(field) - .map_err(|_| last_error::malformed("the field name is not UTF-8"))?; - let refuse = |reason| { - fail_dynamic(&dynamic::Error::Plan { - field: Some(field.to_owned()), - reason, - }) - }; - let field = plan - .fields() - .iter() - .find(|candidate| candidate.name() == field) - .ok_or_else(|| refuse(dynamic::Reason::NoSuchField))?; - if field.target().is_none() { - return Err(refuse(dynamic::Reason::NotATarget)); - } - let value = decode_value(value)?; - if field.field_type().is_some_and(|kind| !kind.holds(&value)) { - return Err(fail_dynamic(&dynamic::Error::Source { - field: Some(field.name().to_owned()), - reason: dynamic::Reason::FieldType, - })); - } - Ok(()) + /// A query's inputs, as [`query_term`](super::query_term) takes them: + /// the plan parses, the field is one of its fields, the operator is one + /// the engine knows and the field answers, and the input reads as the + /// field's kind and passes the refusals the typed path makes before a + /// term exists ([`dynamic::check_query`]). + pub fn query_term(input: &[u8], plan: &[u8], field: &[u8], operator: &[u8]) -> Result<(), u32> { + let (plan, field, operator) = parse_query(plan, field, operator)?; + dynamic::check_query( + &plan, + field, + operator, + checked_plaintext(input)?, + &resolver(), + ) + .map_err(|e| fail_dynamic(&e)) } /// A record tree against its plan, as [`decrypt_record`] takes them: @@ -373,32 +516,60 @@ pub mod validate { // Codec glue // ============================================================================= -fn decode_value(bytes: &[u8]) -> Result { - codec::decode_value(&mut codec::Reader::new(bytes)) - .map_err(|_| last_error::malformed("an input is not a value in the transport codec")) +/// Plaintext from a frame: checked against the engine's value model and +/// held as the frame, in a buffer wiped when it drops. The same frames are +/// refused as control data's, so a malformed input fails the same way on +/// every path. +fn decode_plaintext(bytes: &[u8]) -> Result { + ValueFrame::from_bytes(bytes).map_err(|_| not_a_frame()) } -/// Encode a value tree into a buffer sized **before** the first byte is -/// written. -/// -/// This buffer is plaintext on the decrypt path, and a `Vec` grown by the -/// codec's pushes would leave partial plaintext in every abandoned -/// allocation a reallocation could not extend in place — memory nothing -/// wipes, undercutting the guarantee the registry makes about the buffer it -/// eventually hands the host. Reserving the exact encoded length up front -/// means the encoder never reallocates, and `register`'s `into_boxed_slice` -/// (capacity == length) does not copy either. -fn encode_value(value: FfiValue) -> Result, u32> { - let mut out = exact_buffer(value_encoded_len(&value))?; - codec::encode_value(value, &mut out).map_err(|_| STATUS_ENCODING)?; +/// Plaintext checked as [`decode_plaintext`] checks it, and walked in place +/// from the host's buffer: for a source or an input that is walked once, +/// where a frame of its own would be one more copy of the plaintext. The +/// record split copies each field into its own wiped frame, and a query's +/// input is read straight into its term. +fn checked_plaintext(bytes: &[u8]) -> Result, u32> { + ValueFrame::check(bytes).map_err(|_| not_a_frame()) +} + +/// An opened value as the output buffer: its frame, copied into a buffer +/// whose capacity is exactly its length (see [`encode_tree`]); the frame +/// is wiped when it drops. +fn encode_frame(frame: ValueFrame) -> Result, u32> { + let mut out = exact_buffer(Some(frame.risky_ref().len()))?; + out.extend_from_slice(frame.risky_ref()); + Ok(out) +} + +/// One frame, parsed and checked but not decoded: what a term's value is +/// walked from, in place. +fn parse_frame(bytes: &[u8]) -> Result, u32> { + Frame::parse(bytes).map_err(|_| not_a_frame()) +} + +fn not_a_frame() -> u32 { + last_error::malformed("an input is not a value in the transport frame") +} + +/// One of the engine's answers (a query term, a page of verdicts, a limit, a +/// catalog), written to its frame by serde, in a buffer whose capacity is +/// exactly its length. An answer is not plaintext, but the registry's +/// `into_boxed_slice` would reallocate a buffer with spare capacity, so it +/// is sized as every other output is. A query term's search tokens are +/// derived from plaintext, so the staging buffer is wiped once copied. +fn encode_answer(answer: &T) -> Result, u32> { + let frame = + Zeroizing::new(vitaminc_aead_frame::serde::to_vec(answer).map_err(|_| STATUS_ENCODING)?); + let mut out = exact_buffer(Some(frame.len()))?; + out.extend_from_slice(&frame); Ok(out) } fn decode_tree(bytes: &[u8]) -> Result { - let tree: BytesTree = - codec::decode_ciphertext_boxed(&mut codec::Reader::new(bytes)).map_err(|_| { - last_error::malformed("the record is not a ciphertext tree in the transport codec") - })?; + let tree: BytesTree = decode_ciphertext(bytes).map_err(|_| { + last_error::malformed("the record is not a ciphertext tree in the transport codec") + })?; // Structural only — a decoded leaf proves nothing until its AEAD opens // (see the `SealedValue` docs). map_leaves(tree, &mut |l: Vec| { @@ -408,25 +579,25 @@ fn decode_tree(bytes: &[u8]) -> Result { } /// The encode twin of [`decode_tree`]. Passthrough nodes can carry caller -/// plaintext, so this is sized up front for the same reason -/// [`encode_value`] is. +/// plaintext, so this is sized up front: the registry wipes only the buffer +/// it holds, and `register`'s `into_boxed_slice` reallocates a `Vec` whose +/// capacity exceeds its length, leaving the old allocation unwiped. fn encode_tree(tree: StackCipherText) -> Result, u32> { let tree = map_leaves(tree, &mut |l: SealedValue| Ok::<_, u32>(l.to_bytes()))?; let mut out = exact_buffer(tree_encoded_len(&tree))?; - codec::encode_ciphertext_boxed(tree, &mut out).map_err(|_| STATUS_ENCODING)?; + encode_ciphertext(&tree, &mut out).map_err(|_| STATUS_ENCODING)?; Ok(out) } /// Rebuild a ciphertext tree with every leaf run through `leaf`, keeping the -/// structure (and the passthrough payloads) untouched. One definition for +/// structure (and the passthrough frames) untouched. One definition for /// both directions of the frozen [`SealedValue`] leaf encoding. fn map_leaves( - tree: CipherText, + tree: Node, leaf: &mut impl FnMut(A) -> Result, -) -> Result, E> { +) -> Result, E> { Ok(match tree { CipherText::Single(l) => CipherText::Single(leaf(l)?), - CipherText::None(l) => CipherText::None(leaf(l)?), CipherText::EmptySequence(l) => CipherText::EmptySequence(leaf(l)?), CipherText::EmptyMap(l) => CipherText::EmptyMap(leaf(l)?), CipherText::Sequence(items) => CipherText::Sequence( @@ -460,41 +631,14 @@ fn exact_buffer(len: Option) -> Result, u32> { Ok(out) } -/// Exact byte length of the codec's encoding of `value`. `None` on overflow -/// or on a length the codec's `u32` frames cannot express — the encode would -/// fail on those anyway, so the caller reports an encoding error. -fn value_encoded_len(value: &FfiValue) -> Option { - // tag byte + fixed payload, or tag + u32 length prefix + payload. - let framed = |len: usize| u32::try_from(len).ok().and_then(|_| len.checked_add(5)); - match value { - FfiValue::Null | FfiValue::Undefined | FfiValue::Bool(_) => Some(1), - FfiValue::Int32(_) | FfiValue::UInt32(_) | FfiValue::Float32(_) => Some(5), - FfiValue::Int64(_) | FfiValue::UInt64(_) | FfiValue::Float64(_) => Some(9), - FfiValue::String(s) => framed(s.risky_ref().len()), - FfiValue::Bytes(b) => framed(b.risky_ref().len()), - FfiValue::Array(items) => items.iter().try_fold(5usize, |acc, item| { - acc.checked_add(value_encoded_len(item)?) - }), - FfiValue::Object(entries) => entries.iter().try_fold(5usize, |acc, (key, value)| { - acc.checked_add(framed(key.len())?.checked_sub(1)?)? - .checked_add(value_encoded_len(value)?) - }), - FfiValue::Passthrough(inner) => value_encoded_len(inner)?.checked_add(1), - } -} - -/// Exact byte length of the codec's encoding of a ciphertext tree. The -/// passthrough payloads are read (not consumed) through `Any::downcast_ref`, -/// matching what `encode_ciphertext_boxed` will re-home them to; a payload -/// that is not an [`FfiValue`] is `None`, which is the same rejection the -/// encoder would make. +/// Exact byte length of the codec's encoding of a ciphertext tree. A +/// passthrough is its node byte and its frame, written as it is. fn tree_encoded_len(tree: &BytesTree) -> Option { let framed = |len: usize| u32::try_from(len).ok().and_then(|_| len.checked_add(5)); match tree { - CipherText::Single(l) - | CipherText::None(l) - | CipherText::EmptySequence(l) - | CipherText::EmptyMap(l) => framed(l.len()), + CipherText::Single(l) | CipherText::EmptySequence(l) | CipherText::EmptyMap(l) => { + framed(l.len()) + } CipherText::Sequence(items) => items .iter() .try_fold(5usize, |acc, item| acc.checked_add(tree_encoded_len(item)?)), @@ -502,63 +646,39 @@ fn tree_encoded_len(tree: &BytesTree) -> Option { acc.checked_add(framed(key.len())?.checked_sub(1)?)? .checked_add(tree_encoded_len(value)?) }), - CipherText::Passthrough(p) => { - value_encoded_len((**p).downcast_ref::()?)?.checked_add(1) - } + CipherText::Passthrough(p) => p.0.len().checked_add(1), } } #[cfg(test)] mod tests { use super::*; - use vitaminc_protected::Protected; - - // `value_encoded_len` / `tree_encoded_len` re-derive the codec's framing - // arithmetic; the codec exports no `encoded_len` of its own, so these - // pins are the only thing that fails if the two drift. Drift is not a - // cosmetic bug: an undersized reservation makes `encode_value` - // reallocate mid-encode, leaving unwiped partial plaintext in the - // abandoned allocation — silently. - - fn every_value_shape() -> Vec { - vec![ - FfiValue::Null, - FfiValue::Undefined, - FfiValue::Bool(true), - FfiValue::Int32(-5), - FfiValue::UInt32(5), - FfiValue::Float32(1.5), - FfiValue::Int64(-9), - FfiValue::UInt64(9), - FfiValue::Float64(2.5), - FfiValue::String("".into()), - FfiValue::String("héllo".into()), - FfiValue::Bytes(Protected::new(Vec::new())), - FfiValue::Bytes(Protected::new(vec![0u8; 300])), - FfiValue::Array(Vec::new()), - FfiValue::Array(vec![FfiValue::Bool(false), FfiValue::String("x".into())]), - FfiValue::Object(Vec::new()), - FfiValue::Object(vec![ - ("a".to_string(), FfiValue::Int32(1)), - ( - "nested".to_string(), - FfiValue::Object(vec![("b".to_string(), FfiValue::Null)]), - ), - ]), - FfiValue::Passthrough(Box::new(FfiValue::Int64(7))), - FfiValue::Passthrough(Box::new(FfiValue::Array(vec![FfiValue::String( - "deep".into(), - )]))), - ] - } - + use crate::support::Tree; + use vitaminc_protected::Controlled; + + // `tree_encoded_len` re-derives the codec's arithmetic; the codec exports + // no `encoded_len` of its own, so this pin is the only thing that fails if + // the two drift. Drift is not a cosmetic bug: an undersized reservation + // makes `encode_tree` reallocate mid-encode, leaving unwiped partial + // plaintext in the abandoned allocation — silently. + + /// The registry wipes the buffer it is handed, and `into_boxed_slice` + /// reallocates one whose capacity exceeds its length. So every answer's + /// capacity is its length. #[test] - fn value_encoded_len_matches_the_codec_exactly() { - for (i, value) in every_value_shape().into_iter().enumerate() { - let expected = value_encoded_len(&value).expect("encodable shape"); - let mut out = Vec::new(); - codec::encode_value(value, &mut out).expect("codec encode"); - assert_eq!(out.len(), expected, "shape {i}"); + fn an_encoded_answer_has_no_spare_capacity() { + let entries = [ + dynamic::OperatorEntry::new(dynamic::OpSpec::Eq, &IndexSpec::Equality), + dynamic::OperatorEntry::new(dynamic::OpSpec::Between, &IndexSpec::Ope), + ]; + for out in [ + encode_answer(&vec![true, false, true]), + encode_answer(&Some(11u64)), + encode_answer(&None::), + encode_answer(&entries), + ] { + let out = out.expect("encodes"); + assert_eq!(out.capacity(), out.len()); } } @@ -568,10 +688,9 @@ mod tests { let trees: Vec = vec![ leaf(b""), leaf(&[7u8; 40]), - CipherText::None(vec![1, 2]), CipherText::EmptySequence(vec![3]), CipherText::EmptyMap(Vec::new()), - CipherText::Sequence(vec![leaf(b"a"), CipherText::None(vec![9])]), + CipherText::Sequence(vec![leaf(b"a"), CipherText::EmptyMap(vec![9])]), CipherText::Map(vec![ ("name".to_string(), leaf(b"ct")), ( @@ -579,32 +698,33 @@ mod tests { CipherText::Map(vec![("x".to_string(), leaf(b"y"))]), ), ]), - CipherText::Passthrough( - Box::new(FfiValue::Bytes(Protected::new(vec![1, 2, 3]))) as BoxedPassthrough - ), + CipherText::Passthrough(stack_encrypt::PassthroughFrame( + Tree::Bytes(vec![1, 2, 3]) + .to_frame() + .expect("a frame") + .risky_unwrap(), + )), ]; for (i, tree) in trees.into_iter().enumerate() { let expected = tree_encoded_len(&tree).expect("encodable shape"); let mut out = Vec::new(); - codec::encode_ciphertext_boxed(tree, &mut out).expect("codec encode"); + encode_ciphertext(&tree, &mut out).expect("codec encode"); assert_eq!(out.len(), expected, "tree {i}"); } } // ---- the EQL target path, native, over FakeDataKeySource ----------------- - fn encoded(value: FfiValue) -> Vec { - let mut out = Vec::new(); - codec::encode_value(value, &mut out).expect("encode"); - out + fn encoded(value: Tree) -> Vec { + value.to_frame().expect("encode").risky_unwrap() } - fn text(value: &str) -> FfiValue { - FfiValue::String(value.into()) + fn text(value: &str) -> Tree { + Tree::String(value.into()) } - fn object(entries: Vec<(&str, FfiValue)>) -> FfiValue { - FfiValue::Object( + fn object(entries: Vec<(&str, Tree)>) -> Tree { + Tree::Object( entries .into_iter() .map(|(k, v)| (k.to_string(), v)) @@ -619,10 +739,7 @@ mod tests { ( "email", object(vec![ - ( - "context", - FfiValue::Array(vec![text("users"), text("email")]), - ), + ("context", Tree::Array(vec![text("users"), text("email")])), ("target", text("TextEq")), ("type", text("string")), ]), @@ -630,11 +747,8 @@ mod tests { ( "notes", object(vec![ - ( - "context", - FfiValue::Array(vec![text("users"), text("notes")]), - ), - ("outputs", FfiValue::Array(vec![text("c")])), + ("context", Tree::Array(vec![text("users"), text("notes")])), + ("outputs", Tree::Array(vec![text("c")])), ("type", text("string")), ]), ), @@ -644,22 +758,22 @@ mod tests { #[test] fn targets_lists_what_this_build_holds_in_the_wire_shape() { let out = targets().expect("encodes"); - let FfiValue::Object(entries) = decode_value(&out).expect("decodes") else { + let Tree::Object(entries) = Tree::from_frame(&out).expect("decodes") else { panic!("an object"); }; assert_eq!(entries.len(), 1); assert_eq!(entries[0].0, "targets"); - let FfiValue::Array(items) = &entries[0].1 else { + let Tree::Array(items) = &entries[0].1 else { panic!("a list"); }; if !crate::targets::HOLDS_EQL { assert!(items.is_empty(), "the build without EQL types lists none"); return; } - let FfiValue::Object(text_eq) = items + let Tree::Object(text_eq) = items .iter() .find(|item| { - matches!(item, FfiValue::Object(fields) if fields.iter().any(|(k, v)| k == "name" && matches!(v, FfiValue::String(s) if s.risky_ref() == b"TextEq"))) + matches!(item, Tree::Object(fields) if fields.iter().any(|(k, v)| k == "name" && matches!(v, Tree::String(s) if s.as_bytes() == b"TextEq"))) }) .expect("TextEq is listed") else { @@ -678,10 +792,15 @@ mod tests { "query", "query_sql_domain", "producible", - "reason" + "reason", + "operators" ] ); - assert!(matches!(text_eq[8].1, FfiValue::Bool(true))); + assert!(matches!(text_eq[8].1, Tree::Bool(true))); + assert!( + matches!(&text_eq[10].1, Tree::Array(operators) if operators.len() == 1), + "TextEq answers one operator" + ); } #[test] @@ -691,14 +810,11 @@ mod tests { assert_eq!(plan_check(&plan), Ok(()), "the eql build runs TextEq"); // A type the engine cannot produce, and one that does not exist, // are refused at plan_check. - for name in ["TextOrdOre", "Nope"] { + for name in ["DateOrdOre", "Nope"] { let refused = encoded(object(vec![( "email", object(vec![ - ( - "context", - FfiValue::Array(vec![text("users"), text("email")]), - ), + ("context", Tree::Array(vec![text("users"), text("email")])), ("target", text(name)), ]), )])); @@ -731,7 +847,7 @@ mod tests { validate::record(&source, &plan).expect("the source fits"); let sealed = block_on(encrypt_record(&keyset, &source, &plan)).expect("seals"); - // The stored tree carries the EQL JSON under "eql": the v3 envelope + // The stored tree carries the EQL JSON under "eql": the v4 envelope // with the column as its i and a stack-encrypt ciphertext. let tree = decode_tree(&sealed).expect("a tree"); let CipherText::Map(mut fields) = tree else { @@ -749,11 +865,11 @@ mod tests { let CipherText::Passthrough(payload) = &outputs[0].1 else { panic!("a passthrough") }; - let FfiValue::Bytes(bytes) = payload.downcast_ref::().expect("a value") else { + let Tree::Bytes(bytes) = Tree::from_frame(&payload.0).expect("a value") else { panic!("bytes") }; - let eql: serde_json::Value = serde_json::from_slice(bytes.risky_ref()).expect("JSON"); - assert_eq!(eql["v"], 3); + let eql: serde_json::Value = serde_json::from_slice(bytes.as_slice()).expect("JSON"); + assert_eq!(eql["v"], 4, "Stack Encrypt writes EQL v4"); assert_eq!(eql["i"], serde_json::json!({"t": "users", "c": "email"})); assert!(eql["c"].as_str().unwrap().starts_with("stack-encrypt:1:")); assert_eq!(eql["hm"].as_str().unwrap().len(), 64); @@ -762,19 +878,26 @@ mod tests { validate::record_tree(&sealed, &plan, None).expect("the tree fits"); let opened = block_on(decrypt_record(Scope::Client(&cipher), &sealed, &plan, None)).expect("opens"); - let FfiValue::Object(values) = decode_value(&opened).expect("a value") else { + let Tree::Object(values) = Tree::from_frame(&opened).expect("a value") else { panic!("an object") }; - assert!( - matches!(&values[0].1, FfiValue::String(s) if s.risky_ref() == b"alice@example.com") - ); - assert!(matches!(&values[1].1, FfiValue::String(s) if s.risky_ref() == b"likes cats")); + assert!(matches!(&values[0].1, Tree::String(s) if s.as_bytes() == b"alice@example.com")); + assert!(matches!(&values[1].1, Tree::String(s) if s.as_bytes() == b"likes cats")); - // The query value matches the stored term. + // The eq query's term is the type's query value, and it matches the + // stored term. let probe = encoded(text("alice@example.com")); - validate::query(&probe, &plan, b"email").expect("a target field"); - let query_bytes = block_on(query(&keyset, &probe, &plan, b"email")).expect("derives"); - let probe_json: serde_json::Value = serde_json::from_slice(&query_bytes).expect("JSON"); + validate::query_term(&probe, &plan, b"email", b"eq").expect("a target field answers eq"); + let term = block_on(query_term(&keyset, &probe, &plan, b"email", b"eq")).expect("derives"); + let Tree::Object(term) = Tree::from_frame(&term).expect("a value") else { + panic!("a query term is an object") + }; + assert!(matches!(&term[3].1, Tree::String(s) if s.as_bytes() == b"eql_v4.query_text_eq")); + let Tree::Bytes(query_bytes) = &term[4].1 else { + panic!("the term is the query value's bytes") + }; + let probe_json: serde_json::Value = + serde_json::from_slice(query_bytes.as_slice()).expect("JSON"); assert_eq!( probe_json["hm"], eql["hm"], "the query matches the stored value" @@ -786,15 +909,15 @@ mod tests { // A query on a sealed field, an unknown field, or a wrong kind is // refused at validation. assert_eq!( - validate::query(&probe, &plan, b"notes"), + validate::query_term(&probe, &plan, b"notes", b"eq"), Err(STATUS_ENCODING) ); assert_eq!( - validate::query(&probe, &plan, b"nope"), + validate::query_term(&probe, &plan, b"nope", b"eq"), Err(STATUS_ENCODING) ); assert_eq!( - validate::query(&encoded(FfiValue::UInt32(1)), &plan, b"email"), + validate::query_term(&encoded(Tree::UInt32(1)), &plan, b"email", b"eq"), Err(STATUS_ENCODING) ); } diff --git a/languages/golang/encrypt/guest/src/options.rs b/languages/golang/encrypt/guest/src/options.rs index 02db276e1..d3bb9f5fb 100644 --- a/languages/golang/encrypt/guest/src/options.rs +++ b/languages/golang/encrypt/guest/src/options.rs @@ -1,7 +1,7 @@ //! The per-call options object, and the keyset selector it carries. //! //! Every export that touches a keyset takes one more codec-encoded -//! argument: an [`FfiValue::Object`] with one required key, `keyset`, whose +//! argument: a frame object with one required key, `keyset`, whose //! value is a tagged object naming the keyset the call binds to, and on //! `se_decrypt_record` one optional key, `context` (below): //! @@ -45,11 +45,10 @@ use stack_encrypt::dynamic::Scope; use stack_encrypt::{KeysetCipher, Label, StackCipher}; use stack_kms::{IdentifiedBy, IndexKeySource}; use uuid::Uuid; -use vitaminc_aead_value::FfiValue; -use vitaminc_protected::Controlled; use zerokms_protocol::Name; use stack_guest_abi::last_error::malformed; +use vitaminc_aead_frame::serde::from_slice; use crate::status::fail_error; #[cfg(any(doc, test))] @@ -88,84 +87,133 @@ pub struct Options { pub context: Option