diff --git a/Cargo.lock b/Cargo.lock index 89550f2f36..a9586ea2e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6594,9 +6594,11 @@ dependencies = [ name = "trusted-server-adapter-axum" version = "0.1.0" dependencies = [ + "anyhow", "async-trait", "axum", "base64", + "bytes", "edgezero-adapter-axum", "edgezero-core", "error-stack", @@ -6606,6 +6608,7 @@ dependencies = [ "serde_json", "simple_logger", "temp-env", + "tempfile", "tokio", "tower 0.4.13", "trusted-server-ad-server-iabtl", @@ -6633,6 +6636,7 @@ dependencies = [ "trusted-server-permission-signal-mtm", "trusted-server-permission-signal-tcf", "trusted-server-permission-signal-us-privacy", + "trusted-server-provenance-encypher", "trusted-server-shared-aws", "trusted-server-tag-google-tag", "trusted-server-tag-google-tag-manager", @@ -6683,6 +6687,7 @@ dependencies = [ "trusted-server-permission-signal-mtm", "trusted-server-permission-signal-tcf", "trusted-server-permission-signal-us-privacy", + "trusted-server-provenance-encypher", "trusted-server-shared-aws", "trusted-server-tag-google-tag", "trusted-server-tag-google-tag-manager", @@ -6739,6 +6744,7 @@ dependencies = [ "trusted-server-permission-signal-mtm", "trusted-server-permission-signal-tcf", "trusted-server-permission-signal-us-privacy", + "trusted-server-provenance-encypher", "trusted-server-shared-aws", "trusted-server-tag-google-tag", "trusted-server-tag-google-tag-manager", @@ -6791,6 +6797,7 @@ dependencies = [ "trusted-server-permission-signal-mtm", "trusted-server-permission-signal-tcf", "trusted-server-permission-signal-us-privacy", + "trusted-server-provenance-encypher", "trusted-server-shared-aws", "trusted-server-tag-google-tag", "trusted-server-tag-google-tag-manager", @@ -7017,6 +7024,7 @@ dependencies = [ "trusted-server-identity-lockr", "trusted-server-loading-cache-51degrees", "trusted-server-middleware-51degrees", + "trusted-server-provenance-encypher", "trusted-server-shared-aws", "trusted-server-tag-google-tag", "trusted-server-tag-google-tag-manager", @@ -7349,6 +7357,27 @@ dependencies = [ "trusted-server-permission-signal-gpc", ] +[[package]] +name = "trusted-server-provenance-encypher" +version = "0.1.0" +dependencies = [ + "async-trait", + "bytes", + "edgezero-core", + "error-stack", + "futures", + "http", + "log", + "regex", + "serde", + "serde_json", + "sha2 0.10.9", + "trusted-server-core", + "url", + "validator", + "web-time", +] + [[package]] name = "trusted-server-shared-51degrees" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 4f1e312212..1f27b93efb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ members = [ "crates/ads-txt/iabtl", "crates/analytics/tinybird", "crates/headers/geo", + "crates/provenance/encypher", "crates/loading-cache/51degrees", "crates/shared/aws", "crates/shared/51degrees", @@ -176,6 +177,7 @@ trusted-server-cmp-inmobi = { path = "crates/cmp/inmobi" } trusted-server-ads-txt-iabtl = { path = "crates/ads-txt/iabtl" } trusted-server-analytics-tinybird = { path = "crates/analytics/tinybird" } trusted-server-headers-geo = { path = "crates/headers/geo" } +trusted-server-provenance-encypher = { path = "crates/provenance/encypher" } trusted-server-loading-cache-51degrees = { path = "crates/loading-cache/51degrees" } trusted-server-shared-aws = { path = "crates/shared/aws" } trusted-server-shared-51degrees = { path = "crates/shared/51degrees" } diff --git a/PRDs/ARCHIVE/trusted-server-v2-encypher.md b/PRDs/ARCHIVE/trusted-server-v2-encypher.md new file mode 100644 index 0000000000..331b8a32e0 --- /dev/null +++ b/PRDs/ARCHIVE/trusted-server-v2-encypher.md @@ -0,0 +1,133 @@ +# Trusted Server v2: Encypher provenance as a module + +## Status + +Complete, as a draft pull request: +https://github.com/IABTechLab/trusted-server/pull/1448, branch +`encypher/v2-provenance-module` on the `erik-sv` fork, based on +`feature/55-scripts-in-crates` at 5be114773 (row 55 of the stack tracked in +#777). Signing against the live Encypher API has not been exercised. + +## Current Goal + +None open. The port is implemented, verified locally and published for review. + +## Overview + +Encypher's text provenance contribution signed article pages from inside core, +through an HTML stream processor factory, a request context type and core +changes of its own. The stack in #777 moves every vendor out of core and lets a +module change a page only through a middleware. This work moves the +contribution onto those interfaces as `crates/provenance/encypher` +(`provenance.encypher`) and adds the neutral core seams it needs. Design: +`docs/superpowers/specs/2026-10-24-encypher-provenance-module-design.md`. + +## Objectives + +1. Port every existing Encypher behavior (gates, policy and plan caches, sign + call, render, fail-open, logging) to a vendor-owned module crate. +2. Add only vendor-neutral core seams: an awaited asynchronous finish for + middleware stream processors, an optional platform timer, response facts + for middleware, runtime KV on the adapters that lacked it. +3. Remove obsolete paths in the port: the `enabled` flag, `path_prefixes`, + `[integrations.encypher]`, the stream processor factory, the shared-template + refusal and the synchronous publisher body path. +4. Keep the permission contract: no permission required, no gated value read, + no page the origin marks private sent. +5. Publish a reviewable pull request against the stack head without changing + any upstream branch. + +## Work Breakdown + +- [x] Inspect epic #777, PR #1423 head and the existing Encypher branches +- [x] Create an isolated worktree and branch from 5be114773 +- [x] `PlatformTimer` on `RuntimeServices`, with neutral documentation +- [x] Fastly opens the KV store linked as `trusted_server_kv` +- [x] `StreamProcessor::async_finish` and `finish`, composed in order by every + composite processor +- [x] Every awaiting publisher path ends its processor with `finish`; the + synchronous `stream_publisher_body` path removed +- [x] `MiddlewareContext::status` and `origin_private`, read before core's + cache policy +- [x] Axum runtime KV from `TRUSTED_SERVER_KV_PATH`, on the blocking pool +- [x] Timers on Cloudflare, Spin and Axum +- [x] `crates/provenance/encypher`: config, secret setting, middleware, + ported processor and tests, publisher-path tests +- [x] Module offered on all four adapters and the CLI +- [x] Auction-hold regression test with an asynchronously finished document +- [x] Documentation: configuration guide, Axum guide, example config, design +- [x] Data-handling wording narrowed to what the gates prove +- [x] All repository CI gates run locally +- [x] Axum runtime smoke with a controlled origin +- [x] Draft pull request opened and its body updated with evidence + +## Success Criteria + +| Criterion | Result | +| ----------------------------------------- | ----------------------------------------------------------------------------------------- | +| Signing works through the publisher path | Met in process: buffered and streaming finalizers sign against a stub origin and stub API | +| Every failure passes the page through | Met: ported failure tests; runtime smoke `reason=http_error` with the page intact | +| A private origin page is never sent | Met: in-process test and runtime smoke `gate=origin_private` | +| Core platform-name neutrality lint passes | Met: `scripts/lint-core-platform-names.py` exit 0 | +| CI gates green | Met locally, see Completion Notes | +| Live API signing | Not met: not exercised, see limits | + +## Completion Notes + +Tests were ported from the earlier Encypher branches and written alongside the +port. The work was not strictly test-first, and no baseline run of the stack +was recorded before the change. + +Commands run in the worktree, with exit codes: + +- `cargo fmt --all -- --check`: 0 +- `cargo clippy-fastly`, `clippy-axum`, `clippy-cloudflare`, + `clippy-cloudflare-wasm`, `clippy-spin-native`, `clippy-spin-wasm`, + `clippy-cli`, `clippy-codegen`: 0 each +- `cargo test -p trusted-server-provenance-encypher`: 65 passed +- `cargo test -p trusted-server-core --lib`: 3020 passed; the auction-hold + regression test added after it: 1 passed +- `cargo test-fastly`, `test-axum`, `test-cloudflare`, `test-spin`: no + failures (run before the rustfmt-only change and the auction-hold test) +- `cargo check-cloudflare`, `check-spin`, `build-fastly`: no errors +- `cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity`: 0 +- `scripts/lint-wasm-clock.sh`: 0 +- `python3 scripts/lint-core-platform-names.py`: 0 +- `cd crates/trusted-server-js/lib && npx vitest run`: 0; `npm run format`: 0 +- `cd docs && npm run format`: 0; prettier check of markdown outside `docs/`: 0 +- `cargo test -p trusted-server-core --test template_build_digest`: 0; its + clippy with `-D warnings`: 0 +- docs-proposal: `uvx --from shellcheck-py shellcheck scripts/docs-proposal/*.sh` + (shellcheck is not installed on the host): 0; + `node --test scripts/docs-proposal/hunks.test.mjs`: 0; + `scripts/docs-proposal/test.sh`: 0 + +Runtime smoke, Axum dev server, throwaway script and mock origin, no real +credentials. The module was selected with a `[[serve]]` entry on `/news/`, +runtime KV on, and `[geo] assume_single_jurisdiction = true`, which the stack +requires once `[ec] module` is set. Observed: + +```text +[edgezero] config validate (typed): edgezero.toml + trusted-server.toml OK (strict) +page=public status=200 harbor_text=1 marker=0 +page=private status=200 harbor_text=1 marker=0 +INFO [trusted_server_provenance_encypher] encypher: registered for domain=publisher.example.com +INFO [trusted_server_provenance_encypher::processor] encypher: outcome=passed_through reason=http_error +DEBUG [trusted_server_provenance_encypher] encypher: document not held, gate=origin_private +``` + +`scripts/smoke-axum.sh` itself does not set the `[geo]` value and stops at +that check. This branch leaves `scripts/` and the core settings, config and +Edge Cookie code unchanged from 5be114773; the script was not run on the base +commit. + +Limits: + +- Live API: the runtime smoke pointed the module at a signing endpoint that + was not available, to exercise fail-open. Successful signing was tested in + process only, against a stub API. The live Encypher API was not exercised. +- Personalized pages: the module skips pages whose origin set a cookie or + forbade shared caching, and cannot detect others. A deployment places it on + public article routes. +- Spin's declared KV store also holds the settings; the module's keys are + namespaced. diff --git a/crates/middleware/51degrees/src/lib.rs b/crates/middleware/51degrees/src/lib.rs index 4b5a557ef8..bd5bf849f8 100644 --- a/crates/middleware/51degrees/src/lib.rs +++ b/crates/middleware/51degrees/src/lib.rs @@ -658,6 +658,8 @@ mod tests { request_host: "www.example.com", request_scheme: "https", origin_host: "origin.example.com", + status: http::StatusCode::OK, + origin_private: false, platform: test_support::platform(), document_state: state, max_buffered_script_bytes: 1 << 20, diff --git a/crates/provenance/encypher/Cargo.toml b/crates/provenance/encypher/Cargo.toml new file mode 100644 index 0000000000..5c596e1370 --- /dev/null +++ b/crates/provenance/encypher/Cargo.toml @@ -0,0 +1,39 @@ +[package] +name = "trusted-server-provenance-encypher" +description = "Encypher text provenance: signs the article on each page a [[serve]] entry names it for, embedding invisible markers and a C2PA manifest link, and passes every page it cannot sign through unchanged." +authors = { workspace = true } +edition = { workspace = true } +license = { workspace = true } +publish = { workspace = true } +version = { workspace = true } + +[lib] +doctest = false + +[lints] +workspace = true + +[dependencies] +bytes = { workspace = true } +edgezero-core = { workspace = true } +error-stack = { workspace = true } +futures = { workspace = true } +http = { workspace = true } +log = { workspace = true } +regex = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +sha2 = { workspace = true } +trusted-server-core = { workspace = true } +url = { workspace = true } +validator = { workspace = true } +web-time = { workspace = true } + +[dev-dependencies] +async-trait = { workspace = true } +trusted-server-core = { workspace = true, features = ["test-utils"] } + +# The visible owner of this vendor crate. +[package.metadata.maintainers] +owner = "Encypher" +status = "vendor owned" diff --git a/crates/provenance/encypher/src/api.rs b/crates/provenance/encypher/src/api.rs new file mode 100644 index 0000000000..ddf89b92a7 --- /dev/null +++ b/crates/provenance/encypher/src/api.rs @@ -0,0 +1,303 @@ +use serde::{Deserialize, Serialize}; + +/// A 2 MiB cap accommodates roughly 600 KiB surrogate-escaped plans at the 50,000-scalar input limit. +pub const MAX_RESPONSE_BYTES: usize = 2 * 1024 * 1024; +/// Maximum visible-text length sent for signing, measured in Unicode scalars. +pub const MAX_TEXT_SCALARS: usize = 50_000; +/// Policy responses contain only one digest; cap them independently of plans. +pub const MAX_POLICY_RESPONSE_BYTES: usize = 1024; + +/// JSON request body for the Encypher CDN signing endpoint. +#[derive(Serialize)] +pub struct SignRequest<'a> { + /// Canonical visible article text. + pub text: &'a str, + /// Absolute page URL without its query string. + pub page_url: &'a str, + /// Optional page title capped at 500 Unicode scalars. + #[serde(skip_serializing_if = "Option::is_none")] + pub document_title: Option<&'a str>, + /// Boundary detection label used for the article. + pub boundary_selector: &'a str, + /// Server policy used to select this article's cache namespace. + pub expected_policy_digest: &'a str, +} + +/// One invisible-marker insertion returned by the signing endpoint. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct PlanOperation { + /// Visible codepoint index after which to insert, or -1 for before the first. + pub insert_after_index: i64, + /// Invisible marker characters to insert. + pub marker: String, +} + +/// A complete marker embedding plan. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct EmbeddingPlan { + /// Index unit, which must be `codepoint`. + pub index_unit: String, + /// Ordered marker insertion operations. + pub operations: Vec, +} + +/// Parsed response from the Encypher CDN signing endpoint. +#[derive(Debug, Deserialize)] +pub struct SignResponse { + /// Whether signing succeeded. + pub success: bool, + /// Marker plan when signing succeeded. + #[serde(default)] + pub embedding_plan: Option, + /// C2PA manifest URL when signing succeeded. + #[serde(default)] + pub manifest_url: Option, + /// Stable failure reason when signing did not succeed. + #[serde(default)] + pub error: Option, + /// Actual policy used by the server; required before a successful plan is used. + #[serde(default)] + pub policy_digest: Option, +} + +/// Fail-closed signing-response parse errors. +#[derive(Debug, PartialEq)] +pub enum ApiError { + /// The response exceeded [`MAX_RESPONSE_BYTES`]. + TooLarge, + /// The response was not valid JSON for [`SignResponse`]. + Malformed, +} + +/// Parses a bounded signing response. +/// +/// # Errors +/// +/// Returns [`ApiError::TooLarge`] above the response limit and +/// [`ApiError::Malformed`] when JSON decoding fails. +pub fn parse_sign_response(body: &[u8]) -> Result { + if body.len() > MAX_RESPONSE_BYTES { + return Err(ApiError::TooLarge); + } + serde_json::from_slice(body).map_err(|_| ApiError::Malformed) +} + +/// A policy digest is exactly a lowercase SHA-256 hex value. +pub fn valid_policy_digest(digest: &str) -> bool { + digest.len() == 64 + && digest + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +/// Parse the policy endpoint's strict, bounded response. +/// +/// # Errors +/// +/// Rejects oversized JSON, extra or duplicate fields, and invalid digests. +pub fn parse_policy_response(body: &[u8]) -> Result { + #[derive(Deserialize)] + #[serde(deny_unknown_fields)] + struct PolicyResponse { + policy_digest: String, + } + if body.len() > MAX_POLICY_RESPONSE_BYTES { + return Err(ApiError::TooLarge); + } + let response: PolicyResponse = serde_json::from_slice(body).map_err(|_| ApiError::Malformed)?; + if !valid_policy_digest(&response.policy_digest) { + return Err(ApiError::Malformed); + } + Ok(response.policy_digest) +} + +/// Reasons an embedding plan is unsafe or cannot address the visible text. +#[derive(Debug, PartialEq)] +pub enum PlanError { + /// The plan does not use Unicode codepoint indices. + IndexUnit, + /// An insertion index is outside the visible-text range. + IndexOutOfRange, + /// A marker is empty. + EmptyMarker, + /// A marker contains a character outside the edge-core invisible set. + DisallowedMarkerChar, + /// The plan has no operations. + Empty, + /// The manifest URL is present but not an absolute `https` URL. + ManifestUrl, +} + +/// Validates a signing result before use: index bounds and the invisible marker +/// alphabet for every operation, and a manifest URL that is absent or `https`. +/// +/// This is the only check between the API or KV and the page, so callers must +/// pass the scalar count of the exact text that was signed. +/// +/// # Errors +/// +/// Returns a [`PlanError`] for the first invalid property. +pub fn validate_plan( + plan: &EmbeddingPlan, + manifest_url: Option<&str>, + visible_scalars: usize, +) -> Result<(), PlanError> { + if plan.index_unit != "codepoint" { + return Err(PlanError::IndexUnit); + } + if plan.operations.is_empty() { + return Err(PlanError::Empty); + } + for operation in &plan.operations { + if operation.insert_after_index < -1 { + return Err(PlanError::IndexOutOfRange); + } + if operation.insert_after_index >= 0 { + let Ok(index) = usize::try_from(operation.insert_after_index) else { + return Err(PlanError::IndexOutOfRange); + }; + if index >= visible_scalars { + return Err(PlanError::IndexOutOfRange); + } + } + if operation.marker.is_empty() { + return Err(PlanError::EmptyMarker); + } + if !operation.marker.chars().all(is_allowed_marker_char) { + return Err(PlanError::DisallowedMarkerChar); + } + } + if let Some(url) = manifest_url { + let https = url::Url::parse(url).is_ok_and(|url| { + url.scheme() == "https" && url.host_str().is_some_and(|h| !h.is_empty()) + }); + if !https { + return Err(PlanError::ManifestUrl); + } + } + Ok(()) +} + +fn is_allowed_marker_char(ch: char) -> bool { + matches!(ch, '\u{fe00}'..='\u{fe0f}' | '\u{e0100}'..='\u{e01ef}' | '\u{feff}') +} + +#[cfg(test)] +mod tests { + use super::*; + + fn single_op_plan() -> EmbeddingPlan { + EmbeddingPlan { + index_unit: "codepoint".to_string(), + operations: vec![PlanOperation { + insert_after_index: 0, + marker: "\u{fe00}".to_string(), + }], + } + } + + #[test] + fn policy_response_requires_only_a_lowercase_sha256_digest() { + let digest = "ab".repeat(32); + let body = format!(r#"{{"policy_digest":"{digest}"}}"#); + assert_eq!(parse_policy_response(body.as_bytes()), Ok(digest.clone())); + for body in [ + "{}".to_owned(), + r#"{"policy_digest":null}"#.to_owned(), + r#"{"policy_digest":42}"#.to_owned(), + format!(r#"{{"policy_digest":"{}"}}"#, digest.to_uppercase()), + format!(r#"{{"policy_digest":"{}"}}"#, "g".repeat(64)), + format!(r#"{{"policy_digest":"{}"}}"#, "a".repeat(63)), + format!(r#"{{"policy_digest":"{digest}","extra":true}}"#), + format!(r#"{{"policy_digest":"{digest}","policy_digest":"{digest}"}}"#), + format!(r#"{{"policy_digest":"{digest}"}} trailing"#), + ] { + assert_eq!( + parse_policy_response(body.as_bytes()), + Err(ApiError::Malformed) + ); + } + assert_eq!( + parse_policy_response(&vec![b' '; MAX_POLICY_RESPONSE_BYTES + 1]), + Err(ApiError::TooLarge) + ); + } + + #[test] + fn parses_and_validates_a_response_plan() { + let body = r#"{"success":true,"embedding_plan":{"index_unit":"codepoint","operations":[{"insert_after_index":0,"marker":"︀"}]},"manifest_url":"https://manifest.example.com/x"}"#; + let response = parse_sign_response(body.as_bytes()).expect("should parse"); + let plan = response.embedding_plan.expect("should contain plan"); + assert_eq!( + validate_plan(&plan, response.manifest_url.as_deref(), 1), + Ok(()) + ); + } + + #[test] + fn manifest_url_must_be_absent_or_https() { + let plan = single_op_plan(); + assert_eq!(validate_plan(&plan, None, 1), Ok(())); + for url in [ + "", + "http://manifest.example.com/x", + "javascript:alert(1)", + "data:text/html,", + "/api/v1/public/cdn/manifest/1", + "https://", + ] { + assert_eq!( + validate_plan(&plan, Some(url), 1), + Err(PlanError::ManifestUrl), + "{url} should be rejected" + ); + } + } + + #[test] + fn rejects_markup_and_bad_ranges() { + let markup = EmbeddingPlan { + index_unit: "codepoint".to_string(), + operations: vec![PlanOperation { + insert_after_index: 0, + marker: "".to_string(), + }], + }; + assert_eq!( + validate_plan(&markup, None, 1), + Err(PlanError::DisallowedMarkerChar) + ); + assert_eq!( + validate_plan(&single_op_plan(), None, 0), + Err(PlanError::IndexOutOfRange) + ); + } + + #[test] + fn parses_realistic_large_marker_response() { + let escaped_marker = r"\uDB40\uDD00".repeat(50_000); + let body = format!( + r#"{{"success":true,"embedding_plan":{{"index_unit":"codepoint","operations":[{{"insert_after_index":0,"marker":"{escaped_marker}"}}]}}}}"# + ); + assert!(body.len() > 600_000); + let response = + parse_sign_response(body.as_bytes()).expect("large valid response should parse"); + let plan = response + .embedding_plan + .expect("response should contain a plan"); + assert_eq!(plan.operations[0].marker.chars().count(), 50_000); + } + + #[test] + fn enforces_size_and_empty_plan_limits() { + assert_eq!( + parse_sign_response(&vec![b' '; MAX_RESPONSE_BYTES + 1]).map(|_| ()), + Err(ApiError::TooLarge) + ); + let empty = EmbeddingPlan { + index_unit: "codepoint".to_string(), + operations: Vec::new(), + }; + assert_eq!(validate_plan(&empty, None, 1), Err(PlanError::Empty)); + } +} diff --git a/crates/provenance/encypher/src/boundary.rs b/crates/provenance/encypher/src/boundary.rs new file mode 100644 index 0000000000..b2b78c749d --- /dev/null +++ b/crates/provenance/encypher/src/boundary.rs @@ -0,0 +1,272 @@ +use std::sync::LazyLock; + +use regex::Regex; + +const CMS_CONTENT_CLASSES: [&str; 13] = [ + "entry-content", + "wp-block-post-content", + "post-content", + "article-body", + "article-content", + "story-body", + "story-body-text", + "gh-content", + "w-richtext", + "sqs-block-content", + "body markup", + "e-content", + "RichTextStoryBody", +]; + +static ARTICLE: LazyLock> = LazyLock::new(|| tag_opening("article")); +static MAIN: LazyLock> = LazyLock::new(|| tag_opening("main")); +static BODY: LazyLock> = LazyLock::new(|| tag_opening("body")); +static CLUSTER_CONTAINERS: LazyLock> = LazyLock::new(|| { + ["div", "section", "main"] + .into_iter() + .filter_map(tag_opening) + .collect() +}); +static PARAGRAPH: LazyLock> = LazyLock::new(|| Regex::new(r"(?i)> = LazyLock::new(|| { + CMS_CONTENT_CLASSES + .into_iter() + .filter_map(|class_name| Some((class_name, class_opening(class_name)?))) + .collect() +}); +static ITEMPROP_ARTICLE_BODY: LazyLock> = + LazyLock::new(|| attribute_opening("itemprop", "articleBody")); +static ROLE_MAIN: LazyLock> = LazyLock::new(|| attribute_opening("role", "main")); +static SELECTOR_ATTRIBUTE: LazyLock> = + LazyLock::new(|| Regex::new(r#"\[([A-Za-z0-9_]+)=["']?([^"'\]]+)["']?\]"#).ok()); + +/// The UTF-8 byte range and detection label of an article's outer HTML. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Boundary { + /// Inclusive byte offset of the opening element. + pub start: usize, + /// Exclusive byte offset after the first matching closing element. + pub end: usize, + /// Detection-chain selector label. + pub selector: String, +} + +/// Finds an article boundary using the edge-core priority chain, excluding JSON-LD P5. +#[must_use] +pub fn find_article_boundary(html: &str, override_selector: Option<&str>) -> Option { + if let Some(selector) = override_selector.filter(|selector| !selector.is_empty()) + && let Some(range) = match_selector(html, selector) + { + return Some(Boundary { + start: range.0, + end: range.1, + selector: format!("override:{selector}"), + }); + } + + let articles = ARTICLE + .as_ref() + .map_or_else(Vec::new, |opening| match_all(html, opening)); + if let Some(first) = articles.first().copied() { + let mut selected = first; + let mut selected_count = count_paragraphs(html.get(first.0..first.1).unwrap_or("")); + for candidate in articles.iter().copied().skip(1) { + let candidate_count = + count_paragraphs(html.get(candidate.0..candidate.1).unwrap_or("")); + if candidate_count > selected_count { + selected = candidate; + selected_count = candidate_count; + } + } + return Some(boundary(selected, "article")); + } + + for (class_name, opening) in CMS_CLASSES.iter() { + if let Some(range) = match_first(html, opening) { + return Some(boundary(range, &format!("class:{class_name}"))); + } + } + + if let Some(range) = match_fixed(html, &ITEMPROP_ARTICLE_BODY) { + return Some(boundary(range, "itemprop:articleBody")); + } + if let Some(range) = match_fixed(html, &MAIN) { + return Some(boundary(range, "main")); + } + if let Some(range) = match_fixed(html, &ROLE_MAIN) { + return Some(boundary(range, "role:main")); + } + if let Some(range) = find_largest_p_cluster(html) { + return Some(boundary(range, "p-cluster")); + } + match_fixed(html, &BODY).map(|range| boundary(range, "body")) +} + +fn boundary((start, end): (usize, usize), selector: &str) -> Boundary { + Boundary { + start, + end, + selector: selector.to_owned(), + } +} + +// Every opening pattern captures the element's tag name as group 1. +fn tag_opening(tag_name: &str) -> Option { + Regex::new(&format!( + r"(?i)<({})(?-u:\b)[^>]*>", + regex::escape(tag_name) + )) + .ok() +} + +fn class_opening(class_name: &str) -> Option { + Regex::new(&format!( + r#"(?i)<([A-Za-z0-9_]+)(?-u:\b)[^>]*class\s*=\s*["'][^"']*(?-u:\b){}(?-u:\b)[^"']*["'][^>]*>"#, + regex::escape(class_name) + )) + .ok() +} + +fn attribute_opening(attribute: &str, value: &str) -> Option { + Regex::new(&format!( + r#"(?i)<([A-Za-z0-9_]+)(?-u:\b)[^>]*{}\s*=\s*["']{}["'][^>]*>"#, + regex::escape(attribute), + regex::escape(value) + )) + .ok() +} + +fn match_fixed(html: &str, opening: &LazyLock>) -> Option<(usize, usize)> { + match_first(html, opening.as_ref()?) +} + +fn match_first(html: &str, opening: &Regex) -> Option<(usize, usize)> { + let captures = opening.captures(html)?; + let whole = captures.get(0)?; + close_first_element(html, whole.start(), whole.end(), captures.get(1)?.as_str()) +} + +fn match_all(html: &str, opening: &Regex) -> Vec<(usize, usize)> { + let mut matches = Vec::new(); + let mut cursor = 0_usize; + while let Some(tail) = html.get(cursor..) { + let Some(captures) = opening.captures(tail) else { + break; + }; + let (Some(whole), Some(tag_name)) = (captures.get(0), captures.get(1)) else { + break; + }; + let start = cursor.saturating_add(whole.start()); + let opening_end = cursor.saturating_add(whole.end()); + let Some(range) = close_first_element(html, start, opening_end, tag_name.as_str()) else { + break; + }; + matches.push(range); + cursor = range.1; + } + matches +} + +// An operator-configured selector is compiled per document. +fn match_selector(html: &str, selector: &str) -> Option<(usize, usize)> { + if let Some(class_name) = selector.strip_prefix('.') { + return match_first(html, &class_opening(class_name)?); + } + if selector.starts_with('[') + && let Some(captures) = SELECTOR_ATTRIBUTE.as_ref()?.captures(selector) + { + let opening = attribute_opening(captures.get(1)?.as_str(), captures.get(2)?.as_str())?; + return match_first(html, &opening); + } + match_first(html, &tag_opening(selector)?) +} + +/// Extends an opening tag to the first `` after it, ASCII case-insensitively. +fn close_first_element( + html: &str, + opening_start: usize, + opening_end: usize, + tag_name: &str, +) -> Option<(usize, usize)> { + let closing = format!(""); + let tail = html.as_bytes().get(opening_end..)?; + let offset = tail + .windows(closing.len()) + .position(|window| window.eq_ignore_ascii_case(closing.as_bytes()))?; + Some(( + opening_start, + opening_end + .saturating_add(offset) + .saturating_add(closing.len()), + )) +} + +fn count_paragraphs(html: &str) -> usize { + PARAGRAPH + .as_ref() + .map_or(0, |regex| regex.find_iter(html).count()) +} + +fn find_largest_p_cluster(html: &str) -> Option<(usize, usize)> { + let mut best = None; + let mut best_count = 4_usize; + for opening in CLUSTER_CONTAINERS.iter() { + for range in match_all(html, opening) { + let count = count_paragraphs(html.get(range.0..range.1).unwrap_or("")); + if count > best_count { + best = Some(range); + best_count = count; + } + } + } + best +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn follows_priority_chain_and_byte_offsets() { + let html = "🐟

Main.

Chosen.

"; + let boundary = find_article_boundary(html, None).expect("should find article"); + assert_eq!(boundary.selector, "article"); + assert_eq!( + html.get(boundary.start..boundary.end), + Some("

Chosen.

") + ); + } + + #[test] + fn nested_same_name_uses_first_closing_tag() { + let html = "
outer
inner
tail
"; + let boundary = find_article_boundary(html, None).expect("should find article"); + assert_eq!( + html.get(boundary.start..boundary.end), + Some("
outer
inner
") + ); + } + + #[test] + fn closing_tags_match_ascii_case_insensitively() { + let html = "

One.

x
"; + let boundary = find_article_boundary(html, None).expect("should find article"); + assert_eq!( + html.get(boundary.start..boundary.end), + Some("

One.

") + ); + let boundary = + find_article_boundary(html, Some(".entry-content")).expect("should find override"); + assert_eq!( + html.get(boundary.start..boundary.end), + Some("
x
") + ); + } + + #[test] + fn selects_largest_paragraph_cluster_at_five() { + let html = "

1

2

3

4

5

"; + let boundary = find_article_boundary(html, None).expect("should find cluster"); + assert_eq!(boundary.selector, "p-cluster"); + } +} diff --git a/crates/provenance/encypher/src/embed.rs b/crates/provenance/encypher/src/embed.rs new file mode 100644 index 0000000000..b5195ba58b --- /dev/null +++ b/crates/provenance/encypher/src/embed.rs @@ -0,0 +1,477 @@ +use std::sync::LazyLock; + +use regex::Regex; + +use super::api::EmbeddingPlan; +use super::boundary::Boundary; +use super::text::{ + assemble_text, decode_html_entities, extract_fragments, is_js_whitespace, structural_close_tag, +}; + +const MAGIC_PREFIX: &str = "\u{e01df}\u{e01e0}\u{e01e1}\u{e01e2}"; + +static MANIFEST_META: LazyLock> = LazyLock::new(|| { + Regex::new(r#"(?i)]*name\s*=\s*['"]c2pa-manifest-url['"][^>]*>"#).ok() +}); + +struct MappedChar { + ch: char, + start: usize, + end: usize, +} + +/// Reports whether a string contains the four-character Encypher magic prefix. +#[must_use] +pub fn has_existing_markers(s: &str) -> bool { + s.contains(MAGIC_PREFIX) +} + +/// Reports whether HTML already declares a C2PA manifest URL meta element. +#[must_use] +pub fn has_manifest_meta(html: &str) -> bool { + MANIFEST_META + .as_ref() + .is_some_and(|regex| regex.is_match(html)) +} + +/// Applies a codepoint embedding plan within an article boundary. +/// +/// The plan must already have passed [`super::api::validate_plan`]; markers are +/// inserted verbatim. Returns `None` when the boundary has no visible text, an +/// index has no text position, or text-to-byte alignment fails. +#[must_use] +pub fn apply_plan(html: &str, boundary: &Boundary, plan: &EmbeddingPlan) -> Option { + let boundary_html = html.get(boundary.start..boundary.end)?; + let fragments = extract_fragments(boundary_html); + let visible = assemble_text(boundary_html, &fragments); + let visible_chars: Vec = visible.chars().collect(); + if visible_chars.is_empty() { + return None; + } + + let mut insertion_offsets = vec![None; visible_chars.len()]; + let mut first_visible_start = None; + let mut previous_visible_end = None; + let mut visible_cursor = 0_usize; + + for fragment in fragments { + let raw = boundary_html.get(fragment.start..fragment.end)?; + let normalized = normalize_mapped(decode_with_offsets(raw)); + if normalized.is_empty() { + continue; + } + + while visible_chars.get(visible_cursor) == Some(&' ') { + if let Some(offset) = previous_visible_end + && let Some(slot) = insertion_offsets.get_mut(visible_cursor) + { + *slot = Some(offset); + } + visible_cursor = visible_cursor.saturating_add(1); + } + + for item in normalized { + let target = visible_chars.get(visible_cursor)?; + if !equivalent(item.ch, *target) { + return None; + } + let start = fragment.start.checked_add(item.start)?; + let end = fragment.start.checked_add(item.end)?; + if first_visible_start.is_none() { + first_visible_start = Some(start); + } + let slot = insertion_offsets.get_mut(visible_cursor)?; + *slot = Some(end); + previous_visible_end = Some(end); + visible_cursor = visible_cursor.saturating_add(1); + } + } + + while visible_chars.get(visible_cursor) == Some(&' ') { + if let Some(offset) = previous_visible_end { + let slot = insertion_offsets.get_mut(visible_cursor)?; + *slot = Some(offset); + } + visible_cursor = visible_cursor.saturating_add(1); + } + if visible_cursor != visible_chars.len() { + return None; + } + let first_visible_start = first_visible_start?; + + let mut insertions = Vec::with_capacity(plan.operations.len()); + let mut output_len = html.len(); + for operation in &plan.operations { + let relative = if operation.insert_after_index == -1 { + first_visible_start + } else { + let index = usize::try_from(operation.insert_after_index).ok()?; + insertion_offsets.get(index).copied().flatten()? + }; + let absolute = boundary.start.checked_add(relative)?; + output_len = output_len.checked_add(operation.marker.len())?; + insertions.push((absolute, operation.marker.as_str())); + } + // Stable sorting preserves API order for multiple markers at the same offset. + insertions.sort_by_key(|(offset, _)| *offset); + let mut result = String::with_capacity(output_len); + let mut cursor = 0; + for (offset, marker) in insertions { + result.push_str(html.get(cursor..offset)?); + result.push_str(marker); + cursor = offset; + } + result.push_str(html.get(cursor..)?); + Some(result) +} + +/// Injects an escaped manifest meta element before the structural closing head tag. +/// +/// Returns the input unchanged when the URL is empty, a manifest meta already exists, +/// or no structural closing head tag exists. +#[must_use] +pub fn inject_manifest_meta(html: &str, manifest_url: &str) -> String { + if html.is_empty() || manifest_url.is_empty() || has_manifest_meta(html) { + return html.to_owned(); + } + let Some(offset) = structural_close_tag(html, "head") else { + return html.to_owned(); + }; + let safe_url = manifest_url + .replace('&', "&") + .replace('"', """) + .replace('<', "<") + .replace('>', ">"); + let meta = format!(r#""#); + let Some(before) = html.get(..offset) else { + return html.to_owned(); + }; + let Some(after) = html.get(offset..) else { + return html.to_owned(); + }; + let mut output = String::with_capacity(html.len().saturating_add(meta.len())); + output.push_str(before); + output.push_str(&meta); + output.push_str(after); + output +} + +fn decode_with_offsets(raw: &str) -> Vec { + let mut mapped = Vec::new(); + let mut cursor = 0_usize; + while cursor < raw.len() { + if raw.as_bytes().get(cursor) == Some(&b'&') + && let Some(relative_semicolon) = raw + .as_bytes() + .get(cursor.saturating_add(1)..) + // Entity names/numbers cannot contain another ampersand or markup. + // Stop there rather than rescanning the suffix for every literal '&'. + .and_then(|tail| { + let end = tail + .iter() + .position(|byte| !byte.is_ascii_alphanumeric() && *byte != b'#')?; + (tail[end] == b';').then_some(end) + }) + { + let end = cursor + .saturating_add(1) + .saturating_add(relative_semicolon) + .saturating_add(1); + if let Some(token) = raw.get(cursor..end) { + let decoded = decode_html_entities(token); + if decoded != token { + mapped.extend(decoded.chars().map(|ch| MappedChar { + ch, + start: cursor, + end, + })); + cursor = end; + continue; + } + } + } + + let Some(ch) = raw.get(cursor..).and_then(|tail| tail.chars().next()) else { + break; + }; + let end = cursor.saturating_add(ch.len_utf8()); + mapped.push(MappedChar { + ch, + start: cursor, + end, + }); + cursor = end; + } + mapped +} + +fn normalize_mapped(mapped: Vec) -> Vec { + let mut normalized: Vec = Vec::with_capacity(mapped.len()); + for item in mapped { + if is_js_whitespace(item.ch) { + let Some(last) = normalized.last_mut() else { + continue; + }; + if last.ch == ' ' { + last.end = item.end; + } else { + normalized.push(MappedChar { + ch: ' ', + start: item.start, + end: item.end, + }); + } + } else { + normalized.push(item); + } + } + while normalized.last().is_some_and(|item| item.ch == ' ') { + normalized.pop(); + } + normalized +} + +fn equivalent(left: char, right: char) -> bool { + left == right || matches!((left, right), ('\u{00a0}', ' ') | (' ', '\u{00a0}')) +} + +#[cfg(test)] +mod tests { + use serde::Deserialize; + + use super::*; + use crate::api::{EmbeddingPlan, PlanOperation}; + use crate::boundary::{Boundary, find_article_boundary}; + use crate::text::{assemble_text, extract_fragments}; + + #[test] + fn adversarial_ampersands_have_bounded_embedding_work() { + let html = format!("
{}
", "&".repeat(100_000)); + let boundary = find_article_boundary(&html, None).expect("article"); + let plan = EmbeddingPlan { + index_unit: "codepoint".to_owned(), + operations: vec![PlanOperation { + insert_after_index: 0, + marker: "\u{fe00}".to_owned(), + }], + }; + let output = apply_plan(&html, &boundary, &plan).expect("valid plan"); + assert_eq!( + output, + format!("
&\u{fe00}{}
", "&".repeat(99_999)) + ); + } + + #[test] + fn dense_plan_on_large_document_preserves_operation_order() { + let html = format!( + "
{}
", + "a".repeat(50_000), + "x".repeat(4_900_000) + ); + let boundary = find_article_boundary(&html, None).expect("article"); + let plan = EmbeddingPlan { + index_unit: "codepoint".to_owned(), + operations: (0..20_000) + .rev() + .map(|index| PlanOperation { + insert_after_index: index, + marker: "\u{fe00}".to_owned(), + }) + .collect(), + }; + let output = apply_plan(&html, &boundary, &plan).expect("valid dense plan"); + assert_eq!( + output, + format!( + "
{}{}
", + "a\u{fe00}".repeat(20_000), + "a".repeat(30_000), + "x".repeat(4_900_000) + ) + ); + } + + #[derive(Deserialize)] + struct FixtureFile { + cases: Vec, + } + + #[derive(Deserialize)] + struct FixtureCase { + id: String, + html: String, + override_selector: Option, + boundary: FixtureBoundary, + text: String, + text_rust: String, + text_divergence: Option, + plan: EmbeddingPlan, + expected_output: String, + invalid_plan: Option, + expected_invalid_output: Option, + manifest_url: String, + expected_meta_output_js: String, + expected_meta_output_rust: String, + meta_divergence: Option, + } + + #[derive(Deserialize)] + struct FixtureBoundary { + selector: String, + start: usize, + end: usize, + } + + fn fixtures() -> FixtureFile { + serde_json::from_str(include_str!("fixtures/parity.json")) + .expect("parity fixtures should deserialize") + } + + #[test] + fn groups_same_offset_in_plan_order() { + let html = "🐟
A B
"; + let start = html.find("
").expect("should find boundary"); + let boundary = Boundary { + start, + end: html.len(), + selector: "article".to_string(), + }; + let plan = EmbeddingPlan { + index_unit: "codepoint".to_string(), + operations: vec![ + PlanOperation { + insert_after_index: 0, + marker: "\u{fe00}".to_string(), + }, + PlanOperation { + insert_after_index: 0, + marker: "\u{fe01}".to_string(), + }, + ], + }; + assert_eq!( + apply_plan(html, &boundary, &plan), + Some("🐟
A\u{fe00}\u{fe01} B
".to_string()) + ); + } + + #[test] + fn manifest_meta_uses_structural_head_close_and_escapes() { + let html = ""; + let output = inject_manifest_meta(html, "https://example.com/?a=1&b=\"x\""); + assert!(output.contains(""#; + let html = format!("{prefix}
Visible
"); + assert_eq!( + inject_manifest_meta(&html, "javascript:alert(1)"), + format!( + "{prefix}
Visible
" + ) + ); + // The URL is inert metadata, not an href, script source or refresh target. + let script = + r#""#; + assert_eq!( + inject_manifest_meta( + &format!("{script}"), + "data:text/html," + ), + format!( + "{script}" + ) + ); + } + + #[test] + fn manifest_meta_ignores_quotes_inside_unquoted_values() { + // The browser reads `x"y` as an unquoted value, so the second meta's quoted + // content, including its ``, is attribute text. + let prefix = + r#""#; + let html = format!("{prefix}Visible"); + assert_eq!( + inject_manifest_meta(&html, "https://manifest.example.com/m"), + format!( + "{prefix}Visible" + ) + ); + } + + #[test] + fn edge_core_parity_fixtures_match() { + for fixture in fixtures().cases { + let boundary = + find_article_boundary(&fixture.html, fixture.override_selector.as_deref()) + .unwrap_or_else(|| panic!("{} should have a boundary", fixture.id)); + assert_eq!( + boundary.selector, fixture.boundary.selector, + "{}", + fixture.id + ); + assert_eq!(boundary.start, fixture.boundary.start, "{}", fixture.id); + assert_eq!(boundary.end, fixture.boundary.end, "{}", fixture.id); + let boundary_html = fixture + .html + .get(boundary.start..boundary.end) + .expect("fixture boundary should be valid"); + let fragments = extract_fragments(boundary_html); + let text = assemble_text(boundary_html, &fragments); + assert_eq!(text, fixture.text_rust, "{}", fixture.id); + if fixture.text_divergence.is_some() { + assert_ne!(text, fixture.text, "{}", fixture.id); + } else { + assert_eq!(text, fixture.text, "{}", fixture.id); + } + assert_eq!( + apply_plan(&fixture.html, &boundary, &fixture.plan), + Some(fixture.expected_output), + "{}", + fixture.id + ); + if let Some(invalid_plan) = fixture.invalid_plan { + assert_eq!( + apply_plan(&fixture.html, &boundary, &invalid_plan), + fixture.expected_invalid_output, + "{}", + fixture.id + ); + } + let meta = inject_manifest_meta(&fixture.html, &fixture.manifest_url); + assert_eq!(meta, fixture.expected_meta_output_rust, "{}", fixture.id); + if fixture.meta_divergence.is_some() { + assert_ne!(meta, fixture.expected_meta_output_js, "{}", fixture.id); + } else { + assert_eq!(meta, fixture.expected_meta_output_js, "{}", fixture.id); + } + } + } + + #[test] + fn every_fixture_byte_prefix_is_panic_free() { + for fixture in fixtures().cases { + for end in 0..=fixture.html.len() { + let prefix = String::from_utf8_lossy( + fixture + .html + .as_bytes() + .get(..end) + .expect("prefix range should be valid"), + ); + let fragments = extract_fragments(&prefix); + let _ = assemble_text(&prefix, &fragments); + if let Some(boundary) = + find_article_boundary(&prefix, fixture.override_selector.as_deref()) + { + let _ = apply_plan(&prefix, &boundary, &fixture.plan); + } + let _ = inject_manifest_meta(&prefix, &fixture.manifest_url); + } + } + } +} diff --git a/crates/provenance/encypher/src/fixtures/parity.json b/crates/provenance/encypher/src/fixtures/parity.json new file mode 100644 index 0000000000..8487676317 --- /dev/null +++ b/crates/provenance/encypher/src/fixtures/parity.json @@ -0,0 +1,848 @@ +{ + "generated_from": [ + "integrations/edge-core/boundary.js", + "integrations/edge-core/fragments.js", + "integrations/edge-core/embed.js" + ], + "fixed_plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "cases": [ + { + "id": "override-multibyte-prefix", + "html": "Café & News🙂前

Alpha & beta.

", + "override_selector": "[data=story]", + "boundary": { + "selector": "override:[data=story]", + "start": 78, + "end": 126 + }, + "text": "Alpha & beta.", + "text_rust": "Alpha & beta.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "Café & News🙂前

A︀︁lpha & beta.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "Café & News🙂前

Alpha & beta.

", + "expected_meta_output_rust": "Café & News🙂前

Alpha & beta.

", + "meta_divergence": null + }, + { + "id": "article-most-paragraphs", + "html": "Two articles

Short.

First paragraph.

Second paragraph.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 84, + "end": 159 + }, + "text": "First paragraph. Second paragraph .", + "text_rust": "First paragraph. Second paragraph .", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "Two articles

Short.

F︀︁irst paragraph.

Second paragraph.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "Two articles

Short.

First paragraph.

Second paragraph.

", + "expected_meta_output_rust": "Two articles

Short.

First paragraph.

Second paragraph.

", + "meta_divergence": null + }, + { + "id": "cms-class-entities", + "html": "

Fish & chips — © 🐟.

", + "override_selector": null, + "boundary": { + "selector": "class:entry-content", + "start": 25, + "end": 123 + }, + "text": "Fish & chips — © 🐟.", + "text_rust": "Fish & chips — © 🐟.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

F︀︁ish & chips — © 🐟.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Fish & chips — © 🐟.

", + "expected_meta_output_rust": "

Fish & chips — © 🐟.

", + "meta_divergence": null + }, + { + "id": "itemprop-nbsp", + "html": "

One two three.

", + "override_selector": null, + "boundary": { + "selector": "itemprop:articleBody", + "start": 25, + "end": 94 + }, + "text": "One two three.", + "text_rust": "One two three.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

O︀︁ne two three.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

One two three.

", + "expected_meta_output_rust": "

One two three.

", + "meta_divergence": null + }, + { + "id": "main-inline-tags", + "html": "

Inline

Hello fictional example.com.

", + "override_selector": null, + "boundary": { + "selector": "main", + "start": 25, + "end": 105 + }, + "text": "Inline Hello fictional example.com.", + "text_rust": "Inline Hello fictional example.com.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

I︀︁nline

Hello fictional example.com.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Inline

Hello fictional example.com.

", + "expected_meta_output_rust": "

Inline

Hello fictional example.com.

", + "meta_divergence": null + }, + { + "id": "role-main", + "html": "

Role based article boundary.

", + "override_selector": null, + "boundary": { + "selector": "role:main", + "start": 25, + "end": 83 + }, + "text": "Role based article boundary.", + "text_rust": "Role based article boundary.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

R︀︁ole based article boundary.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Role based article boundary.

", + "expected_meta_output_rust": "

Role based article boundary.

", + "meta_divergence": null + }, + { + "id": "paragraph-cluster", + "html": "

One.

Two.

Three.

Four.

Five.

", + "override_selector": null, + "boundary": { + "selector": "p-cluster", + "start": 25, + "end": 109 + }, + "text": "One. Two. Three. Four. Five.", + "text_rust": "One. Two. Three. Four. Five.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

O︀︁ne.

Two.

Three.

Four.

Five.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

One.

Two.

Three.

Four.

Five.

", + "expected_meta_output_rust": "

One.

Two.

Three.

Four.

Five.

", + "meta_divergence": null + }, + { + "id": "body-fallback", + "html": "

Fallback

Body article text.", + "override_selector": null, + "boundary": { + "selector": "body", + "start": 19, + "end": 80 + }, + "text": "Fallback Body article text.", + "text_rust": "Fallback Body article text.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

F︀︁allback

Body article text.", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Fallback

Body article text.", + "expected_meta_output_rust": "

Fallback

Body article text.", + "meta_divergence": null + }, + { + "id": "nested-same-name-first-close", + "html": "

Outer start.

Inner text.

Outer tail omitted by parity.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 90 + }, + "text": "Outer start. Inner text.", + "text_rust": "Outer start. Inner text.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

O︀︁uter start.

Inner text.

Outer tail omitted by parity.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Outer start.

Inner text.

Outer tail omitted by parity.

", + "expected_meta_output_rust": "

Outer start.

Inner text.

Outer tail omitted by parity.

", + "meta_divergence": null + }, + { + "id": "head-close-inside-script", + "html": "Script case

Script boundary fallback.

", + "override_selector": null, + "boundary": { + "selector": "body", + "start": 86, + "end": 131 + }, + "text": "Script boundary fallback.", + "text_rust": "Script boundary fallback.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "Script case

S︀︁cript boundary fallback.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "Script case

Script boundary fallback.

", + "expected_meta_output_rust": "Script case

Script boundary fallback.

", + "meta_divergence": "JS inserts at a closing-head sequence inside script text; Rust inserts before the structural closing head tag." + }, + { + "id": "head-close-inside-comment", + "html": "Comment case

Comment boundary fallback.

", + "override_selector": null, + "boundary": { + "selector": "body", + "start": 80, + "end": 126 + }, + "text": "Comment boundary fallback.", + "text_rust": "Comment boundary fallback.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "Comment case

C︀︁omment boundary fallback.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "Comment case

Comment boundary fallback.

", + "expected_meta_output_rust": "Comment case

Comment boundary fallback.

", + "meta_divergence": "JS inserts at a closing-head sequence inside a comment; Rust inserts before the structural closing head tag." + }, + { + "id": "head-close-inside-title", + "html": "Headline </head> text

Title boundary fallback.

", + "override_selector": null, + "boundary": { + "selector": "body", + "start": 55, + "end": 99 + }, + "text": "Title boundary fallback.", + "text_rust": "Title boundary fallback.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "Headline </head> text

T︀︁itle boundary fallback.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "Headline <meta name=\"c2pa-manifest-url\" content=\"https://manifest.example.com/c2pa?id=fictional&kind="article"\"></head> text

Title boundary fallback.

", + "expected_meta_output_rust": "Headline </head> text

Title boundary fallback.

", + "meta_divergence": "JS inserts at a closing-head sequence inside title text; Rust never searches raw-text or RCDATA content for the closing head tag." + }, + { + "id": "javascript-whitespace-set", + "html": "

Alpha 
beta\t gamma.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 77 + }, + "text": "Alpha beta gamma.", + "text_rust": "Alpha beta gamma.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

A︀︁lpha 
beta\t gamma.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Alpha 
beta\t gamma.

", + "expected_meta_output_rust": "

Alpha 
beta\t gamma.

", + "meta_divergence": null + }, + { + "id": "skip-raw-elements-and-numeric-entities", + "html": "

Visible A.

Hidden code.
Hidden pre.

Tail B.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 162 + }, + "text": "Visible A. Tail B.", + "text_rust": "Visible A. Tail B.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

V︀︁isible A.

Hidden code.
Hidden pre.

Tail B.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Visible A.

Hidden code.
Hidden pre.

Tail B.

", + "expected_meta_output_rust": "

Visible A.

Hidden code.
Hidden pre.

Tail B.

", + "meta_divergence": null + }, + { + "id": "only-the-seven-skip-elements-are-hidden", + "html": "

Lead text.

Inline title.

After.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 166 + }, + "text": "Lead text. Draft note. Frame fallback. Inline title. After.", + "text_rust": "Lead text. Draft note. Frame fallback. Inline title. After.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

L︀︁ead text.

Inline title.

After.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Lead text.

Inline title.

After.

", + "expected_meta_output_rust": "

Lead text.

Inline title.

After.

", + "meta_divergence": null + }, + { + "id": "self-closing-svg-hides-nothing", + "html": "

Lead text.

After icon.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 103 + }, + "text": "Lead text. After icon.", + "text_rust": "Lead text. After icon.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

L︀︁ead text.

After icon.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Lead text.

After icon.

", + "expected_meta_output_rust": "

Lead text.

After icon.

", + "meta_divergence": null + }, + { + "id": "quoted-gt-in-attribute", + "html": "

Lead text.

\"a b\">Tail text.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 95 + }, + "text": "Lead text. b\">Tail text.", + "text_rust": "Lead text. Tail text.", + "text_divergence": "JS ends a tag at the first \">\", so the rest of the alt value becomes text; Rust ends it at the first \">\" outside a quoted attribute value.", + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

L︀︁ead text.

\"a b\">Tail text.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Lead text.

\"a b\">Tail text.

", + "expected_meta_output_rust": "

Lead text.

\"a b\">Tail text.

", + "meta_divergence": null + }, + { + "id": "nested-same-name-ignored-element", + "html": "

Lead text.

outer inner tail

After.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 116 + }, + "text": "Lead text. After.", + "text_rust": "Lead text. tail After.", + "text_divergence": "JS counts nested start and end tags and drops the outer tail; Rust closes the ignored region at the first matching end tag.", + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

L︀︁ead text.

outer inner tail

After.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Lead text.

outer inner tail

After.

", + "expected_meta_output_rust": "

Lead text.

outer inner tail

After.

", + "meta_divergence": null + }, + { + "id": "self-closing-script-stays-raw-text", + "html": "

Lead text.

After.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 107 + }, + "text": "Lead text. var hidden = 1; After.", + "text_rust": "Lead text. After.", + "text_divergence": "JS treats \".", + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

L︀︁ead text.

After.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Lead text.

After.

", + "expected_meta_output_rust": "

Lead text.

After.

", + "meta_divergence": null + }, + { + "id": "end-tag-with-attributes-closes-ignored-element", + "html": "

Lead text.

After.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 132 + }, + "text": "Lead text. After.", + "text_rust": "Lead text. After.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

L︀︁ead text.

After.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Lead text.

After.

", + "expected_meta_output_rust": "

Lead text.

After.

", + "meta_divergence": null + }, + { + "id": "abrupt-and-bang-comment-ends", + "html": "

Lead text.

Shown one.Shown two.Shown three.

After.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 131 + }, + "text": "Lead text. Shown two.", + "text_rust": "Lead text. Shown one. Shown two. Shown three. After.", + "text_divergence": "JS ends every comment at the next \"-->\"; Rust, like browsers, treats \"\" and \"\" as whole comments and also ends a comment at \"--!>\".", + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

L︀︁ead text.

Shown one.Shown two.Shown three.

After.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Lead text.

Shown one.Shown two.Shown three.

After.

", + "expected_meta_output_rust": "

Lead text.

Shown one.Shown two.Shown three.

After.

", + "meta_divergence": null + }, + { + "id": "out-of-range-index", + "html": "

Small boundary.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 66 + }, + "text": "Small boundary.", + "text_rust": "Small boundary.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

S︀︁mall boundary.

", + "invalid_plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": 15, + "marker": "︀" + } + ] + }, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Small boundary.

", + "expected_meta_output_rust": "

Small boundary.

", + "meta_divergence": null + } + ] +} diff --git a/crates/provenance/encypher/src/lib.rs b/crates/provenance/encypher/src/lib.rs new file mode 100644 index 0000000000..aa5ce0b0ee --- /dev/null +++ b/crates/provenance/encypher/src/lib.rs @@ -0,0 +1,354 @@ +//! Encypher text provenance for article pages, `provenance.encypher`. +//! +//! A serve middleware finds the article in each HTML page a `[[serve]]` entry +//! names it for, sends its visible text to the Encypher signing API, and +//! embeds the returned invisible markers and a manifest meta tag. Plans are +//! cached in the runtime KV store, bound to a short-lived server policy. +//! Every failure passes the page through unchanged. +//! +//! The middleware's preparation applies the request gates with no I/O: only +//! a `GET` of a `200` page on the configured domain, whose origin did not +//! mark it as one reader's, is held. The signing work runs in the stream +//! processor's asynchronous finish, which core awaits at the end of the +//! document. The module requires no permission and reads no gated value. It +//! sends the visible text of the page's article and the page address without +//! its query. A page whose origin set a cookie or forbade shared caching is +//! never sent, but the module cannot tell a personalized article that carries +//! neither, so a deployment places it only on public article routes and has +//! its origin mark every personalized response private. + +use std::any::Any; +use std::sync::Arc; + +use error_stack::Report; +use serde::Deserialize; +use validator::{Validate, ValidationError}; + +use trusted_server_core::error::TrustedServerError; +use trusted_server_core::integrations::{ + IntegrationBuilder, IntegrationRegistration, ModuleSecretSetting, SuppliedMiddleware, +}; +use trusted_server_core::middleware::{ + HTML_MEDIA_TYPE, Middleware, MiddlewareAction, MiddlewareContext, MiddlewarePhase, Preparation, +}; +use trusted_server_core::module_context::{FromModuleContext as _, ModuleCall, ModuleRequest}; +use trusted_server_core::redacted::Redacted; +use trusted_server_core::settings::{IntegrationConfig, Settings}; + +mod api; +mod boundary; +mod embed; +mod processor; +mod text; + +use processor::{PageRequest, Signer, document_processor, is_excluded_path}; + +/// The id the module's registration and its API backend carry. +const INTEGRATION_ID: &str = "provenance_encypher"; + +/// The name `[provenance] module` selects the module by and a `[[serve]]` +/// entry names its middleware by, from this crate's folder. +pub const MODULE: &str = "provenance.encypher"; + +/// `[provenance.encypher]`, the module's settings. +#[derive(Debug, Clone, Deserialize, Validate)] +#[serde(deny_unknown_fields)] +#[validate(schema(function = "validate_hold_exceeds_timeout"))] +pub struct EncypherConfig { + /// Encypher API origin, for example `https://api.example.com`. Must be https. + #[validate(custom(function = validate_api_base_url))] + pub api_base_url: String, + /// Claimed publisher domain: a bare lowercase hostname without `www.`. + #[validate(custom(function = validate_domain))] + pub domain: String, + /// The name of the secret-store entry holding the `cwk_` worker + /// credential, resolved to the credential itself when settings load. + pub worker_token: Redacted, + /// Optional article selector override: `.class`, `[attr=value]` or a tag + /// name. At most 91 characters, so `override:{selector}` fits the API's + /// 100-character `boundary_selector` limit. + #[serde(default)] + #[validate(length(max = 91))] + pub article_selector: String, + /// Minimum visible article text, in Unicode scalars. + #[serde(default = "default_min_text_length")] + #[validate(range(min = 50, max = 50_000))] + pub min_text_length: usize, + /// Largest decoded document held for signing. Capped at + /// `publisher.max_buffered_body_bytes`. + #[serde(default = "default_max_document_bytes")] + #[validate(range(min = 1024))] + pub max_document_bytes: usize, + /// First-byte timeout for each policy or sign call, in milliseconds. The + /// only bound on a hung API on a runtime without a timer. + #[serde(default = "default_timeout_ms")] + #[validate(range(min = 250, max = 10_000))] + pub timeout_ms: u64, + /// Longest one document is held for signing, in milliseconds, on a + /// runtime with a timer. Must exceed `timeout_ms`. + #[serde(default = "default_max_hold_ms")] + #[validate(range(min = 500, max = 15_000))] + pub max_hold_ms: u64, + /// Lifetime of a cached signing plan, in seconds. + #[serde(default = "default_plan_ttl_seconds")] + #[validate(range(min = 60, max = 31_536_000))] + pub plan_ttl_seconds: u64, + /// Lifetime of the server-authoritative policy, in seconds. + #[serde(default = "default_policy_ttl_seconds")] + #[validate(range(min = 60, max = 300))] + pub policy_ttl_seconds: u64, + /// Lifetime of a cached signing failure, in seconds. + #[serde(default = "default_negative_ttl_seconds")] + #[validate(range(min = 60, max = 86_400))] + pub negative_ttl_seconds: u64, +} + +impl IntegrationConfig for EncypherConfig {} + +fn default_min_text_length() -> usize { + 50 +} + +fn default_max_document_bytes() -> usize { + 5_000_000 +} + +fn default_timeout_ms() -> u64 { + 3_000 +} + +fn default_max_hold_ms() -> u64 { + 5_000 +} + +fn default_plan_ttl_seconds() -> u64 { + 604_800 +} + +fn default_policy_ttl_seconds() -> u64 { + 300 +} + +fn default_negative_ttl_seconds() -> u64 { + 300 +} + +fn validate_api_base_url(value: &str) -> Result<(), ValidationError> { + let parsed = + url::Url::parse(value).map_err(|_| ValidationError::new("encypher_api_base_url"))?; + if parsed.scheme() != "https" + || parsed.host_str().is_none_or(str::is_empty) + || parsed.query().is_some() + || parsed.fragment().is_some() + { + return Err(ValidationError::new("encypher_api_base_url")); + } + Ok(()) +} + +fn validate_domain(value: &str) -> Result<(), ValidationError> { + let valid_label = |label: &str| { + !label.is_empty() + && label.len() <= 63 + && !label.starts_with('-') + && !label.ends_with('-') + && label + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') + }; + let valid = value.len() <= 253 + && value.contains('.') + && !value.starts_with("www.") + && value.split('.').all(valid_label); + if valid { + Ok(()) + } else { + Err(ValidationError::new("encypher_domain")) + } +} + +fn validate_hold_exceeds_timeout(config: &EncypherConfig) -> Result<(), ValidationError> { + if config.max_hold_ms > config.timeout_ms { + Ok(()) + } else { + Err(ValidationError::new("encypher_max_hold_ms")) + } +} + +/// The worker token, named in `[provenance.encypher]` by its entry in the +/// secret store, which is in use whenever the module is selected. +const SECRET_SETTINGS: &[ModuleSecretSetting] = &[ModuleSecretSetting { + path: &["worker_token"], + in_use: always_in_use, +}]; + +fn always_in_use(_table: &serde_json::Map) -> bool { + true +} + +/// The page change the module supplies, as a `[[serve]]` entry names it. +const SUPPLIED_MIDDLEWARE: &[SuppliedMiddleware] = + &[SuppliedMiddleware::on_html(MODULE, MiddlewarePhase::Serve)]; + +/// The builder a deployment installs to offer this module. +#[must_use] +pub fn builder() -> IntegrationBuilder { + IntegrationBuilder::new(INTEGRATION_ID, env!("CARGO_PKG_NAME"), register, validate) + .with_module_name(MODULE) + .with_secret_settings(SECRET_SETTINGS) + .with_middleware(SUPPLIED_MIDDLEWARE) +} + +/// Refuses a document that selects the module and names its middleware in no +/// `[[serve]]` entry, where no page would ever be signed. +fn require_an_entry(settings: &Settings) -> Result<(), Report> { + let placed = settings + .serve + .entries() + .iter() + .any(|entry| entry.middleware.iter().any(|name| name == MODULE)); + if placed { + return Ok(()); + } + Err(Report::new(TrustedServerError::Settings { + message: format!( + "[provenance] selects `encypher`, but no [[serve]] entry names `{MODULE}`, so no \ + page would be signed. Add `{MODULE}` to the [[serve]] entry that covers the \ + article pages, with a `path` for each section to sign" + ), + })) +} + +/// The module's settings when a section selects it. +fn config(settings: &Settings) -> Result, Report> { + let Some(config) = settings.module_config::(MODULE)? else { + return Ok(None); + }; + require_an_entry(settings)?; + Ok(Some(config)) +} + +/// Validates the module's table and reports whether a section selects the +/// module. +/// +/// # Errors +/// +/// Returns an error when the table cannot be read or fails its validation, or +/// when no `[[serve]]` entry names the middleware. +pub fn validate(settings: &Settings) -> Result> { + Ok(config(settings)?.is_some()) +} + +/// Registers the middleware when a section selects the module and its worker +/// token is not blank. +/// +/// # Errors +/// +/// Returns the errors [`validate`] returns. +pub fn register( + settings: &Settings, +) -> Result, Report> { + let Some(config) = config(settings)? else { + return Ok(None); + }; + if config.worker_token.expose().trim().is_empty() { + log::warn!("encypher: not registered, because the worker token is empty"); + return Ok(None); + } + log::info!("encypher: registered for domain={}", config.domain); + let max_document_bytes = config + .max_document_bytes + .min(settings.publisher.max_buffered_body_bytes); + Ok(Some( + IntegrationRegistration::builder(INTEGRATION_ID) + .without_js() + .with_middleware(Arc::new(Encypher { + signer: Arc::new(Signer::new(config, max_document_bytes)), + })) + .build(), + )) +} + +/// The middleware, which signs the article of each admitted page. +pub struct Encypher { + signer: Arc, +} + +impl std::fmt::Debug for Encypher { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.debug_struct("Encypher").finish_non_exhaustive() + } +} + +/// What the preparation decided for one document: the page to sign, or none. +type Admission = Option; + +impl Middleware for Encypher { + fn middleware_id(&self) -> &'static str { + MODULE + } + + fn handles_media_type(&self, media_type: &str) -> bool { + media_type == HTML_MEDIA_TYPE + } + + /// Every path but an API, feed, admin or well-known path, or an asset, + /// which is never an article. + fn handles_path(&self, path: &str) -> bool { + !is_excluded_path(path) + } + + /// Serve only: the signed page is written on each reader's copy, so a + /// failed signing is never stored for every reader. + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Serve] + } + + /// Applies the request gates, with no I/O, and keeps the page to sign. + fn prepare<'a>( + &'a self, + call: ModuleCall<'a>, + context: &'a MiddlewareContext<'a>, + ) -> Option> { + let admission = admit(&self.signer, &call, context); + Some(Box::pin(async move { + Ok(Box::new(admission) as Box) + })) + } + + fn create(&self, context: &MiddlewareContext<'_>) -> MiddlewareAction { + let Some(page) = context.take_prepared::(MODULE).flatten() else { + return MiddlewareAction::pass(); + }; + MiddlewareAction { + stream: Some(Box::new(document_processor(&self.signer, page))), + ..MiddlewareAction::pass() + } + } +} + +/// The page a document is the body of, when the gates admit it. +fn admit(signer: &Signer, call: &ModuleCall<'_>, context: &MiddlewareContext<'_>) -> Admission { + let request = ModuleRequest::from_module_context(call).ok()?; + let page = PageRequest { + method: request.method().clone(), + page_url: format!( + "{}://{}{}", + request.scheme(), + request.host(), + request.path() + ), + response_status: context.status.as_u16(), + origin_response_is_private: context.origin_private, + }; + match signer.admit(page) { + Ok(page) => Some(page), + Err(gate) => { + log::debug!("encypher: document not held, gate={}", gate.code()); + None + } + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/provenance/encypher/src/processor.rs b/crates/provenance/encypher/src/processor.rs new file mode 100644 index 0000000000..357d4b528c --- /dev/null +++ b/crates/provenance/encypher/src/processor.rs @@ -0,0 +1,2526 @@ +//! Per-document signing flow: gates, hold, KV cache, sign call, outcomes, logging. +//! +//! A document moves through four phases: +//! +//! 1. **Admit** (the middleware's preparation, with no I/O): the request +//! gates decide whether the document is held at all. +//! 2. **Extract** (synchronous): at end of input the held document's article +//! boundary, visible text and text hash are computed. +//! 3. **Resolve** (the asynchronous finish): policy and plan caches, the policy +//! and sign calls, and cache writes. On runtimes with a timer every call is +//! raced against `max_hold_ms`, keeping a reserve for the skip write. +//! 4. **Render** (synchronous): the plan's markers and the manifest meta tag are +//! applied to the held document. +//! +//! Every module failure emits the held document unchanged. + +use std::io; +use std::pin::pin; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use bytes::Bytes; +use edgezero_core::body::Body as EdgeBody; +use futures::future::{Either, LocalBoxFuture, select}; +use http::{HeaderValue, Method, Request, StatusCode, header}; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; + +use crate::api::{self, ApiError, EmbeddingPlan, SignRequest}; +use crate::boundary::{self, Boundary}; +use crate::{EncypherConfig, INTEGRATION_ID, embed, text}; +use trusted_server_core::integrations::ensure_integration_backend_with_timeout; +use trusted_server_core::platform::{ + KvError, PlatformHttpRequest, PlatformKvStore, PlatformTimer, RuntimeServices, +}; +use trusted_server_core::streaming_processor::{AsyncFinish, StreamProcessor}; + +const SIGN_PATH: &str = "/api/v1/public/cdn/sign"; +const POLICY_PATH: &str = "/api/v1/public/cdn/trusted-server/policy"; +const WORKER_TOKEN_HEADER: &str = "x-encypher-worker-token"; +/// Article plan and skip keys. Entries carry their own `expires_at`. +const CACHE_KEY_PREFIX: &str = "encypher:v3"; +/// Policy digest and policy skip keys. Entries carry their own `expires_at`. +const POLICY_KEY_PREFIX: &str = "encypher:policy:v2"; +/// Part of `max_hold_ms` kept back from resolve work so a skip can still be +/// written after the work runs out of time. +const SKIP_WRITE_RESERVE: Duration = Duration::from_millis(200); +/// How far past `max_hold_ms` the core backstop deadline lies. The module +/// bounds itself first; the backstop only catches an overrun. +const BACKSTOP_MARGIN: Duration = Duration::from_millis(1_000); + +/// Path prefixes never signed (ported from the Fastly worker's `shouldSkipPath`). +const SKIP_PATH_PREFIXES: &[&str] = &[ + "/api/", + "/wp-admin/", + "/wp-json/", + "/feed/", + "/sitemap", + "/.well-known/", +]; +/// Asset suffixes never signed (ported from the Fastly worker's `shouldSkipPath`). +const SKIP_PATH_SUFFIXES: &[&str] = &[ + ".xml", ".json", ".rss", ".atom", ".css", ".js", ".woff", ".woff2", ".png", ".jpg", ".gif", + ".svg", ".ico", +]; + +/// Set once this instance has warned that the runtime KV store is unbound. +static KV_UNAVAILABLE_WARNED: AtomicBool = AtomicBool::new(false); + +/// Why a document was not held. Evaluated in the middleware's preparation, +/// with no I/O. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Gate { + Method, + Status, + OriginPrivate, + InvalidPageUrl, + HostMismatch, + ExcludedPath, +} + +impl Gate { + pub(crate) fn code(self) -> &'static str { + match self { + Self::Method => "method", + Self::Status => "status", + Self::OriginPrivate => "origin_private", + Self::InvalidPageUrl => "invalid_page_url", + Self::HostMismatch => "host_mismatch", + Self::ExcludedPath => "excluded_path", + } + } +} + +/// What the gates are told about the page a document is the body of. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct PageRequest { + /// The reader's request method. + pub(crate) method: Method, + /// The scheme, host and path the reader asked for, without the query, + /// which is the address the article is signed for. + pub(crate) page_url: String, + /// The status of the response the document is the body of. + pub(crate) response_status: u16, + /// Whether the origin marked its response as one reader's. + pub(crate) origin_response_is_private: bool, +} + +/// Reason code for a held document's outcome. Sign failures are also stored +/// as the reason of a negative (`skip`) cache entry. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +enum Reason { + Signed, + CacheHit, + DocumentTooLarge, + NotUtf8, + NoBoundary, + AlreadySigned, + ManifestPresent, + TextTooShort, + KvUnavailable, + KvError, + Timeout, + CachedSkip, + CachedEntryInvalid, + CachedPlanInvalid, + RequestBuild, + Backend, + HttpError, + Forbidden, + Unprocessable, + RateLimited, + ServerError, + UnexpectedStatus, + ResponseTooLarge, + MalformedResponse, + PolicyChanged, + PolicyDigestMismatch, + QuotaExceeded, + SignRejected, + NoPlan, + InvalidPlan, + AlignmentFailed, +} + +impl Reason { + fn code(self) -> &'static str { + match self { + Self::Signed => "signed", + Self::CacheHit => "cache_hit", + Self::DocumentTooLarge => "document_too_large", + Self::NotUtf8 => "not_utf8", + Self::NoBoundary => "no_boundary", + Self::AlreadySigned => "already_signed", + Self::ManifestPresent => "manifest_present", + Self::TextTooShort => "text_too_short", + Self::KvUnavailable => "kv_unavailable", + Self::KvError => "kv_error", + Self::Timeout => "timeout", + Self::CachedSkip => "cached_skip", + Self::CachedEntryInvalid => "cached_entry_invalid", + Self::CachedPlanInvalid => "cached_plan_invalid", + Self::RequestBuild => "request_build", + Self::Backend => "backend", + Self::HttpError => "http_error", + Self::Forbidden => "forbidden", + Self::Unprocessable => "unprocessable", + Self::RateLimited => "rate_limited", + Self::ServerError => "server_error", + Self::UnexpectedStatus => "unexpected_status", + Self::ResponseTooLarge => "response_too_large", + Self::MalformedResponse => "malformed_response", + Self::PolicyChanged => "policy_changed", + Self::PolicyDigestMismatch => "policy_digest_mismatch", + Self::QuotaExceeded => "quota_exceeded", + Self::SignRejected => "sign_rejected", + Self::NoPlan => "no_plan", + Self::InvalidPlan => "invalid_plan", + Self::AlignmentFailed => "alignment_failed", + } + } +} + +/// A cache entry that records when it stops being usable. +trait Expiring { + /// Unix seconds at which the entry expires. + fn expires_at(&self) -> u64; + /// Whether the entry was written without a store TTL, so it stays until deleted. + fn ttl_less(&self) -> bool; + /// Record that the entry is written without a store TTL. + fn mark_ttl_less(&mut self); +} + +/// An article cache entry, keyed by credential context, policy and text hash. +#[derive(Debug, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +enum CacheEntry { + Plan { + plan: EmbeddingPlan, + manifest_url: Option, + expires_at: u64, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + ttl_less: bool, + }, + Skip { + reason: Reason, + expires_at: u64, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + ttl_less: bool, + }, +} + +impl Expiring for CacheEntry { + fn expires_at(&self) -> u64 { + match self { + Self::Plan { expires_at, .. } | Self::Skip { expires_at, .. } => *expires_at, + } + } + + fn ttl_less(&self) -> bool { + match self { + Self::Plan { ttl_less, .. } | Self::Skip { ttl_less, .. } => *ttl_less, + } + } + + fn mark_ttl_less(&mut self) { + match self { + Self::Plan { ttl_less, .. } | Self::Skip { ttl_less, .. } => *ttl_less = true, + } + } +} + +#[derive(Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +enum PolicyEntry { + Digest { + policy_digest: String, + expires_at: u64, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + ttl_less: bool, + }, + Skip { + reason: Reason, + expires_at: u64, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + ttl_less: bool, + }, +} + +impl Expiring for PolicyEntry { + fn expires_at(&self) -> u64 { + match self { + Self::Digest { expires_at, .. } | Self::Skip { expires_at, .. } => *expires_at, + } + } + + fn ttl_less(&self) -> bool { + match self { + Self::Digest { ttl_less, .. } | Self::Skip { ttl_less, .. } => *ttl_less, + } + } + + fn mark_ttl_less(&mut self) { + match self { + Self::Digest { ttl_less, .. } | Self::Skip { ttl_less, .. } => *ttl_less = true, + } + } +} + +/// The one freshness predicate applied to every cache read. +fn is_fresh(expires_at: u64, now: u64) -> bool { + now < expires_at +} + +/// Current Unix time in seconds, from a clock that works on every adapter. +fn unix_now() -> u64 { + web_time::SystemTime::now() + .duration_since(web_time::UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_secs()) +} + +/// Configuration shared by every document processor. +pub(crate) struct Signer { + config: EncypherConfig, + max_document_bytes: usize, + sign_url: String, + policy_url: String, + policy_key: String, + /// First 16 hex characters of `sha256(api_base_url \n domain \n token)`. + cache_context: String, + /// Set by the first TTL write the store reports as unsupported, or by reading + /// an entry written without a TTL; from then on this instance writes without + /// a TTL and deletes expired entries it reads. + ttl_less: AtomicBool, +} + +impl Signer { + /// The signer for `config`, holding at most `max_document_bytes` of one + /// document. + pub(crate) fn new(config: EncypherConfig, max_document_bytes: usize) -> Self { + let sign_url = format!("{}{SIGN_PATH}", config.api_base_url.trim_end_matches('/')); + let policy_url = format!("{}{POLICY_PATH}", config.api_base_url.trim_end_matches('/')); + let context_hash = text::content_hash_hex(&format!( + "{}\n{}\n{}", + config.api_base_url, + config.domain, + config.worker_token.expose().trim() + )); + let cache_context = context_hash + .get(..16) + .unwrap_or(context_hash.as_str()) + .to_owned(); + Self { + config, + max_document_bytes, + sign_url, + policy_url, + policy_key: format!("{POLICY_KEY_PREFIX}:{cache_context}"), + cache_context, + ttl_less: AtomicBool::new(false), + } + } +} + +/// Holds the document of an admitted page until it ends. +pub(crate) fn document_processor(signer: &Arc, request: PageRequest) -> DocumentProcessor { + DocumentProcessor { + signer: Arc::clone(signer), + request, + held: Vec::new(), + state: DocumentState::Holding, + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum DocumentState { + /// Accumulating the document. + Holding, + /// Over the size limit: chunks pass through and nothing is signed. + PassingThrough, + /// The whole document was emitted as held output; it may be signed in the + /// asynchronous finish. + Complete, +} + +/// Holds one admitted document until end of input, then signs it in the +/// asynchronous finish or passes it through. +pub(crate) struct DocumentProcessor { + signer: Arc, + request: PageRequest, + held: Vec, + state: DocumentState, +} + +impl StreamProcessor for DocumentProcessor { + fn process_chunk(&mut self, chunk: &[u8], is_last: bool) -> Result, io::Error> { + if self.state != DocumentState::Holding { + return Ok(chunk.to_vec()); + } + if self.held.len().saturating_add(chunk.len()) > self.signer.max_document_bytes { + self.state = DocumentState::PassingThrough; + log_outcome(false, Reason::DocumentTooLarge); + let mut output = std::mem::take(&mut self.held); + output.extend_from_slice(chunk); + return Ok(output); + } + self.held.extend_from_slice(chunk); + if !is_last { + return Ok(Vec::new()); + } + // The complete document is the fail-open output; the driver owns it. + self.state = DocumentState::Complete; + Ok(std::mem::take(&mut self.held)) + } + + fn reset(&mut self) { + self.held.clear(); + self.state = DocumentState::Holding; + } + + fn async_finish<'a>( + &'a mut self, + held: &'a [u8], + services: &'a RuntimeServices, + ) -> Option> { + if self.state != DocumentState::Complete { + return None; + } + let extracted = std::str::from_utf8(held) + .map_err(|_| Reason::NotUtf8) + .and_then(|html| Ok((html, self.signer.extract(html)?))); + let (html, article) = match extracted { + Ok(extracted) => extracted, + Err(reason) => { + log_outcome(false, reason); + return None; + } + }; + let signer = &*self.signer; + let request = &self.request; + let deadline = + Duration::from_millis(signer.config.max_hold_ms).saturating_add(BACKSTOP_MARGIN); + Some(AsyncFinish::new(deadline, async move { + match signer.resolve(request, services, html, &article).await { + Ok((signed, reason)) => { + log_outcome(true, reason); + signed.into_bytes() + } + Err(reason) => { + log_outcome(false, reason); + held.to_vec() + } + } + })) + } +} + +/// The article found in a held document. +struct Article { + boundary: Boundary, + /// Visible text sent for signing, truncated to the API limit. + text: String, + /// Unicode scalars in `text`. + scalars: usize, +} + +/// Time a document's resolve may still take, on runtimes with a timer. +enum Deadline<'a> { + /// No timer: calls are bounded only by backend timeouts. + Unbounded, + /// The timer has not fired yet. + Running(LocalBoxFuture<'a, ()>), + /// The timer fired; every further race loses at once. + Expired, +} + +impl<'a> Deadline<'a> { + fn new(timer: Option<&'a dyn PlatformTimer>, budget: Duration) -> Self { + timer.map_or(Self::Unbounded, |timer| Self::Running(timer.sleep(budget))) + } + + /// Start the clock now. A timer's sleep starts on its first poll, which + /// would otherwise be the first call that does not complete at once. + async fn start(&mut self) { + if let Self::Running(sleep) = self + && futures::poll!(sleep.as_mut()).is_ready() + { + *self = Self::Expired; + } + } + + /// Run `work` to completion, or return `None` once the deadline passes. + async fn race(&mut self, work: impl Future) -> Option { + match self { + Self::Unbounded => return Some(work.await), + Self::Expired => return None, + Self::Running(sleep) => { + if let Either::Left((value, _)) = select(pin!(work), sleep.as_mut()).await { + return Some(value); + } + } + } + *self = Self::Expired; + None + } + + /// Whether the deadline has passed, so the last race lost. + fn expired(&self) -> bool { + matches!(self, Self::Expired) + } +} + +fn log_outcome(signed: bool, reason: Reason) { + if reason == Reason::KvUnavailable && !KV_UNAVAILABLE_WARNED.swap(true, Ordering::Relaxed) { + log::warn!( + "encypher: the runtime KV store is unavailable, so pages pass through unsigned; bind the adapter's KV store to enable signing" + ); + } + log::info!( + "encypher: outcome={} reason={}", + if signed { "signed" } else { "passed_through" }, + reason.code() + ); +} + +/// Whether `path` is one this module never signs: an API, feed, admin or +/// well-known path, or an asset. +pub(crate) fn is_excluded_path(path: &str) -> bool { + let lower = path.to_ascii_lowercase(); + SKIP_PATH_PREFIXES + .iter() + .any(|prefix| lower.starts_with(prefix)) + || SKIP_PATH_SUFFIXES + .iter() + .any(|suffix| lower.ends_with(suffix)) +} + +impl Signer { + /// Apply the request gates. Only an admitted document is held. + pub(crate) fn admit(&self, request: PageRequest) -> Result { + if request.method != Method::GET { + return Err(Gate::Method); + } + if request.response_status != StatusCode::OK.as_u16() { + return Err(Gate::Status); + } + if request.origin_response_is_private { + return Err(Gate::OriginPrivate); + } + let page_url = url::Url::parse(&request.page_url).map_err(|_| Gate::InvalidPageUrl)?; + let host = page_url.host_str().ok_or(Gate::InvalidPageUrl)?; + if host.strip_prefix("www.").unwrap_or(host) != self.config.domain { + return Err(Gate::HostMismatch); + } + if is_excluded_path(page_url.path()) { + return Err(Gate::ExcludedPath); + } + Ok(request) + } + + /// Find the article and its signable text, with no I/O. + fn extract(&self, html: &str) -> Result { + let boundary = boundary::find_article_boundary(html, Some(&self.config.article_selector)) + .ok_or(Reason::NoBoundary)?; + let article = html + .get(boundary.start..boundary.end) + .ok_or(Reason::NoBoundary)?; + if embed::has_existing_markers(article) { + return Err(Reason::AlreadySigned); + } + if embed::has_manifest_meta(html) { + return Err(Reason::ManifestPresent); + } + let mut visible = text::visible_text(article); + let sign_len = text::truncate_scalars(&visible, api::MAX_TEXT_SCALARS).len(); + visible.truncate(sign_len); + let scalars = text::scalar_count(&visible); + if scalars < self.config.min_text_length { + return Err(Reason::TextTooShort); + } + Ok(Article { + boundary, + text: visible, + scalars, + }) + } + + /// Unix time at which an entry written now with `ttl_seconds` expires. + fn expires_at(ttl_seconds: u64) -> u64 { + unix_now().saturating_add(ttl_seconds) + } + + /// Bound for skip writes made after resolve work stopped. + fn skip_deadline<'a>(services: &'a RuntimeServices) -> Deadline<'a> { + Deadline::new(services.timer(), SKIP_WRITE_RESERVE) + } + + /// Resolve the plan for `article` and render it into `html`. + async fn resolve( + &self, + request: &PageRequest, + services: &RuntimeServices, + html: &str, + article: &Article, + ) -> Result<(String, Reason), Reason> { + let kv = services.kv_store(); + let mut work = Deadline::new( + services.timer(), + Duration::from_millis(self.config.max_hold_ms).saturating_sub(SKIP_WRITE_RESERVE), + ); + work.start().await; + + let policy_digest = self.policy_digest(request, services, kv, &mut work).await?; + let key = format!( + "{CACHE_KEY_PREFIX}:{}:{policy_digest}:{}", + self.cache_context, + text::content_hash_hex(&article.text) + ); + let outcome = match self.read::(kv, &key, &mut work).await { + Ok(Some(CacheEntry::Plan { + plan, manifest_url, .. + })) => { + api::validate_plan(&plan, manifest_url.as_deref(), article.scalars) + .map_err(|_| Reason::CachedPlanInvalid)?; + return render(html, &article.boundary, &plan, manifest_url.as_deref()) + .map(|signed| (signed, Reason::CacheHit)) + .ok_or(Reason::CachedPlanInvalid); + } + Ok(Some(CacheEntry::Skip { .. })) => return Err(Reason::CachedSkip), + Err(Reason::Timeout) => Err(Reason::Timeout), + Err(reason) => return Err(reason), + Ok(None) => self + .request_plan(request, services, html, article, &policy_digest, &mut work) + .await + .and_then(|(plan, manifest_url)| { + api::validate_plan(&plan, manifest_url.as_deref(), article.scalars) + .map_err(|_| Reason::InvalidPlan)?; + let signed = render(html, &article.boundary, &plan, manifest_url.as_deref()) + .ok_or(Reason::AlignmentFailed)?; + Ok((signed, plan, manifest_url)) + }), + }; + match outcome { + Ok((signed, plan, manifest_url)) => { + // A plan write that runs out of time is abandoned; the page is still signed. + let entry = CacheEntry::Plan { + plan, + manifest_url, + expires_at: Self::expires_at(self.config.plan_ttl_seconds), + ttl_less: false, + }; + self.write(kv, &key, entry, self.config.plan_ttl_seconds, &mut work) + .await; + Ok((signed, Reason::Signed)) + } + Err(reason) => { + let mut deadline = Self::skip_deadline(services); + if matches!(reason, Reason::PolicyChanged | Reason::PolicyDigestMismatch) { + self.evict_policy(kv, &mut deadline).await; + } + self.write_skip(kv, &key, reason, article.scalars, &mut deadline) + .await; + Err(reason) + } + } + } + + async fn policy_digest( + &self, + request: &PageRequest, + services: &RuntimeServices, + kv: &dyn PlatformKvStore, + work: &mut Deadline<'_>, + ) -> Result { + let result = match self.read::(kv, &self.policy_key, work).await { + Ok(Some(PolicyEntry::Digest { policy_digest, .. })) + if api::valid_policy_digest(&policy_digest) => + { + return Ok(policy_digest); + } + Ok(Some(PolicyEntry::Skip { .. })) => return Err(Reason::CachedSkip), + Ok(Some(PolicyEntry::Digest { .. })) => Err(Reason::CachedEntryInvalid), + // Nothing can be cached without a store. + Err(Reason::KvUnavailable) => return Err(Reason::KvUnavailable), + Err(reason) => Err(reason), + Ok(None) => self.request_policy(request, services, work).await, + }; + let reason = match result { + Ok(policy_digest) => { + let entry = PolicyEntry::Digest { + policy_digest: policy_digest.clone(), + expires_at: Self::expires_at(self.config.policy_ttl_seconds), + ttl_less: false, + }; + self.write( + kv, + &self.policy_key, + entry, + self.config.policy_ttl_seconds, + work, + ) + .await; + if !work.expired() { + return Ok(policy_digest); + } + // The digest is not cached, so the next view could not find a + // plan skip written under it; skip at the policy level instead. + Reason::Timeout + } + Err(reason) => reason, + }; + let ttl = self.policy_skip_ttl(); + let entry = PolicyEntry::Skip { + reason, + expires_at: Self::expires_at(ttl), + ttl_less: false, + }; + self.write( + kv, + &self.policy_key, + entry, + ttl, + &mut Self::skip_deadline(services), + ) + .await; + Err(reason) + } + + fn policy_skip_ttl(&self) -> u64 { + self.config + .policy_ttl_seconds + .min(self.config.negative_ttl_seconds) + } + + async fn request_policy( + &self, + request: &PageRequest, + services: &RuntimeServices, + work: &mut Deadline<'_>, + ) -> Result { + let mut url = url::Url::parse(&self.policy_url).map_err(|_| Reason::RequestBuild)?; + url.query_pairs_mut() + .append_pair("page_url", &request.page_url); + let body = self + .fetch( + services, + Method::GET, + url.as_str(), + EdgeBody::empty(), + api::MAX_POLICY_RESPONSE_BYTES, + work, + ) + .await?; + api::parse_policy_response(&body).map_err(|error| parse_error_reason(&error)) + } + + async fn evict_policy(&self, kv: &dyn PlatformKvStore, deadline: &mut Deadline<'_>) { + if matches!( + deadline.race(kv.delete(&self.policy_key)).await, + Some(Ok(())) + ) { + return; + } + // If deletion fails, try to replace the stale digest with a bounded skip. + let ttl = self.policy_skip_ttl(); + let entry = PolicyEntry::Skip { + reason: Reason::PolicyChanged, + expires_at: Self::expires_at(ttl), + ttl_less: false, + }; + self.write(kv, &self.policy_key, entry, ttl, deadline).await; + log::debug!("encypher: policy eviction failed"); + } + + /// Call the sign API once. Every failure maps to a reason code. + async fn request_plan( + &self, + request: &PageRequest, + services: &RuntimeServices, + html: &str, + article: &Article, + policy_digest: &str, + work: &mut Deadline<'_>, + ) -> Result<(EmbeddingPlan, Option), Reason> { + let title = text::extract_title(html); + let body = serde_json::to_vec(&SignRequest { + text: &article.text, + page_url: &request.page_url, + document_title: title.as_deref(), + boundary_selector: &article.boundary.selector, + expected_policy_digest: policy_digest, + }) + .map_err(|_| Reason::RequestBuild)?; + let body = self + .fetch( + services, + Method::POST, + &self.sign_url, + EdgeBody::from(body), + api::MAX_RESPONSE_BYTES, + work, + ) + .await?; + let parsed = api::parse_sign_response(&body).map_err(|error| parse_error_reason(&error))?; + if !parsed.success { + return Err(if parsed.error.as_deref() == Some("quota_exceeded") { + Reason::QuotaExceeded + } else { + Reason::SignRejected + }); + } + if parsed.policy_digest.as_deref() != Some(policy_digest) { + return Err(Reason::PolicyDigestMismatch); + } + let plan = parsed.embedding_plan.ok_or(Reason::NoPlan)?; + Ok((plan, parsed.manifest_url)) + } + + /// One bounded HTTP call; never retried. + async fn fetch( + &self, + services: &RuntimeServices, + method: Method, + url: &str, + body: EdgeBody, + max_response_bytes: usize, + work: &mut Deadline<'_>, + ) -> Result { + let mut token = HeaderValue::from_str(self.config.worker_token.expose().trim()) + .map_err(|_| Reason::RequestBuild)?; + token.set_sensitive(true); + let http_request = Request::builder() + .method(method) + .uri(url) + .header(header::CONTENT_TYPE, "application/json") + .header(WORKER_TOKEN_HEADER, token) + .body(body) + .map_err(|_| Reason::RequestBuild)?; + let backend = ensure_integration_backend_with_timeout( + services, + &self.config.api_base_url, + INTEGRATION_ID, + Duration::from_millis(self.config.timeout_ms), + ) + .map_err(|_| Reason::Backend)?; + + let response = match work + .race( + services + .http_client() + .send(PlatformHttpRequest::new(http_request, backend)), + ) + .await + { + None => return Err(Reason::Timeout), + Some(Err(_)) => return Err(Reason::HttpError), + Some(Ok(response)) => response.response, + }; + let status = response.status(); + if status != StatusCode::OK { + return Err(match status.as_u16() { + 403 => Reason::Forbidden, + 409 => Reason::PolicyChanged, + 422 => Reason::Unprocessable, + 429 => Reason::RateLimited, + 500..=599 => Reason::ServerError, + _ => Reason::UnexpectedStatus, + }); + } + // One byte over the limit is enough for the parser to report TooLarge. + match work + .race( + response + .into_body() + .into_bytes_bounded(max_response_bytes.saturating_add(1)), + ) + .await + { + None => Err(Reason::Timeout), + Some(Err(_)) => Err(Reason::ResponseTooLarge), + Some(Ok(body)) => Ok(body), + } + } + + /// Read a fresh entry. Expired entries are misses; in TTL-less write mode + /// they are also deleted, best effort, so the store cleans itself. + async fn read( + &self, + kv: &dyn PlatformKvStore, + key: &str, + deadline: &mut Deadline<'_>, + ) -> Result, Reason> { + let bytes = match deadline.race(kv.get_bytes(key)).await { + None => return Err(Reason::Timeout), + Some(Err(KvError::Unavailable)) => return Err(Reason::KvUnavailable), + Some(Err(_)) => return Err(Reason::KvError), + Some(Ok(None)) => return Ok(None), + Some(Ok(Some(bytes))) => bytes, + }; + let entry: T = serde_json::from_slice(&bytes).map_err(|_| Reason::CachedEntryInvalid)?; + // An entry written without a TTL shows the mode to an instance that has + // not written yet, as on Spin, which instantiates the module per request. + if entry.ttl_less() { + self.ttl_less.store(true, Ordering::Relaxed); + } + if is_fresh(entry.expires_at(), unix_now()) { + return Ok(Some(entry)); + } + if self.ttl_less.load(Ordering::Relaxed) + && !matches!(deadline.race(kv.delete(key)).await, Some(Ok(()))) + { + log::debug!("encypher: expired cache entry could not be deleted"); + } + Ok(None) + } + + /// Best-effort cache write; failures are logged and ignored. + /// + /// Writes carry a TTL until the store first reports TTL writes as + /// unsupported; from then on this instance writes without one, and marks + /// the entry so a later instance that reads it knows the mode too. + async fn write( + &self, + kv: &dyn PlatformKvStore, + key: &str, + mut entry: E, + ttl_seconds: u64, + deadline: &mut Deadline<'_>, + ) { + if !self.ttl_less.load(Ordering::Relaxed) { + let Some(value) = encode(&entry) else { + return; + }; + let ttl = Duration::from_secs(ttl_seconds); + match deadline.race(kv.put_bytes_with_ttl(key, value, ttl)).await { + Some(Ok(())) => return, + Some(Err(KvError::Unsupported { .. })) => { + self.ttl_less.store(true, Ordering::Relaxed); + log::info!( + "encypher: the KV store does not support TTL writes; entries expire by their expires_at" + ); + } + Some(Err(_)) => { + log::debug!("encypher: cache write failed"); + return; + } + None => { + log::debug!("encypher: cache write timed out"); + return; + } + } + } + entry.mark_ttl_less(); + let Some(value) = encode(&entry) else { + return; + }; + match deadline.race(kv.put_bytes(key, value)).await { + Some(Ok(())) => {} + Some(Err(_)) => log::debug!("encypher: cache write failed"), + None => log::debug!("encypher: cache write timed out"), + } + } + + /// Write a negative entry unless a fresh, valid plan is already cached for the key. + /// + /// The KV trait has no insert-if-absent, so this re-reads first: a concurrent + /// successful sign for the same text keeps its plan. + async fn write_skip( + &self, + kv: &dyn PlatformKvStore, + key: &str, + reason: Reason, + scalars: usize, + deadline: &mut Deadline<'_>, + ) { + if let Ok(Some(CacheEntry::Plan { + plan, manifest_url, .. + })) = self.read::(kv, key, deadline).await + && api::validate_plan(&plan, manifest_url.as_deref(), scalars).is_ok() + { + return; + } + let ttl = self.config.negative_ttl_seconds; + let entry = CacheEntry::Skip { + reason, + expires_at: Self::expires_at(ttl), + ttl_less: false, + }; + self.write(kv, key, entry, ttl, deadline).await; + } +} + +/// Serialize a cache entry; a failure is logged and the write skipped. +fn encode(entry: &impl Serialize) -> Option { + match serde_json::to_vec(entry) { + Ok(value) => Some(Bytes::from(value)), + Err(_) => { + log::debug!("encypher: cache entry could not be serialized"); + None + } + } +} + +/// Apply a plan that passed [`api::validate_plan`] against the signed text, and +/// add the manifest meta tag when a manifest URL was returned. +fn render( + html: &str, + boundary: &Boundary, + plan: &EmbeddingPlan, + manifest_url: Option<&str>, +) -> Option { + let signed = embed::apply_plan(html, boundary, plan)?; + Some(match manifest_url { + Some(url) => embed::inject_manifest_meta(&signed, url), + None => signed, + }) +} + +fn parse_error_reason(error: &ApiError) -> Reason { + match error { + ApiError::TooLarge => Reason::ResponseTooLarge, + ApiError::Malformed => Reason::MalformedResponse, + } +} + +#[cfg(test)] +mod tests { + use std::cell::RefCell; + use std::collections::HashMap; + use std::sync::atomic::AtomicUsize; + use std::sync::{LazyLock, Mutex}; + use std::task::Poll; + + use error_stack::Report; + use serde_json::json; + + use super::*; + use trusted_server_core::platform::test_support::{ + StoppedTimer, StubHttpClient, build_services_with_http_client, + }; + use trusted_server_core::platform::{ + PlatformError, PlatformHttpClient, PlatformPendingRequest, PlatformResponse, + PlatformSelectResult, + }; + + const TOKEN: &str = "cwk_test_worker_token_value"; + const PAGE_URL: &str = "https://www.publisher.example.com/news/harbor-vote"; + const MARKER: &str = "\u{E0100}\u{E0101}\u{FE01}"; + const MANIFEST_URL: &str = "https://api.example.com/manifests/doc-1"; + const POLICY_DIGEST: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + const ARTICLE: &str = "Harbor Council Votes\ +

Harbor council votes on the ferry schedule

\ +

The harbor council met on Tuesday evening and approved a revised ferry schedule \ + for the coming season.

Residents asked for earlier departures on weekdays.

\ +
Footer
"; + /// Simulated time that passes each time a [`TickTimer`] sleep is polled. + const TICK: Duration = Duration::from_millis(100); + + // --- test doubles ------------------------------------------------------- + + /// Return `Pending` `times` times, waking the task each time, then `Ready`. + async fn yield_pending(times: usize) { + for _ in 0..times { + let mut yielded = false; + futures::future::poll_fn(|cx| { + if yielded { + Poll::Ready(()) + } else { + yielded = true; + cx.waker().wake_by_ref(); + Poll::Pending + } + }) + .await; + } + } + + /// A timer on simulated time: each poll of a sleep advances it by [`TICK`]. + /// Sleeps only advance while the work they race is pending, as on a real + /// runtime where the clock runs while I/O is outstanding. + #[derive(Default)] + struct TickTimer { + requested: Mutex>, + } + + #[async_trait::async_trait(?Send)] + impl PlatformTimer for TickTimer { + async fn sleep(&self, duration: Duration) { + self.requested + .lock() + .expect("should lock requested durations") + .push(duration); + let ticks = duration.as_millis().div_ceil(TICK.as_millis()); + yield_pending(usize::try_from(ticks).expect("ticks should fit")).await; + } + } + + /// In-memory KV store that counts calls and can fail, stall or refuse TTLs. + #[derive(Default)] + struct MemoryKv { + entries: Mutex>, + ttls: Mutex>, + deletes: Mutex>, + reads: AtomicUsize, + ttl_writes: AtomicUsize, + /// Plan-key reads that report a miss before reads see the stored entry. + hidden_reads: AtomicUsize, + /// Every read fails with [`KvError::Unavailable`], as an unbound store does. + unavailable: bool, + /// Every read fails with an error other than unavailability. + fail_reads: bool, + /// Every read never completes. + hung_reads: bool, + /// Writes to keys with this prefix never complete. + hung_writes_to: Option<&'static str>, + /// The first write to this exact key never completes; later writes succeed. + hung_first_write_to: Option, + hung_first_write_taken: AtomicBool, + /// TTL writes fail with [`KvError::Unsupported`], as on Spin. + ttl_unsupported: bool, + /// `Pending` polls before every operation completes. + pending_polls: usize, + } + + impl MemoryKv { + fn entries(&self) -> HashMap { + self.entries.lock().expect("should lock entries").clone() + } + + fn entry(&self, key: &str) -> serde_json::Value { + serde_json::from_slice(&self.entries()[key]).expect("should store JSON") + } + + fn insert(&self, key: &str, value: &serde_json::Value) { + self.entries.lock().expect("should lock entries").insert( + key.to_owned(), + Bytes::from(serde_json::to_vec(value).expect("should serialize")), + ); + } + + fn remove(&self, key: &str) { + self.entries + .lock() + .expect("should lock entries") + .remove(key); + } + + /// The single article entry under the current plan namespace. + fn only_entry(&self) -> (String, serde_json::Value) { + let entries: Vec<_> = self + .entries() + .into_keys() + .filter(|key| key.starts_with("encypher:v3:")) + .collect(); + assert_eq!(entries.len(), 1, "should hold exactly one article entry"); + let key = entries.into_iter().next().expect("should have an entry"); + let value = self.entry(&key); + (key, value) + } + + fn reads(&self) -> usize { + self.reads.load(Ordering::SeqCst) + } + + async fn store(&self, key: &str, value: Bytes) -> Result<(), KvError> { + yield_pending(self.pending_polls).await; + if self + .hung_writes_to + .is_some_and(|prefix| key.starts_with(prefix)) + || (self.hung_first_write_to.as_deref() == Some(key) + && !self.hung_first_write_taken.swap(true, Ordering::SeqCst)) + { + futures::future::pending::<()>().await; + } + self.entries + .lock() + .expect("should lock entries") + .insert(key.to_owned(), value); + Ok(()) + } + } + + #[async_trait::async_trait(?Send)] + impl PlatformKvStore for MemoryKv { + async fn get_bytes(&self, key: &str) -> Result, KvError> { + self.reads.fetch_add(1, Ordering::SeqCst); + yield_pending(self.pending_polls).await; + if self.hung_reads { + futures::future::pending::<()>().await; + } + if self.unavailable { + return Err(KvError::Unavailable); + } + if self.fail_reads { + return Err(KvError::LimitExceeded { + message: "test failure".to_owned(), + }); + } + if key.starts_with(CACHE_KEY_PREFIX) + && self + .hidden_reads + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |n| n.checked_sub(1)) + .is_ok() + { + return Ok(None); + } + Ok(self + .entries + .lock() + .expect("should lock entries") + .get(key) + .cloned()) + } + + async fn put_bytes(&self, key: &str, value: Bytes) -> Result<(), KvError> { + self.store(key, value).await + } + + async fn put_bytes_with_ttl( + &self, + key: &str, + value: Bytes, + ttl: Duration, + ) -> Result<(), KvError> { + self.ttl_writes.fetch_add(1, Ordering::SeqCst); + if self.ttl_unsupported { + return Err(KvError::Unsupported { + operation: "put_bytes_with_ttl".to_owned(), + }); + } + self.ttls + .lock() + .expect("should lock TTLs") + .insert(key.to_owned(), ttl); + self.store(key, value).await + } + + async fn delete(&self, key: &str) -> Result<(), KvError> { + yield_pending(self.pending_polls).await; + self.deletes + .lock() + .expect("should lock deletes") + .push(key.to_owned()); + self.remove(key); + Ok(()) + } + + async fn list_keys_page( + &self, + _prefix: &str, + _cursor: Option<&str>, + _limit: usize, + ) -> Result { + Ok(edgezero_core::key_value_store::KvPage::default()) + } + } + + /// HTTP client whose send never completes. + #[derive(Default)] + struct HungHttpClient { + sends: AtomicUsize, + } + + #[async_trait::async_trait(?Send)] + impl PlatformHttpClient for HungHttpClient { + async fn send( + &self, + _request: PlatformHttpRequest, + ) -> Result> { + self.sends.fetch_add(1, Ordering::SeqCst); + futures::future::pending().await + } + + async fn send_async( + &self, + _request: PlatformHttpRequest, + ) -> Result> { + Err(Report::new(PlatformError::HttpClient)) + } + + async fn select( + &self, + _pending_requests: Vec, + ) -> Result> { + Err(Report::new(PlatformError::HttpClient)) + } + } + + /// Wraps [`StubHttpClient`] so the send and the response body each return + /// `Pending` before they are ready, as on Cloudflare, Spin and Axum. + struct YieldingHttpClient { + inner: Arc, + pending_polls: usize, + } + + #[async_trait::async_trait(?Send)] + impl PlatformHttpClient for YieldingHttpClient { + async fn send( + &self, + request: PlatformHttpRequest, + ) -> Result> { + yield_pending(self.pending_polls).await; + let response = self.inner.send(request).await?; + let (parts, body) = response.response.into_parts(); + let bytes = body + .into_bytes_bounded(usize::MAX) + .await + .expect("should read the stub body"); + let polls = self.pending_polls; + let body = EdgeBody::stream(futures::stream::once(async move { + yield_pending(polls).await; + bytes + })); + Ok(PlatformResponse::new(http::Response::from_parts( + parts, body, + ))) + } + + async fn send_async( + &self, + _request: PlatformHttpRequest, + ) -> Result> { + Err(Report::new(PlatformError::HttpClient)) + } + + async fn select( + &self, + _pending_requests: Vec, + ) -> Result> { + Err(Report::new(PlatformError::HttpClient)) + } + } + + /// Records log lines, with their level, emitted on the current thread. + struct ThreadLogger; + + thread_local! { + static LOGS: RefCell> = const { RefCell::new(Vec::new()) }; + } + + impl log::Log for ThreadLogger { + fn enabled(&self, _metadata: &log::Metadata<'_>) -> bool { + true + } + + fn log(&self, record: &log::Record<'_>) { + LOGS.with(|logs| { + logs.borrow_mut() + .push(format!("{} {}", record.level(), record.args())); + }); + } + + fn flush(&self) {} + } + + static LOGGER: ThreadLogger = ThreadLogger; + + /// A sign-failure case: name and the API response, `None` for a transport error. + type FailureCase = (&'static str, Option<(u16, Vec)>); + /// A gate case: name, signer, and the request mutation that trips the gate. + type GateCase<'a> = (&'static str, &'a Arc, fn(&mut PageRequest)); + + /// Runs `run` with this thread's log lines captured. Returns `None`, after + /// skipping `run`, when another logger already owns the process. + fn capture_logs(run: impl FnOnce()) -> Option> { + static INSTALLED: LazyLock = LazyLock::new(|| { + let installed = log::set_logger(&LOGGER).is_ok(); + if installed { + log::set_max_level(log::LevelFilter::Trace); + } + installed + }); + if !*INSTALLED { + return None; + } + LOGS.with(|logs| logs.borrow_mut().clear()); + run(); + Some(LOGS.with(RefCell::take)) + } + + // --- fixtures ----------------------------------------------------------- + + fn config_json(token: &str) -> serde_json::Value { + json!({ + "api_base_url": "https://api.example.com", + "domain": "publisher.example.com", + "worker_token": token, + }) + } + + fn factory_with(config: serde_json::Value, max_document_bytes: usize) -> Arc { + let config: EncypherConfig = serde_json::from_value(config).expect("should parse config"); + Arc::new(Signer::new(config, max_document_bytes)) + } + + fn factory() -> Arc { + factory_with(config_json(TOKEN), 5_000_000) + } + + fn policy_key() -> String { + factory().policy_key.clone() + } + + /// Services without a timer, as on Fastly. + fn services(http: Arc, kv: &Arc) -> RuntimeServices { + build_services_with_http_client(http) + .with_kv_store(Arc::clone(kv) as Arc) + } + + /// Services with a [`TickTimer`], as on Cloudflare, Spin and Axum. + fn timed_services(http: Arc, kv: &Arc) -> RuntimeServices { + services(http, kv).with_timer(Arc::new(TickTimer::default())) + } + + fn request() -> PageRequest { + PageRequest { + method: Method::GET, + page_url: PAGE_URL.to_owned(), + response_status: 200, + origin_response_is_private: false, + } + } + + fn plan_json(marker: &str) -> serde_json::Value { + json!({ + "index_unit": "codepoint", + "operations": [{ "insert_after_index": 5, "marker": marker }], + }) + } + + fn sign_response(marker: &str) -> Vec { + sign_response_for_policy(marker, POLICY_DIGEST) + } + + fn policy_response(digest: &str) -> Vec { + serde_json::to_vec(&json!({"policy_digest": digest})).expect("should serialize policy") + } + + fn sign_response_for_policy(marker: &str, digest: &str) -> Vec { + serde_json::to_vec(&json!({ + "success": true, + "policy_digest": digest, + "embedding_plan": plan_json(marker), + "manifest_url": MANIFEST_URL, + "verification_url": "https://api.example.com/verify/doc-1", + "document_id": "doc-1", + })) + .expect("should serialize sign response") + } + + fn signing_http() -> Arc { + let http = Arc::new(StubHttpClient::new()); + http.push_response(200, policy_response(POLICY_DIGEST)); + http.push_response(200, sign_response(MARKER)); + http + } + + fn seed_plan(kv: &Arc) -> String { + view(&factory(), &services(signing_http(), kv)) + } + + /// Drive one document the way the middleware and the publisher driver do: + /// the gates on the request, when there is one, then two chunks, then the + /// asynchronous finish with the core backstop. A request the gates refuse, + /// or none, holds nothing, so the document passes through. + fn run( + factory: &Arc, + request: Option, + services: &RuntimeServices, + html: &str, + ) -> String { + let Some(request) = request.and_then(|request| factory.admit(request).ok()) else { + return html.to_owned(); + }; + futures::executor::block_on(async { + let mut processor = document_processor(factory, request); + let (first, second) = html.as_bytes().split_at(html.len() / 2); + let mut output = processor + .process_chunk(first, false) + .expect("should process first chunk"); + output.extend( + processor + .finish(second, services) + .await + .expect("should finish the document"), + ); + String::from_utf8(output).expect("should emit UTF-8") + }) + } + + /// One view of [`ARTICLE`] at [`PAGE_URL`]. + fn view(factory: &Arc, services: &RuntimeServices) -> String { + run(factory, Some(request()), services, ARTICLE) + } + + fn assert_signed(output: &str) { + assert!(output.contains(MARKER), "should embed the plan marker"); + assert!( + output.contains("c2pa-manifest-url") && output.contains(MANIFEST_URL), + "should inject the manifest meta tag" + ); + assert_eq!( + output.matches(MARKER).count(), + 1, + "should add the marker once" + ); + } + + /// Rewrite the `expires_at` of the entry at `key` to `expires_at`. + fn set_expiry(kv: &MemoryKv, key: &str, expires_at: u64) { + let mut entry = kv.entry(key); + entry["expires_at"] = json!(expires_at); + kv.insert(key, &entry); + } + + fn expired() -> u64 { + unix_now().saturating_sub(1) + } + + // --- policy ------------------------------------------------------------- + + #[test] + fn policy_failures_are_negatively_cached_without_using_an_old_plan() { + let cases: Vec = vec![ + ("unavailable", Some((503, Vec::new()))), + ("forbidden", Some((403, Vec::new()))), + ("transport", None), + ("malformed", Some((200, b"not json".to_vec()))), + ("missing digest", Some((200, b"{}".to_vec()))), + ( + "uppercase", + Some((200, policy_response(&POLICY_DIGEST.to_uppercase()))), + ), + ( + "oversized", + Some((200, vec![b' '; api::MAX_POLICY_RESPONSE_BYTES + 2])), + ), + ]; + for (case, response) in cases { + let kv = Arc::new(MemoryKv::default()); + seed_plan(&kv); + kv.remove(&policy_key()); + let http = Arc::new(StubHttpClient::new()); + if let Some((status, body)) = response { + http.push_response(status, body); + } + let services = services(Arc::clone(&http) as Arc, &kv); + for _ in 0..2 { + assert_eq!(view(&factory(), &services), ARTICLE, "{case}"); + } + assert_eq!(http.recorded_request_methods(), ["GET"], "{case}"); + assert_eq!(kv.entry(&policy_key())["kind"], "skip", "{case}"); + assert_eq!( + kv.ttls.lock().expect("lock")[&policy_key()], + Duration::from_secs(300) + ); + // Once the skip is gone, refresh may reuse only the matching old plan. + kv.remove(&policy_key()); + http.push_response(200, policy_response(POLICY_DIGEST)); + assert_signed(&view(&factory(), &services)); + assert_eq!(http.recorded_request_methods(), ["GET", "GET"]); + } + } + + #[test] + fn malformed_cached_policy_cannot_unlock_a_cached_article() { + let future = unix_now() + 300; + for value in [ + json!("not an entry"), + json!({"kind": "digest", "policy_digest": null, "expires_at": future}), + json!({"kind": "digest", "policy_digest": "bad", "expires_at": future}), + json!({ + "kind": "digest", "policy_digest": POLICY_DIGEST, "expires_at": future, + "extra": true, + }), + // `expires_at` is required. + json!({"kind": "digest", "policy_digest": POLICY_DIGEST}), + ] { + let kv = Arc::new(MemoryKv::default()); + seed_plan(&kv); + kv.insert(&policy_key(), &value); + let http = Arc::new(StubHttpClient::new()); + let services = services(Arc::clone(&http) as Arc, &kv); + for _ in 0..2 { + assert_eq!(view(&factory(), &services), ARTICLE, "{value}"); + } + assert!(http.recorded_request_methods().is_empty(), "{value}"); + } + } + + #[test] + fn policy_cache_hit_and_article_or_policy_changes_select_the_correct_plan() { + let kv = Arc::new(MemoryKv::default()); + seed_plan(&kv); + let http = Arc::new(StubHttpClient::new()); + http.push_response(200, sign_response(MARKER)); + let services = services(Arc::clone(&http) as Arc, &kv); + let changed = ARTICLE.replace("Tuesday", "Wednesday"); + assert_signed(&run(&factory(), Some(request()), &services, &changed)); + assert_eq!(http.recorded_request_methods(), ["POST"]); + + kv.remove(&policy_key()); + let next_digest = "b".repeat(64); + http.push_response(200, policy_response(&next_digest)); + http.push_response(200, sign_response_for_policy(MARKER, &next_digest)); + assert_signed(&view(&factory(), &services)); + assert_eq!(http.recorded_request_methods(), ["POST", "GET", "POST"]); + let count = http.recorded_request_methods().len(); + assert_signed(&view(&factory(), &services)); + assert_eq!(http.recorded_request_methods().len(), count); + // Returning to the original policy selects its original plan, not the intervening one. + kv.remove(&policy_key()); + http.push_response(200, policy_response(POLICY_DIGEST)); + assert_signed(&view(&factory(), &services)); + assert_eq!( + http.recorded_request_methods(), + ["POST", "GET", "POST", "GET"] + ); + } + + #[test] + fn conflict_or_response_digest_mismatch_evicts_policy_without_retrying() { + let mut missing: serde_json::Value = + serde_json::from_slice(&sign_response(MARKER)).expect("JSON"); + missing + .as_object_mut() + .expect("object") + .remove("policy_digest"); + for (status, body) in [ + ( + 409, + br#"{"detail":"TRUSTED_SERVER_POLICY_CHANGED"}"#.to_vec(), + ), + (409, b"unparseable conflict".to_vec()), + (200, sign_response_for_policy(MARKER, &"b".repeat(64))), + (200, sign_response_for_policy(MARKER, "invalid")), + (200, serde_json::to_vec(&missing).expect("JSON")), + ] { + let kv = Arc::new(MemoryKv::default()); + let http = Arc::new(StubHttpClient::new()); + http.push_response(200, policy_response(POLICY_DIGEST)); + http.push_response(status, body); + let services = services(Arc::clone(&http) as Arc, &kv); + assert_eq!(view(&factory(), &services), ARTICLE); + assert_eq!(http.recorded_request_methods(), ["GET", "POST"]); + assert!(!kv.entries().contains_key(&policy_key())); + assert_eq!(kv.only_entry().1["kind"], "skip"); + // Next request refetches policy, but the failed article remains negatively cached. + http.push_response(200, policy_response(POLICY_DIGEST)); + assert_eq!(view(&factory(), &services), ARTICLE); + assert_eq!(http.recorded_request_methods(), ["GET", "POST", "GET"]); + } + } + + #[test] + fn entries_expire_at_their_ttl_and_the_store_ttl_matches() { + let kv = Arc::new(MemoryKv::default()); + let before = unix_now(); + seed_plan(&kv); + let after = unix_now(); + let ttls = kv.ttls.lock().expect("lock").clone(); + let (plan_key, plan) = kv.only_entry(); + for (key, entry, ttl) in [ + (policy_key(), kv.entry(&policy_key()), 300), + (plan_key, plan, 604_800), + ] { + assert_eq!(ttls[&key], Duration::from_secs(ttl)); + let expires_at = entry["expires_at"] + .as_u64() + .expect("should carry expires_at"); + assert!( + (before + ttl..=after + ttl).contains(&expires_at), + "{key} should expire {ttl}s after it was written" + ); + } + for (policy_ttl, negative_ttl, expected) in [(60, 300, 60), (300, 60, 60)] { + let kv = Arc::new(MemoryKv::default()); + let http = Arc::new(StubHttpClient::new()); + http.push_response(503, Vec::new()); + let mut config = config_json(TOKEN); + config["policy_ttl_seconds"] = json!(policy_ttl); + config["negative_ttl_seconds"] = json!(negative_ttl); + let factory = factory_with(config, 5_000_000); + assert_eq!(view(&factory, &services(http, &kv)), ARTICLE); + assert_eq!( + kv.ttls.lock().expect("lock")[&factory.policy_key], + Duration::from_secs(expected) + ); + } + } + + // --- sign path and cache ------------------------------------------------ + + #[test] + fn signs_on_a_miss_with_the_worker_token_and_caches_the_plan() { + let http = signing_http(); + let kv = Arc::new(MemoryKv::default()); + let services = services(Arc::clone(&http) as Arc, &kv); + + let output = view(&factory(), &services); + + assert_signed(&output); + assert_eq!(http.recorded_request_methods(), ["GET", "POST"]); + assert_eq!( + http.recorded_request_uris(), + [ + format!( + "https://api.example.com/api/v1/public/cdn/trusted-server/policy?page_url={}", + url::form_urlencoded::byte_serialize(PAGE_URL.as_bytes()).collect::() + ), + "https://api.example.com/api/v1/public/cdn/sign".to_owned(), + ] + ); + let headers = &http.recorded_request_headers()[1]; + let header = |name: &str| { + headers + .iter() + .find(|(key, _)| key == name) + .map(|(_, value)| value.as_str()) + }; + assert_eq!(header("x-encypher-worker-token"), Some(TOKEN)); + assert_eq!(header("content-type"), Some("application/json")); + assert_eq!( + header("idempotency-key"), + None, + "should send no idempotency key" + ); + let body: serde_json::Value = serde_json::from_slice(&http.recorded_request_bodies()[1]) + .expect("should send a JSON body"); + assert_eq!(body["page_url"], PAGE_URL); + assert_eq!(body["expected_policy_digest"], POLICY_DIGEST); + assert_eq!(body["document_title"], "Harbor Council Votes"); + assert_eq!(body["boundary_selector"], "article"); + let sent_text = body["text"].as_str().expect("should send text"); + assert!( + sent_text.starts_with("Harbor council votes on the ferry schedule The harbor council"), + "should send the article's visible text, got {sent_text}" + ); + assert!(!sent_text.contains("Footer") && !sent_text.contains("Home")); + let (key, entry) = kv.only_entry(); + assert!( + key.starts_with("encypher:v3:") && key.contains(POLICY_DIGEST), + "unexpected cache key {key}" + ); + assert!(!key.contains(TOKEN), "cache key must not contain the token"); + assert!( + policy_key().starts_with("encypher:policy:v2:"), + "unexpected policy key" + ); + assert_eq!(entry["kind"], "plan"); + assert_eq!(entry["plan"], plan_json(MARKER)); + } + + #[test] + fn a_cached_plan_signs_without_calling_the_api() { + let kv = Arc::new(MemoryKv::default()); + let first = seed_plan(&kv); + + let http = Arc::new(StubHttpClient::new()); + let second = view( + &factory(), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_eq!( + second, first, + "a cache hit should serve the same signed page" + ); + assert!( + http.recorded_backend_names().is_empty(), + "should make no API call" + ); + } + + #[test] + fn a_cached_skip_passes_through_without_calling_the_api() { + let kv = Arc::new(MemoryKv::default()); + let first_http = Arc::new(StubHttpClient::new()); + first_http.push_response(200, policy_response(POLICY_DIGEST)); + first_http.push_response(403, b"{}".to_vec()); + view(&factory(), &services(first_http, &kv)); + assert_eq!(kv.only_entry().1["kind"], "skip"); + + let http = Arc::new(StubHttpClient::new()); + let output = view( + &factory(), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_eq!(output, ARTICLE); + assert!( + http.recorded_backend_names().is_empty(), + "should make no API call" + ); + } + + #[test] + fn a_failure_does_not_replace_a_fresh_plan_written_since_the_first_read() { + let seeded = Arc::new(MemoryKv::default()); + seed_plan(&seeded); + // The first read misses, as if a concurrent request wrote the plan after it. + let kv = Arc::new(MemoryKv { + entries: Mutex::new(seeded.entries()), + hidden_reads: AtomicUsize::new(1), + ..MemoryKv::default() + }); + let http = Arc::new(StubHttpClient::new()); + http.push_response(503, Vec::new()); + + let output = view( + &factory(), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_eq!(output, ARTICLE, "the failed request should pass through"); + assert_eq!(http.recorded_backend_names().len(), 1); + assert_eq!(kv.only_entry().1["kind"], "plan", "the plan should survive"); + } + + #[test] + fn sign_failures_pass_through_and_write_a_skip() { + let oversized = { + let mut body = sign_response(MARKER); + body.resize(api::MAX_RESPONSE_BYTES + 2, b' '); + body + }; + let quota = serde_json::to_vec(&json!({ "success": false, "error": "quota_exceeded" })) + .expect("should serialize"); + let insecure_manifest = serde_json::to_vec(&json!({ + "success": true, + "policy_digest": POLICY_DIGEST, + "embedding_plan": plan_json(MARKER), + "manifest_url": "javascript:alert(1)", + })) + .expect("should serialize"); + let cases: Vec = vec![ + ("403", Some((403, b"{}".to_vec()))), + ("422", Some((422, b"{}".to_vec()))), + ("429", Some((429, b"{}".to_vec()))), + ("5xx", Some((502, Vec::new()))), + ("transport error or first-byte timeout", None), + ("oversized body", Some((200, oversized))), + ("malformed body", Some((200, b"not json".to_vec()))), + ( + "invalid marker", + Some((200, sign_response(""))), + ), + ("quota", Some((200, quota))), + ("non-https manifest url", Some((200, insecure_manifest))), + ( + "null plan", + Some(( + 200, + serde_json::to_vec(&json!({ + "success": true, "policy_digest": POLICY_DIGEST, "embedding_plan": null + })) + .expect("JSON"), + )), + ), + ]; + + for (case, response) in cases { + let http = Arc::new(StubHttpClient::new()); + http.push_response(200, policy_response(POLICY_DIGEST)); + if let Some((status, body)) = response { + http.push_response(status, body); + } + let kv = Arc::new(MemoryKv::default()); + + let output = view( + &factory(), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_eq!( + output, ARTICLE, + "{case}: should pass the page through unchanged" + ); + assert_eq!( + http.recorded_backend_names().len(), + 2, + "{case}: one policy call and one sign call" + ); + assert_eq!( + kv.only_entry().1["kind"], + "skip", + "{case}: should cache a skip" + ); + } + } + + #[test] + fn a_tampered_cached_plan_is_rejected_without_calling_the_api() { + for (field, value) in [ + ("/plan/operations/0/marker", ""), + ("/manifest_url", "http://manifest.example.com/doc-1"), + ] { + let kv = Arc::new(MemoryKv::default()); + seed_plan(&kv); + let (key, mut entry) = kv.only_entry(); + *entry + .pointer_mut(field) + .expect("seeded entry should have the field") = json!(value); + kv.insert(&key, &entry); + + let http = Arc::new(StubHttpClient::new()); + let output = view( + &factory(), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_eq!(output, ARTICLE, "{field}: should pass through"); + assert!( + http.recorded_backend_names().is_empty(), + "{field}: should make no API call" + ); + } + } + + #[test] + fn a_changed_credential_misses_the_cache() { + let kv = Arc::new(MemoryKv::default()); + seed_plan(&kv); + + let http = signing_http(); + let output = view( + &factory_with(config_json("cwk_rotated_token"), 5_000_000), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_signed(&output); + assert_eq!(http.recorded_request_methods(), ["GET", "POST"]); + assert_eq!( + kv.entries().len(), + 4, + "each credential should have its own policy and article entries" + ); + } + + // --- asynchronous resolve and the hold deadline ------------------------- + + #[test] + fn calls_that_are_pending_before_ready_are_awaited_and_sign() { + let timers: [Option>; 3] = [ + None, + Some(Arc::new(StoppedTimer)), + Some(Arc::new(TickTimer::default())), + ]; + for timer in timers { + let label = if timer.is_some() { "timer" } else { "no timer" }; + let kv = Arc::new(MemoryKv { + pending_polls: 2, + ..MemoryKv::default() + }); + let stub = signing_http(); + let http = Arc::new(YieldingHttpClient { + inner: Arc::clone(&stub), + pending_polls: 2, + }); + let mut services = services(http, &kv); + if let Some(timer) = timer { + services = services.with_timer(timer); + } + + let first = view(&factory(), &services); + let second = view(&factory(), &services); + + assert_signed(&first); + assert_eq!( + second, first, + "{label}: the second view should hit the cache" + ); + assert_eq!(stub.recorded_request_methods(), ["GET", "POST"], "{label}"); + assert_eq!(kv.only_entry().1["kind"], "plan", "{label}"); + } + } + + #[test] + fn a_hung_policy_call_times_out_writes_a_skip_and_the_next_view_makes_no_call() { + let kv = Arc::new(MemoryKv::default()); + let hung = Arc::new(HungHttpClient::default()); + + let output = view( + &factory(), + &timed_services(Arc::clone(&hung) as Arc, &kv), + ); + + assert_eq!(output, ARTICLE, "should release the page unchanged"); + assert_eq!(hung.sends.load(Ordering::SeqCst), 1); + let skip = kv.entry(&policy_key()); + assert_eq!(skip["kind"], "skip"); + assert_eq!(skip["reason"], "timeout"); + + let http = Arc::new(StubHttpClient::new()); + let next = view( + &factory(), + &timed_services(Arc::clone(&http) as Arc, &kv), + ); + assert_eq!(next, ARTICLE); + assert!( + http.recorded_backend_names().is_empty(), + "the next view should be a cached skip" + ); + } + + #[test] + fn a_hung_sign_call_or_body_times_out_writes_a_plan_skip_and_the_next_view_makes_no_call() { + let kv = Arc::new(MemoryKv::default()); + seed_plan(&kv); + let article_key = kv.only_entry().0; + kv.remove(&article_key); + let hung = Arc::new(HungHttpClient::default()); + let services = timed_services(Arc::clone(&hung) as Arc, &kv); + + for _ in 0..2 { + assert_eq!(view(&factory(), &services), ARTICLE); + } + + assert_eq!( + hung.sends.load(Ordering::SeqCst), + 1, + "one sign call in total" + ); + let (_, skip) = kv.only_entry(); + assert_eq!(skip["kind"], "skip"); + assert_eq!(skip["reason"], "timeout"); + } + + #[test] + fn a_hung_kv_read_times_out_without_calling_the_api() { + let kv = Arc::new(MemoryKv { + hung_reads: true, + ..MemoryKv::default() + }); + let http = Arc::new(StubHttpClient::new()); + + let output = view( + &factory(), + &timed_services(Arc::clone(&http) as Arc, &kv), + ); + + assert_eq!(output, ARTICLE); + assert_eq!(kv.reads(), 1); + assert!(http.recorded_backend_names().is_empty()); + } + + #[test] + fn a_plan_write_that_runs_out_of_time_still_returns_the_signed_page() { + let kv = Arc::new(MemoryKv { + hung_writes_to: Some(CACHE_KEY_PREFIX), + ..MemoryKv::default() + }); + let http = signing_http(); + + let output = view( + &factory(), + &timed_services(Arc::clone(&http) as Arc, &kv), + ); + + assert_signed(&output); + assert_eq!(http.recorded_request_methods(), ["GET", "POST"]); + assert!( + kv.entries() + .keys() + .all(|key| !key.starts_with(CACHE_KEY_PREFIX)), + "the abandoned plan write should store nothing" + ); + } + + #[test] + fn a_policy_digest_write_that_runs_out_of_time_caches_a_policy_skip() { + let kv = Arc::new(MemoryKv { + hung_first_write_to: Some(policy_key()), + ..MemoryKv::default() + }); + let http = Arc::new(StubHttpClient::new()); + http.push_response(200, policy_response(POLICY_DIGEST)); + + let output = view( + &factory(), + &timed_services(Arc::clone(&http) as Arc, &kv), + ); + + assert_eq!(output, ARTICLE, "should release the page unchanged"); + assert_eq!( + http.recorded_request_methods(), + ["GET"], + "no sign call after the digest write ran out of time" + ); + let skip = kv.entry(&policy_key()); + assert_eq!(skip["kind"], "skip"); + assert_eq!(skip["reason"], "timeout"); + assert!( + kv.entries() + .keys() + .all(|key| !key.starts_with(CACHE_KEY_PREFIX)), + "no plan skip under a digest the next view cannot know" + ); + + // A hung client turns any API call into a full hold, so zero sends + // after one KV read shows the next view is a cached skip. + let reads = kv.reads(); + let hung = Arc::new(HungHttpClient::default()); + let next = view( + &factory(), + &timed_services(Arc::clone(&hung) as Arc, &kv), + ); + assert_eq!(next, ARTICLE); + assert_eq!(hung.sends.load(Ordering::SeqCst), 0, "no API call"); + assert_eq!(kv.reads() - reads, 1, "only the policy entry is read"); + } + + #[test] + fn the_core_backstop_lies_beyond_the_module_hold_bound() { + let kv = Arc::new(MemoryKv::default()); + let timer = Arc::new(TickTimer::default()); + let services = services(Arc::new(HungHttpClient::default()), &kv) + .with_timer(Arc::clone(&timer) as Arc); + + assert_eq!(view(&factory(), &services), ARTICLE); + + let max_hold = Duration::from_millis(5_000); + let requested = timer.requested.lock().expect("lock").clone(); + let (backstop, module): (Vec<_>, Vec<_>) = + requested.into_iter().partition(|sleep| *sleep > max_hold); + assert_eq!(backstop.len(), 1, "one core backstop beyond max_hold_ms"); + // The skip-write reserve starts only after the work deadline, so the + // module's own sleeps run back to back. + assert!( + !module.is_empty() && module.iter().sum::() <= max_hold, + "module sleeps {module:?} should fit within max_hold_ms" + ); + assert_eq!( + kv.entry(&policy_key())["reason"], + "timeout", + "the module deadline, not the backstop, should release the page" + ); + } + + // --- expiry, write mode and key namespaces ------------------------------ + + #[test] + fn expired_entries_are_misses() { + // (entry to expire, API calls the next view makes) + let cases: [(&str, &[&str]); 4] = [ + ("policy digest", &["GET"]), + ("policy skip", &["GET"]), + ("plan", &["POST"]), + ("plan skip", &["POST"]), + ]; + for (case, expected_calls) in cases { + let kv = Arc::new(MemoryKv::default()); + seed_plan(&kv); + let (plan_key, _) = kv.only_entry(); + let key = if case.starts_with("policy") { + policy_key() + } else { + plan_key + }; + if case.ends_with("skip") { + kv.insert( + &key, + &json!({"kind": "skip", "reason": "server_error", "expires_at": expired()}), + ); + } else { + set_expiry(&kv, &key, expired()); + } + let http = Arc::new(StubHttpClient::new()); + if case.starts_with("policy") { + http.push_response(200, policy_response(POLICY_DIGEST)); + } else { + http.push_response(200, sign_response(MARKER)); + } + + let output = view( + &factory(), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_signed(&output); + assert_eq!(http.recorded_request_methods(), expected_calls, "{case}"); + assert!( + kv.entry(&key)["expires_at"].as_u64() > Some(unix_now()), + "{case}: should be replaced by a fresh entry" + ); + } + } + + #[test] + fn an_expired_plan_does_not_block_a_skip_write() { + let seeded = Arc::new(MemoryKv::default()); + seed_plan(&seeded); + let kv = Arc::new(MemoryKv { + entries: Mutex::new(seeded.entries()), + hidden_reads: AtomicUsize::new(1), + ..MemoryKv::default() + }); + let (plan_key, _) = kv.only_entry(); + set_expiry(&kv, &plan_key, expired()); + let http = Arc::new(StubHttpClient::new()); + http.push_response(503, Vec::new()); + + assert_eq!(view(&factory(), &services(http, &kv)), ARTICLE); + + assert_eq!(kv.entry(&plan_key)["kind"], "skip"); + } + + #[test] + fn a_store_without_ttl_writes_switches_once_and_expires_by_value() { + let kv = Arc::new(MemoryKv { + ttl_unsupported: true, + ..MemoryKv::default() + }); + let factory = factory(); + let http = signing_http(); + let services = services(Arc::clone(&http) as Arc, &kv); + + let first = view(&factory, &services); + + assert_signed(&first); + assert_eq!( + kv.ttl_writes.load(Ordering::SeqCst), + 1, + "only the first write should try a TTL" + ); + let (plan_key, plan) = kv.only_entry(); + assert_eq!(plan["kind"], "plan"); + assert!(plan["expires_at"].as_u64() > Some(unix_now())); + assert_eq!(kv.entry(&policy_key())["kind"], "digest"); + assert_eq!(view(&factory, &services), first, "should hit the cache"); + assert_eq!(http.recorded_request_methods(), ["GET", "POST"]); + + // Expired entries are misses and are deleted when read. + set_expiry(&kv, &plan_key, expired()); + http.push_response(200, sign_response(MARKER)); + assert_signed(&view(&factory, &services)); + assert_eq!(*kv.deletes.lock().expect("lock"), [plan_key.as_str()]); + assert_eq!( + kv.entry(&plan_key)["kind"], + "plan", + "should store a new plan" + ); + } + + #[test] + fn a_fresh_instance_deletes_expired_entries_written_without_a_ttl() { + // Spin instantiates the module per request, so the instance that reads + // an expired entry has not written anything yet. + let kv = Arc::new(MemoryKv { + ttl_unsupported: true, + ..MemoryKv::default() + }); + view(&factory(), &services(signing_http(), &kv)); + set_expiry(&kv, &policy_key(), expired()); + let http = Arc::new(StubHttpClient::new()); + http.push_response(503, Vec::new()); + + let output = view( + &factory(), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_eq!(output, ARTICLE); + assert_eq!(http.recorded_request_methods(), ["GET"]); + assert_eq!(*kv.deletes.lock().expect("lock"), [policy_key()]); + } + + #[test] + fn expired_entries_are_not_deleted_on_a_store_with_ttls() { + let kv = Arc::new(MemoryKv::default()); + let http = signing_http(); + let services = services(Arc::clone(&http) as Arc, &kv); + view(&factory(), &services); + let (plan_key, _) = kv.only_entry(); + set_expiry(&kv, &plan_key, expired()); + http.push_response(200, sign_response(MARKER)); + + assert_signed(&view(&factory(), &services)); + + assert!(kv.deletes.lock().expect("lock").is_empty()); + } + + #[test] + fn entries_under_the_previous_key_versions_are_never_read() { + let current = Arc::new(MemoryKv::default()); + seed_plan(¤t); + // The same plan and policy, as the previous release stored them. + let kv = Arc::new(MemoryKv::default()); + for (key, value) in current.entries() { + let mut entry: serde_json::Value = + serde_json::from_slice(&value).expect("should store JSON"); + entry.as_object_mut().expect("object").remove("expires_at"); + let old_key = key + .replace("encypher:v3:", "encypher:v2:") + .replace("encypher:policy:v2:", "encypher:policy:v1:"); + kv.insert(&old_key, &entry); + } + let old_entries = kv.entries(); + let http = signing_http(); + + let output = view( + &factory(), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_signed(&output); + assert_eq!(http.recorded_request_methods(), ["GET", "POST"]); + let entries = kv.entries(); + assert_eq!(entries.len(), old_entries.len() * 2); + for (key, value) in old_entries { + assert_eq!(entries[&key], value, "{key} should be left untouched"); + } + } + + // --- KV failures -------------------------------------------------------- + + #[test] + fn kv_read_errors_pass_through_without_calling_the_api() { + for kv in [ + MemoryKv { + fail_reads: true, + ..MemoryKv::default() + }, + MemoryKv { + unavailable: true, + ..MemoryKv::default() + }, + ] { + let kv = Arc::new(kv); + let http = Arc::new(StubHttpClient::new()); + + let output = view( + &factory(), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_eq!(output, ARTICLE); + assert_eq!(kv.reads(), 1); + assert!( + http.recorded_backend_names().is_empty(), + "should make no API call" + ); + } + } + + #[test] + fn an_unbound_kv_store_logs_kv_unavailable_and_warns_once() { + let Some(logs) = capture_logs(|| { + let http: Arc = Arc::new(StubHttpClient::new()); + let unbound = Arc::new(MemoryKv { + unavailable: true, + ..MemoryKv::default() + }); + for _ in 0..2 { + view(&factory(), &services(Arc::clone(&http), &unbound)); + } + let failing = Arc::new(MemoryKv { + fail_reads: true, + ..MemoryKv::default() + }); + view(&factory(), &services(http, &failing)); + }) else { + // Log lines go to a logger installed elsewhere and cannot be observed here. + return; + }; + + let outcomes: Vec<_> = logs + .iter() + .filter(|line| line.contains("encypher: outcome=")) + .collect(); + assert_eq!( + outcomes, + [ + "INFO encypher: outcome=passed_through reason=kv_unavailable", + "INFO encypher: outcome=passed_through reason=kv_unavailable", + "INFO encypher: outcome=passed_through reason=kv_error", + ] + ); + assert_eq!( + logs.iter() + .filter(|line| line.starts_with("WARN") && line.contains("KV store is unavailable")) + .count(), + 1, + "should warn once per instance: {logs:?}" + ); + } + + // --- gates and skips ---------------------------------------------------- + + #[test] + fn gated_documents_pass_through_with_no_kv_or_api_io() { + let signer = factory(); + let cases: Vec<(GateCase<'_>, Gate)> = vec![ + ( + ("method", &signer, |r| r.method = Method::POST), + Gate::Method, + ), + ( + ("status", &signer, |r| r.response_status = 404), + Gate::Status, + ), + ( + ("origin private", &signer, |r| { + r.origin_response_is_private = true; + }), + Gate::OriginPrivate, + ), + ( + ("host mismatch", &signer, |r| { + r.page_url = "https://other.example.com/news/harbor-vote".to_owned(); + }), + Gate::HostMismatch, + ), + ( + ("excluded path", &signer, |r| { + r.page_url = "https://publisher.example.com/WP-Admin/post".to_owned(); + }), + Gate::ExcludedPath, + ), + ( + ("asset suffix", &signer, |r| { + r.page_url = "https://publisher.example.com/feed.XML".to_owned(); + }), + Gate::ExcludedPath, + ), + ( + ("invalid page url", &signer, |r| { + r.page_url = "not a url".to_owned(); + }), + Gate::InvalidPageUrl, + ), + ]; + + for ((case, signer, mutate), gate) in cases { + let http = Arc::new(StubHttpClient::new()); + let kv = Arc::new(MemoryKv::default()); + let services = services(Arc::clone(&http) as Arc, &kv); + let mut request = request(); + mutate(&mut request); + + assert_eq!( + signer.admit(request.clone()), + Err(gate), + "{case}: should be refused by its gate" + ); + let output = run(signer, Some(request), &services, ARTICLE); + + assert_eq!(output, ARTICLE, "{case}: should pass through"); + assert_eq!(kv.reads(), 0, "{case}: should read no KV"); + assert!( + http.recorded_backend_names().is_empty(), + "{case}: no API call" + ); + } + + let kv = Arc::new(MemoryKv::default()); + assert_eq!( + run( + &factory(), + None, + &services(Arc::new(StubHttpClient::new()), &kv), + ARTICLE + ), + ARTICLE, + "nothing prepared" + ); + } + + #[test] + fn a_bare_domain_page_is_signed() { + let kv = Arc::new(MemoryKv::default()); + let mut request = request(); + request.page_url = "https://publisher.example.com/news/harbor-vote".to_owned(); + + let output = run( + &factory(), + Some(request), + &services(signing_http(), &kv), + ARTICLE, + ); + + assert_signed(&output); + } + + #[test] + fn an_oversized_document_is_flushed_and_passed_through() { + let http = Arc::new(StubHttpClient::new()); + let kv = Arc::new(MemoryKv::default()); + + let output = view( + &factory_with(config_json(TOKEN), 64), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_eq!(output, ARTICLE); + assert_eq!(kv.reads(), 0); + assert!(http.recorded_backend_names().is_empty()); + } + + #[test] + fn already_signed_or_short_pages_are_not_sent() { + let magic = "\u{E01DF}\u{E01E0}\u{E01E1}\u{E01E2}"; + let marked = ARTICLE.replace("The harbor council", &format!("The harbor{magic} council")); + let with_meta = ARTICLE.replace( + "", + r#""#, + ); + let short = "

Too short.

"; + + for page in [marked.as_str(), with_meta.as_str(), short] { + let http = Arc::new(StubHttpClient::new()); + let kv = Arc::new(MemoryKv::default()); + + let output = run( + &factory(), + Some(request()), + &services(Arc::clone(&http) as Arc, &kv), + page, + ); + + assert_eq!(output, page); + assert_eq!(kv.reads(), 0, "should read no KV"); + assert!( + http.recorded_backend_names().is_empty(), + "should make no API call" + ); + } + } + + #[test] + fn logs_carry_outcomes_but_never_the_token() { + let kv = Arc::new(MemoryKv::default()); + let Some(logs) = capture_logs(|| { + view(&factory(), &services(signing_http(), &kv)); + view(&factory(), &services(Arc::new(StubHttpClient::new()), &kv)); + let failing = Arc::new(StubHttpClient::new()); + failing.push_response(403, b"{}".to_vec()); + view( + &factory(), + &services(failing, &Arc::new(MemoryKv::default())), + ); + }) else { + // Log lines go to a logger installed elsewhere and cannot be observed here. + return; + }; + + assert!( + logs.iter() + .any(|line| line.ends_with("outcome=signed reason=signed")), + "should log the signing outcome: {logs:?}" + ); + for line in &logs { + assert!(!line.contains(TOKEN), "log line leaks the token: {line}"); + assert!( + !line.contains("encypher:v3:") && !line.contains("encypher:policy:v2:"), + "log line leaks the cache key: {line}" + ); + assert!( + !line.contains("ferry"), + "log line leaks article text: {line}" + ); + } + } +} diff --git a/crates/provenance/encypher/src/tests.rs b/crates/provenance/encypher/src/tests.rs new file mode 100644 index 0000000000..7251c37678 --- /dev/null +++ b/crates/provenance/encypher/src/tests.rs @@ -0,0 +1,421 @@ +use super::*; + +use std::sync::Arc; + +use edgezero_core::body::Body as EdgeBody; +use edgezero_core::key_value_store::KvPage; +use futures::executor::block_on; +use http::{Method, Request, header}; +use serde_json::json; +use trusted_server_core::auction::compile_auction_plan; +use trusted_server_core::auction::orchestrator::AuctionOrchestrator; +use trusted_server_core::cache_policy::EdgeCacheHeader; +use trusted_server_core::ec::EcContext; +use trusted_server_core::html_processor::test_support::page_settings; +use trusted_server_core::integrations::IntegrationRegistry; +use trusted_server_core::platform::test_support::{ + StubHttpClient, build_services_with_http_client, +}; +use trusted_server_core::platform::{ + KvError, PlatformHttpClient, PlatformKvStore, RuntimeServices, +}; +use trusted_server_core::publisher::{ + AppContext, AuctionDispatch, buffer_publisher_response_async, handle_publisher_request, + publisher_response_into_streaming_response, +}; + +const TOKEN: &str = "cwk_test_worker_token_value"; +const MARKER: &str = "\u{E0100}\u{E0101}\u{FE01}"; +const MANIFEST_URL: &str = "https://api.example.com/manifests/doc-1"; +const POLICY_DIGEST: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +const ARTICLE: &str = "Harbor Council Votes\ +

Harbor council votes on the ferry schedule

\ +

The harbor council met on Tuesday evening and approved a revised ferry schedule \ + for the coming season.

Residents asked for earlier departures on weekdays.

\ +
Footer
"; + +fn module_settings(table: &str, serve: bool) -> String { + let entry = if serve { + format!("[[serve]]\nmedia_type = \"text/html\"\nmiddleware = [\"{MODULE}\"]\n") + } else { + String::new() + }; + format!("[provenance]\nmodule = \"encypher\"\n\n[provenance.encypher]\n{table}\n{entry}") +} + +fn table() -> String { + format!( + "api_base_url = \"https://api.example.com\"\ndomain = \"publisher.example.com\"\n\ + worker_token = \"{TOKEN}\"\n" + ) +} + +fn placed() -> Settings { + page_settings(&module_settings(&table(), true)) +} + +fn config_value() -> serde_json::Value { + json!({ + "api_base_url": "https://api.example.com", + "domain": "publisher.example.com", + "worker_token": TOKEN, + }) +} + +fn parsed(value: &serde_json::Value) -> Result { + let config: EncypherConfig = + serde_json::from_value(value.clone()).map_err(|error| error.to_string())?; + config.validate().map_err(|error| error.to_string())?; + Ok(config) +} + +#[test] +fn module_constant_is_the_crate_folder() { + assert_eq!( + MODULE, + trusted_server_core::module_name!(), + "should be named by the folder this crate lives in" + ); +} + +#[test] +fn an_unselected_module_registers_nothing() { + let settings = page_settings(""); + + assert!(register(&settings).expect("should read").is_none()); + assert!(!validate(&settings).expect("should read")); +} + +#[test] +fn a_selected_and_placed_module_registers_its_middleware() { + let settings = placed(); + + assert!(validate(&settings).expect("should validate")); + assert!( + register(&settings).expect("should register").is_some(), + "a selected module with a worker token should register" + ); +} + +#[test] +fn a_module_named_in_no_serve_entry_is_refused() { + let settings = page_settings(&module_settings(&table(), false)); + + let error = validate(&settings).expect_err("should refuse the document"); + + assert!( + format!("{error:?}").contains("no [[serve]] entry names"), + "should name the missing entry: {error:?}" + ); +} + +#[test] +fn a_blank_worker_token_registers_nothing() { + for token in ["", " \t"] { + let table = format!( + "api_base_url = \"https://api.example.com\"\ndomain = \"publisher.example.com\"\n\ + worker_token = \"{token}\"\n" + ); + let settings = page_settings(&module_settings(&table, true)); + + assert!( + register(&settings).expect("should read").is_none(), + "token {token:?} should not register" + ); + } +} + +#[test] +fn required_settings_and_bounds_are_enforced() { + for field in ["domain", "api_base_url", "worker_token"] { + let mut config = config_value(); + config + .as_object_mut() + .expect("should be an object") + .remove(field); + assert!( + parsed(&config).is_err(), + "config without {field} should fail" + ); + } + let defaults = parsed(&config_value()).expect("should parse defaults"); + assert_eq!(defaults.policy_ttl_seconds, 300); + assert_eq!(defaults.plan_ttl_seconds, 604_800); + assert_eq!(defaults.max_hold_ms, 5_000); + for (field, value) in [ + ("api_base_url", json!("http://api.example.com")), + ("api_base_url", json!("https://api.example.com/?q=1")), + ("domain", json!("Publisher.example.com")), + ("domain", json!("www.publisher.example.com")), + ("domain", json!("publisher.example.com/path")), + ("domain", json!("localhost")), + ("timeout_ms", json!(100)), + ("timeout_ms", json!(20_000)), + ("min_text_length", json!(10)), + ("negative_ttl_seconds", json!(10)), + ("policy_ttl_seconds", json!(301)), + ("article_selector", json!("a".repeat(92))), + ("enabled", json!(true)), + ("path_prefixes", json!(["/news/"])), + ] { + let mut config = config_value(); + config[field] = value.clone(); + assert!( + parsed(&config).is_err(), + "{field} = {value} should be rejected" + ); + } + for (max_hold_ms, timeout_ms, valid) in [ + (499, 250, false), + (500, 250, true), + (15_000, 3_000, true), + (15_001, 3_000, false), + (3_000, 3_000, false), + (3_001, 3_000, true), + ] { + let mut config = config_value(); + config["max_hold_ms"] = json!(max_hold_ms); + config["timeout_ms"] = json!(timeout_ms); + assert_eq!( + parsed(&config).is_ok(), + valid, + "max_hold_ms {max_hold_ms} with timeout_ms {timeout_ms}" + ); + } +} + +#[test] +fn excluded_paths_leave_the_middleware_out_of_the_chain() { + let middleware = Encypher { + signer: Arc::new(Signer::new( + parsed(&config_value()).expect("should parse"), + 5_000_000, + )), + }; + + assert!(middleware.handles_path("/news/harbor-vote")); + for path in [ + "/wp-admin/post", + "/feed/", + "/sitemap.xml", + "/app.js", + "/api/v1/x", + ] { + assert!(!middleware.handles_path(path), "{path} should be left out"); + } + assert_eq!(middleware.phases(), [MiddlewarePhase::Serve]); +} + +/// A runtime KV store held in memory. +#[derive(Default)] +struct MemoryKv(std::sync::Mutex>); + +#[async_trait::async_trait(?Send)] +impl PlatformKvStore for MemoryKv { + async fn get_bytes(&self, key: &str) -> Result, KvError> { + Ok(self.0.lock().expect("should lock").get(key).cloned()) + } + + async fn put_bytes(&self, key: &str, value: bytes::Bytes) -> Result<(), KvError> { + self.0 + .lock() + .expect("should lock") + .insert(key.to_owned(), value); + Ok(()) + } + + async fn put_bytes_with_ttl( + &self, + key: &str, + value: bytes::Bytes, + _ttl: std::time::Duration, + ) -> Result<(), KvError> { + self.put_bytes(key, value).await + } + + async fn delete(&self, key: &str) -> Result<(), KvError> { + self.0.lock().expect("should lock").remove(key); + Ok(()) + } + + async fn list_keys_page( + &self, + _prefix: &str, + _cursor: Option<&str>, + _limit: usize, + ) -> Result { + Ok(KvPage::default()) + } +} + +/// An origin answering [`ARTICLE`] with `cache_control`, then the policy and +/// sign answers of the API. +fn stub(cache_control: &str) -> Arc { + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + ARTICLE.as_bytes().to_vec(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("cache-control", cache_control), + ], + ); + stub.push_response( + 200, + serde_json::to_vec(&json!({ "policy_digest": POLICY_DIGEST })).expect("should encode"), + ); + stub.push_response( + 200, + serde_json::to_vec(&json!({ + "success": true, + "policy_digest": POLICY_DIGEST, + "embedding_plan": { + "index_unit": "codepoint", + "operations": [{ "insert_after_index": 5, "marker": MARKER }], + }, + "manifest_url": MANIFEST_URL, + "verification_url": "https://api.example.com/verify/doc-1", + "document_id": "doc-1", + })) + .expect("should encode"), + ); + stub +} + +fn page_request(method: Method) -> Request { + Request::builder() + .method(method) + .uri("https://publisher.example.com/news/harbor-vote?utm_source=feed") + .header(header::HOST, "publisher.example.com") + .body(EdgeBody::empty()) + .expect("should build the page request") +} + +/// The page body a reader receives from the publisher path, through the +/// buffered or the streaming finalizer. +fn served( + settings: &Settings, + services: &RuntimeServices, + request: Request, + streaming: bool, +) -> String { + let registry = IntegrationRegistry::with_registrations(settings, &[builder()]) + .expect("should build the registry"); + let plan = compile_auction_plan(settings).expect("should compile the auction plan"); + let orchestrator = Arc::new(AuctionOrchestrator::from_plan(Arc::new(plan), None)); + let method = request.method().clone(); + block_on(async { + let mut ec_context = + EcContext::read_from_request_resolving_geo(settings, &request, services) + .await + .expect("should read the request's state"); + let publisher = handle_publisher_request( + AppContext { + settings, + integration_registry: ®istry, + }, + services, + None, + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &[], + registry: None, + }, + request, + EdgeCacheHeader::SurrogateControl, + ) + .await + .expect("should fetch the page"); + let response = if streaming { + publisher_response_into_streaming_response( + publisher, + &method, + Arc::new(settings.clone()), + ®istry, + Arc::clone(&orchestrator), + services.clone(), + ) + .await + } else { + buffer_publisher_response_async( + publisher, + &method, + settings, + ®istry, + &orchestrator, + services, + ) + .await + } + .expect("should finish the page"); + let body = response + .into_body() + .into_bytes_bounded(16 * 1024 * 1024) + .await + .expect("should read the body"); + String::from_utf8(body.to_vec()).expect("should be UTF-8") + }) +} + +fn services_for(stub: &Arc) -> RuntimeServices { + build_services_with_http_client(Arc::clone(stub) as Arc) + .with_kv_store(Arc::new(MemoryKv::default())) +} + +#[test] +fn the_publisher_path_signs_an_article_through_both_finalizers() { + for streaming in [false, true] { + let stub = stub("public, max-age=60"); + let page = served( + &placed(), + &services_for(&stub), + page_request(Method::GET), + streaming, + ); + + assert!( + page.contains(MARKER), + "streaming={streaming}: should embed the marker: {page}" + ); + assert!( + page.contains("c2pa-manifest-url") && page.contains(MANIFEST_URL), + "streaming={streaming}: should write the manifest meta tag" + ); + assert_eq!( + stub.recorded_request_methods(), + ["GET", "GET", "POST"], + "streaming={streaming}: origin, policy, then sign" + ); + // The test request carries no TLS signal, so the scheme the publisher + // path detects is `http`. + let policy_uri = &stub.recorded_request_uris()[1]; + assert!( + policy_uri + .ends_with("page_url=http%3A%2F%2Fpublisher.example.com%2Fnews%2Fharbor-vote"), + "should send the page address without its query: {policy_uri}" + ); + } +} + +#[test] +fn a_page_the_origin_marked_private_is_never_sent() { + let stub = stub("private, max-age=0"); + + let page = served( + &placed(), + &services_for(&stub), + page_request(Method::GET), + false, + ); + + assert!( + !page.contains(MARKER), + "should not sign a reader's own page" + ); + assert_eq!( + stub.recorded_request_methods(), + ["GET"], + "should call the origin and nothing else" + ); +} diff --git a/crates/provenance/encypher/src/text.rs b/crates/provenance/encypher/src/text.rs new file mode 100644 index 0000000000..28ebd3450a --- /dev/null +++ b/crates/provenance/encypher/src/text.rs @@ -0,0 +1,500 @@ +use std::sync::LazyLock; + +use regex::Regex; +use sha2::{Digest as _, Sha256}; + +/// Elements whose content canonical text drops: edge-core's set in +/// `integrations/edge-core/fragments.js`. +const TEXT_IGNORED: [&str; 7] = ["script", "style", "noscript", "svg", "math", "code", "pre"]; + +/// Elements whose content browsers never parse as markup (raw text and RCDATA). +/// A `/>` spelling does not close them, and the closing-tag search skips them. +const RAW_TEXT: [&str; 9] = [ + "script", + "style", + "noscript", + "title", + "textarea", + "xmp", + "iframe", + "noembed", + "plaintext", +]; + +static TITLE: LazyLock> = + LazyLock::new(|| Regex::new(r"(?is)]*>(.*?)").ok()); +static DECIMAL_ENTITY: LazyLock> = LazyLock::new(|| Regex::new(r"&#([0-9]+);").ok()); +static HEX_ENTITY: LazyLock> = + LazyLock::new(|| Regex::new(r"&#x([0-9a-fA-F]+);").ok()); + +/// A byte range containing one raw text run in an HTML document. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Fragment { + /// Inclusive UTF-8 byte offset at which the text begins. + pub start: usize, + /// Exclusive UTF-8 byte offset at which the text ends. + pub end: usize, +} + +/// Extracts visible raw-text fragments and their UTF-8 byte ranges. +#[must_use] +pub fn extract_fragments(html: &str) -> Vec { + let mut fragments = Vec::new(); + scan_html( + html, + |name| contains_tag(&TEXT_IGNORED, name), + |start, end| { + if html + .get(start..end) + .is_some_and(|raw| raw.chars().any(|ch| !is_js_whitespace(ch))) + { + fragments.push(Fragment { start, end }); + } + }, + |_, _| {}, + ); + fragments +} + +/// Decodes, normalizes, trims, and joins raw text fragments with single spaces. +#[must_use] +pub fn assemble_text(html: &str, fragments: &[Fragment]) -> String { + let mut output = String::new(); + for fragment in fragments { + if let Some(raw) = html.get(fragment.start..fragment.end) { + let decoded = decode_html_entities(raw); + let normalized = normalize_js_whitespace(&decoded); + if !normalized.is_empty() { + if !output.is_empty() { + output.push(' '); + } + output.push_str(&normalized); + } + } + } + output +} + +/// Returns the canonical visible text of an HTML string. +#[must_use] +pub fn visible_text(html: &str) -> String { + assemble_text(html, &extract_fragments(html)) +} + +/// Returns a lowercase SHA-256 digest of `text` as 64 hexadecimal characters. +#[must_use] +pub fn content_hash_hex(text: &str) -> String { + let digest = Sha256::digest(text.as_bytes()); + let mut output = String::with_capacity(64); + for byte in digest { + if let Some(high) = char::from_digit(u32::from(byte >> 4), 16) { + output.push(high); + } + if let Some(low) = char::from_digit(u32::from(byte & 0x0f), 16) { + output.push(low); + } + } + output +} + +/// Returns at most `max_scalars` Unicode scalar values without splitting one. +#[must_use] +pub fn truncate_scalars(text: &str, max_scalars: usize) -> &str { + let Some((end, _)) = text.char_indices().nth(max_scalars) else { + return text; + }; + text.get(..end).unwrap_or(text) +} + +/// Counts Unicode scalar values in `text`. +#[must_use] +pub fn scalar_count(text: &str) -> usize { + text.chars().count() +} + +/// Extracts the first title, decodes entities, normalizes whitespace, and caps it at 500 scalars. +#[must_use] +pub fn extract_title(html: &str) -> Option { + let regex = TITLE.as_ref()?; + let raw = regex.captures(html)?.get(1)?.as_str(); + let decoded = decode_html_entities(raw); + let mut normalized = normalize_js_whitespace(&decoded); + normalized.truncate(truncate_scalars(&normalized, 500).len()); + Some(normalized) +} + +/// Finds the first closing tag outside comments, the canonical-text ignored +/// elements, and raw-text or RCDATA elements. +#[must_use] +pub fn structural_close_tag(html: &str, tag: &str) -> Option { + let mut found = None; + scan_html( + html, + |name| contains_tag(&TEXT_IGNORED, name) || contains_tag(&RAW_TEXT, name), + |_, _| {}, + |start, token| { + if found.is_none() + && closing_tag_name(token).is_some_and(|name| name.eq_ignore_ascii_case(tag)) + { + found = Some(start); + } + }, + ); + found +} + +/// Scans tags, comments and text runs; both consumers share its comment and +/// quoted-attribute handling. `ignores` picks the elements whose content is +/// skipped: canonical text passes [`TEXT_IGNORED`], and the structural +/// closing-tag search passes [`TEXT_IGNORED`] and [`RAW_TEXT`]. +fn scan_html<'a>( + html: &'a str, + ignores: impl Fn(&str) -> bool, + mut text: impl FnMut(usize, usize), + mut tag: impl FnMut(usize, &'a str), +) { + let bytes = html.as_bytes(); + let mut cursor = 0; + let mut ignored: Option<&str> = None; + while cursor < bytes.len() { + if let Some(name) = ignored { + let Some(open) = find_byte(bytes, b'<', cursor) else { + break; + }; + cursor = open + 1; + if !starts_with_at(bytes, open, b"' || *byte == b'/' + }) + { + continue; + } + let Some(end) = tag_end(bytes, open + 2) else { + break; + }; + // As in browsers, attributes and `/` do not stop an end tag from + // closing. Unlike edge-core's depth count, no other end tag closes. + ignored = None; + cursor = end + 1; + continue; + } + if starts_with_at(bytes, cursor, b"` and `` are whole comments, and `--!>` also ends one. +fn comment_end(bytes: &[u8], start: usize) -> Option { + let body = start + 4; + if starts_with_at(bytes, body, b">") { + return Some(body + 1); + } + if starts_with_at(bytes, body, b"->") { + return Some(body + 2); + } + let dash = find_bytes(bytes, b"-->", body).map(|end| end + 3); + let bang = find_bytes(bytes, b"--!>", body).map(|end| end + 4); + match (dash, bang) { + (Some(dash), Some(bang)) => Some(dash.min(bang)), + (end, None) | (None, end) => end, + } +} + +/// Tokenizer states between a tag's name and its closing `>`. +#[derive(Clone, Copy)] +enum TagState { + Name, + BeforeAttributeName, + AttributeName, + BeforeValue, + Quoted(u8), + Unquoted, +} + +/// Returns the offset of the `>` ending the tag whose name starts at `start`. +/// As in the HTML tokenizer, a quote delimits only a value it begins. +fn tag_end(bytes: &[u8], start: usize) -> Option { + let mut state = TagState::Name; + for (offset, byte) in bytes.get(start..)?.iter().copied().enumerate() { + let space = byte.is_ascii_whitespace(); + state = match state { + TagState::Quoted(quote) => { + if byte == quote { + TagState::BeforeAttributeName + } else { + state + } + } + _ if byte == b'>' => return Some(start + offset), + TagState::Name if space || byte == b'/' => TagState::BeforeAttributeName, + TagState::BeforeAttributeName if !space && byte != b'/' => TagState::AttributeName, + TagState::AttributeName if byte == b'=' => TagState::BeforeValue, + TagState::AttributeName if byte == b'/' => TagState::BeforeAttributeName, + TagState::BeforeValue if byte == b'"' || byte == b'\'' => TagState::Quoted(byte), + TagState::BeforeValue if !space => TagState::Unquoted, + TagState::Unquoted if space => TagState::BeforeAttributeName, + _ => state, + }; + } + None +} + +pub(crate) fn decode_html_entities(input: &str) -> String { + let mut result = input.to_owned(); + for (encoded, decoded) in [ + ("&", "&"), + ("<", "<"), + (">", ">"), + (""", "\""), + ("'", "'"), + ("'", "'"), + ("'", "'"), + ("/", "/"), + (" ", "\u{00a0}"), + ("–", "\u{2013}"), + ("—", "\u{2014}"), + ("‘", "\u{2018}"), + ("’", "\u{2019}"), + ("“", "\u{201c}"), + ("”", "\u{201d}"), + ("…", "\u{2026}"), + ("•", "\u{2022}"), + ] { + if result.contains(encoded) { + result = result.replace(encoded, decoded); + } + } + result = decode_numeric_entities(&result, false); + decode_numeric_entities(&result, true) +} + +pub(crate) fn normalize_js_whitespace(input: &str) -> String { + let mut output = String::with_capacity(input.len()); + let mut pending_space = false; + for ch in input.chars() { + if is_js_whitespace(ch) { + pending_space = !output.is_empty(); + } else { + if pending_space { + output.push(' '); + pending_space = false; + } + output.push(ch); + } + } + output +} + +pub(crate) fn is_js_whitespace(ch: char) -> bool { + matches!( + ch, + '\u{0009}'..='\u{000d}' + | '\u{0020}' + | '\u{00a0}' + | '\u{1680}' + | '\u{2000}'..='\u{200a}' + | '\u{2028}' + | '\u{2029}' + | '\u{202f}' + | '\u{205f}' + | '\u{3000}' + | '\u{feff}' + ) +} + +fn decode_numeric_entities(input: &str, hexadecimal: bool) -> String { + let regex = if hexadecimal { + &*HEX_ENTITY + } else { + &*DECIMAL_ENTITY + }; + let Some(regex) = regex.as_ref() else { + return input.to_owned(); + }; + let radix = if hexadecimal { 16 } else { 10 }; + regex + .replace_all(input, |captures: ®ex::Captures<'_>| { + captures + .get(1) + .and_then(|digits| u32::from_str_radix(digits.as_str(), radix).ok()) + .and_then(char::from_u32) + .map_or_else( + || { + captures + .get(0) + .map_or("", |matched| matched.as_str()) + .to_owned() + }, + |decoded| decoded.to_string(), + ) + }) + .into_owned() +} + +fn find_byte(bytes: &[u8], needle: u8, start: usize) -> Option { + let relative = bytes + .get(start..)? + .iter() + .position(|byte| *byte == needle)?; + Some(start.saturating_add(relative)) +} + +fn find_bytes(bytes: &[u8], needle: &[u8], start: usize) -> Option { + if needle.is_empty() { + return Some(start.min(bytes.len())); + } + let haystack = bytes.get(start..)?; + let relative = haystack + .windows(needle.len()) + .position(|window| window == needle)?; + Some(start.saturating_add(relative)) +} + +fn starts_with_at(bytes: &[u8], start: usize, needle: &[u8]) -> bool { + bytes + .get(start..) + .is_some_and(|tail| tail.starts_with(needle)) +} + +fn opening_tag_name(token: &str) -> Option<&str> { + let content = token + .strip_prefix('<')? + .trim_start_matches(is_js_whitespace); + if content.starts_with('/') { + return None; + } + take_tag_name(content) +} + +fn closing_tag_name(token: &str) -> Option<&str> { + let content = token.strip_prefix("").then_some(name) +} + +fn take_tag_name(content: &str) -> Option<&str> { + let end = content + .char_indices() + .find(|(_, ch)| !(ch.is_ascii_alphanumeric() || *ch == '_')) + .map_or(content.len(), |(index, _)| index); + (end > 0).then(|| content.get(..end)).flatten() +} + +fn contains_tag(tags: &[&str], name: &str) -> bool { + tags.iter() + .any(|candidate| name.eq_ignore_ascii_case(candidate)) +} + +/// Matches edge-core's `/\/\s*>$/` test on a complete tag token. +fn is_self_closing(token: &str) -> bool { + token + .strip_suffix('>') + .is_some_and(|body| body.trim_end_matches(is_js_whitespace).ends_with('/')) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn quoted_markup_and_unmatched_script_tags_stay_inert() { + for html in [ + r#"
Visible
"#, + r#"
Visible
"#, + ] { + assert_eq!(structural_close_tag(html, "head"), html.rfind("")); + assert_eq!(visible_text(html), "Visible"); + } + } + + #[test] + fn browser_end_tag_and_comment_forms_close_where_browsers_do() { + // `` ends the script and `` is a whole comment, so each + // later `` sits inside an attribute value. + for prefix in [ + r#""#, + r#""#, + ] { + let html = format!("{prefix}
Visible
"); + assert_eq!(structural_close_tag(&html, "head"), Some(prefix.len())); + } + assert_eq!( + visible_text("abce