From 99f490e7a4af2866d3f7ac11e1aa54054c3a01a4 Mon Sep 17 00:00:00 2001 From: Erik Svilich Date: Sat, 24 Oct 2026 15:58:59 +0000 Subject: [PATCH 01/12] Add an optional platform timer to runtime services --- .../trusted-server-core/src/platform/mod.rs | 3 ++ .../src/platform/test_support.rs | 29 +++++++++++++++- .../trusted-server-core/src/platform/timer.rs | 20 +++++++++++ .../trusted-server-core/src/platform/types.rs | 34 ++++++++++++++++++- 4 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 crates/trusted-server-core/src/platform/timer.rs diff --git a/crates/trusted-server-core/src/platform/mod.rs b/crates/trusted-server-core/src/platform/mod.rs index 50fd3ca6c..2f40b03e2 100644 --- a/crates/trusted-server-core/src/platform/mod.rs +++ b/crates/trusted-server-core/src/platform/mod.rs @@ -15,6 +15,7 @@ //! - [`PlatformTemplateCache`] — shared transformed-template caching //! - [`PlatformValueCache`], a shared cache of small values with lifetimes //! - [`PlatformLoadingCache`], a cache that loads a missing value once per key +//! - [`PlatformTimer`], an optional timer for bounding asynchronous work //! //! ## Platform-Agnostic Components //! @@ -57,6 +58,7 @@ mod template_cache; reason = "test doubles, offered to other crates' tests under the test-utils feature" )] pub mod test_support; +mod timer; mod traits; mod types; mod value_cache; @@ -101,6 +103,7 @@ pub use template_cache::{ pub use traits::{ GeoModule, PlatformBackend, PlatformConfigStore, PlatformGeo, PlatformSecretStore, }; +pub use timer::PlatformTimer; pub use types::{ ClientInfo, GeoInfo, PlatformBackendSpec, RuntimeServices, RuntimeServicesBuilder, StoreId, StoreName, diff --git a/crates/trusted-server-core/src/platform/test_support.rs b/crates/trusted-server-core/src/platform/test_support.rs index 20f865f0e..711175873 100644 --- a/crates/trusted-server-core/src/platform/test_support.rs +++ b/crates/trusted-server-core/src/platform/test_support.rs @@ -12,7 +12,7 @@ use super::{ ClientInfo, GeoInfo, PlatformBackend, PlatformBackendSpec, PlatformCacheIntent, PlatformConfigStore, PlatformError, PlatformGeo, PlatformHttpClient, PlatformHttpRequest, PlatformImageOptimizerOptions, PlatformPendingRequest, PlatformResponse, PlatformSecretStore, - PlatformSelectResult, RuntimeServices, StoreId, StoreName, + PlatformSelectResult, PlatformTimer, RuntimeServices, StoreId, StoreName, }; pub struct NoopConfigStore; @@ -61,6 +61,33 @@ impl PlatformSecretStore for NoopSecretStore { } } +/// A timer whose sleeps have always already elapsed, recording each requested +/// duration. Work still pending when its deadline is first raced misses it. +#[derive(Default)] +pub struct ElapsedTimer { + pub requested: Mutex>, +} + +#[async_trait::async_trait(?Send)] +impl PlatformTimer for ElapsedTimer { + async fn sleep(&self, duration: Duration) { + self.requested + .lock() + .expect("should lock requested durations") + .push(duration); + } +} + +/// A timer that never fires. +pub struct StoppedTimer; + +#[async_trait::async_trait(?Send)] +impl PlatformTimer for StoppedTimer { + async fn sleep(&self, _duration: Duration) { + futures::future::pending::<()>().await; + } +} + pub struct HashMapConfigStore { data: HashMap, } diff --git a/crates/trusted-server-core/src/platform/timer.rs b/crates/trusted-server-core/src/platform/timer.rs new file mode 100644 index 000000000..e08603588 --- /dev/null +++ b/crates/trusted-server-core/src/platform/timer.rs @@ -0,0 +1,20 @@ +use std::time::Duration; + +/// A timer that can wake a pending future after a delay. +/// +/// Optional on [`crate::platform::RuntimeServices`]: its presence is the +/// capability. An adapter provides one only when its runtime can fire a timer +/// while other work of the request is pending. A runtime whose host calls +/// block the guest could not fire one in the middle of a call, so its adapter +/// provides none, and work there is bounded by the transport timeouts alone. +/// +/// Uses `?Send` futures, matching [`crate::platform::PlatformHttpClient`] and +/// [`crate::platform::PlatformKvStore`], because the WebAssembly timers are +/// `!Send`. +#[async_trait::async_trait(?Send)] +pub trait PlatformTimer: Send + Sync { + /// Completes once `duration` has elapsed. + /// + /// Dropping the returned future before it completes cancels the timer. + async fn sleep(&self, duration: Duration); +} diff --git a/crates/trusted-server-core/src/platform/types.rs b/crates/trusted-server-core/src/platform/types.rs index 8c3b96090..25266a330 100644 --- a/crates/trusted-server-core/src/platform/types.rs +++ b/crates/trusted-server-core/src/platform/types.rs @@ -7,7 +7,7 @@ use crate::auction::telemetry::{AuctionTelemetrySink, NoopAuctionTelemetrySink}; use super::{ GeoModule, PlatformBackend, PlatformConfigStore, PlatformHttpClient, PlatformKvStore, - PlatformSecretStore, + PlatformSecretStore, PlatformTimer, }; use crate::ec::device::DeviceModule; use crate::ec::identifiers::IdentifierModule; @@ -203,6 +203,9 @@ pub struct RuntimeServices { pub(crate) geo: Arc, /// Auction telemetry sink. pub(crate) auction_telemetry_sink: Arc, + /// Wall-clock timer, when the runtime can fire one while request work is + /// pending. `None` means the platform has no such timer. + pub(crate) timer: Option>, /// Per-request client metadata extracted at the entry point. pub(crate) client_info: ClientInfo, /// Host-computed client signals (TLS JA4, HTTP/2), when the host @@ -350,6 +353,13 @@ impl RuntimeServices { &*self.auction_telemetry_sink } + /// Returns the platform timer, or `None` when the runtime cannot fire a + /// timer while request work is pending. + #[must_use] + pub fn timer(&self) -> Option<&dyn PlatformTimer> { + self.timer.as_deref() + } + /// Returns per-request client metadata (IP address, TLS details). #[must_use] pub fn client_info(&self) -> &ClientInfo { @@ -599,6 +609,15 @@ impl RuntimeServices { ..self } } + + /// Returns a clone of this instance with `timer` as the platform timer. + #[must_use] + pub fn with_timer(self, timer: Arc) -> Self { + Self { + timer: Some(timer), + ..self + } + } } impl fmt::Debug for RuntimeServices { @@ -625,6 +644,7 @@ pub struct RuntimeServicesBuilder { http_client: Option>, geo: Option>, auction_telemetry_sink: Option>, + timer: Option>, client_info: Option, host_signals: Option>, resolved_ec_module: Option>, @@ -648,6 +668,7 @@ impl RuntimeServicesBuilder { http_client: None, geo: None, auction_telemetry_sink: None, + timer: None, client_info: None, host_signals: None, resolved_ec_module: None, @@ -741,6 +762,16 @@ impl RuntimeServicesBuilder { self } + /// Set the platform timer, or `None` when the runtime has none. + /// + /// Only adapters whose runtime can fire a timer while request work is + /// pending provide one. Unset means no timer. + #[must_use] + pub fn timer(mut self, timer: Option>) -> Self { + self.timer = timer; + self + } + /// Set the per-request client metadata. #[must_use] pub fn client_info(mut self, client_info: ClientInfo) -> Self { @@ -856,6 +887,7 @@ impl RuntimeServicesBuilder { auction_telemetry_sink: self .auction_telemetry_sink .unwrap_or_else(|| Arc::new(NoopAuctionTelemetrySink)), + timer: self.timer, client_info: self .client_info .expect("should set client_info before building RuntimeServices"), From 561f0df68f036955fc3ac8f5ef28e6855c952dd3 Mon Sep 17 00:00:00 2001 From: Erik Svilich Date: Sat, 24 Oct 2026 16:08:50 +0000 Subject: [PATCH 02/12] Open the Fastly runtime KV store when it is linked --- .../trusted-server-adapter-fastly/src/app.rs | 44 +++++++++- .../src/streaming_processor.rs | 84 +++++++++++++++++++ fastly.toml | 6 ++ 3 files changed, 131 insertions(+), 3 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 7955c6cae..47f4c8edd 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -85,6 +85,7 @@ use std::sync::Arc; use crate::rate_limiter::{FastlyRateLimiter, RATE_COUNTER_NAME}; use edgezero_adapter_fastly::context::FastlyRequestContext; +use edgezero_adapter_fastly::key_value_store::FastlyKvStore; use edgezero_core::app::{App, Hooks, StoreMetadata, StoresMetadata}; use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; @@ -158,6 +159,23 @@ use crate::platform::{ // AppState // --------------------------------------------------------------------------- +/// The logical id the deployment's KV store is linked to the service under, +/// as `edgezero.toml` declares it. +const DEFAULT_KV_STORE_ID: &str = "trusted_server_kv"; + +/// Opens the KV store linked under [`DEFAULT_KV_STORE_ID`], or +/// [`UnavailableKvStore`] when the service links none, so a module that +/// needs the store passes its work by rather than failing the request. +fn open_default_kv_store() -> Arc { + match FastlyKvStore::open(DEFAULT_KV_STORE_ID) { + Ok(store) => Arc::new(store), + Err(err) => { + log::debug!("The KV store `{DEFAULT_KV_STORE_ID}` is not linked: {err}"); + Arc::new(UnavailableKvStore) + } + } +} + /// The stores a service reads and the key of its app-config blob. /// /// [`Self::logical`] gives the logical stores and key. The entry point puts @@ -398,7 +416,7 @@ pub(crate) fn build_state_with_registrations( Arc::new(trusted_server_core::auction::NoopAuctionTelemetrySink) as Arc }); - let default_kv_store = Arc::new(UnavailableKvStore) as Arc; + let default_kv_store = open_default_kv_store(); let loading_caches = trusted_server_core::integrations::loading_caches_from(&settings, integrations)?; @@ -1762,8 +1780,8 @@ impl Hooks for TrustedServerApp { ids: &[DEFAULT_CONFIG_STORE_ID], }), kv: Some(StoreMetadata { - default: "trusted_server_kv", - ids: &["trusted_server_kv"], + default: DEFAULT_KV_STORE_ID, + ids: &[DEFAULT_KV_STORE_ID], }), secrets: Some(StoreMetadata { default: DEFAULT_SECRET_STORE_ID, @@ -1866,6 +1884,26 @@ mod tests { assert_eq!(stores.secret_store_name.as_ref(), "trusted_server_secrets"); } + #[test] + fn app_state_opens_the_linked_kv_store() { + let state = app_state_for_settings(test_settings()); + + block_on( + state + .default_kv_store + .put_bytes("default-kv-open-test", Bytes::from_static(b"value")), + ) + .expect("should write to the linked KV store"); + let value = block_on(state.default_kv_store.get_bytes("default-kv-open-test")) + .expect("should read from the linked KV store"); + + assert_eq!( + value, + Some(Bytes::from_static(b"value")), + "should read back what was written to the store linked as trusted_server_kv" + ); + } + fn app_state_for_settings(settings: Settings) -> Arc { build_state_from_settings(settings).expect("should build app state from settings") } diff --git a/crates/trusted-server-core/src/streaming_processor.rs b/crates/trusted-server-core/src/streaming_processor.rs index ec5ba1ac8..4c751ebdc 100644 --- a/crates/trusted-server-core/src/streaming_processor.rs +++ b/crates/trusted-server-core/src/streaming_processor.rs @@ -22,6 +22,7 @@ use std::cell::{Cell, RefCell}; use std::io::{self, Read, Write}; use std::rc::Rc; +use std::time::Duration; use brotli::Decompressor; use brotli::enc::BrotliEncoderParams; @@ -29,8 +30,10 @@ use brotli::enc::writer::CompressorWriter; use error_stack::{Report, ResultExt as _}; use flate2::read::ZlibDecoder; use flate2::write::{GzEncoder, ZlibEncoder}; +use futures::future::{Either, LocalBoxFuture}; use crate::error::TrustedServerError; +use crate::platform::RuntimeServices; /// Trait for streaming content processors pub trait StreamProcessor { @@ -50,6 +53,87 @@ pub trait StreamProcessor { /// Reset the processor state (useful for reuse) fn reset(&mut self) {} + + /// Start an optional asynchronous finish of this processor's final output. + /// + /// `held` is exactly what the final [`process_chunk`](Self::process_chunk) + /// call (`is_last = true`) returned: the bytes the processor emits when it + /// does no asynchronous work. The driver owns `held` and calls this at most + /// once, right after that final call, with the request's + /// [`RuntimeServices`]. + /// + /// Returning `None` (the default) emits `held` unchanged; no future is + /// created. Returning a finish makes the driver await it and emit its + /// output in place of `held`. The output is infallible: a processor absorbs + /// its own failures and returns the bytes to emit, typically `held` itself. + /// On a runtime with a [`PlatformTimer`](crate::platform::PlatformTimer), + /// the driver drops the finish once the deadline given to + /// [`AsyncFinish::new`] elapses and emits `held` instead. + /// + /// Only drivers that can await call this. A synchronous driver, such as + /// [`StreamingPipeline`], emits `held`. + fn async_finish<'a>( + &'a mut self, + held: &'a [u8], + services: &'a RuntimeServices, + ) -> Option> { + let _ = (held, services); + None + } +} + +/// Asynchronous work a [`StreamProcessor`] completes after its final chunk. +/// +/// Created by [`StreamProcessor::async_finish`]. The future is `?Send`, like +/// the platform services it awaits. +pub struct AsyncFinish<'a> { + deadline: Duration, + output: LocalBoxFuture<'a, Vec>, +} + +impl<'a> AsyncFinish<'a> { + /// Wrap `output`, which resolves to the bytes to emit, with the time the + /// driver may wait for it. + /// + /// `deadline` is measured from when the driver starts awaiting the finish. + /// It is a backstop: a processor should bound its own calls well within it. + #[must_use] + pub fn new(deadline: Duration, output: impl Future> + 'a) -> Self { + Self { + deadline, + output: Box::pin(output), + } + } +} + +/// Run `processor`'s optional asynchronous finish over `held`, its final output. +/// +/// Returns the finish output, or `held` when the processor has no finish. When +/// `services` has a timer, the finish races its deadline; on expiry the finish +/// future is dropped and `held` is returned. Without a timer the finish is +/// awaited to completion. +pub(crate) async fn finish_held( + processor: &mut dyn StreamProcessor, + held: Vec, + services: &RuntimeServices, +) -> Vec { + let deadline = { + let Some(AsyncFinish { deadline, output }) = processor.async_finish(&held, services) else { + return held; + }; + let Some(timer) = services.timer() else { + return output.await; + }; + match futures::future::select(output, timer.sleep(deadline)).await { + Either::Left((output, _)) => return output, + Either::Right(((), _)) => deadline, + } + }; + log::warn!( + "stream processor finish exceeded its {} ms deadline; emitting its held output", + deadline.as_millis() + ); + held } /// Compression type for the stream diff --git a/fastly.toml b/fastly.toml index 020b15ea2..bc8f06a7d 100644 --- a/fastly.toml +++ b/fastly.toml @@ -23,6 +23,12 @@ build = """ [local_server.backends] [local_server.kv_stores] + # The runtime KV store, linked as the logical `trusted_server_kv` that + # edgezero.toml declares. + [[local_server.kv_stores.trusted_server_kv]] + key = "placeholder" + data = "placeholder" + [[local_server.kv_stores.counter_store]] key = "placeholder" data = "placeholder" From 589838405c01e81e135bf6bd8dfb8fe0506bf703 Mon Sep 17 00:00:00 2001 From: Erik Svilich Date: Sat, 24 Oct 2026 16:08:50 +0000 Subject: [PATCH 03/12] End middleware stream processors with an awaited asynchronous finish A stream processor may implement StreamProcessor::async_finish over the bytes its last process_chunk call returned. StreamProcessor::finish ends a processor once; a processor made of others ends each in order, so each is handed what the one before it finished with. Every publisher path that can await now ends its processor this way: the no-hold pipeline, both auction-hold loops, the serve pass on a shared template and the lazy stream. A body read whole takes the same chunk pipeline, and the synchronous stream_publisher_body path is removed. A middleware is also told the response status and whether the origin marked the response as one reader's, read before core writes its own cache policy. --- crates/middleware/51degrees/src/lib.rs | 2 + .../benches/html_processor_bench.rs | 2 + .../trusted-server-core/src/html_processor.rs | 66 +++ .../src/integrations/js_asset_proxy.rs | 2 + crates/trusted-server-core/src/middleware.rs | 12 + .../src/middleware/tests.rs | 2 + crates/trusted-server-core/src/publisher.rs | 523 +++++++++--------- .../src/streaming_processor.rs | 44 +- 8 files changed, 381 insertions(+), 272 deletions(-) diff --git a/crates/middleware/51degrees/src/lib.rs b/crates/middleware/51degrees/src/lib.rs index 4b5a557ef..bd5bf849f 100644 --- a/crates/middleware/51degrees/src/lib.rs +++ b/crates/middleware/51degrees/src/lib.rs @@ -658,6 +658,8 @@ mod tests { request_host: "www.example.com", request_scheme: "https", origin_host: "origin.example.com", + status: http::StatusCode::OK, + origin_private: false, platform: test_support::platform(), document_state: state, max_buffered_script_bytes: 1 << 20, diff --git a/crates/trusted-server-core/benches/html_processor_bench.rs b/crates/trusted-server-core/benches/html_processor_bench.rs index e650dc078..962001fc2 100644 --- a/crates/trusted-server-core/benches/html_processor_bench.rs +++ b/crates/trusted-server-core/benches/html_processor_bench.rs @@ -18,6 +18,8 @@ fn make_config() -> HtmlProcessorConfig { ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, prepared: std::sync::Arc::new(trusted_server_core::middleware::Prepared::default()), + status: http::StatusCode::OK, + origin_private: false, request_state: trusted_server_core::integrations::IntegrationRequestState::default(), // The benchmark measures URL rewriting, not ad injection, and // `ad_slots_script` is `None` here — matching the previous behaviour, diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index 723e55a23..44929b782 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -7,6 +7,8 @@ use std::rc::Rc; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; +use futures::future::LocalBoxFuture; +use http::StatusCode; use lol_html::{ EndTagHandler, Settings as RewriterSettings, element, end, html_content::{ContentType, EndTag, TextChunk, TextType}, @@ -22,6 +24,7 @@ use crate::middleware::{ ElementHandler, MatchedAttribute, MiddlewareChain, MiddlewareContext, MiddlewarePlan, Prepared, ResponseHeader, TextHandler, }; +use crate::platform::RuntimeServices; use crate::prefix::{PUBLIC_PREFIX_ATTRIBUTE, PublicPrefix}; use crate::publisher::build_empty_bids_script; use crate::settings::Settings; @@ -32,6 +35,8 @@ mod script_text; use script_text::ScriptTextGuard; +/// The HTML rewriter followed by the middleware's stream processors, each +/// handed what the one before it produced. struct HtmlWithStreamingProcessors { inner: Box, processors: Vec>, @@ -52,6 +57,24 @@ impl StreamProcessor for HtmlWithStreamingProcessors { processor.reset(); } } + + /// Ends the rewriter once and then each processor in order, so a + /// processor is handed its last chunk only after the one before it has + /// finished, its asynchronous finish included. + fn finish<'a>( + &'a mut self, + chunk: &'a [u8], + services: &'a RuntimeServices, + ) -> LocalBoxFuture<'a, Result, io::Error>> { + Box::pin(async move { + let mut output = self.inner.finish(chunk, services).await?; + for processor in &mut self.processors { + let finished = processor.finish(&output, services).await?; + output = finished; + } + Ok(output) + }) + } } /// What the `` seam injects. @@ -125,6 +148,10 @@ pub struct HtmlProcessorConfig { /// What the middleware prepared for this document, see /// [`Middleware::prepare`](crate::middleware::Middleware::prepare). pub prepared: Arc, + /// See [`MiddlewareContext::status`]. + pub status: StatusCode, + /// See [`MiddlewareContext::origin_private`]. + pub origin_private: bool, } impl HtmlProcessorConfig { @@ -151,6 +178,8 @@ impl HtmlProcessorConfig { csp_nonce_observed: None, public_prefix: None, prepared: Arc::new(Prepared::default()), + status: StatusCode::OK, + origin_private: false, } } @@ -224,6 +253,15 @@ impl HtmlProcessorConfig { self.prepared = prepared; self } + + /// Sets the status of the response the document is the body of, and + /// whether the origin marked that response as one reader's. + #[must_use] + pub fn with_response(mut self, status: StatusCode, origin_private: bool) -> Self { + self.status = status; + self.origin_private = origin_private; + self + } } /// The chunk of text being rewritten, as the last handler to change it left @@ -301,6 +339,8 @@ pub fn create_html_processor_with_middleware( request_host: &config.request_host, request_scheme: &config.request_scheme, origin_host: &config.origin_host, + status: config.status, + origin_private: config.origin_private, platform: config.integrations.platform(), document_state: &document_state, max_buffered_script_bytes: config.max_buffered_body_bytes, @@ -313,6 +353,8 @@ pub fn create_html_processor_with_middleware( request_host: &config.request_host, request_scheme: &config.request_scheme, origin_host: &config.origin_host, + status: config.status, + origin_private: config.origin_private, platform: config.integrations.platform(), request_state: &config.request_state, max_buffered_script_bytes: config.max_buffered_body_bytes, @@ -336,6 +378,10 @@ pub struct ReaderDocument<'a> { /// The host the document names as its own, being the `Host` its source /// sends. pub origin_host: &'a str, + /// See [`MiddlewareContext::status`]. + pub status: StatusCode, + /// See [`MiddlewareContext::origin_private`]. + pub origin_private: bool, /// What the adapter reports about the platform the deployment runs on. pub platform: &'a PlatformIdentity, /// What modules left on this reader's request for their page changes. @@ -372,6 +418,8 @@ pub fn create_serve_processor( request_host: document.request_host, request_scheme: document.request_scheme, origin_host: document.origin_host, + status: document.status, + origin_private: document.origin_private, platform: document.platform, document_state: &document_state, max_buffered_script_bytes: document.max_buffered_script_bytes, @@ -1240,6 +1288,8 @@ mod tests { csp_nonce_observed: None, public_prefix: None, prepared: Arc::new(Prepared::default()), + status: StatusCode::OK, + origin_private: false, body_close: BodyCloseInjection::None, origin_host: "origin.example.com".to_owned(), request_host: "test.example.com".to_owned(), @@ -1991,6 +2041,8 @@ mod tests { csp_nonce_observed: None, public_prefix: None, prepared: Arc::new(Prepared::default()), + status: StatusCode::OK, + origin_private: false, body_close: BodyCloseInjection::None, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), @@ -2071,6 +2123,8 @@ mod tests { csp_nonce_observed: None, public_prefix: None, prepared: Arc::new(Prepared::default()), + status: StatusCode::OK, + origin_private: false, body_close: BodyCloseInjection::InlineBids, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), @@ -2112,6 +2166,8 @@ mod tests { csp_nonce_observed: None, public_prefix: None, prepared: Arc::new(Prepared::default()), + status: StatusCode::OK, + origin_private: false, body_close: BodyCloseInjection::InlineBids, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), @@ -2154,6 +2210,8 @@ mod tests { csp_nonce_observed: None, public_prefix: None, prepared: Arc::new(Prepared::default()), + status: StatusCode::OK, + origin_private: false, body_close: BodyCloseInjection::None, origin_host: "origin.test-publisher.example.com".to_string(), request_host: request_host.to_string(), @@ -2210,6 +2268,8 @@ mod tests { csp_nonce_observed: None, public_prefix: None, prepared: Arc::new(Prepared::default()), + status: StatusCode::OK, + origin_private: false, body_close: BodyCloseInjection::InlineBids, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), @@ -2244,6 +2304,8 @@ mod tests { csp_nonce_observed: None, public_prefix: None, prepared: Arc::new(Prepared::default()), + status: StatusCode::OK, + origin_private: false, body_close: BodyCloseInjection::None, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), @@ -2280,6 +2342,8 @@ mod tests { ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, prepared: Arc::new(Prepared::default()), + status: StatusCode::OK, + origin_private: false, request_state: IntegrationRequestState::default(), } } @@ -2490,6 +2554,8 @@ mod tests { csp_nonce_observed: None, public_prefix: None, prepared: Arc::new(Prepared::default()), + status: StatusCode::OK, + origin_private: false, body_close: BodyCloseInjection::Marker(MARKER.to_string()), origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), diff --git a/crates/trusted-server-core/src/integrations/js_asset_proxy.rs b/crates/trusted-server-core/src/integrations/js_asset_proxy.rs index 69d9fca9a..76e910336 100644 --- a/crates/trusted-server-core/src/integrations/js_asset_proxy.rs +++ b/crates/trusted-server-core/src/integrations/js_asset_proxy.rs @@ -687,6 +687,8 @@ mod tests { csp_nonce_observed: None, public_prefix: None, prepared: std::sync::Arc::new(crate::middleware::Prepared::default()), + status: http::StatusCode::OK, + origin_private: false, body_close: BodyCloseInjection::None, origin_host: "origin.example.com".to_string(), request_host: "publisher.example.com".to_string(), diff --git a/crates/trusted-server-core/src/middleware.rs b/crates/trusted-server-core/src/middleware.rs index bf7717dc6..f8555ff0f 100644 --- a/crates/trusted-server-core/src/middleware.rs +++ b/crates/trusted-server-core/src/middleware.rs @@ -144,6 +144,16 @@ pub struct MiddlewareContext<'a> { /// sends, the same for a page served from the store as for one fetched /// for this request. pub origin_host: &'a str, + /// The status of the response the document is the body of. A page from + /// the store of shared templates was stored from a `200`. + pub status: StatusCode, + /// Whether the origin marked its response as one reader's, by setting a + /// cookie or by a `Cache-Control` that forbids shared storage. Read from + /// the origin's own headers, before core writes its cache policy on the + /// response, so a middleware that sends the document elsewhere can leave + /// a reader's own page alone. A page from the store of shared templates + /// was never one reader's. + pub origin_private: bool, /// What the adapter reports about the platform the deployment runs on, /// as the attestation evidence carries it. pub platform: &'a PlatformIdentity, @@ -1103,6 +1113,8 @@ pub mod test_support { request_host: REQUEST_HOST, request_scheme: "https", origin_host: ORIGIN_HOST, + status: http::StatusCode::OK, + origin_private: false, platform, document_state, max_buffered_script_bytes: 16 * 1024 * 1024, diff --git a/crates/trusted-server-core/src/middleware/tests.rs b/crates/trusted-server-core/src/middleware/tests.rs index 72814b6d2..fc493f74e 100644 --- a/crates/trusted-server-core/src/middleware/tests.rs +++ b/crates/trusted-server-core/src/middleware/tests.rs @@ -67,6 +67,8 @@ fn reader(request_state: &IntegrationRequestState) -> ReaderDocument<'_> { request_host: REQUEST_HOST, request_scheme: "https", origin_host: ORIGIN_HOST, + status: StatusCode::OK, + origin_private: false, platform: test_support::platform(), request_state, max_buffered_script_bytes: 16 * 1024 * 1024, diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 8ebe52a4a..94d633b9e 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -11,12 +11,12 @@ //! must gate them out before the body enters the rewrite pipeline. //! //! **Note on platform coupling:** The handler boundaries use portable HTTP -//! types: [`handle_publisher_request`] and [`stream_publisher_body`] take and -//! return `http::Request`/`http::Response` over `EdgeBody`, and platform I/O is -//! reached through `RuntimeServices` rather than a platform SDK directly. The -//! streaming processor itself is generic: `process_response_streaming` writes -//! into any [`Write`] (a `Vec` for buffered routes, a streaming writer for -//! the streaming route). It is not a content-rewriting concern. +//! types: [`handle_publisher_request`] and [`stream_publisher_body_async`] +//! take and return `http::Request`/`http::Response` over `EdgeBody`, and +//! platform I/O is reached through `RuntimeServices` rather than a platform +//! SDK directly. The body pipeline itself is generic: it writes into any +//! [`Write`] (a `Vec` for buffered routes) or yields a lazy stream for the +//! streaming route. It is not a content-rewriting concern. use std::borrow::Cow; use std::io::Write; @@ -32,6 +32,7 @@ use error_stack::{Report, ResultExt}; use flate2::read::ZlibDecoder; use flate2::write::{GzEncoder, ZlibEncoder}; use futures::StreamExt as _; +use futures::future::LocalBoxFuture; use http::{HeaderValue, Method, Request, Response, StatusCode, Uri, header}; use sha2::Digest as _; use web_time::Instant; @@ -96,8 +97,8 @@ use crate::settings::{ use crate::source::{SourceLocation, SourceRequest}; use crate::source_cache::{Found, Lookup, SourceAddress, SourceCache, SourceVary}; use crate::streaming_processor::{ - BodyStreamDecoder, BodyStreamEncoder, Compression, GzipDecodeReader, PipelineConfig, - STREAM_CHUNK_SIZE, StreamProcessor, StreamingPipeline, + BodyStreamDecoder, BodyStreamEncoder, Compression, GzipDecodeReader, STREAM_CHUNK_SIZE, + StreamProcessor, }; use crate::streaming_replacer::create_url_replacer; @@ -261,19 +262,43 @@ impl BodyChunkSource { } } -fn process_and_encode_chunk( +fn process_and_encode_chunk( processor: &mut P, encoder: &mut BodyStreamEncoder, chunk: &[u8], - is_last: bool, process_error: &str, ) -> Result, Report> { let processed = processor - .process_chunk(chunk, is_last) + .process_chunk(chunk, false) .change_context(TrustedServerError::Proxy { message: process_error.to_string(), })?; + encode_processed(encoder, processed) +} + +/// Ends `processor`, awaiting any asynchronous finish of a middleware's +/// processor, and encodes what it emits last. +async fn finish_and_encode( + processor: &mut P, + encoder: &mut BodyStreamEncoder, + services: &RuntimeServices, +) -> Result, Report> { + let processed = + processor + .finish(&[], services) + .await + .change_context(TrustedServerError::Proxy { + message: "Failed to finalize processor".to_string(), + })?; + encode_processed(encoder, processed) +} + +/// Encodes processed bytes, or `None` when there is nothing to emit. +fn encode_processed( + encoder: &mut BodyStreamEncoder, + processed: Vec, +) -> Result, Report> { if processed.is_empty() { return Ok(None); } @@ -635,36 +660,6 @@ fn parse_single_module_filename( registry.js_module_id(stem) } -/// Parameters for processing response streaming. -struct ProcessResponseParams<'a> { - content_encoding: &'a str, - origin_host: &'a str, - origin_url: &'a str, - request_host: &'a str, - request_scheme: &'a str, - /// See [`OwnedProcessResponseParams::request_path`]. - request_path: &'a str, - settings: &'a Settings, - content_type: &'a str, - /// See [`OwnedProcessResponseParams::status`]. - status: StatusCode, - integration_registry: &'a IntegrationRegistry, - /// Head script carrying this request's permission state, or [`None`] under a - /// shared-template mode. See [`template_permissions_script`]. - permissions_script: Option<&'a str>, - ad_slots_script: Option<&'a str>, - ad_bids_state: &'a Arc>>, - /// See [`HtmlStreamProcessorParams::request_state`]. - request_state: &'a IntegrationRequestState, - /// See [`HtmlStreamProcessorParams::shared_template_authorized`]. - shared_template_authorized: bool, - /// See [`HtmlStreamProcessorParams::csp_nonce_observed`]. - csp_nonce_observed: Option<&'a Arc>, - prepared: &'a Arc, - /// See [`OwnedProcessResponseParams::public_prefix`]. - public_prefix: &'a crate::prefix::PublicPrefix, -} - struct PublisherBodyProcessor { inner: Box, /// What the serve middleware write on the response, taken by the @@ -679,6 +674,8 @@ struct TextResponse<'a> { /// The response's `Content-Type`, parameters and all. content_type: &'a str, status: StatusCode, + /// See [`MiddlewareContext::origin_private`](crate::middleware::MiddlewareContext::origin_private). + origin_private: bool, request_path: &'a str, request_host: &'a str, request_scheme: &'a str, @@ -744,6 +741,8 @@ fn middleware_stream_processors( request_host: response.request_host, request_scheme: response.request_scheme, origin_host: response.origin_host, + status: response.status, + origin_private: response.origin_private, platform: integration_registry.platform(), document_state: &document_state, max_buffered_script_bytes: response.max_buffered_script_bytes, @@ -898,6 +897,8 @@ async fn prepare_document( request_host: document.request_host, request_scheme: document.request_scheme, origin_host: document.origin_host, + status: document.status, + origin_private: document.origin_private, platform: integration_registry.platform(), document_state: &document_state, max_buffered_script_bytes: document.max_buffered_script_bytes, @@ -1004,6 +1005,27 @@ impl StreamProcessor for WithStreamProcessors { processor.reset(); } } + + /// Ends core's move of the address and then each processor in order, so + /// a processor is handed its last chunk only after the one before it has + /// finished, its asynchronous finish included. + fn finish<'a>( + &'a mut self, + chunk: &'a [u8], + services: &'a RuntimeServices, + ) -> LocalBoxFuture<'a, Result, std::io::Error>> { + Box::pin(async move { + let mut output = match &mut self.inner { + Some(inner) => inner.finish(chunk, services).await?, + None => chunk.to_vec(), + }; + for processor in &mut self.processors { + let finished = processor.finish(&output, services).await?; + output = finished; + } + Ok(output) + }) + } } impl PublisherBodyProcessor { @@ -1029,6 +1051,8 @@ impl PublisherBodyProcessor { { let (processor, headers) = create_html_stream_processor(HtmlStreamProcessorParams { origin_host: ¶ms.origin_host, + status: params.status, + origin_private: params.origin_private, request_host: ¶ms.request_host, request_scheme: ¶ms.request_scheme, request_path: ¶ms.request_path, @@ -1052,6 +1076,7 @@ impl PublisherBodyProcessor { &TextResponse { content_type: ¶ms.content_type, status: params.status, + origin_private: params.origin_private, request_path: ¶ms.request_path, request_host: ¶ms.request_host, request_scheme: ¶ms.request_scheme, @@ -1093,105 +1118,31 @@ impl StreamProcessor for PublisherBodyProcessor { fn process_chunk(&mut self, chunk: &[u8], is_last: bool) -> Result, std::io::Error> { self.inner.process_chunk(chunk, is_last) } + + fn finish<'a>( + &'a mut self, + chunk: &'a [u8], + services: &'a RuntimeServices, + ) -> LocalBoxFuture<'a, Result, std::io::Error>> { + self.inner.finish(chunk, services) + } } -/// Process response body through the streaming pipeline. -/// -/// Selects the processor by content type, the HTML rewriter or the text -/// pass of the entries' middleware, and pipes chunks from `body` through it -/// into `output`. The caller decides what `output` is, a `Vec` for -/// buffered responses or a `StreamingBody` for streaming. +/// Pipes `body` through the processor its content type selects, the HTML +/// rewriter or the text pass of the entries' middleware, into `output`, and +/// ends the processor with its asynchronous finish. A body read whole and a +/// streamed one take the same chunk pipeline. /// /// # Errors /// /// Returns an error if processor creation or chunk processing fails. -fn process_response_streaming( - body: EdgeBody, - output: &mut W, - params: &ProcessResponseParams, - output_compression: Compression, -) -> Result, Report> { - let is_html = is_html_content_type(params.content_type); - log::debug!( - "process_response_streaming: content_type={}, content_encoding={}, is_html={}", - params.content_type, - params.content_encoding, - is_html - ); - - let compression = Compression::from_content_encoding(params.content_encoding); - let config = PipelineConfig { - input_compression: compression, - output_compression, - chunk_size: 8192, - }; - // Bound how much decoded gzip output may sit in the heap at once, using the - // same ceiling the buffered writer enforces on the rewritten output: a gzip - // bomb is then rejected mid-decode instead of materializing its full - // expansion first. The bound is per-step, so a large honest body still - // streams through and only the buffered writer judges the total — otherwise - // gzip would reject bodies the identity, deflate and brotli paths accept. - let max_pending_decoded_bytes = params.settings.publisher.max_buffered_body_bytes; - - let response_headers = if is_html { - let (processor, headers) = create_html_stream_processor(HtmlStreamProcessorParams { - origin_host: params.origin_host, - request_host: params.request_host, - request_scheme: params.request_scheme, - request_path: params.request_path, - settings: params.settings, - integration_registry: params.integration_registry, - permissions_script: params.permissions_script.map(str::to_string), - ad_slots_script: params.ad_slots_script.map(str::to_string), - ad_bids_state: params.ad_bids_state.clone(), - request_state: params.request_state.clone(), - shared_template_authorized: params.shared_template_authorized, - csp_nonce_observed: params.csp_nonce_observed.cloned(), - prepared: Arc::clone(params.prepared), - public_prefix: params.public_prefix, - deferred_inline_marker: None, - })?; - StreamingPipeline::new(config, processor) - .with_max_pending_decoded_bytes(max_pending_decoded_bytes) - .process(body_as_reader(body)?, output)?; - headers - } else { - let (processor, headers) = middleware_stream_processors( - &TextResponse { - content_type: params.content_type, - status: params.status, - request_path: params.request_path, - request_host: params.request_host, - request_scheme: params.request_scheme, - origin_host: params.origin_host, - request_state: params.request_state, - max_buffered_script_bytes: params.settings.publisher.max_buffered_body_bytes, - prepared: params.prepared, - }, - params.settings, - params.integration_registry, - )? - .into_processor( - params.origin_host, - params.origin_url, - params.request_host, - params.request_scheme, - ); - StreamingPipeline::new(config, processor) - .with_max_pending_decoded_bytes(max_pending_decoded_bytes) - .process(body_as_reader(body)?, output)?; - headers - }; - - Ok(response_headers) -} - async fn process_response_streaming_async( body: EdgeBody, output: &mut W, params: &OwnedProcessResponseParams, settings: &Settings, integration_registry: &IntegrationRegistry, + services: &RuntimeServices, ) -> Result, Report> { log::debug!( "process_response_streaming_async: content_type={}, content_encoding={}", @@ -1228,6 +1179,7 @@ async fn process_response_streaming_async( input_compression, output_compression, settings.publisher.max_buffered_body_bytes, + services, ) .await?; Ok(response_headers) @@ -1239,7 +1191,7 @@ async fn process_response_streaming_async( /// [`passthrough_finish_segments`]. Shared by the write-sink driver /// ([`process_body_chunks_async`]) and the lazy publisher body stream so the /// two no-hold paths cannot drift apart. -async fn passthrough_step( +async fn passthrough_step( source: &mut BodyChunkSource, decoder: &mut BodyStreamDecoder, encoder: &mut BodyStreamEncoder, @@ -1253,25 +1205,22 @@ async fn passthrough_step( return Ok(Some(Vec::new())); } let mut segments = Vec::new(); - if let Some(encoded) = process_and_encode_chunk( - processor, - encoder, - &decoded, - false, - "Failed to process chunk", - )? { + if let Some(encoded) = + process_and_encode_chunk(processor, encoder, &decoded, "Failed to process chunk")? + { segments.push(encoded); } Ok(Some(segments)) } -async fn process_body_chunks_async( +async fn process_body_chunks_async( body: EdgeBody, writer: &mut W, processor: &mut P, input_compression: Compression, output_compression: Compression, max_body_bytes: usize, + services: &RuntimeServices, ) -> Result<(), Report> { let mut decoder = BodyStreamDecoder::new(input_compression, max_body_bytes); let mut encoder = BodyStreamEncoder::new(output_compression); @@ -1285,7 +1234,9 @@ async fn process_body_chunks_async( } } - for encoded in passthrough_finish_segments(processor, &mut decoder, &mut encoder)? { + for encoded in + passthrough_finish_segments(processor, &mut decoder, &mut encoder, services).await? + { write_encoded_segment(writer, &encoded)?; } writer.flush().change_context(TrustedServerError::Proxy { @@ -1308,12 +1259,13 @@ fn write_encoded_segment( } /// Finalize a no-hold chunk pipeline: drain the decoder tail through the -/// processor, signal end-of-stream to the processor, and emit the encoder -/// trailer. Returns the encoded segments for the caller to emit. -fn passthrough_finish_segments( +/// processor, end the processor with its asynchronous finish, and emit the +/// encoder trailer. Returns the encoded segments for the caller to emit. +async fn passthrough_finish_segments( processor: &mut P, decoder: &mut BodyStreamDecoder, encoder: &mut BodyStreamEncoder, + services: &RuntimeServices, ) -> Result, Report> { let mut segments = Vec::new(); let decoded_tail = decoder.finish()?; @@ -1322,19 +1274,12 @@ fn passthrough_finish_segments( processor, encoder, &decoded_tail, - false, "Failed to process decoded tail", )? { segments.push(encoded); } - if let Some(encoded) = process_and_encode_chunk( - processor, - encoder, - &[], - true, - "Failed to finalize processor", - )? { + if let Some(encoded) = finish_and_encode(processor, encoder, services).await? { segments.push(encoded); } let trailer = encoder.finish()?; @@ -1585,8 +1530,9 @@ async fn hold_step_next_chunk( .map(Some) } -/// Drain the decoder tail at end of the origin stream, returning the prefix the -/// caller must emit before [`hold_finish_tail_segments`]. +/// Drain the decoder tail at end of the origin stream and end the processor, +/// returning the prefix the caller must emit before +/// [`hold_finish_tail_segments`]. /// /// A codec can hold document bytes back until its own finalization — the gzip /// decoder releases the remainder of the final member at `finish()` — and that @@ -1594,8 +1540,12 @@ async fn hold_step_next_chunk( /// collection keeps the invariant the mid-stream path already has: only the /// closing `` tail waits for the auction, never renderable content. /// -/// On decoder failure the pending auction is abandoned before the error is -/// returned. +/// The processor's end awaits each middleware processor's asynchronous finish +/// in order, so a seam token a processor held reaches the seam scanner here, +/// once the whole chain has finished. +/// +/// On decoder or processor failure the pending auction is abandoned before +/// the error is returned. async fn hold_finish_ready_segments( processor: &mut P, decoder: &mut BodyStreamDecoder, @@ -1613,18 +1563,18 @@ async fn hold_finish_ready_segments( let mut step = hold_step_decoded_chunk(processor, encoder, &decoded_tail, state, collect_refs).await?; - let final_processed = - match processor - .process_chunk(&[], true) - .change_context(TrustedServerError::Proxy { - message: "Failed to finalize processor".to_string(), - }) { - Ok(processed) => processed, - Err(err) => { - abandon_hold_auction(state, collect_refs.services, "stream_process_error").await; - return Err(err); - } - }; + let final_processed = match processor + .finish(&[], collect_refs.services) + .await + .change_context(TrustedServerError::Proxy { + message: "Failed to finalize processor".to_string(), + }) { + Ok(processed) => processed, + Err(err) => { + abandon_hold_auction(state, collect_refs.services, "stream_process_error").await; + return Err(err); + } + }; let final_ready = match state.hold.as_mut() { Some(seam) => seam.push(&final_processed), None => final_processed, @@ -1703,6 +1653,10 @@ async fn hold_finish_tail_segments( /// registry alive for as long as the processor. struct HtmlStreamProcessorParams<'a> { origin_host: &'a str, + /// See [`OwnedProcessResponseParams::status`]. + status: StatusCode, + /// See [`OwnedProcessResponseParams::origin_private`]. + origin_private: bool, request_host: &'a str, request_scheme: &'a str, /// See [`OwnedProcessResponseParams::request_path`]. @@ -1920,7 +1874,8 @@ fn create_html_stream_processor( .with_body_close(body_close) .with_csp_nonce_observer(csp_nonce_observed) .with_public_prefix(params.public_prefix.clone()) - .with_prepared(params.prepared); + .with_prepared(params.prepared) + .with_response(params.status, params.origin_private); // The middleware of the first [[fetch]] entry covering this path. let fetch = params.integration_registry.middleware_chain( @@ -1975,6 +1930,8 @@ fn reader_serve_processor( request_host: ¶ms.request_host, request_scheme: ¶ms.request_scheme, origin_host, + status: params.status, + origin_private: params.origin_private, platform: integration_registry.platform(), request_state: ¶ms.request_state, max_buffered_script_bytes: settings.publisher.max_buffered_body_bytes, @@ -1985,14 +1942,15 @@ fn reader_serve_processor( } /// Runs the `[[serve]]` entry covering this request over a reader's whole -/// copy, assembled and not yet encoded, and writes its headers on the -/// response. -fn run_serve_pass( +/// copy, assembled and not yet encoded, ending it with its asynchronous +/// finish, and writes its headers on the response. +async fn run_serve_pass( bytes: Vec, params: &OwnedProcessResponseParams, settings: &Settings, integration_registry: &IntegrationRegistry, response: &mut Response, + services: &RuntimeServices, ) -> Result, Report> { let pass = reader_serve_processor(params, settings, integration_registry)?; write_response_headers(response, pass.response_headers); @@ -2000,7 +1958,8 @@ fn run_serve_pass( return Ok(bytes); }; processor - .process_chunk(&bytes, true) + .finish(&bytes, services) + .await .change_context(TrustedServerError::Proxy { message: "a serve middleware could not rewrite the page".to_string(), }) @@ -2011,10 +1970,22 @@ fn run_serve_pass( fn serve_chunk( serve: &mut Option>, bytes: Vec, - is_last: bool, ) -> Result, std::io::Error> { match serve { - Some(processor) => processor.process_chunk(&bytes, is_last), + Some(processor) => processor.process_chunk(&bytes, false), + None => Ok(bytes), + } +} + +/// The last piece of a streamed reader's copy through the `[[serve]]` entry, +/// when one covers the request, ending it with its asynchronous finish. +async fn serve_last_chunk( + serve: &mut Option>, + bytes: Vec, + services: &RuntimeServices, +) -> Result, std::io::Error> { + match serve { + Some(processor) => processor.finish(&bytes, services).await, None => Ok(bytes), } } @@ -2045,7 +2016,7 @@ pub enum PublisherResponse { response: Response, /// Origin body to be piped through the streaming pipeline. body: EdgeBody, - /// Parameters for [`process_response_streaming`]. + /// Parameters for [`stream_publisher_body_async`]. params: Box, }, /// A shared template read from template cache, to be assembled on the way out. @@ -2253,8 +2224,8 @@ fn apply_publisher_asset_cache_policy( Ok(()) } -/// Owned version of [`ProcessResponseParams`] for returning from -/// [`handle_publisher_request`] without lifetime issues. +/// Owned body-processing parameters returned from [`handle_publisher_request`] +/// so the body can be processed after the handler returns. pub struct OwnedProcessResponseParams { /// Where to store the transformed template, or [`None`] to store nothing. /// @@ -2290,6 +2261,9 @@ pub struct OwnedProcessResponseParams { /// The origin's status, which decides whether the entries' middleware /// run on a body that is not HTML. See [`text_entry_media_type`]. pub(crate) status: StatusCode, + /// Whether the origin marked its response as one reader's, see + /// [`MiddlewareContext::origin_private`](crate::middleware::MiddlewareContext::origin_private). + pub(crate) origin_private: bool, pub(crate) ad_slots_script: Option, pub(crate) ad_bids_state: AdBidsState, /// Observation context for the in-flight auction. @@ -2535,7 +2509,9 @@ pub async fn buffer_publisher_response_async( settings, integration_registry, &mut response, - )?; + services, + ) + .await?; encode_complete_body(bytes, response_compression(&response))? } else { bytes @@ -2591,7 +2567,9 @@ pub async fn buffer_publisher_response_async( settings, integration_registry, &mut response, - )?; + services, + ) + .await?; let assembled = encode_complete_body(assembled, response_compression(&response))?; response.headers_mut().insert( http::header::CONTENT_LENGTH, @@ -2803,6 +2781,8 @@ fn build_template_assembly_params( request_path: String::new(), content_type: entry.metadata.content_type.clone(), status: StatusCode::OK, + // A stored template passed the gate that keeps one reader's page out. + origin_private: false, // The template already carries the head seam; re-injecting would duplicate it. permissions_json, ad_slots_script: None, @@ -3158,7 +3138,7 @@ pub async fn publisher_response_into_streaming_response( // Each piece goes through the serve entry as it leaves, so // the head is still written before the auction is awaited. let encoded_head = encoder - .encode_chunk(serve_chunk(&mut serve, head.to_vec(), false)?) + .encode_chunk(serve_chunk(&mut serve, head.to_vec())?) .map_err(publisher_stream_error)?; if !encoded_head.is_empty() { yield bytes::Bytes::from(encoded_head); @@ -3189,14 +3169,14 @@ pub async fn publisher_response_into_streaming_response( let seam = seam_script_for(¶ms); if !seam.is_empty() { let encoded_seam = encoder - .encode_chunk(serve_chunk(&mut serve, seam.into_bytes(), false)?) + .encode_chunk(serve_chunk(&mut serve, seam.into_bytes())?) .map_err(publisher_stream_error)?; if !encoded_seam.is_empty() { yield bytes::Bytes::from(encoded_seam); } } let encoded_tail = encoder - .encode_chunk(serve_chunk(&mut serve, tail.to_vec(), true)?) + .encode_chunk(serve_last_chunk(&mut serve, tail.to_vec(), &services).await?) .map_err(publisher_stream_error)?; if !encoded_tail.is_empty() { yield bytes::Bytes::from(encoded_tail); @@ -3410,9 +3390,14 @@ pub async fn publisher_response_into_streaming_response( yield encoded; } } - for encoded in - passthrough_finish_segments(&mut processor, &mut decoder, &mut encoder) - .map_err(publisher_stream_error)? + for encoded in passthrough_finish_segments( + &mut processor, + &mut decoder, + &mut encoder, + &services, + ) + .await + .map_err(publisher_stream_error)? { yield encoded; } @@ -3571,55 +3556,6 @@ impl Write for BoundedWriter { } } -/// Stream the publisher response body through the processing pipeline. -/// -/// Called by the adapter after `stream_to_client()` has committed the response -/// headers. Runs synchronously against an already-materialised body; the async -/// I/O happened upstream in [`handle_publisher_request`]. Writes processed -/// chunks directly to `output`. -/// -/// # Errors -/// -/// Returns an error if processing fails mid-stream. Since headers are already -/// committed, the caller should log the error and drop the `StreamingBody` -/// (client sees a truncated response). -pub fn stream_publisher_body( - body: EdgeBody, - output: &mut W, - params: &OwnedProcessResponseParams, - settings: &Settings, - integration_registry: &IntegrationRegistry, -) -> Result, Report> { - let permissions_script = permissions_script_for(params, settings); - let borrowed = ProcessResponseParams { - content_encoding: ¶ms.content_encoding, - origin_host: ¶ms.origin_host, - origin_url: ¶ms.origin_url, - request_host: ¶ms.request_host, - request_scheme: ¶ms.request_scheme, - request_path: ¶ms.request_path, - settings, - content_type: ¶ms.content_type, - status: params.status, - integration_registry, - permissions_script: permissions_script.as_deref(), - ad_slots_script: params.ad_slots_script.as_deref(), - ad_bids_state: params.ad_bids_state.script_cell(), - request_state: ¶ms.request_state, - shared_template_authorized: params.template_cache_key.is_some(), - csp_nonce_observed: params.csp_nonce_observed.as_ref(), - prepared: ¶ms.prepared, - public_prefix: ¶ms.public_prefix, - }; - let input_compression = Compression::from_content_encoding(¶ms.content_encoding); - let output_compression = if params.template_cache_key.is_some() { - Compression::None - } else { - input_compression - }; - process_response_streaming(body, output, &borrowed, output_compression) -} - /// Stream publisher body with parser-confirmed live bid injection. /// /// Drives the origin body through the HTML pipeline one chunk at a time. The @@ -3632,9 +3568,11 @@ pub fn stream_publisher_body( /// 3. The generated marker is replaced with the collected bid script. /// /// For non-HTML content types the auction is collected before any body bytes -/// are written (no `` to inject). If `params.dispatched_auction` is -/// `None` the function falls back to the synchronous -/// [`stream_publisher_body`] path. +/// are written (no `` to inject). Without a dispatched auction the body +/// takes the same chunk pipeline with no seam, whether it was read whole or +/// arrives as a stream. Every path ends the processor with its asynchronous +/// finish, so a middleware processor's asynchronous work completes before the +/// output is finalized. /// /// # Errors /// @@ -3650,19 +3588,15 @@ pub async fn stream_publisher_body_async( services: &RuntimeServices, ) -> Result, Report> { let Some(dispatched) = params.dispatched_auction.take() else { - if body.is_stream() { - return process_response_streaming_async( - body, - output, - params, - settings, - integration_registry, - ) - .await; - } - - // No auction and already-buffered body — keep the existing sync pipeline. - return stream_publisher_body(body, output, params, settings, integration_registry); + return process_response_streaming_async( + body, + output, + params, + settings, + integration_registry, + services, + ) + .await; }; let telemetry = AuctionTelemetryCarry { observation: params.auction_observation.take(), @@ -3683,17 +3617,15 @@ pub async fn stream_publisher_body_async( settings, ) .await; - if body.is_stream() { - return process_response_streaming_async( - body, - output, - params, - settings, - integration_registry, - ) - .await; - } - return stream_publisher_body(body, output, params, settings, integration_registry); + return process_response_streaming_async( + body, + output, + params, + settings, + integration_registry, + services, + ) + .await; } // HTML: let lol_html mark the structural body end for the auction seam. @@ -3708,6 +3640,8 @@ pub async fn stream_publisher_body_async( prepared: Arc::clone(¶ms.prepared), public_prefix: ¶ms.public_prefix, origin_host: ¶ms.origin_host, + status: params.status, + origin_private: params.origin_private, request_host: ¶ms.request_host, request_scheme: ¶ms.request_scheme, request_path: ¶ms.request_path, @@ -3913,6 +3847,8 @@ fn write_serve_middleware_headers( request_host: reader.request_host, request_scheme: reader.request_scheme, origin_host: reader.origin_host, + status: response.status(), + origin_private: reader.origin_private, platform: reader.platform, document_state: &document_state, max_buffered_script_bytes: reader.max_buffered_script_bytes, @@ -4848,11 +4784,12 @@ async fn body_close_hold_loop( loop { match reader.read(&mut buffer) { Ok(0) => { - let final_out = match processor.process_chunk(&[], true).change_context( - TrustedServerError::Proxy { + let final_out = match processor + .finish(&[], deps.services) + .await + .change_context(TrustedServerError::Proxy { message: "Failed to finalize processor".to_string(), - }, - ) { + }) { Ok(output) => output, Err(err) => { abandon_reader_auction( @@ -6121,6 +6058,7 @@ pub async fn handle_publisher_request( let document = TextResponse { content_type: crate::middleware::HTML_MEDIA_TYPE, status: StatusCode::OK, + origin_private: false, request_path: &request_path, request_host, request_scheme, @@ -6280,6 +6218,11 @@ pub async fn handle_publisher_request( response = empty_document(&entry.media_type)?; } + // Read before core writes its own cache policy on the response below, so a + // middleware is told what the origin declared. + let origin_private = response.headers().contains_key(header::SET_COOKIE) + || cache_control_forbids_shared_storage(response.headers()); + // Every serve middleware an entry names for this path is asked for the // headers it sets on the response that is sent, whatever it is, so a // redirect with no body reaches a middleware that rewrites its Location. @@ -6288,6 +6231,8 @@ pub async fn handle_publisher_request( request_host, request_scheme, origin_host: &origin_host_header, + status: response.status(), + origin_private, platform: integration_registry.platform(), request_state: &request_state, max_buffered_script_bytes: settings.publisher.max_buffered_body_bytes, @@ -6538,6 +6483,7 @@ pub async fn handle_publisher_request( let document = TextResponse { content_type: &content_type, status, + origin_private, request_path: &request_path, request_host, request_scheme, @@ -6677,6 +6623,7 @@ pub async fn handle_publisher_request( request_path: request_path.clone(), content_type, status, + origin_private, permissions_json, ad_slots_script: ad_slots_script.clone(), ad_bids_state: ad_bids_state.clone(), @@ -9821,6 +9768,26 @@ mod tests { use http::{Method, Request as HttpRequest, StatusCode, header}; use std::sync::Arc; + /// Runs the body pipeline with no auction over `body`, ending the + /// processor with the asynchronous finish on services with no timer, as a + /// buffered adapter does with a body it read whole. + fn stream_publisher_body( + body: EdgeBody, + output: &mut W, + params: &OwnedProcessResponseParams, + settings: &Settings, + integration_registry: &IntegrationRegistry, + ) -> Result, Report> { + futures::executor::block_on(process_response_streaming_async( + body, + output, + params, + settings, + integration_registry, + &noop_services(), + )) + } + /// A resolved permission state as page JSON, for tests that need a payload /// with something in it rather than the empty state. /// @@ -11368,6 +11335,7 @@ mod tests { request_path: "/".to_owned(), content_type: "application/json".to_owned(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -11828,6 +11796,8 @@ mod tests { csp_nonce_observed: None, public_prefix: None, prepared: Arc::new(Prepared::default()), + status: StatusCode::OK, + origin_private: false, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), @@ -23795,6 +23765,7 @@ mod tests { request_path: "/a.txt".to_owned(), content_type: content_type.to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -23911,6 +23882,7 @@ mod tests { request_path: "/a.txt".to_owned(), content_type: fixture::FRAMED_MEDIA_TYPE.to_string(), status: StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -23980,6 +23952,7 @@ mod tests { request_path: "/a.txt".to_owned(), content_type: "text/plain; charset=utf-8".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -24035,6 +24008,7 @@ mod tests { request_path: "/a.txt".to_owned(), content_type: content_type.to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -26760,6 +26734,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/css".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -26826,6 +26801,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -26913,6 +26889,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -27009,6 +26986,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -27120,6 +27098,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -27172,6 +27151,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -27304,6 +27284,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/css".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -27374,6 +27355,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/css".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -27447,6 +27429,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/css".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -27520,6 +27503,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/css".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -27593,6 +27577,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/css".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -27646,6 +27631,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/css".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -27875,6 +27861,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: Some( r#""# @@ -27956,6 +27943,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: Some( r#""# @@ -28039,6 +28027,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/css".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -28116,6 +28105,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/css".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -28275,6 +28265,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: Some( r#""# @@ -28955,6 +28946,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -29156,6 +29148,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: Some( r#""# @@ -29241,6 +29234,7 @@ mod tests { request_path: "/".to_owned(), content_type: "Text/HTML; Charset=utf-8".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: Some( r#""# @@ -29309,6 +29303,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/html".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -29432,6 +29427,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), @@ -29495,6 +29491,7 @@ mod tests { request_path: "/".to_owned(), content_type: "text/html".to_string(), status: http::StatusCode::OK, + origin_private: false, permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), diff --git a/crates/trusted-server-core/src/streaming_processor.rs b/crates/trusted-server-core/src/streaming_processor.rs index 4c751ebdc..1b379ec8c 100644 --- a/crates/trusted-server-core/src/streaming_processor.rs +++ b/crates/trusted-server-core/src/streaming_processor.rs @@ -54,7 +54,7 @@ pub trait StreamProcessor { /// Reset the processor state (useful for reuse) fn reset(&mut self) {} - /// Start an optional asynchronous finish of this processor's final output. + /// Starts an optional asynchronous finish of this processor's final output. /// /// `held` is exactly what the final [`process_chunk`](Self::process_chunk) /// call (`is_last = true`) returned: the bytes the processor emits when it @@ -62,15 +62,16 @@ pub trait StreamProcessor { /// once, right after that final call, with the request's /// [`RuntimeServices`]. /// - /// Returning `None` (the default) emits `held` unchanged; no future is - /// created. Returning a finish makes the driver await it and emit its + /// Returning `None`, the default, emits `held` unchanged and creates no + /// future. Returning a finish makes the driver await it and emit its /// output in place of `held`. The output is infallible: a processor absorbs /// its own failures and returns the bytes to emit, typically `held` itself. /// On a runtime with a [`PlatformTimer`](crate::platform::PlatformTimer), /// the driver drops the finish once the deadline given to /// [`AsyncFinish::new`] elapses and emits `held` instead. /// - /// Only drivers that can await call this. A synchronous driver, such as + /// Only a driver that can await calls this, through + /// [`finish`](Self::finish). A synchronous driver, such as /// [`StreamingPipeline`], emits `held`. fn async_finish<'a>( &'a mut self, @@ -80,6 +81,30 @@ pub trait StreamProcessor { let _ = (held, services); None } + + /// Ends the document with `chunk` as its last input, awaiting any + /// asynchronous finish, for a driver that can await. Called once, in place + /// of `process_chunk(chunk, true)`. + /// + /// The default processes `chunk` as the last chunk and runs + /// [`async_finish`](Self::async_finish) over what that returned. A + /// processor made of others ends each of them in order with their own + /// `finish`, so each is handed what the one before it finished with, and + /// none is handed its last chunk before the one before it has finished. + /// + /// # Errors + /// + /// Returns an error if processing the last chunk fails. + fn finish<'a>( + &'a mut self, + chunk: &'a [u8], + services: &'a RuntimeServices, + ) -> LocalBoxFuture<'a, Result, io::Error>> { + Box::pin(async move { + let held = self.process_chunk(chunk, true)?; + Ok(finish_held(self, held, services).await) + }) + } } /// Asynchronous work a [`StreamProcessor`] completes after its final chunk. @@ -92,7 +117,7 @@ pub struct AsyncFinish<'a> { } impl<'a> AsyncFinish<'a> { - /// Wrap `output`, which resolves to the bytes to emit, with the time the + /// Wraps `output`, which resolves to the bytes to emit, with the time the /// driver may wait for it. /// /// `deadline` is measured from when the driver starts awaiting the finish. @@ -106,14 +131,15 @@ impl<'a> AsyncFinish<'a> { } } -/// Run `processor`'s optional asynchronous finish over `held`, its final output. +/// Runs `processor`'s optional asynchronous finish over `held`, its final +/// output. /// /// Returns the finish output, or `held` when the processor has no finish. When /// `services` has a timer, the finish races its deadline; on expiry the finish /// future is dropped and `held` is returned. Without a timer the finish is /// awaited to completion. -pub(crate) async fn finish_held( - processor: &mut dyn StreamProcessor, +async fn finish_held( + processor: &mut P, held: Vec, services: &RuntimeServices, ) -> Vec { @@ -130,7 +156,7 @@ pub(crate) async fn finish_held( } }; log::warn!( - "stream processor finish exceeded its {} ms deadline; emitting its held output", + "A stream processor's finish ran past its {} ms deadline, so its held output is emitted", deadline.as_millis() ); held From 6ddcfaf2a936d50d25fa1c5f3c46548e5709ee4d Mon Sep 17 00:00:00 2001 From: Erik Svilich Date: Sat, 24 Oct 2026 16:09:37 +0000 Subject: [PATCH 04/12] Wire Axum runtime KV from TRUSTED_SERVER_KV_PATH --- Cargo.lock | 3 + crates/trusted-server-adapter-axum/Cargo.toml | 3 + .../trusted-server-adapter-axum/src/main.rs | 19 +- .../src/platform.rs | 112 ++++++++-- .../tests/runtime_kv.rs | 205 ++++++++++++++++++ docs/guide/axum-dev.md | 10 + 6 files changed, 334 insertions(+), 18 deletions(-) create mode 100644 crates/trusted-server-adapter-axum/tests/runtime_kv.rs diff --git a/Cargo.lock b/Cargo.lock index 89550f2f3..90fda973c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6594,9 +6594,11 @@ dependencies = [ name = "trusted-server-adapter-axum" version = "0.1.0" dependencies = [ + "anyhow", "async-trait", "axum", "base64", + "bytes", "edgezero-adapter-axum", "edgezero-core", "error-stack", @@ -6606,6 +6608,7 @@ dependencies = [ "serde_json", "simple_logger", "temp-env", + "tempfile", "tokio", "tower 0.4.13", "trusted-server-ad-server-iabtl", diff --git a/crates/trusted-server-adapter-axum/Cargo.toml b/crates/trusted-server-adapter-axum/Cargo.toml index 47604fc94..02ca020e3 100644 --- a/crates/trusted-server-adapter-axum/Cargo.toml +++ b/crates/trusted-server-adapter-axum/Cargo.toml @@ -19,7 +19,9 @@ name = "trusted-server-axum" path = "src/main.rs" [dependencies] +anyhow = { workspace = true } async-trait = { workspace = true } +bytes = { workspace = true } edgezero-adapter-axum = { workspace = true, features = ["axum"] } edgezero-core = { workspace = true } error-stack = { workspace = true } @@ -64,6 +66,7 @@ axum = { workspace = true } base64 = { workspace = true } serde_json = { workspace = true } temp-env = { workspace = true } +tempfile = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } tower = { workspace = true, features = ["util"] } # Test fixture only. It proves the integration seam from outside core and must diff --git a/crates/trusted-server-adapter-axum/src/main.rs b/crates/trusted-server-adapter-axum/src/main.rs index 66d592e0b..0a88091dd 100644 --- a/crates/trusted-server-adapter-axum/src/main.rs +++ b/crates/trusted-server-adapter-axum/src/main.rs @@ -1,7 +1,10 @@ +use std::path::Path; + use edgezero_adapter_axum::dev_server::{AxumDevServer, AxumDevServerConfig}; use edgezero_core::app::Hooks as _; use log::LevelFilter; use trusted_server_adapter_axum::app::TrustedServerApp; +use trusted_server_adapter_axum::platform::{KV_PATH_ENV, open_kv_handle}; /// Environment variable that sets the maximum log level. const LOG_LEVEL_ENV: &str = "RUST_LOG"; @@ -39,7 +42,21 @@ fn main() { log::info!("Listening on http://{}", config.addr); let router = TrustedServerApp::routes(); - if let Err(err) = AxumDevServer::with_config(router, config).run() { + let mut server = AxumDevServer::with_config(router, config); + if let Some(path) = std::env::var_os(KV_PATH_ENV).filter(|path| !path.is_empty()) { + // Construct the store once so every request shares one handle. + match open_kv_handle(Path::new(&path)) { + Ok(handle) => { + log::info!("Runtime KV: {}", Path::new(&path).display()); + server = server.with_kv_handle(handle); + } + Err(err) => { + log::error!("{KV_PATH_ENV} is set but the KV store cannot be opened: {err}"); + std::process::exit(1); + } + } + } + if let Err(err) = server.run() { log::error!("trusted-server-adapter-axum failed: {err}"); std::process::exit(1); } diff --git a/crates/trusted-server-adapter-axum/src/platform.rs b/crates/trusted-server-adapter-axum/src/platform.rs index 8d0edfdc6..86c3ecbf0 100644 --- a/crates/trusted-server-adapter-axum/src/platform.rs +++ b/crates/trusted-server-adapter-axum/src/platform.rs @@ -1,18 +1,23 @@ use std::future::Future; use std::net::IpAddr; +use std::path::Path; use std::pin::Pin; use std::sync::Arc; use std::task::{Context, Poll}; use std::time::Duration; use async_trait::async_trait; +use bytes::Bytes; +use edgezero_adapter_axum::key_value_store::PersistentKvStore; use edgezero_core::http::{HeaderMap, HeaderName, HeaderValue, header}; +use edgezero_core::key_value_store::{KvHandle, KvPage, KvStore}; use error_stack::{Report, ResultExt as _}; use trusted_server_core::platform::{ - AuctionTargetCapabilities, AuctionTargetDescriptor, ClientInfo, GeoInfo, PlatformBackend, - PlatformBackendSpec, PlatformConfigStore, PlatformError, PlatformGeo, PlatformHttpClient, - PlatformHttpRequest, PlatformPendingRequest, PlatformResponse, PlatformSecretStore, - PlatformSelectResult, RuntimeServices, StoreId, StoreName, + AuctionTargetCapabilities, AuctionTargetDescriptor, ClientInfo, GeoInfo, KvError, + PlatformBackend, PlatformBackendSpec, PlatformConfigStore, PlatformError, PlatformGeo, + PlatformHttpClient, PlatformHttpRequest, PlatformKvStore, PlatformPendingRequest, + PlatformResponse, PlatformSecretStore, PlatformSelectResult, RuntimeServices, StoreId, + StoreName, UnavailableKvStore, }; // --------------------------------------------------------------------------- @@ -563,17 +568,85 @@ impl PlatformHttpClient for AxumPlatformHttpClient { } } +// --------------------------------------------------------------------------- +// Runtime KV +// --------------------------------------------------------------------------- + +/// Environment variable naming the file that backs runtime KV. +/// +/// When set at startup, the dev server persists runtime KV in that file. When +/// unset, runtime KV is unavailable. +pub const KV_PATH_ENV: &str = "TRUSTED_SERVER_KV_PATH"; + +/// Open the persistent runtime KV store at `path`, creating missing parent +/// directories. +/// +/// # Errors +/// +/// Returns an error when the directory cannot be created or the database +/// cannot be opened (corrupted file, locked by another process). +pub fn open_kv_handle(path: &Path) -> Result { + if let Some(parent) = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + std::fs::create_dir_all(parent).map_err(|err| { + KvError::Internal(anyhow::anyhow!( + "failed to create KV directory `{}`: {err}", + parent.display() + )) + })?; + } + Ok(KvHandle::new(Arc::new(PersistentKvStore::new(path)?))) +} + +/// Bridges edgezero's [`KvHandle`] to [`PlatformKvStore`]. +struct KvHandleAdapter(KvHandle); + +#[async_trait(?Send)] +impl KvStore for KvHandleAdapter { + async fn get_bytes(&self, key: &str) -> Result, KvError> { + self.0.get_bytes(key).await + } + + async fn put_bytes(&self, key: &str, value: Bytes) -> Result<(), KvError> { + self.0.put_bytes(key, value).await + } + + async fn put_bytes_with_ttl( + &self, + key: &str, + value: Bytes, + ttl: Duration, + ) -> Result<(), KvError> { + self.0.put_bytes_with_ttl(key, value, ttl).await + } + + async fn delete(&self, key: &str) -> Result<(), KvError> { + self.0.delete(key).await + } + + async fn list_keys_page( + &self, + prefix: &str, + cursor: Option<&str>, + limit: usize, + ) -> Result { + self.0.list_keys_page(prefix, cursor, limit).await + } +} + // --------------------------------------------------------------------------- // build_runtime_services // --------------------------------------------------------------------------- /// Construct [`RuntimeServices`] for an incoming Axum request. /// -/// # Degraded features in dev +/// # Runtime KV /// -/// The generic runtime KV slot uses -/// [`trusted_server_core::platform::UnavailableKvStore`]. A `warn` log is -/// emitted once per process. +/// Runtime KV comes from the KV handle the dev server injects into each +/// request, which `main` builds when [`KV_PATH_ENV`] is set. Without it the +/// slot is [`UnavailableKvStore`] and a `warn` log is emitted once per process. pub fn build_runtime_services( ctx: &edgezero_core::context::RequestContext, settings: &trusted_server_core::settings::Settings, @@ -581,17 +654,25 @@ pub fn build_runtime_services( [Arc], >, ) -> RuntimeServices { - static KV_WARNED: std::sync::OnceLock<()> = std::sync::OnceLock::new(); - KV_WARNED.get_or_init(|| { - log::warn!("Axum dev server: generic runtime KV is unavailable (UnavailableKvStore)."); - }); + let kv_store: Arc = match ctx.kv_store_default() { + Some(handle) => Arc::new(KvHandleAdapter(handle)), + None => { + static KV_WARNED: std::sync::OnceLock<()> = std::sync::OnceLock::new(); + KV_WARNED.get_or_init(|| { + log::warn!( + "Axum dev server: generic runtime KV is unavailable; set {KV_PATH_ENV} to enable it." + ); + }); + Arc::new(UnavailableKvStore) + } + }; let client_ip = edgezero_adapter_axum::context::AxumRequestContext::get(ctx.request()) .and_then(|c| c.remote_addr) .map(|addr| addr.ip()); use trusted_server_core::platform::{ - PlatformBackend, PlatformConfigStore, PlatformGeo, PlatformKvStore, PlatformSecretStore, + PlatformBackend, PlatformConfigStore, PlatformGeo, PlatformSecretStore, }; // Stateless shims are promoted to process-wide statics so callers clone @@ -600,7 +681,6 @@ pub fn build_runtime_services( std::sync::OnceLock::new(); static SECRET_STORE: std::sync::OnceLock> = std::sync::OnceLock::new(); - static KV_STORE: std::sync::OnceLock> = std::sync::OnceLock::new(); static BACKEND: std::sync::OnceLock> = std::sync::OnceLock::new(); static GEO: std::sync::OnceLock> = std::sync::OnceLock::new(); @@ -611,9 +691,7 @@ pub fn build_runtime_services( .secret_store(Arc::clone(SECRET_STORE.get_or_init(|| { Arc::new(AxumPlatformSecretStore) as Arc }))) - .kv_store(Arc::clone(KV_STORE.get_or_init(|| { - Arc::new(trusted_server_core::platform::UnavailableKvStore) as Arc - }))) + .kv_store(kv_store) .backend(Arc::clone(BACKEND.get_or_init(|| { Arc::new(AxumPlatformBackend) as Arc }))) diff --git a/crates/trusted-server-adapter-axum/tests/runtime_kv.rs b/crates/trusted-server-adapter-axum/tests/runtime_kv.rs new file mode 100644 index 000000000..9be72ff12 --- /dev/null +++ b/crates/trusted-server-adapter-axum/tests/runtime_kv.rs @@ -0,0 +1,205 @@ +//! Runtime KV wiring for the Axum dev server. +//! +//! Each test builds the KV handle exactly as `main` does (`open_kv_handle` on +//! the `TRUSTED_SERVER_KV_PATH` value) and attaches it with +//! `EdgeZeroAxumService::with_kv_handle`, the same call the dev server makes. +//! A probe route reads and writes through `build_runtime_services`, so the +//! adapter's own request-to-`RuntimeServices` path is what is exercised. + +use axum::body::{Body as AxumBody, to_bytes}; +use axum::http::Request; +use bytes::Bytes; +use edgezero_adapter_axum::service::EdgeZeroAxumService; +use edgezero_core::body::Body; +use edgezero_core::context::RequestContext; +use edgezero_core::http::{HandlerFuture, StatusCode, response_builder}; +use edgezero_core::router::RouterService; +use tower::{Service as _, ServiceExt as _}; +use trusted_server_adapter_axum::app::TrustedServerApp; +use trusted_server_adapter_axum::platform::{build_runtime_services, open_kv_handle}; + +const PROBE_KEY: &str = "probe-key"; + +fn reply( + status: StatusCode, + body: Body, +) -> Result { + Ok(response_builder() + .status(status) + .body(body) + .expect("should build probe response")) +} + +/// The services the request path builds for `ctx`, with default settings and +/// no permission signal module. +fn services_for(ctx: &RequestContext) -> trusted_server_core::platform::RuntimeServices { + build_runtime_services( + ctx, + &trusted_server_core::settings::Settings::default(), + &std::sync::Arc::default(), + ) +} + +fn probe_router() -> RouterService { + RouterService::builder() + .put("/probe", |ctx: RequestContext| -> HandlerFuture { + Box::pin(async move { + let services = services_for(&ctx); + match services + .kv_store() + .put_bytes(PROBE_KEY, Bytes::from_static(b"stored")) + .await + { + Ok(()) => reply(StatusCode::NO_CONTENT, Body::empty()), + Err(_) => reply(StatusCode::SERVICE_UNAVAILABLE, Body::empty()), + } + }) + }) + .get("/probe", |ctx: RequestContext| -> HandlerFuture { + Box::pin(async move { + let services = services_for(&ctx); + match services.kv_store().get_bytes(PROBE_KEY).await { + Ok(Some(value)) => reply(StatusCode::OK, Body::from(value.to_vec())), + Ok(None) => reply(StatusCode::NOT_FOUND, Body::empty()), + Err(_) => reply(StatusCode::SERVICE_UNAVAILABLE, Body::empty()), + } + }) + }) + .build() +} + +async fn call(service: &mut EdgeZeroAxumService, method: &str) -> (u16, Bytes) { + let request = Request::builder() + .method(method) + .uri("/probe") + .body(AxumBody::empty()) + .expect("should build probe request"); + let response = service + .ready() + .await + .expect("should be ready") + .call(request) + .await + .expect("should serve probe request"); + let status = response.status().as_u16(); + let body = to_bytes(response.into_body(), 1024) + .await + .expect("should read probe body"); + (status, body) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn runtime_kv_persists_across_requests_and_restarts() { + let dir = tempfile::tempdir().expect("should create temp dir"); + let path = dir.path().join("nested").join("kv.redb"); + + { + let handle = open_kv_handle(&path).expect("should open KV and create its directory"); + let mut service = EdgeZeroAxumService::new(probe_router()).with_kv_handle(handle); + assert_eq!(call(&mut service, "PUT").await.0, 204, "should write"); + let (status, body) = call(&mut service, "GET").await; + assert_eq!(status, 200, "a second request should see the write"); + assert_eq!(body, Bytes::from_static(b"stored")); + } + + let handle = open_kv_handle(&path).expect("should reopen the same KV file"); + let mut restarted = EdgeZeroAxumService::new(probe_router()).with_kv_handle(handle); + let (status, body) = call(&mut restarted, "GET").await; + assert_eq!(status, 200, "data should survive a process restart"); + assert_eq!(body, Bytes::from_static(b"stored")); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn runtime_kv_is_isolated_between_server_instances() { + let dir = tempfile::tempdir().expect("should create temp dir"); + let mut first = EdgeZeroAxumService::new(probe_router()) + .with_kv_handle(open_kv_handle(&dir.path().join("first.redb")).expect("should open")); + let mut second = EdgeZeroAxumService::new(probe_router()) + .with_kv_handle(open_kv_handle(&dir.path().join("second.redb")).expect("should open")); + + assert_eq!( + call(&mut first, "PUT").await.0, + 204, + "should write to first" + ); + + assert_eq!( + call(&mut second, "GET").await.0, + 404, + "a write to one instance must not be visible to another" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn runtime_kv_is_unavailable_without_a_configured_path() { + let mut service = EdgeZeroAxumService::new(probe_router()); + + assert_eq!( + call(&mut service, "GET").await.0, + 503, + "with no KV handle injected, KV should report unavailable" + ); +} + +#[test] +fn open_kv_handle_rejects_a_path_that_cannot_be_created() { + let dir = tempfile::tempdir().expect("should create temp dir"); + let blocker = dir.path().join("file"); + std::fs::write(&blocker, b"not a directory").expect("should write blocker file"); + + let result = open_kv_handle(&blocker.join("kv.redb")); + + assert!(result.is_err(), "a path under a regular file should fail"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn application_serves_requests_with_and_without_runtime_kv() { + let settings = trusted_server_core::settings::Settings::from_toml( + r#" + [[handlers]] + path = "^/_ts/admin" + username = "admin" + password = "admin-pass" + + [publisher] + domain = "test-publisher.example.com" + cookie_domain = ".test-publisher.example.com" + origin_url = "https://origin.test-publisher.example.com" + proxy_secret = "integration-test-proxy-secret" + + [ec] + passphrase = "test-secret-key-32-bytes-minimum" + "#, + ) + .expect("should parse settings"); + let dir = tempfile::tempdir().expect("should create temp dir"); + let handle = open_kv_handle(&dir.path().join("kv.redb")).expect("should open KV"); + + for kv in [None, Some(handle)] { + let router = + TrustedServerApp::routes_with_settings(settings.clone()).expect("should build router"); + let mut service = EdgeZeroAxumService::new(router); + if let Some(handle) = kv { + service = service.with_kv_handle(handle); + } + let request = Request::builder() + .method("POST") + .uri("/_ts/admin/keys/rotate") + .body(AxumBody::empty()) + .expect("should build request"); + + let response = service + .ready() + .await + .expect("should be ready") + .call(request) + .await + .expect("should serve request"); + + assert_eq!( + response.status().as_u16(), + 401, + "the app should answer the same with or without runtime KV" + ); + } +} diff --git a/docs/guide/axum-dev.md b/docs/guide/axum-dev.md index 73988b96b..d7ce5d880 100644 --- a/docs/guide/axum-dev.md +++ b/docs/guide/axum-dev.md @@ -59,6 +59,16 @@ its `ext.debug` echo of the resolved request) when the `[demand.]` table of a Prebid Server demand source sets `debug = true`. Enable it only for short local debugging sessions. +## Runtime KV + +Generic runtime KV (`RuntimeServices::kv_store`) is off by default. Set +`TRUSTED_SERVER_KV_PATH` to a file path before starting the server and the dev +server opens one persistent store at that path, creating missing directories, +and shares it across all requests. Data survives restarts; two servers pointed +at different paths do not see each other's data. If the variable is set but the +store cannot be opened, the server exits with an error. When it is unset, +runtime KV reports unavailable and a warning is logged once. + ## Runtime boundaries Axum sanitizes forwarded client-IP headers but does not resolve From 47afad3562346c7e726cbaa0c62601b2da788f09 Mon Sep 17 00:00:00 2001 From: Erik Svilich Date: Sat, 24 Oct 2026 16:09:55 +0000 Subject: [PATCH 05/12] Run Axum runtime KV operations on the blocking pool --- .../src/platform.rs | 166 +++++++++++++++++- 1 file changed, 165 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-adapter-axum/src/platform.rs b/crates/trusted-server-adapter-axum/src/platform.rs index 86c3ecbf0..0afbe8900 100644 --- a/crates/trusted-server-adapter-axum/src/platform.rs +++ b/crates/trusted-server-adapter-axum/src/platform.rs @@ -597,7 +597,88 @@ pub fn open_kv_handle(path: &Path) -> Result { )) })?; } - Ok(KvHandle::new(Arc::new(PersistentKvStore::new(path)?))) + Ok(KvHandle::new(Arc::new(BlockingPoolKvStore(Arc::new( + PersistentKvStore::new(path)?, + ))))) +} + +/// Runs each operation of a synchronous [`KvStore`] on Tokio's blocking pool. +/// +/// [`PersistentKvStore`] does its `redb` transactions inside its `async fn`s +/// without yielding, so a slow commit would hold the request task and keep +/// any timer raced against it (an integration deadline, for example) from +/// firing. Here the request task only awaits the blocking task and stays +/// free to observe its deadline. +/// +/// A caller that stops waiting drops only the join handle: the operation +/// keeps running to its end in the background. That is safe for `redb`. A +/// write either commits atomically or fails and its transaction aborts on +/// drop, and `redb` admits one write transaction at a time, so a later +/// operation waits for the abandoned one instead of interleaving with it. +/// Dropping the Tokio runtime waits for running blocking tasks, so a commit in +/// flight is not cut short when the server stops. +struct BlockingPoolKvStore(Arc); + +impl BlockingPoolKvStore { + async fn run(&self, op: F) -> Result + where + T: Send + 'static, + F: FnOnce(Arc) -> Fut + Send + 'static, + Fut: Future>, + { + let store = Arc::clone(&self.0); + tokio::task::spawn_blocking(move || futures::executor::block_on(op(store))) + .await + .map_err(|err| KvError::Internal(anyhow::anyhow!("KV task failed: {err}")))? + } +} + +#[async_trait(?Send)] +impl KvStore for BlockingPoolKvStore { + async fn delete(&self, key: &str) -> Result<(), KvError> { + let key = key.to_owned(); + self.run(move |store| async move { store.delete(&key).await }) + .await + } + + async fn get_bytes(&self, key: &str) -> Result, KvError> { + let key = key.to_owned(); + self.run(move |store| async move { store.get_bytes(&key).await }) + .await + } + + async fn list_keys_page( + &self, + prefix: &str, + cursor: Option<&str>, + limit: usize, + ) -> Result { + let prefix = prefix.to_owned(); + let cursor = cursor.map(str::to_owned); + self.run(move |store| async move { + store + .list_keys_page(&prefix, cursor.as_deref(), limit) + .await + }) + .await + } + + async fn put_bytes(&self, key: &str, value: Bytes) -> Result<(), KvError> { + let key = key.to_owned(); + self.run(move |store| async move { store.put_bytes(&key, value).await }) + .await + } + + async fn put_bytes_with_ttl( + &self, + key: &str, + value: Bytes, + ttl: Duration, + ) -> Result<(), KvError> { + let key = key.to_owned(); + self.run(move |store| async move { store.put_bytes_with_ttl(&key, value, ttl).await }) + .await + } } /// Bridges edgezero's [`KvHandle`] to [`PlatformKvStore`]. @@ -1072,6 +1153,89 @@ mod tests { ); } + /// Stands in for a slow `redb` commit: `put_bytes` blocks its thread + /// until the test releases it. + struct GatedStore { + gate: std::sync::Mutex>, + finished: std::sync::atomic::AtomicBool, + } + + #[async_trait(?Send)] + impl KvStore for GatedStore { + async fn delete(&self, _key: &str) -> Result<(), KvError> { + Err(KvError::Unavailable) + } + + async fn get_bytes(&self, _key: &str) -> Result, KvError> { + Err(KvError::Unavailable) + } + + async fn list_keys_page( + &self, + _prefix: &str, + _cursor: Option<&str>, + _limit: usize, + ) -> Result { + Err(KvError::Unavailable) + } + + async fn put_bytes(&self, _key: &str, _value: Bytes) -> Result<(), KvError> { + let _ = self + .gate + .lock() + .expect("should lock gate") + .recv_timeout(Duration::from_secs(5)); + self.finished + .store(true, std::sync::atomic::Ordering::SeqCst); + Ok(()) + } + + async fn put_bytes_with_ttl( + &self, + _key: &str, + _value: Bytes, + _ttl: Duration, + ) -> Result<(), KvError> { + Err(KvError::Unavailable) + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn blocking_kv_op_does_not_delay_a_racing_deadline() { + let (release, gate) = std::sync::mpsc::channel(); + let store = Arc::new(GatedStore { + gate: std::sync::Mutex::new(gate), + finished: std::sync::atomic::AtomicBool::new(false), + }); + let handle = KvHandle::new(Arc::new(BlockingPoolKvStore(Arc::clone(&store)))); + + let started = std::time::Instant::now(); + let raced = tokio::time::timeout( + Duration::from_millis(50), + handle.put_bytes("key", Bytes::from_static(b"value")), + ) + .await; + let elapsed = started.elapsed(); + + assert!( + raced.is_err(), + "the deadline should fire while the store op is still blocked" + ); + assert!( + elapsed < Duration::from_secs(2), + "the deadline should fire on time, fired after {elapsed:?}" + ); + + release.send(()).expect("should release the blocked op"); + tokio::time::timeout(Duration::from_secs(5), async { + while !store.finished.load(std::sync::atomic::Ordering::SeqCst) { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("the abandoned op should still run to completion"); + } + async fn serve_raw_response(response: &'static [u8]) -> String { let listener = tokio::net::TcpListener::bind("127.0.0.1:0") .await From d5f250fbdcb25e07d719028f5f7c80341aa943d8 Mon Sep 17 00:00:00 2001 From: Erik Svilich Date: Sat, 24 Oct 2026 16:10:32 +0000 Subject: [PATCH 06/12] Provide platform timers on Cloudflare, Spin and Axum --- .../src/platform.rs | 39 ++++++++++++++++++- .../src/platform.rs | 34 +++++++++++++++- .../src/platform.rs | 31 ++++++++++++++- 3 files changed, 98 insertions(+), 6 deletions(-) diff --git a/crates/trusted-server-adapter-axum/src/platform.rs b/crates/trusted-server-adapter-axum/src/platform.rs index 0afbe8900..b97eeef21 100644 --- a/crates/trusted-server-adapter-axum/src/platform.rs +++ b/crates/trusted-server-adapter-axum/src/platform.rs @@ -16,8 +16,8 @@ use trusted_server_core::platform::{ AuctionTargetCapabilities, AuctionTargetDescriptor, ClientInfo, GeoInfo, KvError, PlatformBackend, PlatformBackendSpec, PlatformConfigStore, PlatformError, PlatformGeo, PlatformHttpClient, PlatformHttpRequest, PlatformKvStore, PlatformPendingRequest, - PlatformResponse, PlatformSecretStore, PlatformSelectResult, RuntimeServices, StoreId, - StoreName, UnavailableKvStore, + PlatformResponse, PlatformSecretStore, PlatformSelectResult, PlatformTimer, RuntimeServices, + StoreId, StoreName, UnavailableKvStore, }; // --------------------------------------------------------------------------- @@ -764,6 +764,7 @@ pub fn build_runtime_services( std::sync::OnceLock::new(); static BACKEND: std::sync::OnceLock> = std::sync::OnceLock::new(); static GEO: std::sync::OnceLock> = std::sync::OnceLock::new(); + static TIMER: std::sync::OnceLock> = std::sync::OnceLock::new(); RuntimeServices::builder() .config_store(Arc::clone(CONFIG_STORE.get_or_init(|| { @@ -791,6 +792,9 @@ pub fn build_runtime_services( // crates this adapter links, so every request asks exactly the ones // configuration named, in that order. .permission_signal_modules(Arc::clone(permission_signal_modules)) + .timer(Some(Arc::clone(TIMER.get_or_init(|| { + Arc::new(AxumPlatformTimer) as Arc + })))) .client_info(ClientInfo { client_ip, tls_protocol: None, @@ -800,6 +804,20 @@ pub fn build_runtime_services( .build() } +// --------------------------------------------------------------------------- +// AxumPlatformTimer +// --------------------------------------------------------------------------- + +/// Wall-clock timer backed by the tokio runtime's time driver. +pub struct AxumPlatformTimer; + +#[async_trait(?Send)] +impl PlatformTimer for AxumPlatformTimer { + async fn sleep(&self, duration: Duration) { + tokio::time::sleep(duration).await; + } +} + // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- @@ -850,6 +868,23 @@ mod tests { ); } + #[tokio::test] + async fn runtime_timer_releases_work_that_never_completes() { + let services = test_services(); + let timer = services.timer().expect("should provide a runtime timer"); + + let raced = futures::future::select( + Box::pin(futures::future::pending::<()>()), + timer.sleep(Duration::from_millis(5)), + ) + .await; + + assert!( + matches!(raced, futures::future::Either::Right(_)), + "should fire while the other future stays pending" + ); + } + #[test] fn the_backend_name_and_transport_timer_are_pinned() { let backend = AxumPlatformBackend; diff --git a/crates/trusted-server-adapter-cloudflare/src/platform.rs b/crates/trusted-server-adapter-cloudflare/src/platform.rs index 8cbb0a93c..d8c387ec0 100644 --- a/crates/trusted-server-adapter-cloudflare/src/platform.rs +++ b/crates/trusted-server-adapter-cloudflare/src/platform.rs @@ -11,8 +11,8 @@ use error_stack::ResultExt as _; use trusted_server_core::platform::{ AuctionTargetCapabilities, AuctionTargetDescriptor, ClientInfo, GeoInfo, KvError, PlatformBackend, PlatformBackendSpec, PlatformConfigStore, PlatformError, PlatformGeo, - PlatformHttpClient, PlatformKvStore, PlatformSecretStore, RuntimeServices, StoreId, StoreName, - UnavailableKvStore, + PlatformHttpClient, PlatformKvStore, PlatformSecretStore, PlatformTimer, RuntimeServices, + StoreId, StoreName, UnavailableKvStore, }; #[cfg(not(target_arch = "wasm32"))] @@ -614,6 +614,35 @@ impl PlatformSecretStore for CloudflareSecretStoreAdapter { } } +// --------------------------------------------------------------------------- +// Timer — wasm32 only (Workers `setTimeout`) +// --------------------------------------------------------------------------- + +/// Wall-clock timer backed by the Workers `setTimeout`. +/// +/// workers-rs [`worker::Delay`] clears its timeout when dropped, so a +/// cancelled sleep leaves no timer behind in the isolate. +#[cfg(target_arch = "wasm32")] +struct CloudflareTimer; + +#[cfg(target_arch = "wasm32")] +#[async_trait::async_trait(?Send)] +impl PlatformTimer for CloudflareTimer { + async fn sleep(&self, duration: Duration) { + worker::Delay::from(duration).await; + } +} + +/// The Workers timer on wasm32; none on the native host, which has no +/// Workers event loop. +fn runtime_timer() -> Option> { + #[cfg(target_arch = "wasm32")] + let timer: Option> = Some(Arc::new(CloudflareTimer)); + #[cfg(not(target_arch = "wasm32"))] + let timer: Option> = None; + timer +} + // --------------------------------------------------------------------------- // build_runtime_services // --------------------------------------------------------------------------- @@ -693,6 +722,7 @@ pub fn build_runtime_services( // crates this adapter links, so every request asks exactly the ones // configuration named, in that order. .permission_signal_modules(Arc::clone(permission_signal_modules)) + .timer(runtime_timer()) .client_info(ClientInfo { client_ip, tls_protocol: None, diff --git a/crates/trusted-server-adapter-spin/src/platform.rs b/crates/trusted-server-adapter-spin/src/platform.rs index 9a3a160a3..f9efe3f96 100644 --- a/crates/trusted-server-adapter-spin/src/platform.rs +++ b/crates/trusted-server-adapter-spin/src/platform.rs @@ -13,8 +13,8 @@ use http_body_util::BodyExt as _; use trusted_server_core::platform::{ AuctionTargetCapabilities, AuctionTargetDescriptor, ClientInfo, GeoInfo, KvError, PlatformBackend, PlatformBackendSpec, PlatformConfigStore, PlatformError, PlatformGeo, - PlatformHttpClient, PlatformKvStore, PlatformSecretStore, RuntimeServices, StoreId, StoreName, - UnavailableKvStore, + PlatformHttpClient, PlatformKvStore, PlatformSecretStore, PlatformTimer, RuntimeServices, + StoreId, StoreName, UnavailableKvStore, }; #[cfg(not(all(feature = "spin", target_arch = "wasm32")))] @@ -757,6 +757,32 @@ impl PlatformSecretStore for SpinSecretStoreAdapter { } } +// --------------------------------------------------------------------------- +// Timer — Spin wasm32 only (WASI monotonic clock) +// --------------------------------------------------------------------------- + +/// Wall-clock timer backed by the WASI monotonic clock through +/// [`spin_sdk::time::sleep`]. +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +struct SpinPlatformTimer; + +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +#[async_trait::async_trait(?Send)] +impl PlatformTimer for SpinPlatformTimer { + async fn sleep(&self, duration: Duration) { + spin_sdk::time::sleep(duration).await; + } +} + +/// The WASI timer in the Spin component; none on the native host. +fn runtime_timer() -> Option> { + #[cfg(all(feature = "spin", target_arch = "wasm32"))] + let timer: Option> = Some(Arc::new(SpinPlatformTimer)); + #[cfg(not(all(feature = "spin", target_arch = "wasm32")))] + let timer: Option> = None; + timer +} + // --------------------------------------------------------------------------- // build_runtime_services // --------------------------------------------------------------------------- @@ -816,6 +842,7 @@ pub fn build_runtime_services( // crates this adapter links, so every request asks exactly the ones // configuration named, in that order. .permission_signal_modules(Arc::clone(permission_signal_modules)) + .timer(runtime_timer()) .client_info(ClientInfo { client_ip, tls_protocol: None, From a0eb046ce90bf0693146668ccb14c600963f293d Mon Sep 17 00:00:00 2001 From: Erik Svilich Date: Sat, 24 Oct 2026 16:18:51 +0000 Subject: [PATCH 07/12] Move Encypher text provenance into the provenance.encypher module crate The Encypher signing integration becomes a serve middleware in crates/provenance/encypher, selected by [provenance] module = "encypher" and placed by a [[serve]] entry. Its preparation applies the request gates with no I/O, and its stream processor signs the article in the asynchronous finish core now awaits. The worker token is a module secret setting. The enabled flag and path_prefixes give way to the selection and the entry's path, and the module is offered on every adapter and the CLI. --- Cargo.lock | 26 + Cargo.toml | 2 + crates/provenance/encypher/Cargo.toml | 39 + crates/provenance/encypher/src/api.rs | 303 ++ crates/provenance/encypher/src/boundary.rs | 272 ++ crates/provenance/encypher/src/embed.rs | 477 ++++ .../encypher/src/fixtures/parity.json | 848 ++++++ crates/provenance/encypher/src/lib.rs | 346 +++ crates/provenance/encypher/src/processor.rs | 2520 +++++++++++++++++ crates/provenance/encypher/src/tests.rs | 399 +++ crates/provenance/encypher/src/text.rs | 500 ++++ crates/trusted-server-adapter-axum/Cargo.toml | 1 + crates/trusted-server-adapter-axum/src/app.rs | 1 + .../Cargo.toml | 1 + .../src/app.rs | 1 + .../trusted-server-adapter-fastly/Cargo.toml | 1 + .../trusted-server-adapter-fastly/src/app.rs | 1 + crates/trusted-server-adapter-spin/Cargo.toml | 1 + crates/trusted-server-adapter-spin/src/app.rs | 1 + crates/trusted-server-cli/Cargo.toml | 1 + crates/trusted-server-cli/src/app_config.rs | 1 + crates/trusted-server-core/src/publisher.rs | 93 +- 22 files changed, 5830 insertions(+), 5 deletions(-) create mode 100644 crates/provenance/encypher/Cargo.toml create mode 100644 crates/provenance/encypher/src/api.rs create mode 100644 crates/provenance/encypher/src/boundary.rs create mode 100644 crates/provenance/encypher/src/embed.rs create mode 100644 crates/provenance/encypher/src/fixtures/parity.json create mode 100644 crates/provenance/encypher/src/lib.rs create mode 100644 crates/provenance/encypher/src/processor.rs create mode 100644 crates/provenance/encypher/src/tests.rs create mode 100644 crates/provenance/encypher/src/text.rs diff --git a/Cargo.lock b/Cargo.lock index 90fda973c..a9586ea2e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6636,6 +6636,7 @@ dependencies = [ "trusted-server-permission-signal-mtm", "trusted-server-permission-signal-tcf", "trusted-server-permission-signal-us-privacy", + "trusted-server-provenance-encypher", "trusted-server-shared-aws", "trusted-server-tag-google-tag", "trusted-server-tag-google-tag-manager", @@ -6686,6 +6687,7 @@ dependencies = [ "trusted-server-permission-signal-mtm", "trusted-server-permission-signal-tcf", "trusted-server-permission-signal-us-privacy", + "trusted-server-provenance-encypher", "trusted-server-shared-aws", "trusted-server-tag-google-tag", "trusted-server-tag-google-tag-manager", @@ -6742,6 +6744,7 @@ dependencies = [ "trusted-server-permission-signal-mtm", "trusted-server-permission-signal-tcf", "trusted-server-permission-signal-us-privacy", + "trusted-server-provenance-encypher", "trusted-server-shared-aws", "trusted-server-tag-google-tag", "trusted-server-tag-google-tag-manager", @@ -6794,6 +6797,7 @@ dependencies = [ "trusted-server-permission-signal-mtm", "trusted-server-permission-signal-tcf", "trusted-server-permission-signal-us-privacy", + "trusted-server-provenance-encypher", "trusted-server-shared-aws", "trusted-server-tag-google-tag", "trusted-server-tag-google-tag-manager", @@ -7020,6 +7024,7 @@ dependencies = [ "trusted-server-identity-lockr", "trusted-server-loading-cache-51degrees", "trusted-server-middleware-51degrees", + "trusted-server-provenance-encypher", "trusted-server-shared-aws", "trusted-server-tag-google-tag", "trusted-server-tag-google-tag-manager", @@ -7352,6 +7357,27 @@ dependencies = [ "trusted-server-permission-signal-gpc", ] +[[package]] +name = "trusted-server-provenance-encypher" +version = "0.1.0" +dependencies = [ + "async-trait", + "bytes", + "edgezero-core", + "error-stack", + "futures", + "http", + "log", + "regex", + "serde", + "serde_json", + "sha2 0.10.9", + "trusted-server-core", + "url", + "validator", + "web-time", +] + [[package]] name = "trusted-server-shared-51degrees" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 4f1e31221..1f27b93ef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ members = [ "crates/ads-txt/iabtl", "crates/analytics/tinybird", "crates/headers/geo", + "crates/provenance/encypher", "crates/loading-cache/51degrees", "crates/shared/aws", "crates/shared/51degrees", @@ -176,6 +177,7 @@ trusted-server-cmp-inmobi = { path = "crates/cmp/inmobi" } trusted-server-ads-txt-iabtl = { path = "crates/ads-txt/iabtl" } trusted-server-analytics-tinybird = { path = "crates/analytics/tinybird" } trusted-server-headers-geo = { path = "crates/headers/geo" } +trusted-server-provenance-encypher = { path = "crates/provenance/encypher" } trusted-server-loading-cache-51degrees = { path = "crates/loading-cache/51degrees" } trusted-server-shared-aws = { path = "crates/shared/aws" } trusted-server-shared-51degrees = { path = "crates/shared/51degrees" } diff --git a/crates/provenance/encypher/Cargo.toml b/crates/provenance/encypher/Cargo.toml new file mode 100644 index 000000000..5c596e137 --- /dev/null +++ b/crates/provenance/encypher/Cargo.toml @@ -0,0 +1,39 @@ +[package] +name = "trusted-server-provenance-encypher" +description = "Encypher text provenance: signs the article on each page a [[serve]] entry names it for, embedding invisible markers and a C2PA manifest link, and passes every page it cannot sign through unchanged." +authors = { workspace = true } +edition = { workspace = true } +license = { workspace = true } +publish = { workspace = true } +version = { workspace = true } + +[lib] +doctest = false + +[lints] +workspace = true + +[dependencies] +bytes = { workspace = true } +edgezero-core = { workspace = true } +error-stack = { workspace = true } +futures = { workspace = true } +http = { workspace = true } +log = { workspace = true } +regex = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +sha2 = { workspace = true } +trusted-server-core = { workspace = true } +url = { workspace = true } +validator = { workspace = true } +web-time = { workspace = true } + +[dev-dependencies] +async-trait = { workspace = true } +trusted-server-core = { workspace = true, features = ["test-utils"] } + +# The visible owner of this vendor crate. +[package.metadata.maintainers] +owner = "Encypher" +status = "vendor owned" diff --git a/crates/provenance/encypher/src/api.rs b/crates/provenance/encypher/src/api.rs new file mode 100644 index 000000000..ddf89b92a --- /dev/null +++ b/crates/provenance/encypher/src/api.rs @@ -0,0 +1,303 @@ +use serde::{Deserialize, Serialize}; + +/// A 2 MiB cap accommodates roughly 600 KiB surrogate-escaped plans at the 50,000-scalar input limit. +pub const MAX_RESPONSE_BYTES: usize = 2 * 1024 * 1024; +/// Maximum visible-text length sent for signing, measured in Unicode scalars. +pub const MAX_TEXT_SCALARS: usize = 50_000; +/// Policy responses contain only one digest; cap them independently of plans. +pub const MAX_POLICY_RESPONSE_BYTES: usize = 1024; + +/// JSON request body for the Encypher CDN signing endpoint. +#[derive(Serialize)] +pub struct SignRequest<'a> { + /// Canonical visible article text. + pub text: &'a str, + /// Absolute page URL without its query string. + pub page_url: &'a str, + /// Optional page title capped at 500 Unicode scalars. + #[serde(skip_serializing_if = "Option::is_none")] + pub document_title: Option<&'a str>, + /// Boundary detection label used for the article. + pub boundary_selector: &'a str, + /// Server policy used to select this article's cache namespace. + pub expected_policy_digest: &'a str, +} + +/// One invisible-marker insertion returned by the signing endpoint. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct PlanOperation { + /// Visible codepoint index after which to insert, or -1 for before the first. + pub insert_after_index: i64, + /// Invisible marker characters to insert. + pub marker: String, +} + +/// A complete marker embedding plan. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct EmbeddingPlan { + /// Index unit, which must be `codepoint`. + pub index_unit: String, + /// Ordered marker insertion operations. + pub operations: Vec, +} + +/// Parsed response from the Encypher CDN signing endpoint. +#[derive(Debug, Deserialize)] +pub struct SignResponse { + /// Whether signing succeeded. + pub success: bool, + /// Marker plan when signing succeeded. + #[serde(default)] + pub embedding_plan: Option, + /// C2PA manifest URL when signing succeeded. + #[serde(default)] + pub manifest_url: Option, + /// Stable failure reason when signing did not succeed. + #[serde(default)] + pub error: Option, + /// Actual policy used by the server; required before a successful plan is used. + #[serde(default)] + pub policy_digest: Option, +} + +/// Fail-closed signing-response parse errors. +#[derive(Debug, PartialEq)] +pub enum ApiError { + /// The response exceeded [`MAX_RESPONSE_BYTES`]. + TooLarge, + /// The response was not valid JSON for [`SignResponse`]. + Malformed, +} + +/// Parses a bounded signing response. +/// +/// # Errors +/// +/// Returns [`ApiError::TooLarge`] above the response limit and +/// [`ApiError::Malformed`] when JSON decoding fails. +pub fn parse_sign_response(body: &[u8]) -> Result { + if body.len() > MAX_RESPONSE_BYTES { + return Err(ApiError::TooLarge); + } + serde_json::from_slice(body).map_err(|_| ApiError::Malformed) +} + +/// A policy digest is exactly a lowercase SHA-256 hex value. +pub fn valid_policy_digest(digest: &str) -> bool { + digest.len() == 64 + && digest + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +/// Parse the policy endpoint's strict, bounded response. +/// +/// # Errors +/// +/// Rejects oversized JSON, extra or duplicate fields, and invalid digests. +pub fn parse_policy_response(body: &[u8]) -> Result { + #[derive(Deserialize)] + #[serde(deny_unknown_fields)] + struct PolicyResponse { + policy_digest: String, + } + if body.len() > MAX_POLICY_RESPONSE_BYTES { + return Err(ApiError::TooLarge); + } + let response: PolicyResponse = serde_json::from_slice(body).map_err(|_| ApiError::Malformed)?; + if !valid_policy_digest(&response.policy_digest) { + return Err(ApiError::Malformed); + } + Ok(response.policy_digest) +} + +/// Reasons an embedding plan is unsafe or cannot address the visible text. +#[derive(Debug, PartialEq)] +pub enum PlanError { + /// The plan does not use Unicode codepoint indices. + IndexUnit, + /// An insertion index is outside the visible-text range. + IndexOutOfRange, + /// A marker is empty. + EmptyMarker, + /// A marker contains a character outside the edge-core invisible set. + DisallowedMarkerChar, + /// The plan has no operations. + Empty, + /// The manifest URL is present but not an absolute `https` URL. + ManifestUrl, +} + +/// Validates a signing result before use: index bounds and the invisible marker +/// alphabet for every operation, and a manifest URL that is absent or `https`. +/// +/// This is the only check between the API or KV and the page, so callers must +/// pass the scalar count of the exact text that was signed. +/// +/// # Errors +/// +/// Returns a [`PlanError`] for the first invalid property. +pub fn validate_plan( + plan: &EmbeddingPlan, + manifest_url: Option<&str>, + visible_scalars: usize, +) -> Result<(), PlanError> { + if plan.index_unit != "codepoint" { + return Err(PlanError::IndexUnit); + } + if plan.operations.is_empty() { + return Err(PlanError::Empty); + } + for operation in &plan.operations { + if operation.insert_after_index < -1 { + return Err(PlanError::IndexOutOfRange); + } + if operation.insert_after_index >= 0 { + let Ok(index) = usize::try_from(operation.insert_after_index) else { + return Err(PlanError::IndexOutOfRange); + }; + if index >= visible_scalars { + return Err(PlanError::IndexOutOfRange); + } + } + if operation.marker.is_empty() { + return Err(PlanError::EmptyMarker); + } + if !operation.marker.chars().all(is_allowed_marker_char) { + return Err(PlanError::DisallowedMarkerChar); + } + } + if let Some(url) = manifest_url { + let https = url::Url::parse(url).is_ok_and(|url| { + url.scheme() == "https" && url.host_str().is_some_and(|h| !h.is_empty()) + }); + if !https { + return Err(PlanError::ManifestUrl); + } + } + Ok(()) +} + +fn is_allowed_marker_char(ch: char) -> bool { + matches!(ch, '\u{fe00}'..='\u{fe0f}' | '\u{e0100}'..='\u{e01ef}' | '\u{feff}') +} + +#[cfg(test)] +mod tests { + use super::*; + + fn single_op_plan() -> EmbeddingPlan { + EmbeddingPlan { + index_unit: "codepoint".to_string(), + operations: vec![PlanOperation { + insert_after_index: 0, + marker: "\u{fe00}".to_string(), + }], + } + } + + #[test] + fn policy_response_requires_only_a_lowercase_sha256_digest() { + let digest = "ab".repeat(32); + let body = format!(r#"{{"policy_digest":"{digest}"}}"#); + assert_eq!(parse_policy_response(body.as_bytes()), Ok(digest.clone())); + for body in [ + "{}".to_owned(), + r#"{"policy_digest":null}"#.to_owned(), + r#"{"policy_digest":42}"#.to_owned(), + format!(r#"{{"policy_digest":"{}"}}"#, digest.to_uppercase()), + format!(r#"{{"policy_digest":"{}"}}"#, "g".repeat(64)), + format!(r#"{{"policy_digest":"{}"}}"#, "a".repeat(63)), + format!(r#"{{"policy_digest":"{digest}","extra":true}}"#), + format!(r#"{{"policy_digest":"{digest}","policy_digest":"{digest}"}}"#), + format!(r#"{{"policy_digest":"{digest}"}} trailing"#), + ] { + assert_eq!( + parse_policy_response(body.as_bytes()), + Err(ApiError::Malformed) + ); + } + assert_eq!( + parse_policy_response(&vec![b' '; MAX_POLICY_RESPONSE_BYTES + 1]), + Err(ApiError::TooLarge) + ); + } + + #[test] + fn parses_and_validates_a_response_plan() { + let body = r#"{"success":true,"embedding_plan":{"index_unit":"codepoint","operations":[{"insert_after_index":0,"marker":"︀"}]},"manifest_url":"https://manifest.example.com/x"}"#; + let response = parse_sign_response(body.as_bytes()).expect("should parse"); + let plan = response.embedding_plan.expect("should contain plan"); + assert_eq!( + validate_plan(&plan, response.manifest_url.as_deref(), 1), + Ok(()) + ); + } + + #[test] + fn manifest_url_must_be_absent_or_https() { + let plan = single_op_plan(); + assert_eq!(validate_plan(&plan, None, 1), Ok(())); + for url in [ + "", + "http://manifest.example.com/x", + "javascript:alert(1)", + "data:text/html,", + "/api/v1/public/cdn/manifest/1", + "https://", + ] { + assert_eq!( + validate_plan(&plan, Some(url), 1), + Err(PlanError::ManifestUrl), + "{url} should be rejected" + ); + } + } + + #[test] + fn rejects_markup_and_bad_ranges() { + let markup = EmbeddingPlan { + index_unit: "codepoint".to_string(), + operations: vec![PlanOperation { + insert_after_index: 0, + marker: "".to_string(), + }], + }; + assert_eq!( + validate_plan(&markup, None, 1), + Err(PlanError::DisallowedMarkerChar) + ); + assert_eq!( + validate_plan(&single_op_plan(), None, 0), + Err(PlanError::IndexOutOfRange) + ); + } + + #[test] + fn parses_realistic_large_marker_response() { + let escaped_marker = r"\uDB40\uDD00".repeat(50_000); + let body = format!( + r#"{{"success":true,"embedding_plan":{{"index_unit":"codepoint","operations":[{{"insert_after_index":0,"marker":"{escaped_marker}"}}]}}}}"# + ); + assert!(body.len() > 600_000); + let response = + parse_sign_response(body.as_bytes()).expect("large valid response should parse"); + let plan = response + .embedding_plan + .expect("response should contain a plan"); + assert_eq!(plan.operations[0].marker.chars().count(), 50_000); + } + + #[test] + fn enforces_size_and_empty_plan_limits() { + assert_eq!( + parse_sign_response(&vec![b' '; MAX_RESPONSE_BYTES + 1]).map(|_| ()), + Err(ApiError::TooLarge) + ); + let empty = EmbeddingPlan { + index_unit: "codepoint".to_string(), + operations: Vec::new(), + }; + assert_eq!(validate_plan(&empty, None, 1), Err(PlanError::Empty)); + } +} diff --git a/crates/provenance/encypher/src/boundary.rs b/crates/provenance/encypher/src/boundary.rs new file mode 100644 index 000000000..b2b78c749 --- /dev/null +++ b/crates/provenance/encypher/src/boundary.rs @@ -0,0 +1,272 @@ +use std::sync::LazyLock; + +use regex::Regex; + +const CMS_CONTENT_CLASSES: [&str; 13] = [ + "entry-content", + "wp-block-post-content", + "post-content", + "article-body", + "article-content", + "story-body", + "story-body-text", + "gh-content", + "w-richtext", + "sqs-block-content", + "body markup", + "e-content", + "RichTextStoryBody", +]; + +static ARTICLE: LazyLock> = LazyLock::new(|| tag_opening("article")); +static MAIN: LazyLock> = LazyLock::new(|| tag_opening("main")); +static BODY: LazyLock> = LazyLock::new(|| tag_opening("body")); +static CLUSTER_CONTAINERS: LazyLock> = LazyLock::new(|| { + ["div", "section", "main"] + .into_iter() + .filter_map(tag_opening) + .collect() +}); +static PARAGRAPH: LazyLock> = LazyLock::new(|| Regex::new(r"(?i)> = LazyLock::new(|| { + CMS_CONTENT_CLASSES + .into_iter() + .filter_map(|class_name| Some((class_name, class_opening(class_name)?))) + .collect() +}); +static ITEMPROP_ARTICLE_BODY: LazyLock> = + LazyLock::new(|| attribute_opening("itemprop", "articleBody")); +static ROLE_MAIN: LazyLock> = LazyLock::new(|| attribute_opening("role", "main")); +static SELECTOR_ATTRIBUTE: LazyLock> = + LazyLock::new(|| Regex::new(r#"\[([A-Za-z0-9_]+)=["']?([^"'\]]+)["']?\]"#).ok()); + +/// The UTF-8 byte range and detection label of an article's outer HTML. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Boundary { + /// Inclusive byte offset of the opening element. + pub start: usize, + /// Exclusive byte offset after the first matching closing element. + pub end: usize, + /// Detection-chain selector label. + pub selector: String, +} + +/// Finds an article boundary using the edge-core priority chain, excluding JSON-LD P5. +#[must_use] +pub fn find_article_boundary(html: &str, override_selector: Option<&str>) -> Option { + if let Some(selector) = override_selector.filter(|selector| !selector.is_empty()) + && let Some(range) = match_selector(html, selector) + { + return Some(Boundary { + start: range.0, + end: range.1, + selector: format!("override:{selector}"), + }); + } + + let articles = ARTICLE + .as_ref() + .map_or_else(Vec::new, |opening| match_all(html, opening)); + if let Some(first) = articles.first().copied() { + let mut selected = first; + let mut selected_count = count_paragraphs(html.get(first.0..first.1).unwrap_or("")); + for candidate in articles.iter().copied().skip(1) { + let candidate_count = + count_paragraphs(html.get(candidate.0..candidate.1).unwrap_or("")); + if candidate_count > selected_count { + selected = candidate; + selected_count = candidate_count; + } + } + return Some(boundary(selected, "article")); + } + + for (class_name, opening) in CMS_CLASSES.iter() { + if let Some(range) = match_first(html, opening) { + return Some(boundary(range, &format!("class:{class_name}"))); + } + } + + if let Some(range) = match_fixed(html, &ITEMPROP_ARTICLE_BODY) { + return Some(boundary(range, "itemprop:articleBody")); + } + if let Some(range) = match_fixed(html, &MAIN) { + return Some(boundary(range, "main")); + } + if let Some(range) = match_fixed(html, &ROLE_MAIN) { + return Some(boundary(range, "role:main")); + } + if let Some(range) = find_largest_p_cluster(html) { + return Some(boundary(range, "p-cluster")); + } + match_fixed(html, &BODY).map(|range| boundary(range, "body")) +} + +fn boundary((start, end): (usize, usize), selector: &str) -> Boundary { + Boundary { + start, + end, + selector: selector.to_owned(), + } +} + +// Every opening pattern captures the element's tag name as group 1. +fn tag_opening(tag_name: &str) -> Option { + Regex::new(&format!( + r"(?i)<({})(?-u:\b)[^>]*>", + regex::escape(tag_name) + )) + .ok() +} + +fn class_opening(class_name: &str) -> Option { + Regex::new(&format!( + r#"(?i)<([A-Za-z0-9_]+)(?-u:\b)[^>]*class\s*=\s*["'][^"']*(?-u:\b){}(?-u:\b)[^"']*["'][^>]*>"#, + regex::escape(class_name) + )) + .ok() +} + +fn attribute_opening(attribute: &str, value: &str) -> Option { + Regex::new(&format!( + r#"(?i)<([A-Za-z0-9_]+)(?-u:\b)[^>]*{}\s*=\s*["']{}["'][^>]*>"#, + regex::escape(attribute), + regex::escape(value) + )) + .ok() +} + +fn match_fixed(html: &str, opening: &LazyLock>) -> Option<(usize, usize)> { + match_first(html, opening.as_ref()?) +} + +fn match_first(html: &str, opening: &Regex) -> Option<(usize, usize)> { + let captures = opening.captures(html)?; + let whole = captures.get(0)?; + close_first_element(html, whole.start(), whole.end(), captures.get(1)?.as_str()) +} + +fn match_all(html: &str, opening: &Regex) -> Vec<(usize, usize)> { + let mut matches = Vec::new(); + let mut cursor = 0_usize; + while let Some(tail) = html.get(cursor..) { + let Some(captures) = opening.captures(tail) else { + break; + }; + let (Some(whole), Some(tag_name)) = (captures.get(0), captures.get(1)) else { + break; + }; + let start = cursor.saturating_add(whole.start()); + let opening_end = cursor.saturating_add(whole.end()); + let Some(range) = close_first_element(html, start, opening_end, tag_name.as_str()) else { + break; + }; + matches.push(range); + cursor = range.1; + } + matches +} + +// An operator-configured selector is compiled per document. +fn match_selector(html: &str, selector: &str) -> Option<(usize, usize)> { + if let Some(class_name) = selector.strip_prefix('.') { + return match_first(html, &class_opening(class_name)?); + } + if selector.starts_with('[') + && let Some(captures) = SELECTOR_ATTRIBUTE.as_ref()?.captures(selector) + { + let opening = attribute_opening(captures.get(1)?.as_str(), captures.get(2)?.as_str())?; + return match_first(html, &opening); + } + match_first(html, &tag_opening(selector)?) +} + +/// Extends an opening tag to the first `` after it, ASCII case-insensitively. +fn close_first_element( + html: &str, + opening_start: usize, + opening_end: usize, + tag_name: &str, +) -> Option<(usize, usize)> { + let closing = format!(""); + let tail = html.as_bytes().get(opening_end..)?; + let offset = tail + .windows(closing.len()) + .position(|window| window.eq_ignore_ascii_case(closing.as_bytes()))?; + Some(( + opening_start, + opening_end + .saturating_add(offset) + .saturating_add(closing.len()), + )) +} + +fn count_paragraphs(html: &str) -> usize { + PARAGRAPH + .as_ref() + .map_or(0, |regex| regex.find_iter(html).count()) +} + +fn find_largest_p_cluster(html: &str) -> Option<(usize, usize)> { + let mut best = None; + let mut best_count = 4_usize; + for opening in CLUSTER_CONTAINERS.iter() { + for range in match_all(html, opening) { + let count = count_paragraphs(html.get(range.0..range.1).unwrap_or("")); + if count > best_count { + best = Some(range); + best_count = count; + } + } + } + best +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn follows_priority_chain_and_byte_offsets() { + let html = "🐟

Main.

Chosen.

"; + let boundary = find_article_boundary(html, None).expect("should find article"); + assert_eq!(boundary.selector, "article"); + assert_eq!( + html.get(boundary.start..boundary.end), + Some("

Chosen.

") + ); + } + + #[test] + fn nested_same_name_uses_first_closing_tag() { + let html = "
outer
inner
tail
"; + let boundary = find_article_boundary(html, None).expect("should find article"); + assert_eq!( + html.get(boundary.start..boundary.end), + Some("
outer
inner
") + ); + } + + #[test] + fn closing_tags_match_ascii_case_insensitively() { + let html = "

One.

x
"; + let boundary = find_article_boundary(html, None).expect("should find article"); + assert_eq!( + html.get(boundary.start..boundary.end), + Some("

One.

") + ); + let boundary = + find_article_boundary(html, Some(".entry-content")).expect("should find override"); + assert_eq!( + html.get(boundary.start..boundary.end), + Some("
x
") + ); + } + + #[test] + fn selects_largest_paragraph_cluster_at_five() { + let html = "

1

2

3

4

5

"; + let boundary = find_article_boundary(html, None).expect("should find cluster"); + assert_eq!(boundary.selector, "p-cluster"); + } +} diff --git a/crates/provenance/encypher/src/embed.rs b/crates/provenance/encypher/src/embed.rs new file mode 100644 index 000000000..b5195ba58 --- /dev/null +++ b/crates/provenance/encypher/src/embed.rs @@ -0,0 +1,477 @@ +use std::sync::LazyLock; + +use regex::Regex; + +use super::api::EmbeddingPlan; +use super::boundary::Boundary; +use super::text::{ + assemble_text, decode_html_entities, extract_fragments, is_js_whitespace, structural_close_tag, +}; + +const MAGIC_PREFIX: &str = "\u{e01df}\u{e01e0}\u{e01e1}\u{e01e2}"; + +static MANIFEST_META: LazyLock> = LazyLock::new(|| { + Regex::new(r#"(?i)]*name\s*=\s*['"]c2pa-manifest-url['"][^>]*>"#).ok() +}); + +struct MappedChar { + ch: char, + start: usize, + end: usize, +} + +/// Reports whether a string contains the four-character Encypher magic prefix. +#[must_use] +pub fn has_existing_markers(s: &str) -> bool { + s.contains(MAGIC_PREFIX) +} + +/// Reports whether HTML already declares a C2PA manifest URL meta element. +#[must_use] +pub fn has_manifest_meta(html: &str) -> bool { + MANIFEST_META + .as_ref() + .is_some_and(|regex| regex.is_match(html)) +} + +/// Applies a codepoint embedding plan within an article boundary. +/// +/// The plan must already have passed [`super::api::validate_plan`]; markers are +/// inserted verbatim. Returns `None` when the boundary has no visible text, an +/// index has no text position, or text-to-byte alignment fails. +#[must_use] +pub fn apply_plan(html: &str, boundary: &Boundary, plan: &EmbeddingPlan) -> Option { + let boundary_html = html.get(boundary.start..boundary.end)?; + let fragments = extract_fragments(boundary_html); + let visible = assemble_text(boundary_html, &fragments); + let visible_chars: Vec = visible.chars().collect(); + if visible_chars.is_empty() { + return None; + } + + let mut insertion_offsets = vec![None; visible_chars.len()]; + let mut first_visible_start = None; + let mut previous_visible_end = None; + let mut visible_cursor = 0_usize; + + for fragment in fragments { + let raw = boundary_html.get(fragment.start..fragment.end)?; + let normalized = normalize_mapped(decode_with_offsets(raw)); + if normalized.is_empty() { + continue; + } + + while visible_chars.get(visible_cursor) == Some(&' ') { + if let Some(offset) = previous_visible_end + && let Some(slot) = insertion_offsets.get_mut(visible_cursor) + { + *slot = Some(offset); + } + visible_cursor = visible_cursor.saturating_add(1); + } + + for item in normalized { + let target = visible_chars.get(visible_cursor)?; + if !equivalent(item.ch, *target) { + return None; + } + let start = fragment.start.checked_add(item.start)?; + let end = fragment.start.checked_add(item.end)?; + if first_visible_start.is_none() { + first_visible_start = Some(start); + } + let slot = insertion_offsets.get_mut(visible_cursor)?; + *slot = Some(end); + previous_visible_end = Some(end); + visible_cursor = visible_cursor.saturating_add(1); + } + } + + while visible_chars.get(visible_cursor) == Some(&' ') { + if let Some(offset) = previous_visible_end { + let slot = insertion_offsets.get_mut(visible_cursor)?; + *slot = Some(offset); + } + visible_cursor = visible_cursor.saturating_add(1); + } + if visible_cursor != visible_chars.len() { + return None; + } + let first_visible_start = first_visible_start?; + + let mut insertions = Vec::with_capacity(plan.operations.len()); + let mut output_len = html.len(); + for operation in &plan.operations { + let relative = if operation.insert_after_index == -1 { + first_visible_start + } else { + let index = usize::try_from(operation.insert_after_index).ok()?; + insertion_offsets.get(index).copied().flatten()? + }; + let absolute = boundary.start.checked_add(relative)?; + output_len = output_len.checked_add(operation.marker.len())?; + insertions.push((absolute, operation.marker.as_str())); + } + // Stable sorting preserves API order for multiple markers at the same offset. + insertions.sort_by_key(|(offset, _)| *offset); + let mut result = String::with_capacity(output_len); + let mut cursor = 0; + for (offset, marker) in insertions { + result.push_str(html.get(cursor..offset)?); + result.push_str(marker); + cursor = offset; + } + result.push_str(html.get(cursor..)?); + Some(result) +} + +/// Injects an escaped manifest meta element before the structural closing head tag. +/// +/// Returns the input unchanged when the URL is empty, a manifest meta already exists, +/// or no structural closing head tag exists. +#[must_use] +pub fn inject_manifest_meta(html: &str, manifest_url: &str) -> String { + if html.is_empty() || manifest_url.is_empty() || has_manifest_meta(html) { + return html.to_owned(); + } + let Some(offset) = structural_close_tag(html, "head") else { + return html.to_owned(); + }; + let safe_url = manifest_url + .replace('&', "&") + .replace('"', """) + .replace('<', "<") + .replace('>', ">"); + let meta = format!(r#""#); + let Some(before) = html.get(..offset) else { + return html.to_owned(); + }; + let Some(after) = html.get(offset..) else { + return html.to_owned(); + }; + let mut output = String::with_capacity(html.len().saturating_add(meta.len())); + output.push_str(before); + output.push_str(&meta); + output.push_str(after); + output +} + +fn decode_with_offsets(raw: &str) -> Vec { + let mut mapped = Vec::new(); + let mut cursor = 0_usize; + while cursor < raw.len() { + if raw.as_bytes().get(cursor) == Some(&b'&') + && let Some(relative_semicolon) = raw + .as_bytes() + .get(cursor.saturating_add(1)..) + // Entity names/numbers cannot contain another ampersand or markup. + // Stop there rather than rescanning the suffix for every literal '&'. + .and_then(|tail| { + let end = tail + .iter() + .position(|byte| !byte.is_ascii_alphanumeric() && *byte != b'#')?; + (tail[end] == b';').then_some(end) + }) + { + let end = cursor + .saturating_add(1) + .saturating_add(relative_semicolon) + .saturating_add(1); + if let Some(token) = raw.get(cursor..end) { + let decoded = decode_html_entities(token); + if decoded != token { + mapped.extend(decoded.chars().map(|ch| MappedChar { + ch, + start: cursor, + end, + })); + cursor = end; + continue; + } + } + } + + let Some(ch) = raw.get(cursor..).and_then(|tail| tail.chars().next()) else { + break; + }; + let end = cursor.saturating_add(ch.len_utf8()); + mapped.push(MappedChar { + ch, + start: cursor, + end, + }); + cursor = end; + } + mapped +} + +fn normalize_mapped(mapped: Vec) -> Vec { + let mut normalized: Vec = Vec::with_capacity(mapped.len()); + for item in mapped { + if is_js_whitespace(item.ch) { + let Some(last) = normalized.last_mut() else { + continue; + }; + if last.ch == ' ' { + last.end = item.end; + } else { + normalized.push(MappedChar { + ch: ' ', + start: item.start, + end: item.end, + }); + } + } else { + normalized.push(item); + } + } + while normalized.last().is_some_and(|item| item.ch == ' ') { + normalized.pop(); + } + normalized +} + +fn equivalent(left: char, right: char) -> bool { + left == right || matches!((left, right), ('\u{00a0}', ' ') | (' ', '\u{00a0}')) +} + +#[cfg(test)] +mod tests { + use serde::Deserialize; + + use super::*; + use crate::api::{EmbeddingPlan, PlanOperation}; + use crate::boundary::{Boundary, find_article_boundary}; + use crate::text::{assemble_text, extract_fragments}; + + #[test] + fn adversarial_ampersands_have_bounded_embedding_work() { + let html = format!("
{}
", "&".repeat(100_000)); + let boundary = find_article_boundary(&html, None).expect("article"); + let plan = EmbeddingPlan { + index_unit: "codepoint".to_owned(), + operations: vec![PlanOperation { + insert_after_index: 0, + marker: "\u{fe00}".to_owned(), + }], + }; + let output = apply_plan(&html, &boundary, &plan).expect("valid plan"); + assert_eq!( + output, + format!("
&\u{fe00}{}
", "&".repeat(99_999)) + ); + } + + #[test] + fn dense_plan_on_large_document_preserves_operation_order() { + let html = format!( + "
{}
", + "a".repeat(50_000), + "x".repeat(4_900_000) + ); + let boundary = find_article_boundary(&html, None).expect("article"); + let plan = EmbeddingPlan { + index_unit: "codepoint".to_owned(), + operations: (0..20_000) + .rev() + .map(|index| PlanOperation { + insert_after_index: index, + marker: "\u{fe00}".to_owned(), + }) + .collect(), + }; + let output = apply_plan(&html, &boundary, &plan).expect("valid dense plan"); + assert_eq!( + output, + format!( + "
{}{}
", + "a\u{fe00}".repeat(20_000), + "a".repeat(30_000), + "x".repeat(4_900_000) + ) + ); + } + + #[derive(Deserialize)] + struct FixtureFile { + cases: Vec, + } + + #[derive(Deserialize)] + struct FixtureCase { + id: String, + html: String, + override_selector: Option, + boundary: FixtureBoundary, + text: String, + text_rust: String, + text_divergence: Option, + plan: EmbeddingPlan, + expected_output: String, + invalid_plan: Option, + expected_invalid_output: Option, + manifest_url: String, + expected_meta_output_js: String, + expected_meta_output_rust: String, + meta_divergence: Option, + } + + #[derive(Deserialize)] + struct FixtureBoundary { + selector: String, + start: usize, + end: usize, + } + + fn fixtures() -> FixtureFile { + serde_json::from_str(include_str!("fixtures/parity.json")) + .expect("parity fixtures should deserialize") + } + + #[test] + fn groups_same_offset_in_plan_order() { + let html = "🐟
A B
"; + let start = html.find("
").expect("should find boundary"); + let boundary = Boundary { + start, + end: html.len(), + selector: "article".to_string(), + }; + let plan = EmbeddingPlan { + index_unit: "codepoint".to_string(), + operations: vec![ + PlanOperation { + insert_after_index: 0, + marker: "\u{fe00}".to_string(), + }, + PlanOperation { + insert_after_index: 0, + marker: "\u{fe01}".to_string(), + }, + ], + }; + assert_eq!( + apply_plan(html, &boundary, &plan), + Some("🐟
A\u{fe00}\u{fe01} B
".to_string()) + ); + } + + #[test] + fn manifest_meta_uses_structural_head_close_and_escapes() { + let html = ""; + let output = inject_manifest_meta(html, "https://example.com/?a=1&b=\"x\""); + assert!(output.contains(""#; + let html = format!("{prefix}
Visible
"); + assert_eq!( + inject_manifest_meta(&html, "javascript:alert(1)"), + format!( + "{prefix}
Visible
" + ) + ); + // The URL is inert metadata, not an href, script source or refresh target. + let script = + r#""#; + assert_eq!( + inject_manifest_meta( + &format!("{script}"), + "data:text/html," + ), + format!( + "{script}" + ) + ); + } + + #[test] + fn manifest_meta_ignores_quotes_inside_unquoted_values() { + // The browser reads `x"y` as an unquoted value, so the second meta's quoted + // content, including its ``, is attribute text. + let prefix = + r#""#; + let html = format!("{prefix}Visible"); + assert_eq!( + inject_manifest_meta(&html, "https://manifest.example.com/m"), + format!( + "{prefix}Visible" + ) + ); + } + + #[test] + fn edge_core_parity_fixtures_match() { + for fixture in fixtures().cases { + let boundary = + find_article_boundary(&fixture.html, fixture.override_selector.as_deref()) + .unwrap_or_else(|| panic!("{} should have a boundary", fixture.id)); + assert_eq!( + boundary.selector, fixture.boundary.selector, + "{}", + fixture.id + ); + assert_eq!(boundary.start, fixture.boundary.start, "{}", fixture.id); + assert_eq!(boundary.end, fixture.boundary.end, "{}", fixture.id); + let boundary_html = fixture + .html + .get(boundary.start..boundary.end) + .expect("fixture boundary should be valid"); + let fragments = extract_fragments(boundary_html); + let text = assemble_text(boundary_html, &fragments); + assert_eq!(text, fixture.text_rust, "{}", fixture.id); + if fixture.text_divergence.is_some() { + assert_ne!(text, fixture.text, "{}", fixture.id); + } else { + assert_eq!(text, fixture.text, "{}", fixture.id); + } + assert_eq!( + apply_plan(&fixture.html, &boundary, &fixture.plan), + Some(fixture.expected_output), + "{}", + fixture.id + ); + if let Some(invalid_plan) = fixture.invalid_plan { + assert_eq!( + apply_plan(&fixture.html, &boundary, &invalid_plan), + fixture.expected_invalid_output, + "{}", + fixture.id + ); + } + let meta = inject_manifest_meta(&fixture.html, &fixture.manifest_url); + assert_eq!(meta, fixture.expected_meta_output_rust, "{}", fixture.id); + if fixture.meta_divergence.is_some() { + assert_ne!(meta, fixture.expected_meta_output_js, "{}", fixture.id); + } else { + assert_eq!(meta, fixture.expected_meta_output_js, "{}", fixture.id); + } + } + } + + #[test] + fn every_fixture_byte_prefix_is_panic_free() { + for fixture in fixtures().cases { + for end in 0..=fixture.html.len() { + let prefix = String::from_utf8_lossy( + fixture + .html + .as_bytes() + .get(..end) + .expect("prefix range should be valid"), + ); + let fragments = extract_fragments(&prefix); + let _ = assemble_text(&prefix, &fragments); + if let Some(boundary) = + find_article_boundary(&prefix, fixture.override_selector.as_deref()) + { + let _ = apply_plan(&prefix, &boundary, &fixture.plan); + } + let _ = inject_manifest_meta(&prefix, &fixture.manifest_url); + } + } + } +} diff --git a/crates/provenance/encypher/src/fixtures/parity.json b/crates/provenance/encypher/src/fixtures/parity.json new file mode 100644 index 000000000..848767631 --- /dev/null +++ b/crates/provenance/encypher/src/fixtures/parity.json @@ -0,0 +1,848 @@ +{ + "generated_from": [ + "integrations/edge-core/boundary.js", + "integrations/edge-core/fragments.js", + "integrations/edge-core/embed.js" + ], + "fixed_plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "cases": [ + { + "id": "override-multibyte-prefix", + "html": "Café & News🙂前

Alpha & beta.

", + "override_selector": "[data=story]", + "boundary": { + "selector": "override:[data=story]", + "start": 78, + "end": 126 + }, + "text": "Alpha & beta.", + "text_rust": "Alpha & beta.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "Café & News🙂前

A︀︁lpha & beta.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "Café & News🙂前

Alpha & beta.

", + "expected_meta_output_rust": "Café & News🙂前

Alpha & beta.

", + "meta_divergence": null + }, + { + "id": "article-most-paragraphs", + "html": "Two articles

Short.

First paragraph.

Second paragraph.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 84, + "end": 159 + }, + "text": "First paragraph. Second paragraph .", + "text_rust": "First paragraph. Second paragraph .", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "Two articles

Short.

F︀︁irst paragraph.

Second paragraph.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "Two articles

Short.

First paragraph.

Second paragraph.

", + "expected_meta_output_rust": "Two articles

Short.

First paragraph.

Second paragraph.

", + "meta_divergence": null + }, + { + "id": "cms-class-entities", + "html": "

Fish & chips — © 🐟.

", + "override_selector": null, + "boundary": { + "selector": "class:entry-content", + "start": 25, + "end": 123 + }, + "text": "Fish & chips — © 🐟.", + "text_rust": "Fish & chips — © 🐟.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

F︀︁ish & chips — © 🐟.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Fish & chips — © 🐟.

", + "expected_meta_output_rust": "

Fish & chips — © 🐟.

", + "meta_divergence": null + }, + { + "id": "itemprop-nbsp", + "html": "

One two three.

", + "override_selector": null, + "boundary": { + "selector": "itemprop:articleBody", + "start": 25, + "end": 94 + }, + "text": "One two three.", + "text_rust": "One two three.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

O︀︁ne two three.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

One two three.

", + "expected_meta_output_rust": "

One two three.

", + "meta_divergence": null + }, + { + "id": "main-inline-tags", + "html": "

Inline

Hello fictional example.com.

", + "override_selector": null, + "boundary": { + "selector": "main", + "start": 25, + "end": 105 + }, + "text": "Inline Hello fictional example.com.", + "text_rust": "Inline Hello fictional example.com.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

I︀︁nline

Hello fictional example.com.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Inline

Hello fictional example.com.

", + "expected_meta_output_rust": "

Inline

Hello fictional example.com.

", + "meta_divergence": null + }, + { + "id": "role-main", + "html": "

Role based article boundary.

", + "override_selector": null, + "boundary": { + "selector": "role:main", + "start": 25, + "end": 83 + }, + "text": "Role based article boundary.", + "text_rust": "Role based article boundary.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

R︀︁ole based article boundary.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Role based article boundary.

", + "expected_meta_output_rust": "

Role based article boundary.

", + "meta_divergence": null + }, + { + "id": "paragraph-cluster", + "html": "

One.

Two.

Three.

Four.

Five.

", + "override_selector": null, + "boundary": { + "selector": "p-cluster", + "start": 25, + "end": 109 + }, + "text": "One. Two. Three. Four. Five.", + "text_rust": "One. Two. Three. Four. Five.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

O︀︁ne.

Two.

Three.

Four.

Five.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

One.

Two.

Three.

Four.

Five.

", + "expected_meta_output_rust": "

One.

Two.

Three.

Four.

Five.

", + "meta_divergence": null + }, + { + "id": "body-fallback", + "html": "

Fallback

Body article text.", + "override_selector": null, + "boundary": { + "selector": "body", + "start": 19, + "end": 80 + }, + "text": "Fallback Body article text.", + "text_rust": "Fallback Body article text.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

F︀︁allback

Body article text.", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Fallback

Body article text.", + "expected_meta_output_rust": "

Fallback

Body article text.", + "meta_divergence": null + }, + { + "id": "nested-same-name-first-close", + "html": "

Outer start.

Inner text.

Outer tail omitted by parity.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 90 + }, + "text": "Outer start. Inner text.", + "text_rust": "Outer start. Inner text.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

O︀︁uter start.

Inner text.

Outer tail omitted by parity.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Outer start.

Inner text.

Outer tail omitted by parity.

", + "expected_meta_output_rust": "

Outer start.

Inner text.

Outer tail omitted by parity.

", + "meta_divergence": null + }, + { + "id": "head-close-inside-script", + "html": "Script case

Script boundary fallback.

", + "override_selector": null, + "boundary": { + "selector": "body", + "start": 86, + "end": 131 + }, + "text": "Script boundary fallback.", + "text_rust": "Script boundary fallback.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "Script case

S︀︁cript boundary fallback.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "Script case

Script boundary fallback.

", + "expected_meta_output_rust": "Script case

Script boundary fallback.

", + "meta_divergence": "JS inserts at a closing-head sequence inside script text; Rust inserts before the structural closing head tag." + }, + { + "id": "head-close-inside-comment", + "html": "Comment case

Comment boundary fallback.

", + "override_selector": null, + "boundary": { + "selector": "body", + "start": 80, + "end": 126 + }, + "text": "Comment boundary fallback.", + "text_rust": "Comment boundary fallback.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "Comment case

C︀︁omment boundary fallback.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "Comment case

Comment boundary fallback.

", + "expected_meta_output_rust": "Comment case

Comment boundary fallback.

", + "meta_divergence": "JS inserts at a closing-head sequence inside a comment; Rust inserts before the structural closing head tag." + }, + { + "id": "head-close-inside-title", + "html": "Headline </head> text

Title boundary fallback.

", + "override_selector": null, + "boundary": { + "selector": "body", + "start": 55, + "end": 99 + }, + "text": "Title boundary fallback.", + "text_rust": "Title boundary fallback.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "Headline </head> text

T︀︁itle boundary fallback.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "Headline <meta name=\"c2pa-manifest-url\" content=\"https://manifest.example.com/c2pa?id=fictional&kind="article"\"></head> text

Title boundary fallback.

", + "expected_meta_output_rust": "Headline </head> text

Title boundary fallback.

", + "meta_divergence": "JS inserts at a closing-head sequence inside title text; Rust never searches raw-text or RCDATA content for the closing head tag." + }, + { + "id": "javascript-whitespace-set", + "html": "

Alpha 
beta\t gamma.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 77 + }, + "text": "Alpha beta gamma.", + "text_rust": "Alpha beta gamma.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

A︀︁lpha 
beta\t gamma.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Alpha 
beta\t gamma.

", + "expected_meta_output_rust": "

Alpha 
beta\t gamma.

", + "meta_divergence": null + }, + { + "id": "skip-raw-elements-and-numeric-entities", + "html": "

Visible A.

Hidden code.
Hidden pre.

Tail B.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 162 + }, + "text": "Visible A. Tail B.", + "text_rust": "Visible A. Tail B.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

V︀︁isible A.

Hidden code.
Hidden pre.

Tail B.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Visible A.

Hidden code.
Hidden pre.

Tail B.

", + "expected_meta_output_rust": "

Visible A.

Hidden code.
Hidden pre.

Tail B.

", + "meta_divergence": null + }, + { + "id": "only-the-seven-skip-elements-are-hidden", + "html": "

Lead text.

Inline title.

After.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 166 + }, + "text": "Lead text. Draft note. Frame fallback. Inline title. After.", + "text_rust": "Lead text. Draft note. Frame fallback. Inline title. After.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

L︀︁ead text.

Inline title.

After.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Lead text.

Inline title.

After.

", + "expected_meta_output_rust": "

Lead text.

Inline title.

After.

", + "meta_divergence": null + }, + { + "id": "self-closing-svg-hides-nothing", + "html": "

Lead text.

After icon.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 103 + }, + "text": "Lead text. After icon.", + "text_rust": "Lead text. After icon.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

L︀︁ead text.

After icon.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Lead text.

After icon.

", + "expected_meta_output_rust": "

Lead text.

After icon.

", + "meta_divergence": null + }, + { + "id": "quoted-gt-in-attribute", + "html": "

Lead text.

\"a b\">Tail text.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 95 + }, + "text": "Lead text. b\">Tail text.", + "text_rust": "Lead text. Tail text.", + "text_divergence": "JS ends a tag at the first \">\", so the rest of the alt value becomes text; Rust ends it at the first \">\" outside a quoted attribute value.", + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

L︀︁ead text.

\"a b\">Tail text.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Lead text.

\"a b\">Tail text.

", + "expected_meta_output_rust": "

Lead text.

\"a b\">Tail text.

", + "meta_divergence": null + }, + { + "id": "nested-same-name-ignored-element", + "html": "

Lead text.

outer inner tail

After.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 116 + }, + "text": "Lead text. After.", + "text_rust": "Lead text. tail After.", + "text_divergence": "JS counts nested start and end tags and drops the outer tail; Rust closes the ignored region at the first matching end tag.", + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

L︀︁ead text.

outer inner tail

After.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Lead text.

outer inner tail

After.

", + "expected_meta_output_rust": "

Lead text.

outer inner tail

After.

", + "meta_divergence": null + }, + { + "id": "self-closing-script-stays-raw-text", + "html": "

Lead text.

After.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 107 + }, + "text": "Lead text. var hidden = 1; After.", + "text_rust": "Lead text. After.", + "text_divergence": "JS treats \".", + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

L︀︁ead text.

After.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Lead text.

After.

", + "expected_meta_output_rust": "

Lead text.

After.

", + "meta_divergence": null + }, + { + "id": "end-tag-with-attributes-closes-ignored-element", + "html": "

Lead text.

After.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 132 + }, + "text": "Lead text. After.", + "text_rust": "Lead text. After.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

L︀︁ead text.

After.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Lead text.

After.

", + "expected_meta_output_rust": "

Lead text.

After.

", + "meta_divergence": null + }, + { + "id": "abrupt-and-bang-comment-ends", + "html": "

Lead text.

Shown one.Shown two.Shown three.

After.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 131 + }, + "text": "Lead text. Shown two.", + "text_rust": "Lead text. Shown one. Shown two. Shown three. After.", + "text_divergence": "JS ends every comment at the next \"-->\"; Rust, like browsers, treats \"\" and \"\" as whole comments and also ends a comment at \"--!>\".", + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

L︀︁ead text.

Shown one.Shown two.Shown three.

After.

", + "invalid_plan": null, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Lead text.

Shown one.Shown two.Shown three.

After.

", + "expected_meta_output_rust": "

Lead text.

Shown one.Shown two.Shown three.

After.

", + "meta_divergence": null + }, + { + "id": "out-of-range-index", + "html": "

Small boundary.

", + "override_selector": null, + "boundary": { + "selector": "article", + "start": 25, + "end": 66 + }, + "text": "Small boundary.", + "text_rust": "Small boundary.", + "text_divergence": null, + "plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": -1, + "marker": "" + }, + { + "insert_after_index": 0, + "marker": "︀" + }, + { + "insert_after_index": 0, + "marker": "︁" + } + ] + }, + "expected_output": "

S︀︁mall boundary.

", + "invalid_plan": { + "index_unit": "codepoint", + "operations": [ + { + "insert_after_index": 15, + "marker": "︀" + } + ] + }, + "expected_invalid_output": null, + "manifest_url": "https://manifest.example.com/c2pa?id=fictional&kind=\"article\"", + "expected_meta_output_js": "

Small boundary.

", + "expected_meta_output_rust": "

Small boundary.

", + "meta_divergence": null + } + ] +} diff --git a/crates/provenance/encypher/src/lib.rs b/crates/provenance/encypher/src/lib.rs new file mode 100644 index 000000000..e633f6ee0 --- /dev/null +++ b/crates/provenance/encypher/src/lib.rs @@ -0,0 +1,346 @@ +//! Encypher text provenance for article pages, `provenance.encypher`. +//! +//! A serve middleware finds the article in each HTML page a `[[serve]]` entry +//! names it for, sends its visible text to the Encypher signing API, and +//! embeds the returned invisible markers and a manifest meta tag. Plans are +//! cached in the runtime KV store, bound to a short-lived server policy. +//! Every failure passes the page through unchanged. +//! +//! The middleware's preparation applies the request gates with no I/O: only +//! a `GET` of a `200` page on the configured domain, whose origin did not +//! mark it as one reader's, is held. The signing work runs in the stream +//! processor's asynchronous finish, which core awaits at the end of the +//! document. The module requires no permission: it sends the publisher's own +//! article text and page address, never anything of the reader's, and a page +//! the origin marked as one reader's is never sent. + +use std::any::Any; +use std::sync::Arc; + +use error_stack::Report; +use serde::Deserialize; +use validator::{Validate, ValidationError}; + +use trusted_server_core::error::TrustedServerError; +use trusted_server_core::integrations::{ + IntegrationBuilder, IntegrationRegistration, ModuleSecretSetting, SuppliedMiddleware, +}; +use trusted_server_core::middleware::{ + HTML_MEDIA_TYPE, Middleware, MiddlewareAction, MiddlewareContext, MiddlewarePhase, Preparation, +}; +use trusted_server_core::module_context::{FromModuleContext as _, ModuleCall, ModuleRequest}; +use trusted_server_core::redacted::Redacted; +use trusted_server_core::settings::{IntegrationConfig, Settings}; + +mod api; +mod boundary; +mod embed; +mod processor; +mod text; + +use processor::{PageRequest, Signer, document_processor, is_excluded_path}; + +/// The id the module's registration and its API backend carry. +const INTEGRATION_ID: &str = "provenance_encypher"; + +/// The name `[provenance] module` selects the module by and a `[[serve]]` +/// entry names its middleware by, from this crate's folder. +pub const MODULE: &str = "provenance.encypher"; + +/// `[provenance.encypher]`, the module's settings. +#[derive(Debug, Clone, Deserialize, Validate)] +#[serde(deny_unknown_fields)] +#[validate(schema(function = "validate_hold_exceeds_timeout"))] +pub struct EncypherConfig { + /// Encypher API origin, for example `https://api.example.com`. Must be https. + #[validate(custom(function = validate_api_base_url))] + pub api_base_url: String, + /// Claimed publisher domain: a bare lowercase hostname without `www.`. + #[validate(custom(function = validate_domain))] + pub domain: String, + /// The name of the secret-store entry holding the `cwk_` worker + /// credential, resolved to the credential itself when settings load. + pub worker_token: Redacted, + /// Optional article selector override: `.class`, `[attr=value]` or a tag + /// name. At most 91 characters, so `override:{selector}` fits the API's + /// 100-character `boundary_selector` limit. + #[serde(default)] + #[validate(length(max = 91))] + pub article_selector: String, + /// Minimum visible article text, in Unicode scalars. + #[serde(default = "default_min_text_length")] + #[validate(range(min = 50, max = 50_000))] + pub min_text_length: usize, + /// Largest decoded document held for signing. Capped at + /// `publisher.max_buffered_body_bytes`. + #[serde(default = "default_max_document_bytes")] + #[validate(range(min = 1024))] + pub max_document_bytes: usize, + /// First-byte timeout for each policy or sign call, in milliseconds. The + /// only bound on a hung API on a runtime without a timer. + #[serde(default = "default_timeout_ms")] + #[validate(range(min = 250, max = 10_000))] + pub timeout_ms: u64, + /// Longest one document is held for signing, in milliseconds, on a + /// runtime with a timer. Must exceed `timeout_ms`. + #[serde(default = "default_max_hold_ms")] + #[validate(range(min = 500, max = 15_000))] + pub max_hold_ms: u64, + /// Lifetime of a cached signing plan, in seconds. + #[serde(default = "default_plan_ttl_seconds")] + #[validate(range(min = 60, max = 31_536_000))] + pub plan_ttl_seconds: u64, + /// Lifetime of the server-authoritative policy, in seconds. + #[serde(default = "default_policy_ttl_seconds")] + #[validate(range(min = 60, max = 300))] + pub policy_ttl_seconds: u64, + /// Lifetime of a cached signing failure, in seconds. + #[serde(default = "default_negative_ttl_seconds")] + #[validate(range(min = 60, max = 86_400))] + pub negative_ttl_seconds: u64, +} + +impl IntegrationConfig for EncypherConfig {} + +fn default_min_text_length() -> usize { + 50 +} + +fn default_max_document_bytes() -> usize { + 5_000_000 +} + +fn default_timeout_ms() -> u64 { + 3_000 +} + +fn default_max_hold_ms() -> u64 { + 5_000 +} + +fn default_plan_ttl_seconds() -> u64 { + 604_800 +} + +fn default_policy_ttl_seconds() -> u64 { + 300 +} + +fn default_negative_ttl_seconds() -> u64 { + 300 +} + +fn validate_api_base_url(value: &str) -> Result<(), ValidationError> { + let parsed = + url::Url::parse(value).map_err(|_| ValidationError::new("encypher_api_base_url"))?; + if parsed.scheme() != "https" + || parsed.host_str().is_none_or(str::is_empty) + || parsed.query().is_some() + || parsed.fragment().is_some() + { + return Err(ValidationError::new("encypher_api_base_url")); + } + Ok(()) +} + +fn validate_domain(value: &str) -> Result<(), ValidationError> { + let valid_label = |label: &str| { + !label.is_empty() + && label.len() <= 63 + && !label.starts_with('-') + && !label.ends_with('-') + && label + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') + }; + let valid = value.len() <= 253 + && value.contains('.') + && !value.starts_with("www.") + && value.split('.').all(valid_label); + if valid { + Ok(()) + } else { + Err(ValidationError::new("encypher_domain")) + } +} + +fn validate_hold_exceeds_timeout(config: &EncypherConfig) -> Result<(), ValidationError> { + if config.max_hold_ms > config.timeout_ms { + Ok(()) + } else { + Err(ValidationError::new("encypher_max_hold_ms")) + } +} + +/// The worker token, named in `[provenance.encypher]` by its entry in the +/// secret store, which is in use whenever the module is selected. +const SECRET_SETTINGS: &[ModuleSecretSetting] = &[ModuleSecretSetting { + path: &["worker_token"], + in_use: always_in_use, +}]; + +fn always_in_use(_table: &serde_json::Map) -> bool { + true +} + +/// The page change the module supplies, as a `[[serve]]` entry names it. +const SUPPLIED_MIDDLEWARE: &[SuppliedMiddleware] = + &[SuppliedMiddleware::on_html(MODULE, MiddlewarePhase::Serve)]; + +/// The builder a deployment installs to offer this module. +#[must_use] +pub fn builder() -> IntegrationBuilder { + IntegrationBuilder::new(INTEGRATION_ID, env!("CARGO_PKG_NAME"), register, validate) + .with_module_name(MODULE) + .with_secret_settings(SECRET_SETTINGS) + .with_middleware(SUPPLIED_MIDDLEWARE) +} + +/// Refuses a document that selects the module and names its middleware in no +/// `[[serve]]` entry, where no page would ever be signed. +fn require_an_entry(settings: &Settings) -> Result<(), Report> { + let placed = settings + .serve + .entries() + .iter() + .any(|entry| entry.middleware.iter().any(|name| name == MODULE)); + if placed { + return Ok(()); + } + Err(Report::new(TrustedServerError::Settings { + message: format!( + "[provenance] selects `encypher`, but no [[serve]] entry names `{MODULE}`, so no \ + page would be signed. Add `{MODULE}` to the [[serve]] entry that covers the \ + article pages, with a `path` for each section to sign" + ), + })) +} + +/// The module's settings when a section selects it. +fn config(settings: &Settings) -> Result, Report> { + let Some(config) = settings.module_config::(MODULE)? else { + return Ok(None); + }; + require_an_entry(settings)?; + Ok(Some(config)) +} + +/// Validates the module's table and reports whether a section selects the +/// module. +/// +/// # Errors +/// +/// Returns an error when the table cannot be read or fails its validation, or +/// when no `[[serve]]` entry names the middleware. +pub fn validate(settings: &Settings) -> Result> { + Ok(config(settings)?.is_some()) +} + +/// Registers the middleware when a section selects the module and its worker +/// token is not blank. +/// +/// # Errors +/// +/// Returns the errors [`validate`] returns. +pub fn register( + settings: &Settings, +) -> Result, Report> { + let Some(config) = config(settings)? else { + return Ok(None); + }; + if config.worker_token.expose().trim().is_empty() { + log::warn!("encypher: not registered, because the worker token is empty"); + return Ok(None); + } + log::info!("encypher: registered for domain={}", config.domain); + let max_document_bytes = config + .max_document_bytes + .min(settings.publisher.max_buffered_body_bytes); + Ok(Some( + IntegrationRegistration::builder(INTEGRATION_ID) + .without_js() + .with_middleware(Arc::new(Encypher { + signer: Arc::new(Signer::new(config, max_document_bytes)), + })) + .build(), + )) +} + +/// The middleware, which signs the article of each admitted page. +pub struct Encypher { + signer: Arc, +} + +impl std::fmt::Debug for Encypher { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.debug_struct("Encypher").finish_non_exhaustive() + } +} + +/// What the preparation decided for one document: the page to sign, or none. +type Admission = Option; + +impl Middleware for Encypher { + fn middleware_id(&self) -> &'static str { + MODULE + } + + fn handles_media_type(&self, media_type: &str) -> bool { + media_type == HTML_MEDIA_TYPE + } + + /// Every path but an API, feed, admin or well-known path, or an asset, + /// which is never an article. + fn handles_path(&self, path: &str) -> bool { + !is_excluded_path(path) + } + + /// Serve only: the signed page is written on each reader's copy, so a + /// failed signing is never stored for every reader. + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Serve] + } + + /// Applies the request gates, with no I/O, and keeps the page to sign. + fn prepare<'a>( + &'a self, + call: ModuleCall<'a>, + context: &'a MiddlewareContext<'a>, + ) -> Option> { + let admission = admit(&self.signer, &call, context); + Some(Box::pin(async move { + Ok(Box::new(admission) as Box) + })) + } + + fn create(&self, context: &MiddlewareContext<'_>) -> MiddlewareAction { + let Some(page) = context.take_prepared::(MODULE).flatten() else { + return MiddlewareAction::pass(); + }; + MiddlewareAction { + stream: Some(Box::new(document_processor(&self.signer, page))), + ..MiddlewareAction::pass() + } + } +} + +/// The page a document is the body of, when the gates admit it. +fn admit(signer: &Signer, call: &ModuleCall<'_>, context: &MiddlewareContext<'_>) -> Admission { + let request = ModuleRequest::from_module_context(call).ok()?; + let page = PageRequest { + method: request.method().clone(), + page_url: format!("{}://{}{}", request.scheme(), request.host(), request.path()), + response_status: context.status.as_u16(), + origin_response_is_private: context.origin_private, + }; + match signer.admit(page) { + Ok(page) => Some(page), + Err(gate) => { + log::debug!("encypher: document not held, gate={}", gate.code()); + None + } + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/provenance/encypher/src/processor.rs b/crates/provenance/encypher/src/processor.rs new file mode 100644 index 000000000..c97ea3701 --- /dev/null +++ b/crates/provenance/encypher/src/processor.rs @@ -0,0 +1,2520 @@ +//! Per-document signing flow: gates, hold, KV cache, sign call, outcomes, logging. +//! +//! A document moves through four phases: +//! +//! 1. **Admit** (the middleware's preparation, with no I/O): the request +//! gates decide whether the document is held at all. +//! 2. **Extract** (synchronous): at end of input the held document's article +//! boundary, visible text and text hash are computed. +//! 3. **Resolve** (the asynchronous finish): policy and plan caches, the policy +//! and sign calls, and cache writes. On runtimes with a timer every call is +//! raced against `max_hold_ms`, keeping a reserve for the skip write. +//! 4. **Render** (synchronous): the plan's markers and the manifest meta tag are +//! applied to the held document. +//! +//! Every module failure emits the held document unchanged. + +use std::io; +use std::pin::pin; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use bytes::Bytes; +use edgezero_core::body::Body as EdgeBody; +use futures::future::{Either, LocalBoxFuture, select}; +use http::{HeaderValue, Method, Request, StatusCode, header}; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; + +use crate::api::{self, ApiError, EmbeddingPlan, SignRequest}; +use crate::boundary::{self, Boundary}; +use crate::{EncypherConfig, INTEGRATION_ID, embed, text}; +use trusted_server_core::integrations::ensure_integration_backend_with_timeout; +use trusted_server_core::platform::{ + KvError, PlatformHttpRequest, PlatformKvStore, PlatformTimer, RuntimeServices, +}; +use trusted_server_core::streaming_processor::{AsyncFinish, StreamProcessor}; + +const SIGN_PATH: &str = "/api/v1/public/cdn/sign"; +const POLICY_PATH: &str = "/api/v1/public/cdn/trusted-server/policy"; +const WORKER_TOKEN_HEADER: &str = "x-encypher-worker-token"; +/// Article plan and skip keys. Entries carry their own `expires_at`. +const CACHE_KEY_PREFIX: &str = "encypher:v3"; +/// Policy digest and policy skip keys. Entries carry their own `expires_at`. +const POLICY_KEY_PREFIX: &str = "encypher:policy:v2"; +/// Part of `max_hold_ms` kept back from resolve work so a skip can still be +/// written after the work runs out of time. +const SKIP_WRITE_RESERVE: Duration = Duration::from_millis(200); +/// How far past `max_hold_ms` the core backstop deadline lies. The module +/// bounds itself first; the backstop only catches an overrun. +const BACKSTOP_MARGIN: Duration = Duration::from_millis(1_000); + +/// Path prefixes never signed (ported from the Fastly worker's `shouldSkipPath`). +const SKIP_PATH_PREFIXES: &[&str] = &[ + "/api/", + "/wp-admin/", + "/wp-json/", + "/feed/", + "/sitemap", + "/.well-known/", +]; +/// Asset suffixes never signed (ported from the Fastly worker's `shouldSkipPath`). +const SKIP_PATH_SUFFIXES: &[&str] = &[ + ".xml", ".json", ".rss", ".atom", ".css", ".js", ".woff", ".woff2", ".png", ".jpg", ".gif", + ".svg", ".ico", +]; + +/// Set once this instance has warned that the runtime KV store is unbound. +static KV_UNAVAILABLE_WARNED: AtomicBool = AtomicBool::new(false); + +/// Why a document was not held. Evaluated in the middleware's preparation, +/// with no I/O. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Gate { + Method, + Status, + OriginPrivate, + InvalidPageUrl, + HostMismatch, + ExcludedPath, +} + +impl Gate { + pub(crate) fn code(self) -> &'static str { + match self { + Self::Method => "method", + Self::Status => "status", + Self::OriginPrivate => "origin_private", + Self::InvalidPageUrl => "invalid_page_url", + Self::HostMismatch => "host_mismatch", + Self::ExcludedPath => "excluded_path", + } + } +} + +/// What the gates are told about the page a document is the body of. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct PageRequest { + /// The reader's request method. + pub(crate) method: Method, + /// The scheme, host and path the reader asked for, without the query, + /// which is the address the article is signed for. + pub(crate) page_url: String, + /// The status of the response the document is the body of. + pub(crate) response_status: u16, + /// Whether the origin marked its response as one reader's. + pub(crate) origin_response_is_private: bool, +} + +/// Reason code for a held document's outcome. Sign failures are also stored +/// as the reason of a negative (`skip`) cache entry. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +enum Reason { + Signed, + CacheHit, + DocumentTooLarge, + NotUtf8, + NoBoundary, + AlreadySigned, + ManifestPresent, + TextTooShort, + KvUnavailable, + KvError, + Timeout, + CachedSkip, + CachedEntryInvalid, + CachedPlanInvalid, + RequestBuild, + Backend, + HttpError, + Forbidden, + Unprocessable, + RateLimited, + ServerError, + UnexpectedStatus, + ResponseTooLarge, + MalformedResponse, + PolicyChanged, + PolicyDigestMismatch, + QuotaExceeded, + SignRejected, + NoPlan, + InvalidPlan, + AlignmentFailed, +} + +impl Reason { + fn code(self) -> &'static str { + match self { + Self::Signed => "signed", + Self::CacheHit => "cache_hit", + Self::DocumentTooLarge => "document_too_large", + Self::NotUtf8 => "not_utf8", + Self::NoBoundary => "no_boundary", + Self::AlreadySigned => "already_signed", + Self::ManifestPresent => "manifest_present", + Self::TextTooShort => "text_too_short", + Self::KvUnavailable => "kv_unavailable", + Self::KvError => "kv_error", + Self::Timeout => "timeout", + Self::CachedSkip => "cached_skip", + Self::CachedEntryInvalid => "cached_entry_invalid", + Self::CachedPlanInvalid => "cached_plan_invalid", + Self::RequestBuild => "request_build", + Self::Backend => "backend", + Self::HttpError => "http_error", + Self::Forbidden => "forbidden", + Self::Unprocessable => "unprocessable", + Self::RateLimited => "rate_limited", + Self::ServerError => "server_error", + Self::UnexpectedStatus => "unexpected_status", + Self::ResponseTooLarge => "response_too_large", + Self::MalformedResponse => "malformed_response", + Self::PolicyChanged => "policy_changed", + Self::PolicyDigestMismatch => "policy_digest_mismatch", + Self::QuotaExceeded => "quota_exceeded", + Self::SignRejected => "sign_rejected", + Self::NoPlan => "no_plan", + Self::InvalidPlan => "invalid_plan", + Self::AlignmentFailed => "alignment_failed", + } + } +} + +/// A cache entry that records when it stops being usable. +trait Expiring { + /// Unix seconds at which the entry expires. + fn expires_at(&self) -> u64; + /// Whether the entry was written without a store TTL, so it stays until deleted. + fn ttl_less(&self) -> bool; + /// Record that the entry is written without a store TTL. + fn mark_ttl_less(&mut self); +} + +/// An article cache entry, keyed by credential context, policy and text hash. +#[derive(Debug, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +enum CacheEntry { + Plan { + plan: EmbeddingPlan, + manifest_url: Option, + expires_at: u64, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + ttl_less: bool, + }, + Skip { + reason: Reason, + expires_at: u64, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + ttl_less: bool, + }, +} + +impl Expiring for CacheEntry { + fn expires_at(&self) -> u64 { + match self { + Self::Plan { expires_at, .. } | Self::Skip { expires_at, .. } => *expires_at, + } + } + + fn ttl_less(&self) -> bool { + match self { + Self::Plan { ttl_less, .. } | Self::Skip { ttl_less, .. } => *ttl_less, + } + } + + fn mark_ttl_less(&mut self) { + match self { + Self::Plan { ttl_less, .. } | Self::Skip { ttl_less, .. } => *ttl_less = true, + } + } +} + +#[derive(Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +enum PolicyEntry { + Digest { + policy_digest: String, + expires_at: u64, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + ttl_less: bool, + }, + Skip { + reason: Reason, + expires_at: u64, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + ttl_less: bool, + }, +} + +impl Expiring for PolicyEntry { + fn expires_at(&self) -> u64 { + match self { + Self::Digest { expires_at, .. } | Self::Skip { expires_at, .. } => *expires_at, + } + } + + fn ttl_less(&self) -> bool { + match self { + Self::Digest { ttl_less, .. } | Self::Skip { ttl_less, .. } => *ttl_less, + } + } + + fn mark_ttl_less(&mut self) { + match self { + Self::Digest { ttl_less, .. } | Self::Skip { ttl_less, .. } => *ttl_less = true, + } + } +} + +/// The one freshness predicate applied to every cache read. +fn is_fresh(expires_at: u64, now: u64) -> bool { + now < expires_at +} + +/// Current Unix time in seconds, from a clock that works on every adapter. +fn unix_now() -> u64 { + web_time::SystemTime::now() + .duration_since(web_time::UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_secs()) +} + +/// Configuration shared by every document processor. +pub(crate) struct Signer { + config: EncypherConfig, + max_document_bytes: usize, + sign_url: String, + policy_url: String, + policy_key: String, + /// First 16 hex characters of `sha256(api_base_url \n domain \n token)`. + cache_context: String, + /// Set by the first TTL write the store reports as unsupported, or by reading + /// an entry written without a TTL; from then on this instance writes without + /// a TTL and deletes expired entries it reads. + ttl_less: AtomicBool, +} + +impl Signer { + /// The signer for `config`, holding at most `max_document_bytes` of one + /// document. + pub(crate) fn new(config: EncypherConfig, max_document_bytes: usize) -> Self { + let sign_url = format!("{}{SIGN_PATH}", config.api_base_url.trim_end_matches('/')); + let policy_url = format!("{}{POLICY_PATH}", config.api_base_url.trim_end_matches('/')); + let context_hash = text::content_hash_hex(&format!( + "{}\n{}\n{}", + config.api_base_url, + config.domain, + config.worker_token.expose().trim() + )); + let cache_context = context_hash + .get(..16) + .unwrap_or(context_hash.as_str()) + .to_owned(); + Self { + config, + max_document_bytes, + sign_url, + policy_url, + policy_key: format!("{POLICY_KEY_PREFIX}:{cache_context}"), + cache_context, + ttl_less: AtomicBool::new(false), + } + } +} + +/// Holds the document of an admitted page until it ends. +pub(crate) fn document_processor(signer: &Arc, request: PageRequest) -> DocumentProcessor { + DocumentProcessor { + signer: Arc::clone(signer), + request, + held: Vec::new(), + state: DocumentState::Holding, + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum DocumentState { + /// Accumulating the document. + Holding, + /// Over the size limit: chunks pass through and nothing is signed. + PassingThrough, + /// The whole document was emitted as held output; it may be signed in the + /// asynchronous finish. + Complete, +} + +/// Holds one admitted document until end of input, then signs it in the +/// asynchronous finish or passes it through. +pub(crate) struct DocumentProcessor { + signer: Arc, + request: PageRequest, + held: Vec, + state: DocumentState, +} + +impl StreamProcessor for DocumentProcessor { + fn process_chunk(&mut self, chunk: &[u8], is_last: bool) -> Result, io::Error> { + if self.state != DocumentState::Holding { + return Ok(chunk.to_vec()); + } + if self.held.len().saturating_add(chunk.len()) > self.signer.max_document_bytes { + self.state = DocumentState::PassingThrough; + log_outcome(false, Reason::DocumentTooLarge); + let mut output = std::mem::take(&mut self.held); + output.extend_from_slice(chunk); + return Ok(output); + } + self.held.extend_from_slice(chunk); + if !is_last { + return Ok(Vec::new()); + } + // The complete document is the fail-open output; the driver owns it. + self.state = DocumentState::Complete; + Ok(std::mem::take(&mut self.held)) + } + + fn reset(&mut self) { + self.held.clear(); + self.state = DocumentState::Holding; + } + + fn async_finish<'a>( + &'a mut self, + held: &'a [u8], + services: &'a RuntimeServices, + ) -> Option> { + if self.state != DocumentState::Complete { + return None; + } + let extracted = std::str::from_utf8(held) + .map_err(|_| Reason::NotUtf8) + .and_then(|html| Ok((html, self.signer.extract(html)?))); + let (html, article) = match extracted { + Ok(extracted) => extracted, + Err(reason) => { + log_outcome(false, reason); + return None; + } + }; + let signer = &*self.signer; + let request = &self.request; + let deadline = + Duration::from_millis(signer.config.max_hold_ms).saturating_add(BACKSTOP_MARGIN); + Some(AsyncFinish::new(deadline, async move { + match signer.resolve(request, services, html, &article).await { + Ok((signed, reason)) => { + log_outcome(true, reason); + signed.into_bytes() + } + Err(reason) => { + log_outcome(false, reason); + held.to_vec() + } + } + })) + } +} + +/// The article found in a held document. +struct Article { + boundary: Boundary, + /// Visible text sent for signing, truncated to the API limit. + text: String, + /// Unicode scalars in `text`. + scalars: usize, +} + +/// Time a document's resolve may still take, on runtimes with a timer. +enum Deadline<'a> { + /// No timer: calls are bounded only by backend timeouts. + Unbounded, + /// The timer has not fired yet. + Running(LocalBoxFuture<'a, ()>), + /// The timer fired; every further race loses at once. + Expired, +} + +impl<'a> Deadline<'a> { + fn new(timer: Option<&'a dyn PlatformTimer>, budget: Duration) -> Self { + timer.map_or(Self::Unbounded, |timer| Self::Running(timer.sleep(budget))) + } + + /// Start the clock now. A timer's sleep starts on its first poll, which + /// would otherwise be the first call that does not complete at once. + async fn start(&mut self) { + if let Self::Running(sleep) = self + && futures::poll!(sleep.as_mut()).is_ready() + { + *self = Self::Expired; + } + } + + /// Run `work` to completion, or return `None` once the deadline passes. + async fn race(&mut self, work: impl Future) -> Option { + match self { + Self::Unbounded => return Some(work.await), + Self::Expired => return None, + Self::Running(sleep) => { + if let Either::Left((value, _)) = select(pin!(work), sleep.as_mut()).await { + return Some(value); + } + } + } + *self = Self::Expired; + None + } + + /// Whether the deadline has passed, so the last race lost. + fn expired(&self) -> bool { + matches!(self, Self::Expired) + } +} + +fn log_outcome(signed: bool, reason: Reason) { + if reason == Reason::KvUnavailable && !KV_UNAVAILABLE_WARNED.swap(true, Ordering::Relaxed) { + log::warn!( + "encypher: the runtime KV store is unavailable, so pages pass through unsigned; bind the adapter's KV store to enable signing" + ); + } + log::info!( + "encypher: outcome={} reason={}", + if signed { "signed" } else { "passed_through" }, + reason.code() + ); +} + +/// Whether `path` is one this module never signs: an API, feed, admin or +/// well-known path, or an asset. +pub(crate) fn is_excluded_path(path: &str) -> bool { + let lower = path.to_ascii_lowercase(); + SKIP_PATH_PREFIXES + .iter() + .any(|prefix| lower.starts_with(prefix)) + || SKIP_PATH_SUFFIXES + .iter() + .any(|suffix| lower.ends_with(suffix)) +} + +impl Signer { + /// Apply the request gates. Only an admitted document is held. + pub(crate) fn admit(&self, request: PageRequest) -> Result { + if request.method != Method::GET { + return Err(Gate::Method); + } + if request.response_status != StatusCode::OK.as_u16() { + return Err(Gate::Status); + } + if request.origin_response_is_private { + return Err(Gate::OriginPrivate); + } + let page_url = url::Url::parse(&request.page_url).map_err(|_| Gate::InvalidPageUrl)?; + let host = page_url.host_str().ok_or(Gate::InvalidPageUrl)?; + if host.strip_prefix("www.").unwrap_or(host) != self.config.domain { + return Err(Gate::HostMismatch); + } + if is_excluded_path(page_url.path()) { + return Err(Gate::ExcludedPath); + } + Ok(request) + } + + /// Find the article and its signable text, with no I/O. + fn extract(&self, html: &str) -> Result { + let boundary = boundary::find_article_boundary(html, Some(&self.config.article_selector)) + .ok_or(Reason::NoBoundary)?; + let article = html + .get(boundary.start..boundary.end) + .ok_or(Reason::NoBoundary)?; + if embed::has_existing_markers(article) { + return Err(Reason::AlreadySigned); + } + if embed::has_manifest_meta(html) { + return Err(Reason::ManifestPresent); + } + let mut visible = text::visible_text(article); + let sign_len = text::truncate_scalars(&visible, api::MAX_TEXT_SCALARS).len(); + visible.truncate(sign_len); + let scalars = text::scalar_count(&visible); + if scalars < self.config.min_text_length { + return Err(Reason::TextTooShort); + } + Ok(Article { + boundary, + text: visible, + scalars, + }) + } + + /// Unix time at which an entry written now with `ttl_seconds` expires. + fn expires_at(ttl_seconds: u64) -> u64 { + unix_now().saturating_add(ttl_seconds) + } + + /// Bound for skip writes made after resolve work stopped. + fn skip_deadline<'a>(services: &'a RuntimeServices) -> Deadline<'a> { + Deadline::new(services.timer(), SKIP_WRITE_RESERVE) + } + + /// Resolve the plan for `article` and render it into `html`. + async fn resolve( + &self, + request: &PageRequest, + services: &RuntimeServices, + html: &str, + article: &Article, + ) -> Result<(String, Reason), Reason> { + let kv = services.kv_store(); + let mut work = Deadline::new( + services.timer(), + Duration::from_millis(self.config.max_hold_ms).saturating_sub(SKIP_WRITE_RESERVE), + ); + work.start().await; + + let policy_digest = self.policy_digest(request, services, kv, &mut work).await?; + let key = format!( + "{CACHE_KEY_PREFIX}:{}:{policy_digest}:{}", + self.cache_context, + text::content_hash_hex(&article.text) + ); + let outcome = match self.read::(kv, &key, &mut work).await { + Ok(Some(CacheEntry::Plan { + plan, manifest_url, .. + })) => { + api::validate_plan(&plan, manifest_url.as_deref(), article.scalars) + .map_err(|_| Reason::CachedPlanInvalid)?; + return render(html, &article.boundary, &plan, manifest_url.as_deref()) + .map(|signed| (signed, Reason::CacheHit)) + .ok_or(Reason::CachedPlanInvalid); + } + Ok(Some(CacheEntry::Skip { .. })) => return Err(Reason::CachedSkip), + Err(Reason::Timeout) => Err(Reason::Timeout), + Err(reason) => return Err(reason), + Ok(None) => self + .request_plan(request, services, html, article, &policy_digest, &mut work) + .await + .and_then(|(plan, manifest_url)| { + api::validate_plan(&plan, manifest_url.as_deref(), article.scalars) + .map_err(|_| Reason::InvalidPlan)?; + let signed = render(html, &article.boundary, &plan, manifest_url.as_deref()) + .ok_or(Reason::AlignmentFailed)?; + Ok((signed, plan, manifest_url)) + }), + }; + match outcome { + Ok((signed, plan, manifest_url)) => { + // A plan write that runs out of time is abandoned; the page is still signed. + let entry = CacheEntry::Plan { + plan, + manifest_url, + expires_at: Self::expires_at(self.config.plan_ttl_seconds), + ttl_less: false, + }; + self.write(kv, &key, entry, self.config.plan_ttl_seconds, &mut work) + .await; + Ok((signed, Reason::Signed)) + } + Err(reason) => { + let mut deadline = Self::skip_deadline(services); + if matches!(reason, Reason::PolicyChanged | Reason::PolicyDigestMismatch) { + self.evict_policy(kv, &mut deadline).await; + } + self.write_skip(kv, &key, reason, article.scalars, &mut deadline) + .await; + Err(reason) + } + } + } + + async fn policy_digest( + &self, + request: &PageRequest, + services: &RuntimeServices, + kv: &dyn PlatformKvStore, + work: &mut Deadline<'_>, + ) -> Result { + let result = match self.read::(kv, &self.policy_key, work).await { + Ok(Some(PolicyEntry::Digest { policy_digest, .. })) + if api::valid_policy_digest(&policy_digest) => + { + return Ok(policy_digest); + } + Ok(Some(PolicyEntry::Skip { .. })) => return Err(Reason::CachedSkip), + Ok(Some(PolicyEntry::Digest { .. })) => Err(Reason::CachedEntryInvalid), + // Nothing can be cached without a store. + Err(Reason::KvUnavailable) => return Err(Reason::KvUnavailable), + Err(reason) => Err(reason), + Ok(None) => self.request_policy(request, services, work).await, + }; + let reason = match result { + Ok(policy_digest) => { + let entry = PolicyEntry::Digest { + policy_digest: policy_digest.clone(), + expires_at: Self::expires_at(self.config.policy_ttl_seconds), + ttl_less: false, + }; + self.write( + kv, + &self.policy_key, + entry, + self.config.policy_ttl_seconds, + work, + ) + .await; + if !work.expired() { + return Ok(policy_digest); + } + // The digest is not cached, so the next view could not find a + // plan skip written under it; skip at the policy level instead. + Reason::Timeout + } + Err(reason) => reason, + }; + let ttl = self.policy_skip_ttl(); + let entry = PolicyEntry::Skip { + reason, + expires_at: Self::expires_at(ttl), + ttl_less: false, + }; + self.write( + kv, + &self.policy_key, + entry, + ttl, + &mut Self::skip_deadline(services), + ) + .await; + Err(reason) + } + + fn policy_skip_ttl(&self) -> u64 { + self.config + .policy_ttl_seconds + .min(self.config.negative_ttl_seconds) + } + + async fn request_policy( + &self, + request: &PageRequest, + services: &RuntimeServices, + work: &mut Deadline<'_>, + ) -> Result { + let mut url = url::Url::parse(&self.policy_url).map_err(|_| Reason::RequestBuild)?; + url.query_pairs_mut() + .append_pair("page_url", &request.page_url); + let body = self + .fetch( + services, + Method::GET, + url.as_str(), + EdgeBody::empty(), + api::MAX_POLICY_RESPONSE_BYTES, + work, + ) + .await?; + api::parse_policy_response(&body).map_err(|error| parse_error_reason(&error)) + } + + async fn evict_policy(&self, kv: &dyn PlatformKvStore, deadline: &mut Deadline<'_>) { + if matches!( + deadline.race(kv.delete(&self.policy_key)).await, + Some(Ok(())) + ) { + return; + } + // If deletion fails, try to replace the stale digest with a bounded skip. + let ttl = self.policy_skip_ttl(); + let entry = PolicyEntry::Skip { + reason: Reason::PolicyChanged, + expires_at: Self::expires_at(ttl), + ttl_less: false, + }; + self.write(kv, &self.policy_key, entry, ttl, deadline).await; + log::debug!("encypher: policy eviction failed"); + } + + /// Call the sign API once. Every failure maps to a reason code. + async fn request_plan( + &self, + request: &PageRequest, + services: &RuntimeServices, + html: &str, + article: &Article, + policy_digest: &str, + work: &mut Deadline<'_>, + ) -> Result<(EmbeddingPlan, Option), Reason> { + let title = text::extract_title(html); + let body = serde_json::to_vec(&SignRequest { + text: &article.text, + page_url: &request.page_url, + document_title: title.as_deref(), + boundary_selector: &article.boundary.selector, + expected_policy_digest: policy_digest, + }) + .map_err(|_| Reason::RequestBuild)?; + let body = self + .fetch( + services, + Method::POST, + &self.sign_url, + EdgeBody::from(body), + api::MAX_RESPONSE_BYTES, + work, + ) + .await?; + let parsed = api::parse_sign_response(&body).map_err(|error| parse_error_reason(&error))?; + if !parsed.success { + return Err(if parsed.error.as_deref() == Some("quota_exceeded") { + Reason::QuotaExceeded + } else { + Reason::SignRejected + }); + } + if parsed.policy_digest.as_deref() != Some(policy_digest) { + return Err(Reason::PolicyDigestMismatch); + } + let plan = parsed.embedding_plan.ok_or(Reason::NoPlan)?; + Ok((plan, parsed.manifest_url)) + } + + /// One bounded HTTP call; never retried. + async fn fetch( + &self, + services: &RuntimeServices, + method: Method, + url: &str, + body: EdgeBody, + max_response_bytes: usize, + work: &mut Deadline<'_>, + ) -> Result { + let mut token = HeaderValue::from_str(self.config.worker_token.expose().trim()) + .map_err(|_| Reason::RequestBuild)?; + token.set_sensitive(true); + let http_request = Request::builder() + .method(method) + .uri(url) + .header(header::CONTENT_TYPE, "application/json") + .header(WORKER_TOKEN_HEADER, token) + .body(body) + .map_err(|_| Reason::RequestBuild)?; + let backend = ensure_integration_backend_with_timeout( + services, + &self.config.api_base_url, + INTEGRATION_ID, + Duration::from_millis(self.config.timeout_ms), + ) + .map_err(|_| Reason::Backend)?; + + let response = match work + .race( + services + .http_client() + .send(PlatformHttpRequest::new(http_request, backend)), + ) + .await + { + None => return Err(Reason::Timeout), + Some(Err(_)) => return Err(Reason::HttpError), + Some(Ok(response)) => response.response, + }; + let status = response.status(); + if status != StatusCode::OK { + return Err(match status.as_u16() { + 403 => Reason::Forbidden, + 409 => Reason::PolicyChanged, + 422 => Reason::Unprocessable, + 429 => Reason::RateLimited, + 500..=599 => Reason::ServerError, + _ => Reason::UnexpectedStatus, + }); + } + // One byte over the limit is enough for the parser to report TooLarge. + match work + .race( + response + .into_body() + .into_bytes_bounded(max_response_bytes.saturating_add(1)), + ) + .await + { + None => Err(Reason::Timeout), + Some(Err(_)) => Err(Reason::ResponseTooLarge), + Some(Ok(body)) => Ok(body), + } + } + + /// Read a fresh entry. Expired entries are misses; in TTL-less write mode + /// they are also deleted, best effort, so the store cleans itself. + async fn read( + &self, + kv: &dyn PlatformKvStore, + key: &str, + deadline: &mut Deadline<'_>, + ) -> Result, Reason> { + let bytes = match deadline.race(kv.get_bytes(key)).await { + None => return Err(Reason::Timeout), + Some(Err(KvError::Unavailable)) => return Err(Reason::KvUnavailable), + Some(Err(_)) => return Err(Reason::KvError), + Some(Ok(None)) => return Ok(None), + Some(Ok(Some(bytes))) => bytes, + }; + let entry: T = serde_json::from_slice(&bytes).map_err(|_| Reason::CachedEntryInvalid)?; + // An entry written without a TTL shows the mode to an instance that has + // not written yet, as on Spin, which instantiates the module per request. + if entry.ttl_less() { + self.ttl_less.store(true, Ordering::Relaxed); + } + if is_fresh(entry.expires_at(), unix_now()) { + return Ok(Some(entry)); + } + if self.ttl_less.load(Ordering::Relaxed) + && !matches!(deadline.race(kv.delete(key)).await, Some(Ok(()))) + { + log::debug!("encypher: expired cache entry could not be deleted"); + } + Ok(None) + } + + /// Best-effort cache write; failures are logged and ignored. + /// + /// Writes carry a TTL until the store first reports TTL writes as + /// unsupported; from then on this instance writes without one, and marks + /// the entry so a later instance that reads it knows the mode too. + async fn write( + &self, + kv: &dyn PlatformKvStore, + key: &str, + mut entry: E, + ttl_seconds: u64, + deadline: &mut Deadline<'_>, + ) { + if !self.ttl_less.load(Ordering::Relaxed) { + let Some(value) = encode(&entry) else { + return; + }; + let ttl = Duration::from_secs(ttl_seconds); + match deadline.race(kv.put_bytes_with_ttl(key, value, ttl)).await { + Some(Ok(())) => return, + Some(Err(KvError::Unsupported { .. })) => { + self.ttl_less.store(true, Ordering::Relaxed); + log::info!( + "encypher: the KV store does not support TTL writes; entries expire by their expires_at" + ); + } + Some(Err(_)) => { + log::debug!("encypher: cache write failed"); + return; + } + None => { + log::debug!("encypher: cache write timed out"); + return; + } + } + } + entry.mark_ttl_less(); + let Some(value) = encode(&entry) else { + return; + }; + match deadline.race(kv.put_bytes(key, value)).await { + Some(Ok(())) => {} + Some(Err(_)) => log::debug!("encypher: cache write failed"), + None => log::debug!("encypher: cache write timed out"), + } + } + + /// Write a negative entry unless a fresh, valid plan is already cached for the key. + /// + /// The KV trait has no insert-if-absent, so this re-reads first: a concurrent + /// successful sign for the same text keeps its plan. + async fn write_skip( + &self, + kv: &dyn PlatformKvStore, + key: &str, + reason: Reason, + scalars: usize, + deadline: &mut Deadline<'_>, + ) { + if let Ok(Some(CacheEntry::Plan { + plan, manifest_url, .. + })) = self.read::(kv, key, deadline).await + && api::validate_plan(&plan, manifest_url.as_deref(), scalars).is_ok() + { + return; + } + let ttl = self.config.negative_ttl_seconds; + let entry = CacheEntry::Skip { + reason, + expires_at: Self::expires_at(ttl), + ttl_less: false, + }; + self.write(kv, key, entry, ttl, deadline).await; + } +} + +/// Serialize a cache entry; a failure is logged and the write skipped. +fn encode(entry: &impl Serialize) -> Option { + match serde_json::to_vec(entry) { + Ok(value) => Some(Bytes::from(value)), + Err(_) => { + log::debug!("encypher: cache entry could not be serialized"); + None + } + } +} + +/// Apply a plan that passed [`api::validate_plan`] against the signed text, and +/// add the manifest meta tag when a manifest URL was returned. +fn render( + html: &str, + boundary: &Boundary, + plan: &EmbeddingPlan, + manifest_url: Option<&str>, +) -> Option { + let signed = embed::apply_plan(html, boundary, plan)?; + Some(match manifest_url { + Some(url) => embed::inject_manifest_meta(&signed, url), + None => signed, + }) +} + +fn parse_error_reason(error: &ApiError) -> Reason { + match error { + ApiError::TooLarge => Reason::ResponseTooLarge, + ApiError::Malformed => Reason::MalformedResponse, + } +} + +#[cfg(test)] +mod tests { + use std::cell::RefCell; + use std::collections::HashMap; + use std::sync::atomic::AtomicUsize; + use std::sync::{LazyLock, Mutex}; + use std::task::Poll; + + use error_stack::Report; + use serde_json::json; + + use super::*; + use trusted_server_core::platform::test_support::{ + StoppedTimer, StubHttpClient, build_services_with_http_client, + }; + use trusted_server_core::platform::{ + PlatformError, PlatformHttpClient, PlatformPendingRequest, PlatformResponse, + PlatformSelectResult, + }; + + const TOKEN: &str = "cwk_test_worker_token_value"; + const PAGE_URL: &str = "https://www.publisher.example.com/news/harbor-vote"; + const MARKER: &str = "\u{E0100}\u{E0101}\u{FE01}"; + const MANIFEST_URL: &str = "https://api.example.com/manifests/doc-1"; + const POLICY_DIGEST: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + const ARTICLE: &str = "Harbor Council Votes\ +

Harbor council votes on the ferry schedule

\ +

The harbor council met on Tuesday evening and approved a revised ferry schedule \ + for the coming season.

Residents asked for earlier departures on weekdays.

\ +
Footer
"; + /// Simulated time that passes each time a [`TickTimer`] sleep is polled. + const TICK: Duration = Duration::from_millis(100); + + // --- test doubles ------------------------------------------------------- + + /// Return `Pending` `times` times, waking the task each time, then `Ready`. + async fn yield_pending(times: usize) { + for _ in 0..times { + let mut yielded = false; + futures::future::poll_fn(|cx| { + if yielded { + Poll::Ready(()) + } else { + yielded = true; + cx.waker().wake_by_ref(); + Poll::Pending + } + }) + .await; + } + } + + /// A timer on simulated time: each poll of a sleep advances it by [`TICK`]. + /// Sleeps only advance while the work they race is pending, as on a real + /// runtime where the clock runs while I/O is outstanding. + #[derive(Default)] + struct TickTimer { + requested: Mutex>, + } + + #[async_trait::async_trait(?Send)] + impl PlatformTimer for TickTimer { + async fn sleep(&self, duration: Duration) { + self.requested + .lock() + .expect("should lock requested durations") + .push(duration); + let ticks = duration.as_millis().div_ceil(TICK.as_millis()); + yield_pending(usize::try_from(ticks).expect("ticks should fit")).await; + } + } + + /// In-memory KV store that counts calls and can fail, stall or refuse TTLs. + #[derive(Default)] + struct MemoryKv { + entries: Mutex>, + ttls: Mutex>, + deletes: Mutex>, + reads: AtomicUsize, + ttl_writes: AtomicUsize, + /// Plan-key reads that report a miss before reads see the stored entry. + hidden_reads: AtomicUsize, + /// Every read fails with [`KvError::Unavailable`], as an unbound store does. + unavailable: bool, + /// Every read fails with an error other than unavailability. + fail_reads: bool, + /// Every read never completes. + hung_reads: bool, + /// Writes to keys with this prefix never complete. + hung_writes_to: Option<&'static str>, + /// The first write to this exact key never completes; later writes succeed. + hung_first_write_to: Option, + hung_first_write_taken: AtomicBool, + /// TTL writes fail with [`KvError::Unsupported`], as on Spin. + ttl_unsupported: bool, + /// `Pending` polls before every operation completes. + pending_polls: usize, + } + + impl MemoryKv { + fn entries(&self) -> HashMap { + self.entries.lock().expect("should lock entries").clone() + } + + fn entry(&self, key: &str) -> serde_json::Value { + serde_json::from_slice(&self.entries()[key]).expect("should store JSON") + } + + fn insert(&self, key: &str, value: &serde_json::Value) { + self.entries.lock().expect("should lock entries").insert( + key.to_owned(), + Bytes::from(serde_json::to_vec(value).expect("should serialize")), + ); + } + + fn remove(&self, key: &str) { + self.entries + .lock() + .expect("should lock entries") + .remove(key); + } + + /// The single article entry under the current plan namespace. + fn only_entry(&self) -> (String, serde_json::Value) { + let entries: Vec<_> = self + .entries() + .into_keys() + .filter(|key| key.starts_with("encypher:v3:")) + .collect(); + assert_eq!(entries.len(), 1, "should hold exactly one article entry"); + let key = entries.into_iter().next().expect("should have an entry"); + let value = self.entry(&key); + (key, value) + } + + fn reads(&self) -> usize { + self.reads.load(Ordering::SeqCst) + } + + async fn store(&self, key: &str, value: Bytes) -> Result<(), KvError> { + yield_pending(self.pending_polls).await; + if self + .hung_writes_to + .is_some_and(|prefix| key.starts_with(prefix)) + || (self.hung_first_write_to.as_deref() == Some(key) + && !self.hung_first_write_taken.swap(true, Ordering::SeqCst)) + { + futures::future::pending::<()>().await; + } + self.entries + .lock() + .expect("should lock entries") + .insert(key.to_owned(), value); + Ok(()) + } + } + + #[async_trait::async_trait(?Send)] + impl PlatformKvStore for MemoryKv { + async fn get_bytes(&self, key: &str) -> Result, KvError> { + self.reads.fetch_add(1, Ordering::SeqCst); + yield_pending(self.pending_polls).await; + if self.hung_reads { + futures::future::pending::<()>().await; + } + if self.unavailable { + return Err(KvError::Unavailable); + } + if self.fail_reads { + return Err(KvError::LimitExceeded { + message: "test failure".to_owned(), + }); + } + if key.starts_with(CACHE_KEY_PREFIX) + && self + .hidden_reads + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |n| n.checked_sub(1)) + .is_ok() + { + return Ok(None); + } + Ok(self + .entries + .lock() + .expect("should lock entries") + .get(key) + .cloned()) + } + + async fn put_bytes(&self, key: &str, value: Bytes) -> Result<(), KvError> { + self.store(key, value).await + } + + async fn put_bytes_with_ttl( + &self, + key: &str, + value: Bytes, + ttl: Duration, + ) -> Result<(), KvError> { + self.ttl_writes.fetch_add(1, Ordering::SeqCst); + if self.ttl_unsupported { + return Err(KvError::Unsupported { + operation: "put_bytes_with_ttl".to_owned(), + }); + } + self.ttls + .lock() + .expect("should lock TTLs") + .insert(key.to_owned(), ttl); + self.store(key, value).await + } + + async fn delete(&self, key: &str) -> Result<(), KvError> { + yield_pending(self.pending_polls).await; + self.deletes + .lock() + .expect("should lock deletes") + .push(key.to_owned()); + self.remove(key); + Ok(()) + } + + async fn list_keys_page( + &self, + _prefix: &str, + _cursor: Option<&str>, + _limit: usize, + ) -> Result { + Ok(edgezero_core::key_value_store::KvPage::default()) + } + } + + /// HTTP client whose send never completes. + #[derive(Default)] + struct HungHttpClient { + sends: AtomicUsize, + } + + #[async_trait::async_trait(?Send)] + impl PlatformHttpClient for HungHttpClient { + async fn send( + &self, + _request: PlatformHttpRequest, + ) -> Result> { + self.sends.fetch_add(1, Ordering::SeqCst); + futures::future::pending().await + } + + async fn send_async( + &self, + _request: PlatformHttpRequest, + ) -> Result> { + Err(Report::new(PlatformError::HttpClient)) + } + + async fn select( + &self, + _pending_requests: Vec, + ) -> Result> { + Err(Report::new(PlatformError::HttpClient)) + } + } + + /// Wraps [`StubHttpClient`] so the send and the response body each return + /// `Pending` before they are ready, as on Cloudflare, Spin and Axum. + struct YieldingHttpClient { + inner: Arc, + pending_polls: usize, + } + + #[async_trait::async_trait(?Send)] + impl PlatformHttpClient for YieldingHttpClient { + async fn send( + &self, + request: PlatformHttpRequest, + ) -> Result> { + yield_pending(self.pending_polls).await; + let response = self.inner.send(request).await?; + let (parts, body) = response.response.into_parts(); + let bytes = body + .into_bytes_bounded(usize::MAX) + .await + .expect("should read the stub body"); + let polls = self.pending_polls; + let body = EdgeBody::stream(futures::stream::once(async move { + yield_pending(polls).await; + bytes + })); + Ok(PlatformResponse::new(http::Response::from_parts( + parts, body, + ))) + } + + async fn send_async( + &self, + _request: PlatformHttpRequest, + ) -> Result> { + Err(Report::new(PlatformError::HttpClient)) + } + + async fn select( + &self, + _pending_requests: Vec, + ) -> Result> { + Err(Report::new(PlatformError::HttpClient)) + } + } + + /// Records log lines, with their level, emitted on the current thread. + struct ThreadLogger; + + thread_local! { + static LOGS: RefCell> = const { RefCell::new(Vec::new()) }; + } + + impl log::Log for ThreadLogger { + fn enabled(&self, _metadata: &log::Metadata<'_>) -> bool { + true + } + + fn log(&self, record: &log::Record<'_>) { + LOGS.with(|logs| { + logs.borrow_mut() + .push(format!("{} {}", record.level(), record.args())); + }); + } + + fn flush(&self) {} + } + + static LOGGER: ThreadLogger = ThreadLogger; + + /// A sign-failure case: name and the API response, `None` for a transport error. + type FailureCase = (&'static str, Option<(u16, Vec)>); + /// A gate case: name, signer, and the request mutation that trips the gate. + type GateCase<'a> = (&'static str, &'a Arc, fn(&mut PageRequest)); + + /// Runs `run` with this thread's log lines captured. Returns `None`, after + /// skipping `run`, when another logger already owns the process. + fn capture_logs(run: impl FnOnce()) -> Option> { + static INSTALLED: LazyLock = LazyLock::new(|| { + let installed = log::set_logger(&LOGGER).is_ok(); + if installed { + log::set_max_level(log::LevelFilter::Trace); + } + installed + }); + if !*INSTALLED { + return None; + } + LOGS.with(|logs| logs.borrow_mut().clear()); + run(); + Some(LOGS.with(RefCell::take)) + } + + // --- fixtures ----------------------------------------------------------- + + fn config_json(token: &str) -> serde_json::Value { + json!({ + "api_base_url": "https://api.example.com", + "domain": "publisher.example.com", + "worker_token": token, + }) + } + + fn factory_with(config: serde_json::Value, max_document_bytes: usize) -> Arc { + let config: EncypherConfig = serde_json::from_value(config).expect("should parse config"); + Arc::new(Signer::new(config, max_document_bytes)) + } + + fn factory() -> Arc { + factory_with(config_json(TOKEN), 5_000_000) + } + + fn policy_key() -> String { + factory().policy_key.clone() + } + + /// Services without a timer, as on Fastly. + fn services(http: Arc, kv: &Arc) -> RuntimeServices { + build_services_with_http_client(http) + .with_kv_store(Arc::clone(kv) as Arc) + } + + /// Services with a [`TickTimer`], as on Cloudflare, Spin and Axum. + fn timed_services(http: Arc, kv: &Arc) -> RuntimeServices { + services(http, kv).with_timer(Arc::new(TickTimer::default())) + } + + fn request() -> PageRequest { + PageRequest { + method: Method::GET, + page_url: PAGE_URL.to_owned(), + response_status: 200, + origin_response_is_private: false, + } + } + + fn plan_json(marker: &str) -> serde_json::Value { + json!({ + "index_unit": "codepoint", + "operations": [{ "insert_after_index": 5, "marker": marker }], + }) + } + + fn sign_response(marker: &str) -> Vec { + sign_response_for_policy(marker, POLICY_DIGEST) + } + + fn policy_response(digest: &str) -> Vec { + serde_json::to_vec(&json!({"policy_digest": digest})).expect("should serialize policy") + } + + fn sign_response_for_policy(marker: &str, digest: &str) -> Vec { + serde_json::to_vec(&json!({ + "success": true, + "policy_digest": digest, + "embedding_plan": plan_json(marker), + "manifest_url": MANIFEST_URL, + "verification_url": "https://api.example.com/verify/doc-1", + "document_id": "doc-1", + })) + .expect("should serialize sign response") + } + + fn signing_http() -> Arc { + let http = Arc::new(StubHttpClient::new()); + http.push_response(200, policy_response(POLICY_DIGEST)); + http.push_response(200, sign_response(MARKER)); + http + } + + fn seed_plan(kv: &Arc) -> String { + view(&factory(), &services(signing_http(), kv)) + } + + /// Drive one document the way the middleware and the publisher driver do: + /// the gates on the request, when there is one, then two chunks, then the + /// asynchronous finish with the core backstop. A request the gates refuse, + /// or none, holds nothing, so the document passes through. + fn run( + factory: &Arc, + request: Option, + services: &RuntimeServices, + html: &str, + ) -> String { + let Some(request) = request.and_then(|request| factory.admit(request).ok()) else { + return html.to_owned(); + }; + futures::executor::block_on(async { + let mut processor = document_processor(factory, request); + let (first, second) = html.as_bytes().split_at(html.len() / 2); + let mut output = processor + .process_chunk(first, false) + .expect("should process first chunk"); + output.extend( + processor + .finish(second, services) + .await + .expect("should finish the document"), + ); + String::from_utf8(output).expect("should emit UTF-8") + }) + } + + /// One view of [`ARTICLE`] at [`PAGE_URL`]. + fn view(factory: &Arc, services: &RuntimeServices) -> String { + run(factory, Some(request()), services, ARTICLE) + } + + fn assert_signed(output: &str) { + assert!(output.contains(MARKER), "should embed the plan marker"); + assert!( + output.contains("c2pa-manifest-url") && output.contains(MANIFEST_URL), + "should inject the manifest meta tag" + ); + assert_eq!( + output.matches(MARKER).count(), + 1, + "should add the marker once" + ); + } + + /// Rewrite the `expires_at` of the entry at `key` to `expires_at`. + fn set_expiry(kv: &MemoryKv, key: &str, expires_at: u64) { + let mut entry = kv.entry(key); + entry["expires_at"] = json!(expires_at); + kv.insert(key, &entry); + } + + fn expired() -> u64 { + unix_now().saturating_sub(1) + } + + // --- policy ------------------------------------------------------------- + + #[test] + fn policy_failures_are_negatively_cached_without_using_an_old_plan() { + let cases: Vec = vec![ + ("unavailable", Some((503, Vec::new()))), + ("forbidden", Some((403, Vec::new()))), + ("transport", None), + ("malformed", Some((200, b"not json".to_vec()))), + ("missing digest", Some((200, b"{}".to_vec()))), + ( + "uppercase", + Some((200, policy_response(&POLICY_DIGEST.to_uppercase()))), + ), + ( + "oversized", + Some((200, vec![b' '; api::MAX_POLICY_RESPONSE_BYTES + 2])), + ), + ]; + for (case, response) in cases { + let kv = Arc::new(MemoryKv::default()); + seed_plan(&kv); + kv.remove(&policy_key()); + let http = Arc::new(StubHttpClient::new()); + if let Some((status, body)) = response { + http.push_response(status, body); + } + let services = services(Arc::clone(&http) as Arc, &kv); + for _ in 0..2 { + assert_eq!(view(&factory(), &services), ARTICLE, "{case}"); + } + assert_eq!(http.recorded_request_methods(), ["GET"], "{case}"); + assert_eq!(kv.entry(&policy_key())["kind"], "skip", "{case}"); + assert_eq!( + kv.ttls.lock().expect("lock")[&policy_key()], + Duration::from_secs(300) + ); + // Once the skip is gone, refresh may reuse only the matching old plan. + kv.remove(&policy_key()); + http.push_response(200, policy_response(POLICY_DIGEST)); + assert_signed(&view(&factory(), &services)); + assert_eq!(http.recorded_request_methods(), ["GET", "GET"]); + } + } + + #[test] + fn malformed_cached_policy_cannot_unlock_a_cached_article() { + let future = unix_now() + 300; + for value in [ + json!("not an entry"), + json!({"kind": "digest", "policy_digest": null, "expires_at": future}), + json!({"kind": "digest", "policy_digest": "bad", "expires_at": future}), + json!({ + "kind": "digest", "policy_digest": POLICY_DIGEST, "expires_at": future, + "extra": true, + }), + // `expires_at` is required. + json!({"kind": "digest", "policy_digest": POLICY_DIGEST}), + ] { + let kv = Arc::new(MemoryKv::default()); + seed_plan(&kv); + kv.insert(&policy_key(), &value); + let http = Arc::new(StubHttpClient::new()); + let services = services(Arc::clone(&http) as Arc, &kv); + for _ in 0..2 { + assert_eq!(view(&factory(), &services), ARTICLE, "{value}"); + } + assert!(http.recorded_request_methods().is_empty(), "{value}"); + } + } + + #[test] + fn policy_cache_hit_and_article_or_policy_changes_select_the_correct_plan() { + let kv = Arc::new(MemoryKv::default()); + seed_plan(&kv); + let http = Arc::new(StubHttpClient::new()); + http.push_response(200, sign_response(MARKER)); + let services = services(Arc::clone(&http) as Arc, &kv); + let changed = ARTICLE.replace("Tuesday", "Wednesday"); + assert_signed(&run(&factory(), Some(request()), &services, &changed)); + assert_eq!(http.recorded_request_methods(), ["POST"]); + + kv.remove(&policy_key()); + let next_digest = "b".repeat(64); + http.push_response(200, policy_response(&next_digest)); + http.push_response(200, sign_response_for_policy(MARKER, &next_digest)); + assert_signed(&view(&factory(), &services)); + assert_eq!(http.recorded_request_methods(), ["POST", "GET", "POST"]); + let count = http.recorded_request_methods().len(); + assert_signed(&view(&factory(), &services)); + assert_eq!(http.recorded_request_methods().len(), count); + // Returning to the original policy selects its original plan, not the intervening one. + kv.remove(&policy_key()); + http.push_response(200, policy_response(POLICY_DIGEST)); + assert_signed(&view(&factory(), &services)); + assert_eq!( + http.recorded_request_methods(), + ["POST", "GET", "POST", "GET"] + ); + } + + #[test] + fn conflict_or_response_digest_mismatch_evicts_policy_without_retrying() { + let mut missing: serde_json::Value = + serde_json::from_slice(&sign_response(MARKER)).expect("JSON"); + missing + .as_object_mut() + .expect("object") + .remove("policy_digest"); + for (status, body) in [ + ( + 409, + br#"{"detail":"TRUSTED_SERVER_POLICY_CHANGED"}"#.to_vec(), + ), + (409, b"unparseable conflict".to_vec()), + (200, sign_response_for_policy(MARKER, &"b".repeat(64))), + (200, sign_response_for_policy(MARKER, "invalid")), + (200, serde_json::to_vec(&missing).expect("JSON")), + ] { + let kv = Arc::new(MemoryKv::default()); + let http = Arc::new(StubHttpClient::new()); + http.push_response(200, policy_response(POLICY_DIGEST)); + http.push_response(status, body); + let services = services(Arc::clone(&http) as Arc, &kv); + assert_eq!(view(&factory(), &services), ARTICLE); + assert_eq!(http.recorded_request_methods(), ["GET", "POST"]); + assert!(!kv.entries().contains_key(&policy_key())); + assert_eq!(kv.only_entry().1["kind"], "skip"); + // Next request refetches policy, but the failed article remains negatively cached. + http.push_response(200, policy_response(POLICY_DIGEST)); + assert_eq!(view(&factory(), &services), ARTICLE); + assert_eq!(http.recorded_request_methods(), ["GET", "POST", "GET"]); + } + } + + #[test] + fn entries_expire_at_their_ttl_and_the_store_ttl_matches() { + let kv = Arc::new(MemoryKv::default()); + let before = unix_now(); + seed_plan(&kv); + let after = unix_now(); + let ttls = kv.ttls.lock().expect("lock").clone(); + let (plan_key, plan) = kv.only_entry(); + for (key, entry, ttl) in [ + (policy_key(), kv.entry(&policy_key()), 300), + (plan_key, plan, 604_800), + ] { + assert_eq!(ttls[&key], Duration::from_secs(ttl)); + let expires_at = entry["expires_at"] + .as_u64() + .expect("should carry expires_at"); + assert!( + (before + ttl..=after + ttl).contains(&expires_at), + "{key} should expire {ttl}s after it was written" + ); + } + for (policy_ttl, negative_ttl, expected) in [(60, 300, 60), (300, 60, 60)] { + let kv = Arc::new(MemoryKv::default()); + let http = Arc::new(StubHttpClient::new()); + http.push_response(503, Vec::new()); + let mut config = config_json(TOKEN); + config["policy_ttl_seconds"] = json!(policy_ttl); + config["negative_ttl_seconds"] = json!(negative_ttl); + let factory = factory_with(config, 5_000_000); + assert_eq!(view(&factory, &services(http, &kv)), ARTICLE); + assert_eq!( + kv.ttls.lock().expect("lock")[&factory.policy_key], + Duration::from_secs(expected) + ); + } + } + + // --- sign path and cache ------------------------------------------------ + + #[test] + fn signs_on_a_miss_with_the_worker_token_and_caches_the_plan() { + let http = signing_http(); + let kv = Arc::new(MemoryKv::default()); + let services = services(Arc::clone(&http) as Arc, &kv); + + let output = view(&factory(), &services); + + assert_signed(&output); + assert_eq!(http.recorded_request_methods(), ["GET", "POST"]); + assert_eq!( + http.recorded_request_uris(), + [ + format!( + "https://api.example.com/api/v1/public/cdn/trusted-server/policy?page_url={}", + url::form_urlencoded::byte_serialize(PAGE_URL.as_bytes()).collect::() + ), + "https://api.example.com/api/v1/public/cdn/sign".to_owned(), + ] + ); + let headers = &http.recorded_request_headers()[1]; + let header = |name: &str| { + headers + .iter() + .find(|(key, _)| key == name) + .map(|(_, value)| value.as_str()) + }; + assert_eq!(header("x-encypher-worker-token"), Some(TOKEN)); + assert_eq!(header("content-type"), Some("application/json")); + assert_eq!( + header("idempotency-key"), + None, + "should send no idempotency key" + ); + let body: serde_json::Value = serde_json::from_slice(&http.recorded_request_bodies()[1]) + .expect("should send a JSON body"); + assert_eq!(body["page_url"], PAGE_URL); + assert_eq!(body["expected_policy_digest"], POLICY_DIGEST); + assert_eq!(body["document_title"], "Harbor Council Votes"); + assert_eq!(body["boundary_selector"], "article"); + let sent_text = body["text"].as_str().expect("should send text"); + assert!( + sent_text.starts_with("Harbor council votes on the ferry schedule The harbor council"), + "should send the article's visible text, got {sent_text}" + ); + assert!(!sent_text.contains("Footer") && !sent_text.contains("Home")); + let (key, entry) = kv.only_entry(); + assert!( + key.starts_with("encypher:v3:") && key.contains(POLICY_DIGEST), + "unexpected cache key {key}" + ); + assert!(!key.contains(TOKEN), "cache key must not contain the token"); + assert!( + policy_key().starts_with("encypher:policy:v2:"), + "unexpected policy key" + ); + assert_eq!(entry["kind"], "plan"); + assert_eq!(entry["plan"], plan_json(MARKER)); + } + + #[test] + fn a_cached_plan_signs_without_calling_the_api() { + let kv = Arc::new(MemoryKv::default()); + let first = seed_plan(&kv); + + let http = Arc::new(StubHttpClient::new()); + let second = view( + &factory(), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_eq!( + second, first, + "a cache hit should serve the same signed page" + ); + assert!( + http.recorded_backend_names().is_empty(), + "should make no API call" + ); + } + + #[test] + fn a_cached_skip_passes_through_without_calling_the_api() { + let kv = Arc::new(MemoryKv::default()); + let first_http = Arc::new(StubHttpClient::new()); + first_http.push_response(200, policy_response(POLICY_DIGEST)); + first_http.push_response(403, b"{}".to_vec()); + view(&factory(), &services(first_http, &kv)); + assert_eq!(kv.only_entry().1["kind"], "skip"); + + let http = Arc::new(StubHttpClient::new()); + let output = view( + &factory(), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_eq!(output, ARTICLE); + assert!( + http.recorded_backend_names().is_empty(), + "should make no API call" + ); + } + + #[test] + fn a_failure_does_not_replace_a_fresh_plan_written_since_the_first_read() { + let seeded = Arc::new(MemoryKv::default()); + seed_plan(&seeded); + // The first read misses, as if a concurrent request wrote the plan after it. + let kv = Arc::new(MemoryKv { + entries: Mutex::new(seeded.entries()), + hidden_reads: AtomicUsize::new(1), + ..MemoryKv::default() + }); + let http = Arc::new(StubHttpClient::new()); + http.push_response(503, Vec::new()); + + let output = view( + &factory(), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_eq!(output, ARTICLE, "the failed request should pass through"); + assert_eq!(http.recorded_backend_names().len(), 1); + assert_eq!(kv.only_entry().1["kind"], "plan", "the plan should survive"); + } + + #[test] + fn sign_failures_pass_through_and_write_a_skip() { + let oversized = { + let mut body = sign_response(MARKER); + body.resize(api::MAX_RESPONSE_BYTES + 2, b' '); + body + }; + let quota = serde_json::to_vec(&json!({ "success": false, "error": "quota_exceeded" })) + .expect("should serialize"); + let insecure_manifest = serde_json::to_vec(&json!({ + "success": true, + "policy_digest": POLICY_DIGEST, + "embedding_plan": plan_json(MARKER), + "manifest_url": "javascript:alert(1)", + })) + .expect("should serialize"); + let cases: Vec = vec![ + ("403", Some((403, b"{}".to_vec()))), + ("422", Some((422, b"{}".to_vec()))), + ("429", Some((429, b"{}".to_vec()))), + ("5xx", Some((502, Vec::new()))), + ("transport error or first-byte timeout", None), + ("oversized body", Some((200, oversized))), + ("malformed body", Some((200, b"not json".to_vec()))), + ( + "invalid marker", + Some((200, sign_response(""))), + ), + ("quota", Some((200, quota))), + ("non-https manifest url", Some((200, insecure_manifest))), + ( + "null plan", + Some(( + 200, + serde_json::to_vec(&json!({ + "success": true, "policy_digest": POLICY_DIGEST, "embedding_plan": null + })) + .expect("JSON"), + )), + ), + ]; + + for (case, response) in cases { + let http = Arc::new(StubHttpClient::new()); + http.push_response(200, policy_response(POLICY_DIGEST)); + if let Some((status, body)) = response { + http.push_response(status, body); + } + let kv = Arc::new(MemoryKv::default()); + + let output = view( + &factory(), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_eq!( + output, ARTICLE, + "{case}: should pass the page through unchanged" + ); + assert_eq!( + http.recorded_backend_names().len(), + 2, + "{case}: one policy call and one sign call" + ); + assert_eq!( + kv.only_entry().1["kind"], + "skip", + "{case}: should cache a skip" + ); + } + } + + #[test] + fn a_tampered_cached_plan_is_rejected_without_calling_the_api() { + for (field, value) in [ + ("/plan/operations/0/marker", ""), + ("/manifest_url", "http://manifest.example.com/doc-1"), + ] { + let kv = Arc::new(MemoryKv::default()); + seed_plan(&kv); + let (key, mut entry) = kv.only_entry(); + *entry + .pointer_mut(field) + .expect("seeded entry should have the field") = json!(value); + kv.insert(&key, &entry); + + let http = Arc::new(StubHttpClient::new()); + let output = view( + &factory(), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_eq!(output, ARTICLE, "{field}: should pass through"); + assert!( + http.recorded_backend_names().is_empty(), + "{field}: should make no API call" + ); + } + } + + #[test] + fn a_changed_credential_misses_the_cache() { + let kv = Arc::new(MemoryKv::default()); + seed_plan(&kv); + + let http = signing_http(); + let output = view( + &factory_with(config_json("cwk_rotated_token"), 5_000_000), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_signed(&output); + assert_eq!(http.recorded_request_methods(), ["GET", "POST"]); + assert_eq!( + kv.entries().len(), + 4, + "each credential should have its own policy and article entries" + ); + } + + // --- asynchronous resolve and the hold deadline ------------------------- + + #[test] + fn calls_that_are_pending_before_ready_are_awaited_and_sign() { + let timers: [Option>; 3] = [ + None, + Some(Arc::new(StoppedTimer)), + Some(Arc::new(TickTimer::default())), + ]; + for timer in timers { + let label = if timer.is_some() { "timer" } else { "no timer" }; + let kv = Arc::new(MemoryKv { + pending_polls: 2, + ..MemoryKv::default() + }); + let stub = signing_http(); + let http = Arc::new(YieldingHttpClient { + inner: Arc::clone(&stub), + pending_polls: 2, + }); + let mut services = services(http, &kv); + if let Some(timer) = timer { + services = services.with_timer(timer); + } + + let first = view(&factory(), &services); + let second = view(&factory(), &services); + + assert_signed(&first); + assert_eq!( + second, first, + "{label}: the second view should hit the cache" + ); + assert_eq!(stub.recorded_request_methods(), ["GET", "POST"], "{label}"); + assert_eq!(kv.only_entry().1["kind"], "plan", "{label}"); + } + } + + #[test] + fn a_hung_policy_call_times_out_writes_a_skip_and_the_next_view_makes_no_call() { + let kv = Arc::new(MemoryKv::default()); + let hung = Arc::new(HungHttpClient::default()); + + let output = view( + &factory(), + &timed_services(Arc::clone(&hung) as Arc, &kv), + ); + + assert_eq!(output, ARTICLE, "should release the page unchanged"); + assert_eq!(hung.sends.load(Ordering::SeqCst), 1); + let skip = kv.entry(&policy_key()); + assert_eq!(skip["kind"], "skip"); + assert_eq!(skip["reason"], "timeout"); + + let http = Arc::new(StubHttpClient::new()); + let next = view( + &factory(), + &timed_services(Arc::clone(&http) as Arc, &kv), + ); + assert_eq!(next, ARTICLE); + assert!( + http.recorded_backend_names().is_empty(), + "the next view should be a cached skip" + ); + } + + #[test] + fn a_hung_sign_call_or_body_times_out_writes_a_plan_skip_and_the_next_view_makes_no_call() { + let kv = Arc::new(MemoryKv::default()); + seed_plan(&kv); + let article_key = kv.only_entry().0; + kv.remove(&article_key); + let hung = Arc::new(HungHttpClient::default()); + let services = timed_services(Arc::clone(&hung) as Arc, &kv); + + for _ in 0..2 { + assert_eq!(view(&factory(), &services), ARTICLE); + } + + assert_eq!( + hung.sends.load(Ordering::SeqCst), + 1, + "one sign call in total" + ); + let (_, skip) = kv.only_entry(); + assert_eq!(skip["kind"], "skip"); + assert_eq!(skip["reason"], "timeout"); + } + + #[test] + fn a_hung_kv_read_times_out_without_calling_the_api() { + let kv = Arc::new(MemoryKv { + hung_reads: true, + ..MemoryKv::default() + }); + let http = Arc::new(StubHttpClient::new()); + + let output = view( + &factory(), + &timed_services(Arc::clone(&http) as Arc, &kv), + ); + + assert_eq!(output, ARTICLE); + assert_eq!(kv.reads(), 1); + assert!(http.recorded_backend_names().is_empty()); + } + + #[test] + fn a_plan_write_that_runs_out_of_time_still_returns_the_signed_page() { + let kv = Arc::new(MemoryKv { + hung_writes_to: Some(CACHE_KEY_PREFIX), + ..MemoryKv::default() + }); + let http = signing_http(); + + let output = view( + &factory(), + &timed_services(Arc::clone(&http) as Arc, &kv), + ); + + assert_signed(&output); + assert_eq!(http.recorded_request_methods(), ["GET", "POST"]); + assert!( + kv.entries() + .keys() + .all(|key| !key.starts_with(CACHE_KEY_PREFIX)), + "the abandoned plan write should store nothing" + ); + } + + #[test] + fn a_policy_digest_write_that_runs_out_of_time_caches_a_policy_skip() { + let kv = Arc::new(MemoryKv { + hung_first_write_to: Some(policy_key()), + ..MemoryKv::default() + }); + let http = Arc::new(StubHttpClient::new()); + http.push_response(200, policy_response(POLICY_DIGEST)); + + let output = view( + &factory(), + &timed_services(Arc::clone(&http) as Arc, &kv), + ); + + assert_eq!(output, ARTICLE, "should release the page unchanged"); + assert_eq!( + http.recorded_request_methods(), + ["GET"], + "no sign call after the digest write ran out of time" + ); + let skip = kv.entry(&policy_key()); + assert_eq!(skip["kind"], "skip"); + assert_eq!(skip["reason"], "timeout"); + assert!( + kv.entries() + .keys() + .all(|key| !key.starts_with(CACHE_KEY_PREFIX)), + "no plan skip under a digest the next view cannot know" + ); + + // A hung client turns any API call into a full hold, so zero sends + // after one KV read shows the next view is a cached skip. + let reads = kv.reads(); + let hung = Arc::new(HungHttpClient::default()); + let next = view( + &factory(), + &timed_services(Arc::clone(&hung) as Arc, &kv), + ); + assert_eq!(next, ARTICLE); + assert_eq!(hung.sends.load(Ordering::SeqCst), 0, "no API call"); + assert_eq!(kv.reads() - reads, 1, "only the policy entry is read"); + } + + #[test] + fn the_core_backstop_lies_beyond_the_module_hold_bound() { + let kv = Arc::new(MemoryKv::default()); + let timer = Arc::new(TickTimer::default()); + let services = services(Arc::new(HungHttpClient::default()), &kv) + .with_timer(Arc::clone(&timer) as Arc); + + assert_eq!(view(&factory(), &services), ARTICLE); + + let max_hold = Duration::from_millis(5_000); + let requested = timer.requested.lock().expect("lock").clone(); + let (backstop, module): (Vec<_>, Vec<_>) = + requested.into_iter().partition(|sleep| *sleep > max_hold); + assert_eq!(backstop.len(), 1, "one core backstop beyond max_hold_ms"); + // The skip-write reserve starts only after the work deadline, so the + // module's own sleeps run back to back. + assert!( + !module.is_empty() && module.iter().sum::() <= max_hold, + "module sleeps {module:?} should fit within max_hold_ms" + ); + assert_eq!( + kv.entry(&policy_key())["reason"], + "timeout", + "the module deadline, not the backstop, should release the page" + ); + } + + // --- expiry, write mode and key namespaces ------------------------------ + + #[test] + fn expired_entries_are_misses() { + // (entry to expire, API calls the next view makes) + let cases: [(&str, &[&str]); 4] = [ + ("policy digest", &["GET"]), + ("policy skip", &["GET"]), + ("plan", &["POST"]), + ("plan skip", &["POST"]), + ]; + for (case, expected_calls) in cases { + let kv = Arc::new(MemoryKv::default()); + seed_plan(&kv); + let (plan_key, _) = kv.only_entry(); + let key = if case.starts_with("policy") { + policy_key() + } else { + plan_key + }; + if case.ends_with("skip") { + kv.insert( + &key, + &json!({"kind": "skip", "reason": "server_error", "expires_at": expired()}), + ); + } else { + set_expiry(&kv, &key, expired()); + } + let http = Arc::new(StubHttpClient::new()); + if case.starts_with("policy") { + http.push_response(200, policy_response(POLICY_DIGEST)); + } else { + http.push_response(200, sign_response(MARKER)); + } + + let output = view( + &factory(), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_signed(&output); + assert_eq!(http.recorded_request_methods(), expected_calls, "{case}"); + assert!( + kv.entry(&key)["expires_at"].as_u64() > Some(unix_now()), + "{case}: should be replaced by a fresh entry" + ); + } + } + + #[test] + fn an_expired_plan_does_not_block_a_skip_write() { + let seeded = Arc::new(MemoryKv::default()); + seed_plan(&seeded); + let kv = Arc::new(MemoryKv { + entries: Mutex::new(seeded.entries()), + hidden_reads: AtomicUsize::new(1), + ..MemoryKv::default() + }); + let (plan_key, _) = kv.only_entry(); + set_expiry(&kv, &plan_key, expired()); + let http = Arc::new(StubHttpClient::new()); + http.push_response(503, Vec::new()); + + assert_eq!(view(&factory(), &services(http, &kv)), ARTICLE); + + assert_eq!(kv.entry(&plan_key)["kind"], "skip"); + } + + #[test] + fn a_store_without_ttl_writes_switches_once_and_expires_by_value() { + let kv = Arc::new(MemoryKv { + ttl_unsupported: true, + ..MemoryKv::default() + }); + let factory = factory(); + let http = signing_http(); + let services = services(Arc::clone(&http) as Arc, &kv); + + let first = view(&factory, &services); + + assert_signed(&first); + assert_eq!( + kv.ttl_writes.load(Ordering::SeqCst), + 1, + "only the first write should try a TTL" + ); + let (plan_key, plan) = kv.only_entry(); + assert_eq!(plan["kind"], "plan"); + assert!(plan["expires_at"].as_u64() > Some(unix_now())); + assert_eq!(kv.entry(&policy_key())["kind"], "digest"); + assert_eq!(view(&factory, &services), first, "should hit the cache"); + assert_eq!(http.recorded_request_methods(), ["GET", "POST"]); + + // Expired entries are misses and are deleted when read. + set_expiry(&kv, &plan_key, expired()); + http.push_response(200, sign_response(MARKER)); + assert_signed(&view(&factory, &services)); + assert_eq!(*kv.deletes.lock().expect("lock"), [plan_key.as_str()]); + assert_eq!( + kv.entry(&plan_key)["kind"], + "plan", + "should store a new plan" + ); + } + + #[test] + fn a_fresh_instance_deletes_expired_entries_written_without_a_ttl() { + // Spin instantiates the module per request, so the instance that reads + // an expired entry has not written anything yet. + let kv = Arc::new(MemoryKv { + ttl_unsupported: true, + ..MemoryKv::default() + }); + view(&factory(), &services(signing_http(), &kv)); + set_expiry(&kv, &policy_key(), expired()); + let http = Arc::new(StubHttpClient::new()); + http.push_response(503, Vec::new()); + + let output = view( + &factory(), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_eq!(output, ARTICLE); + assert_eq!(http.recorded_request_methods(), ["GET"]); + assert_eq!(*kv.deletes.lock().expect("lock"), [policy_key()]); + } + + #[test] + fn expired_entries_are_not_deleted_on_a_store_with_ttls() { + let kv = Arc::new(MemoryKv::default()); + let http = signing_http(); + let services = services(Arc::clone(&http) as Arc, &kv); + view(&factory(), &services); + let (plan_key, _) = kv.only_entry(); + set_expiry(&kv, &plan_key, expired()); + http.push_response(200, sign_response(MARKER)); + + assert_signed(&view(&factory(), &services)); + + assert!(kv.deletes.lock().expect("lock").is_empty()); + } + + #[test] + fn entries_under_the_previous_key_versions_are_never_read() { + let current = Arc::new(MemoryKv::default()); + seed_plan(¤t); + // The same plan and policy, as the previous release stored them. + let kv = Arc::new(MemoryKv::default()); + for (key, value) in current.entries() { + let mut entry: serde_json::Value = + serde_json::from_slice(&value).expect("should store JSON"); + entry.as_object_mut().expect("object").remove("expires_at"); + let old_key = key + .replace("encypher:v3:", "encypher:v2:") + .replace("encypher:policy:v2:", "encypher:policy:v1:"); + kv.insert(&old_key, &entry); + } + let old_entries = kv.entries(); + let http = signing_http(); + + let output = view( + &factory(), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_signed(&output); + assert_eq!(http.recorded_request_methods(), ["GET", "POST"]); + let entries = kv.entries(); + assert_eq!(entries.len(), old_entries.len() * 2); + for (key, value) in old_entries { + assert_eq!(entries[&key], value, "{key} should be left untouched"); + } + } + + // --- KV failures -------------------------------------------------------- + + #[test] + fn kv_read_errors_pass_through_without_calling_the_api() { + for kv in [ + MemoryKv { + fail_reads: true, + ..MemoryKv::default() + }, + MemoryKv { + unavailable: true, + ..MemoryKv::default() + }, + ] { + let kv = Arc::new(kv); + let http = Arc::new(StubHttpClient::new()); + + let output = view( + &factory(), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_eq!(output, ARTICLE); + assert_eq!(kv.reads(), 1); + assert!( + http.recorded_backend_names().is_empty(), + "should make no API call" + ); + } + } + + #[test] + fn an_unbound_kv_store_logs_kv_unavailable_and_warns_once() { + let Some(logs) = capture_logs(|| { + let http: Arc = Arc::new(StubHttpClient::new()); + let unbound = Arc::new(MemoryKv { + unavailable: true, + ..MemoryKv::default() + }); + for _ in 0..2 { + view(&factory(), &services(Arc::clone(&http), &unbound)); + } + let failing = Arc::new(MemoryKv { + fail_reads: true, + ..MemoryKv::default() + }); + view(&factory(), &services(http, &failing)); + }) else { + // Log lines go to a logger installed elsewhere and cannot be observed here. + return; + }; + + let outcomes: Vec<_> = logs + .iter() + .filter(|line| line.contains("encypher: outcome=")) + .collect(); + assert_eq!( + outcomes, + [ + "INFO encypher: outcome=passed_through reason=kv_unavailable", + "INFO encypher: outcome=passed_through reason=kv_unavailable", + "INFO encypher: outcome=passed_through reason=kv_error", + ] + ); + assert_eq!( + logs.iter() + .filter(|line| line.starts_with("WARN") && line.contains("KV store is unavailable")) + .count(), + 1, + "should warn once per instance: {logs:?}" + ); + } + + // --- gates and skips ---------------------------------------------------- + + #[test] + fn gated_documents_pass_through_with_no_kv_or_api_io() { + let signer = factory(); + let cases: Vec<(GateCase<'_>, Gate)> = vec![ + (("method", &signer, |r| r.method = Method::POST), Gate::Method), + (("status", &signer, |r| r.response_status = 404), Gate::Status), + ( + ("origin private", &signer, |r| { + r.origin_response_is_private = true; + }), + Gate::OriginPrivate, + ), + ( + ("host mismatch", &signer, |r| { + r.page_url = "https://other.example.com/news/harbor-vote".to_owned(); + }), + Gate::HostMismatch, + ), + ( + ("excluded path", &signer, |r| { + r.page_url = "https://publisher.example.com/WP-Admin/post".to_owned(); + }), + Gate::ExcludedPath, + ), + ( + ("asset suffix", &signer, |r| { + r.page_url = "https://publisher.example.com/feed.XML".to_owned(); + }), + Gate::ExcludedPath, + ), + ( + ("invalid page url", &signer, |r| { + r.page_url = "not a url".to_owned(); + }), + Gate::InvalidPageUrl, + ), + ]; + + for ((case, signer, mutate), gate) in cases { + let http = Arc::new(StubHttpClient::new()); + let kv = Arc::new(MemoryKv::default()); + let services = services(Arc::clone(&http) as Arc, &kv); + let mut request = request(); + mutate(&mut request); + + assert_eq!( + signer.admit(request.clone()), + Err(gate), + "{case}: should be refused by its gate" + ); + let output = run(signer, Some(request), &services, ARTICLE); + + assert_eq!(output, ARTICLE, "{case}: should pass through"); + assert_eq!(kv.reads(), 0, "{case}: should read no KV"); + assert!( + http.recorded_backend_names().is_empty(), + "{case}: no API call" + ); + } + + let kv = Arc::new(MemoryKv::default()); + assert_eq!( + run( + &factory(), + None, + &services(Arc::new(StubHttpClient::new()), &kv), + ARTICLE + ), + ARTICLE, + "nothing prepared" + ); + } + + #[test] + fn a_bare_domain_page_is_signed() { + let kv = Arc::new(MemoryKv::default()); + let mut request = request(); + request.page_url = "https://publisher.example.com/news/harbor-vote".to_owned(); + + let output = run( + &factory(), + Some(request), + &services(signing_http(), &kv), + ARTICLE, + ); + + assert_signed(&output); + } + + #[test] + fn an_oversized_document_is_flushed_and_passed_through() { + let http = Arc::new(StubHttpClient::new()); + let kv = Arc::new(MemoryKv::default()); + + let output = view( + &factory_with(config_json(TOKEN), 64), + &services(Arc::clone(&http) as Arc, &kv), + ); + + assert_eq!(output, ARTICLE); + assert_eq!(kv.reads(), 0); + assert!(http.recorded_backend_names().is_empty()); + } + + #[test] + fn already_signed_or_short_pages_are_not_sent() { + let magic = "\u{E01DF}\u{E01E0}\u{E01E1}\u{E01E2}"; + let marked = ARTICLE.replace("The harbor council", &format!("The harbor{magic} council")); + let with_meta = ARTICLE.replace( + "", + r#""#, + ); + let short = "

Too short.

"; + + for page in [marked.as_str(), with_meta.as_str(), short] { + let http = Arc::new(StubHttpClient::new()); + let kv = Arc::new(MemoryKv::default()); + + let output = run( + &factory(), + Some(request()), + &services(Arc::clone(&http) as Arc, &kv), + page, + ); + + assert_eq!(output, page); + assert_eq!(kv.reads(), 0, "should read no KV"); + assert!( + http.recorded_backend_names().is_empty(), + "should make no API call" + ); + } + } + + #[test] + fn logs_carry_outcomes_but_never_the_token() { + let kv = Arc::new(MemoryKv::default()); + let Some(logs) = capture_logs(|| { + view(&factory(), &services(signing_http(), &kv)); + view(&factory(), &services(Arc::new(StubHttpClient::new()), &kv)); + let failing = Arc::new(StubHttpClient::new()); + failing.push_response(403, b"{}".to_vec()); + view( + &factory(), + &services(failing, &Arc::new(MemoryKv::default())), + ); + }) else { + // Log lines go to a logger installed elsewhere and cannot be observed here. + return; + }; + + assert!( + logs.iter() + .any(|line| line.ends_with("outcome=signed reason=signed")), + "should log the signing outcome: {logs:?}" + ); + for line in &logs { + assert!(!line.contains(TOKEN), "log line leaks the token: {line}"); + assert!( + !line.contains("encypher:v3:") && !line.contains("encypher:policy:v2:"), + "log line leaks the cache key: {line}" + ); + assert!( + !line.contains("ferry"), + "log line leaks article text: {line}" + ); + } + } +} diff --git a/crates/provenance/encypher/src/tests.rs b/crates/provenance/encypher/src/tests.rs new file mode 100644 index 000000000..a9cbaf086 --- /dev/null +++ b/crates/provenance/encypher/src/tests.rs @@ -0,0 +1,399 @@ +use super::*; + +use std::sync::Arc; + +use edgezero_core::body::Body as EdgeBody; +use futures::executor::block_on; +use http::{Method, Request, header}; +use serde_json::json; +use trusted_server_core::auction::compile_auction_plan; +use trusted_server_core::auction::orchestrator::AuctionOrchestrator; +use trusted_server_core::cache_policy::EdgeCacheHeader; +use trusted_server_core::ec::EcContext; +use trusted_server_core::html_processor::test_support::page_settings; +use trusted_server_core::integrations::IntegrationRegistry; +use trusted_server_core::platform::test_support::{ + StubHttpClient, build_services_with_http_client, +}; +use edgezero_core::key_value_store::KvPage; +use trusted_server_core::platform::{ + KvError, PlatformHttpClient, PlatformKvStore, RuntimeServices, +}; +use trusted_server_core::publisher::{ + AppContext, AuctionDispatch, buffer_publisher_response_async, handle_publisher_request, + publisher_response_into_streaming_response, +}; + +const TOKEN: &str = "cwk_test_worker_token_value"; +const MARKER: &str = "\u{E0100}\u{E0101}\u{FE01}"; +const MANIFEST_URL: &str = "https://api.example.com/manifests/doc-1"; +const POLICY_DIGEST: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +const ARTICLE: &str = "Harbor Council Votes\ +

Harbor council votes on the ferry schedule

\ +

The harbor council met on Tuesday evening and approved a revised ferry schedule \ + for the coming season.

Residents asked for earlier departures on weekdays.

\ +
Footer
"; + +fn module_settings(table: &str, serve: bool) -> String { + let entry = if serve { + format!("[[serve]]\nmedia_type = \"text/html\"\nmiddleware = [\"{MODULE}\"]\n") + } else { + String::new() + }; + format!("[provenance]\nmodule = \"encypher\"\n\n[provenance.encypher]\n{table}\n{entry}") +} + +fn table() -> String { + format!( + "api_base_url = \"https://api.example.com\"\ndomain = \"publisher.example.com\"\n\ + worker_token = \"{TOKEN}\"\n" + ) +} + +fn placed() -> Settings { + page_settings(&module_settings(&table(), true)) +} + +fn config_value() -> serde_json::Value { + json!({ + "api_base_url": "https://api.example.com", + "domain": "publisher.example.com", + "worker_token": TOKEN, + }) +} + +fn parsed(value: &serde_json::Value) -> Result { + let config: EncypherConfig = + serde_json::from_value(value.clone()).map_err(|error| error.to_string())?; + config.validate().map_err(|error| error.to_string())?; + Ok(config) +} + +#[test] +fn module_constant_is_the_crate_folder() { + assert_eq!( + MODULE, + trusted_server_core::module_name!(), + "should be named by the folder this crate lives in" + ); +} + +#[test] +fn an_unselected_module_registers_nothing() { + let settings = page_settings(""); + + assert!(register(&settings).expect("should read").is_none()); + assert!(!validate(&settings).expect("should read")); +} + +#[test] +fn a_selected_and_placed_module_registers_its_middleware() { + let settings = placed(); + + assert!(validate(&settings).expect("should validate")); + assert!( + register(&settings).expect("should register").is_some(), + "a selected module with a worker token should register" + ); +} + +#[test] +fn a_module_named_in_no_serve_entry_is_refused() { + let settings = page_settings(&module_settings(&table(), false)); + + let error = validate(&settings).expect_err("should refuse the document"); + + assert!( + format!("{error:?}").contains("no [[serve]] entry names"), + "should name the missing entry: {error:?}" + ); +} + +#[test] +fn a_blank_worker_token_registers_nothing() { + for token in ["", " \t"] { + let table = format!( + "api_base_url = \"https://api.example.com\"\ndomain = \"publisher.example.com\"\n\ + worker_token = \"{token}\"\n" + ); + let settings = page_settings(&module_settings(&table, true)); + + assert!( + register(&settings).expect("should read").is_none(), + "token {token:?} should not register" + ); + } +} + +#[test] +fn required_settings_and_bounds_are_enforced() { + for field in ["domain", "api_base_url", "worker_token"] { + let mut config = config_value(); + config + .as_object_mut() + .expect("should be an object") + .remove(field); + assert!(parsed(&config).is_err(), "config without {field} should fail"); + } + let defaults = parsed(&config_value()).expect("should parse defaults"); + assert_eq!(defaults.policy_ttl_seconds, 300); + assert_eq!(defaults.plan_ttl_seconds, 604_800); + assert_eq!(defaults.max_hold_ms, 5_000); + for (field, value) in [ + ("api_base_url", json!("http://api.example.com")), + ("api_base_url", json!("https://api.example.com/?q=1")), + ("domain", json!("Publisher.example.com")), + ("domain", json!("www.publisher.example.com")), + ("domain", json!("publisher.example.com/path")), + ("domain", json!("localhost")), + ("timeout_ms", json!(100)), + ("timeout_ms", json!(20_000)), + ("min_text_length", json!(10)), + ("negative_ttl_seconds", json!(10)), + ("policy_ttl_seconds", json!(301)), + ("article_selector", json!("a".repeat(92))), + ("enabled", json!(true)), + ("path_prefixes", json!(["/news/"])), + ] { + let mut config = config_value(); + config[field] = value.clone(); + assert!(parsed(&config).is_err(), "{field} = {value} should be rejected"); + } + for (max_hold_ms, timeout_ms, valid) in [ + (499, 250, false), + (500, 250, true), + (15_000, 3_000, true), + (15_001, 3_000, false), + (3_000, 3_000, false), + (3_001, 3_000, true), + ] { + let mut config = config_value(); + config["max_hold_ms"] = json!(max_hold_ms); + config["timeout_ms"] = json!(timeout_ms); + assert_eq!( + parsed(&config).is_ok(), + valid, + "max_hold_ms {max_hold_ms} with timeout_ms {timeout_ms}" + ); + } +} + +#[test] +fn excluded_paths_leave_the_middleware_out_of_the_chain() { + let middleware = Encypher { + signer: Arc::new(Signer::new( + parsed(&config_value()).expect("should parse"), + 5_000_000, + )), + }; + + assert!(middleware.handles_path("/news/harbor-vote")); + for path in ["/wp-admin/post", "/feed/", "/sitemap.xml", "/app.js", "/api/v1/x"] { + assert!(!middleware.handles_path(path), "{path} should be left out"); + } + assert_eq!(middleware.phases(), [MiddlewarePhase::Serve]); +} + +/// A runtime KV store held in memory. +#[derive(Default)] +struct MemoryKv(std::sync::Mutex>); + +#[async_trait::async_trait(?Send)] +impl PlatformKvStore for MemoryKv { + async fn get_bytes(&self, key: &str) -> Result, KvError> { + Ok(self.0.lock().expect("should lock").get(key).cloned()) + } + + async fn put_bytes(&self, key: &str, value: bytes::Bytes) -> Result<(), KvError> { + self.0 + .lock() + .expect("should lock") + .insert(key.to_owned(), value); + Ok(()) + } + + async fn put_bytes_with_ttl( + &self, + key: &str, + value: bytes::Bytes, + _ttl: std::time::Duration, + ) -> Result<(), KvError> { + self.put_bytes(key, value).await + } + + async fn delete(&self, key: &str) -> Result<(), KvError> { + self.0.lock().expect("should lock").remove(key); + Ok(()) + } + + async fn list_keys_page( + &self, + _prefix: &str, + _cursor: Option<&str>, + _limit: usize, + ) -> Result { + Ok(KvPage::default()) + } +} + +/// An origin answering [`ARTICLE`] with `cache_control`, then the policy and +/// sign answers of the API. +fn stub(cache_control: &str) -> Arc { + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + ARTICLE.as_bytes().to_vec(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("cache-control", cache_control), + ], + ); + stub.push_response( + 200, + serde_json::to_vec(&json!({ "policy_digest": POLICY_DIGEST })).expect("should encode"), + ); + stub.push_response( + 200, + serde_json::to_vec(&json!({ + "success": true, + "policy_digest": POLICY_DIGEST, + "embedding_plan": { + "index_unit": "codepoint", + "operations": [{ "insert_after_index": 5, "marker": MARKER }], + }, + "manifest_url": MANIFEST_URL, + "verification_url": "https://api.example.com/verify/doc-1", + "document_id": "doc-1", + })) + .expect("should encode"), + ); + stub +} + +fn page_request(method: Method) -> Request { + Request::builder() + .method(method) + .uri("https://publisher.example.com/news/harbor-vote?utm_source=feed") + .header(header::HOST, "publisher.example.com") + .body(EdgeBody::empty()) + .expect("should build the page request") +} + +/// The page body a reader receives from the publisher path, through the +/// buffered or the streaming finalizer. +fn served( + settings: &Settings, + services: &RuntimeServices, + request: Request, + streaming: bool, +) -> String { + let registry = IntegrationRegistry::with_registrations(settings, &[builder()]) + .expect("should build the registry"); + let plan = compile_auction_plan(settings).expect("should compile the auction plan"); + let orchestrator = Arc::new(AuctionOrchestrator::from_plan(Arc::new(plan), None)); + let method = request.method().clone(); + block_on(async { + let mut ec_context = EcContext::read_from_request_resolving_geo(settings, &request, services) + .await + .expect("should read the request's state"); + let publisher = handle_publisher_request( + AppContext { + settings, + integration_registry: ®istry, + }, + services, + None, + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &[], + registry: None, + }, + request, + EdgeCacheHeader::SurrogateControl, + ) + .await + .expect("should fetch the page"); + let response = if streaming { + publisher_response_into_streaming_response( + publisher, + &method, + Arc::new(settings.clone()), + ®istry, + Arc::clone(&orchestrator), + services.clone(), + ) + .await + } else { + buffer_publisher_response_async( + publisher, + &method, + settings, + ®istry, + &orchestrator, + services, + ) + .await + } + .expect("should finish the page"); + let body = response + .into_body() + .into_bytes_bounded(16 * 1024 * 1024) + .await + .expect("should read the body"); + String::from_utf8(body.to_vec()).expect("should be UTF-8") + }) +} + +fn services_for(stub: &Arc) -> RuntimeServices { + build_services_with_http_client(Arc::clone(stub) as Arc) + .with_kv_store(Arc::new(MemoryKv::default())) +} + +#[test] +fn the_publisher_path_signs_an_article_through_both_finalizers() { + for streaming in [false, true] { + let stub = stub("public, max-age=60"); + let page = served( + &placed(), + &services_for(&stub), + page_request(Method::GET), + streaming, + ); + + assert!( + page.contains(MARKER), + "streaming={streaming}: should embed the marker: {page}" + ); + assert!( + page.contains("c2pa-manifest-url") && page.contains(MANIFEST_URL), + "streaming={streaming}: should write the manifest meta tag" + ); + assert_eq!( + stub.recorded_request_methods(), + ["GET", "GET", "POST"], + "streaming={streaming}: origin, policy, then sign" + ); + // The test request carries no TLS signal, so the scheme the publisher + // path detects is `http`. + let policy_uri = &stub.recorded_request_uris()[1]; + assert!( + policy_uri.ends_with("page_url=http%3A%2F%2Fpublisher.example.com%2Fnews%2Fharbor-vote"), + "should send the page address without its query: {policy_uri}" + ); + } +} + +#[test] +fn a_page_the_origin_marked_private_is_never_sent() { + let stub = stub("private, max-age=0"); + + let page = served(&placed(), &services_for(&stub), page_request(Method::GET), false); + + assert!(!page.contains(MARKER), "should not sign a reader's own page"); + assert_eq!( + stub.recorded_request_methods(), + ["GET"], + "should call the origin and nothing else" + ); +} diff --git a/crates/provenance/encypher/src/text.rs b/crates/provenance/encypher/src/text.rs new file mode 100644 index 000000000..28ebd3450 --- /dev/null +++ b/crates/provenance/encypher/src/text.rs @@ -0,0 +1,500 @@ +use std::sync::LazyLock; + +use regex::Regex; +use sha2::{Digest as _, Sha256}; + +/// Elements whose content canonical text drops: edge-core's set in +/// `integrations/edge-core/fragments.js`. +const TEXT_IGNORED: [&str; 7] = ["script", "style", "noscript", "svg", "math", "code", "pre"]; + +/// Elements whose content browsers never parse as markup (raw text and RCDATA). +/// A `/>` spelling does not close them, and the closing-tag search skips them. +const RAW_TEXT: [&str; 9] = [ + "script", + "style", + "noscript", + "title", + "textarea", + "xmp", + "iframe", + "noembed", + "plaintext", +]; + +static TITLE: LazyLock> = + LazyLock::new(|| Regex::new(r"(?is)]*>(.*?)").ok()); +static DECIMAL_ENTITY: LazyLock> = LazyLock::new(|| Regex::new(r"&#([0-9]+);").ok()); +static HEX_ENTITY: LazyLock> = + LazyLock::new(|| Regex::new(r"&#x([0-9a-fA-F]+);").ok()); + +/// A byte range containing one raw text run in an HTML document. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Fragment { + /// Inclusive UTF-8 byte offset at which the text begins. + pub start: usize, + /// Exclusive UTF-8 byte offset at which the text ends. + pub end: usize, +} + +/// Extracts visible raw-text fragments and their UTF-8 byte ranges. +#[must_use] +pub fn extract_fragments(html: &str) -> Vec { + let mut fragments = Vec::new(); + scan_html( + html, + |name| contains_tag(&TEXT_IGNORED, name), + |start, end| { + if html + .get(start..end) + .is_some_and(|raw| raw.chars().any(|ch| !is_js_whitespace(ch))) + { + fragments.push(Fragment { start, end }); + } + }, + |_, _| {}, + ); + fragments +} + +/// Decodes, normalizes, trims, and joins raw text fragments with single spaces. +#[must_use] +pub fn assemble_text(html: &str, fragments: &[Fragment]) -> String { + let mut output = String::new(); + for fragment in fragments { + if let Some(raw) = html.get(fragment.start..fragment.end) { + let decoded = decode_html_entities(raw); + let normalized = normalize_js_whitespace(&decoded); + if !normalized.is_empty() { + if !output.is_empty() { + output.push(' '); + } + output.push_str(&normalized); + } + } + } + output +} + +/// Returns the canonical visible text of an HTML string. +#[must_use] +pub fn visible_text(html: &str) -> String { + assemble_text(html, &extract_fragments(html)) +} + +/// Returns a lowercase SHA-256 digest of `text` as 64 hexadecimal characters. +#[must_use] +pub fn content_hash_hex(text: &str) -> String { + let digest = Sha256::digest(text.as_bytes()); + let mut output = String::with_capacity(64); + for byte in digest { + if let Some(high) = char::from_digit(u32::from(byte >> 4), 16) { + output.push(high); + } + if let Some(low) = char::from_digit(u32::from(byte & 0x0f), 16) { + output.push(low); + } + } + output +} + +/// Returns at most `max_scalars` Unicode scalar values without splitting one. +#[must_use] +pub fn truncate_scalars(text: &str, max_scalars: usize) -> &str { + let Some((end, _)) = text.char_indices().nth(max_scalars) else { + return text; + }; + text.get(..end).unwrap_or(text) +} + +/// Counts Unicode scalar values in `text`. +#[must_use] +pub fn scalar_count(text: &str) -> usize { + text.chars().count() +} + +/// Extracts the first title, decodes entities, normalizes whitespace, and caps it at 500 scalars. +#[must_use] +pub fn extract_title(html: &str) -> Option { + let regex = TITLE.as_ref()?; + let raw = regex.captures(html)?.get(1)?.as_str(); + let decoded = decode_html_entities(raw); + let mut normalized = normalize_js_whitespace(&decoded); + normalized.truncate(truncate_scalars(&normalized, 500).len()); + Some(normalized) +} + +/// Finds the first closing tag outside comments, the canonical-text ignored +/// elements, and raw-text or RCDATA elements. +#[must_use] +pub fn structural_close_tag(html: &str, tag: &str) -> Option { + let mut found = None; + scan_html( + html, + |name| contains_tag(&TEXT_IGNORED, name) || contains_tag(&RAW_TEXT, name), + |_, _| {}, + |start, token| { + if found.is_none() + && closing_tag_name(token).is_some_and(|name| name.eq_ignore_ascii_case(tag)) + { + found = Some(start); + } + }, + ); + found +} + +/// Scans tags, comments and text runs; both consumers share its comment and +/// quoted-attribute handling. `ignores` picks the elements whose content is +/// skipped: canonical text passes [`TEXT_IGNORED`], and the structural +/// closing-tag search passes [`TEXT_IGNORED`] and [`RAW_TEXT`]. +fn scan_html<'a>( + html: &'a str, + ignores: impl Fn(&str) -> bool, + mut text: impl FnMut(usize, usize), + mut tag: impl FnMut(usize, &'a str), +) { + let bytes = html.as_bytes(); + let mut cursor = 0; + let mut ignored: Option<&str> = None; + while cursor < bytes.len() { + if let Some(name) = ignored { + let Some(open) = find_byte(bytes, b'<', cursor) else { + break; + }; + cursor = open + 1; + if !starts_with_at(bytes, open, b"' || *byte == b'/' + }) + { + continue; + } + let Some(end) = tag_end(bytes, open + 2) else { + break; + }; + // As in browsers, attributes and `/` do not stop an end tag from + // closing. Unlike edge-core's depth count, no other end tag closes. + ignored = None; + cursor = end + 1; + continue; + } + if starts_with_at(bytes, cursor, b"` and `` are whole comments, and `--!>` also ends one. +fn comment_end(bytes: &[u8], start: usize) -> Option { + let body = start + 4; + if starts_with_at(bytes, body, b">") { + return Some(body + 1); + } + if starts_with_at(bytes, body, b"->") { + return Some(body + 2); + } + let dash = find_bytes(bytes, b"-->", body).map(|end| end + 3); + let bang = find_bytes(bytes, b"--!>", body).map(|end| end + 4); + match (dash, bang) { + (Some(dash), Some(bang)) => Some(dash.min(bang)), + (end, None) | (None, end) => end, + } +} + +/// Tokenizer states between a tag's name and its closing `>`. +#[derive(Clone, Copy)] +enum TagState { + Name, + BeforeAttributeName, + AttributeName, + BeforeValue, + Quoted(u8), + Unquoted, +} + +/// Returns the offset of the `>` ending the tag whose name starts at `start`. +/// As in the HTML tokenizer, a quote delimits only a value it begins. +fn tag_end(bytes: &[u8], start: usize) -> Option { + let mut state = TagState::Name; + for (offset, byte) in bytes.get(start..)?.iter().copied().enumerate() { + let space = byte.is_ascii_whitespace(); + state = match state { + TagState::Quoted(quote) => { + if byte == quote { + TagState::BeforeAttributeName + } else { + state + } + } + _ if byte == b'>' => return Some(start + offset), + TagState::Name if space || byte == b'/' => TagState::BeforeAttributeName, + TagState::BeforeAttributeName if !space && byte != b'/' => TagState::AttributeName, + TagState::AttributeName if byte == b'=' => TagState::BeforeValue, + TagState::AttributeName if byte == b'/' => TagState::BeforeAttributeName, + TagState::BeforeValue if byte == b'"' || byte == b'\'' => TagState::Quoted(byte), + TagState::BeforeValue if !space => TagState::Unquoted, + TagState::Unquoted if space => TagState::BeforeAttributeName, + _ => state, + }; + } + None +} + +pub(crate) fn decode_html_entities(input: &str) -> String { + let mut result = input.to_owned(); + for (encoded, decoded) in [ + ("&", "&"), + ("<", "<"), + (">", ">"), + (""", "\""), + ("'", "'"), + ("'", "'"), + ("'", "'"), + ("/", "/"), + (" ", "\u{00a0}"), + ("–", "\u{2013}"), + ("—", "\u{2014}"), + ("‘", "\u{2018}"), + ("’", "\u{2019}"), + ("“", "\u{201c}"), + ("”", "\u{201d}"), + ("…", "\u{2026}"), + ("•", "\u{2022}"), + ] { + if result.contains(encoded) { + result = result.replace(encoded, decoded); + } + } + result = decode_numeric_entities(&result, false); + decode_numeric_entities(&result, true) +} + +pub(crate) fn normalize_js_whitespace(input: &str) -> String { + let mut output = String::with_capacity(input.len()); + let mut pending_space = false; + for ch in input.chars() { + if is_js_whitespace(ch) { + pending_space = !output.is_empty(); + } else { + if pending_space { + output.push(' '); + pending_space = false; + } + output.push(ch); + } + } + output +} + +pub(crate) fn is_js_whitespace(ch: char) -> bool { + matches!( + ch, + '\u{0009}'..='\u{000d}' + | '\u{0020}' + | '\u{00a0}' + | '\u{1680}' + | '\u{2000}'..='\u{200a}' + | '\u{2028}' + | '\u{2029}' + | '\u{202f}' + | '\u{205f}' + | '\u{3000}' + | '\u{feff}' + ) +} + +fn decode_numeric_entities(input: &str, hexadecimal: bool) -> String { + let regex = if hexadecimal { + &*HEX_ENTITY + } else { + &*DECIMAL_ENTITY + }; + let Some(regex) = regex.as_ref() else { + return input.to_owned(); + }; + let radix = if hexadecimal { 16 } else { 10 }; + regex + .replace_all(input, |captures: ®ex::Captures<'_>| { + captures + .get(1) + .and_then(|digits| u32::from_str_radix(digits.as_str(), radix).ok()) + .and_then(char::from_u32) + .map_or_else( + || { + captures + .get(0) + .map_or("", |matched| matched.as_str()) + .to_owned() + }, + |decoded| decoded.to_string(), + ) + }) + .into_owned() +} + +fn find_byte(bytes: &[u8], needle: u8, start: usize) -> Option { + let relative = bytes + .get(start..)? + .iter() + .position(|byte| *byte == needle)?; + Some(start.saturating_add(relative)) +} + +fn find_bytes(bytes: &[u8], needle: &[u8], start: usize) -> Option { + if needle.is_empty() { + return Some(start.min(bytes.len())); + } + let haystack = bytes.get(start..)?; + let relative = haystack + .windows(needle.len()) + .position(|window| window == needle)?; + Some(start.saturating_add(relative)) +} + +fn starts_with_at(bytes: &[u8], start: usize, needle: &[u8]) -> bool { + bytes + .get(start..) + .is_some_and(|tail| tail.starts_with(needle)) +} + +fn opening_tag_name(token: &str) -> Option<&str> { + let content = token + .strip_prefix('<')? + .trim_start_matches(is_js_whitespace); + if content.starts_with('/') { + return None; + } + take_tag_name(content) +} + +fn closing_tag_name(token: &str) -> Option<&str> { + let content = token.strip_prefix("").then_some(name) +} + +fn take_tag_name(content: &str) -> Option<&str> { + let end = content + .char_indices() + .find(|(_, ch)| !(ch.is_ascii_alphanumeric() || *ch == '_')) + .map_or(content.len(), |(index, _)| index); + (end > 0).then(|| content.get(..end)).flatten() +} + +fn contains_tag(tags: &[&str], name: &str) -> bool { + tags.iter() + .any(|candidate| name.eq_ignore_ascii_case(candidate)) +} + +/// Matches edge-core's `/\/\s*>$/` test on a complete tag token. +fn is_self_closing(token: &str) -> bool { + token + .strip_suffix('>') + .is_some_and(|body| body.trim_end_matches(is_js_whitespace).ends_with('/')) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn quoted_markup_and_unmatched_script_tags_stay_inert() { + for html in [ + r#"
Visible
"#, + r#"
Visible
"#, + ] { + assert_eq!(structural_close_tag(html, "head"), html.rfind("")); + assert_eq!(visible_text(html), "Visible"); + } + } + + #[test] + fn browser_end_tag_and_comment_forms_close_where_browsers_do() { + // `` ends the script and `` is a whole comment, so each + // later `` sits inside an attribute value. + for prefix in [ + r#""#, + r#""#, + ] { + let html = format!("{prefix}
Visible
"); + assert_eq!(structural_close_tag(&html, "head"), Some(prefix.len())); + } + assert_eq!( + visible_text("abce