diff --git a/crates/trusted-server-cli/Cargo.toml b/crates/trusted-server-cli/Cargo.toml index ce97919c81..8e5cfd60fd 100644 --- a/crates/trusted-server-cli/Cargo.toml +++ b/crates/trusted-server-cli/Cargo.toml @@ -28,24 +28,27 @@ log = { workspace = true } rand = { workspace = true } regex = { workspace = true } rpassword = { workspace = true } -# Also in the macOS/Linux block for the dev proxy; needed on every host so the probe can install -# the process-level crypto provider its `-no-provider` reqwest build expects. -rustls = { workspace = true } # Declared directly rather than inherited from the workspace, to control the rustls # crypto provider. `reqwest`'s plain `rustls-tls` forces `ring`, and this crate already -# links `aws-lc-rs` through `reqwest` 0.13 (via `chromiumoxide` and `edgezero-cli`). +# links `aws-lc-rs` through `reqwest` (via `chromiumoxide` and `edgezero-cli`). # Compiling both providers makes rustls's process-level default ambiguous, and it then # panics on first use — which broke the dev-proxy tests. The `-no-provider` variant uses # whichever provider is already present instead of adding a second. # -# `gzip` so the probe's Accept-Encoding axis compares decoded bytes: reqwest transparently -# decodes a gzip response, which is what "differ only by content coding" must be tested -# against. +# Features: `blocking` for the `ts origin audit-headers` origin client (OriginClient uses +# reqwest::blocking); `gzip`/`json` for the probe's Accept-Encoding axis and JSON bodies. reqwest = { version = "0.12", default-features = false, features = [ + "blocking", "gzip", "json", "rustls-tls-webpki-roots-no-provider", ] } +# The `-no-provider` reqwest variant expects the process-level rustls `CryptoProvider` +# to be installed by the caller. `ts dev proxy` only exists on macOS, so on other hosts +# nothing else pulls a rustls provider; this pins aws-lc-rs (rustls's default) on every +# host target so `ts origin audit-headers` can install it before its first HTTPS fetch. On macOS it +# deduplicates with the proxy's aws-lc-rs — still exactly one provider, so no ambiguity. +rustls = { workspace = true } scraper = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } @@ -116,4 +119,4 @@ flate2 = { workspace = true } predicates = { workspace = true } temp-env = { workspace = true } tempfile = { workspace = true } -tokio = { workspace = true, features = ["test-util"] } +tokio = { workspace = true, features = ["test-util"] } \ No newline at end of file diff --git a/crates/trusted-server-cli/src/commands/origin/audit_headers/analyze.rs b/crates/trusted-server-cli/src/commands/origin/audit_headers/analyze.rs new file mode 100644 index 0000000000..39bb6ba2e6 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/origin/audit_headers/analyze.rs @@ -0,0 +1,360 @@ +//! Groups fetched responses by content type, evaluates each against the +//! cacheability rules, and rolls the per-header verdicts up into a per-type +//! report. + +use std::collections::BTreeMap; + +use serde::Serialize; +use url::Url; + +use super::rules::{ContentTypeGroup, HeaderVerdict, ResponseHeaders, Verdict, evaluate}; +use crate::run::RunOutcome; + +/// One fetched origin response, paired with the URL it came from. +#[derive(Debug, Clone)] +pub(crate) struct FetchedResponse { + /// The URL that was fetched. + pub(crate) url: Url, + /// The cache-relevant headers of the response. + pub(crate) headers: ResponseHeaders, +} + +/// The audit outcome for a single content-type group. +#[derive(Debug, Clone, Serialize)] +pub(crate) struct GroupReport { + /// The content-type family this report covers. + pub(crate) content_type: ContentTypeGroup, + /// The URLs sampled into this group. + pub(crate) urls_sampled: Vec, + /// Group-level (per-type) verdict: worst-of rollup over `verdicts`. + pub(crate) verdict: Verdict, + /// The per-header verdicts, merged worst-of across sampled responses. + pub(crate) verdicts: Vec, +} + +/// Per-content-type-group tallies. `pass + warn + fail == total_groups`. +#[derive(Debug, Clone, Serialize)] +pub(crate) struct AuditSummary { + /// Number of evaluated content-type groups. + pub(crate) total_groups: usize, + /// Groups whose rollup verdict is [`Verdict::Pass`]. + pub(crate) pass: usize, + /// Groups whose rollup verdict is [`Verdict::Warn`]. + pub(crate) warn: usize, + /// Groups whose rollup verdict is [`Verdict::Fail`]. + pub(crate) fail: usize, +} + +/// The full audit report for one origin. +#[derive(Debug, Clone, Serialize)] +pub(crate) struct AuditReport { + /// The origin that was audited. + pub(crate) origin: String, + /// One report per evaluated content-type group. + pub(crate) groups: Vec, + /// Group-level tallies. + pub(crate) summary: AuditSummary, +} + +impl AuditReport { + /// The process outcome per the spec's exit-code contract (§59-69): + /// a `FAIL` verdict — or a `WARN` verdict under `--strict` — is an + /// assertion failure (exit 1); everything else is success (exit 0). + /// Warnings alone exit 0 unless `strict` promotes them. Transport and + /// config errors never reach here: they return `Err` and `main` exits 2, + /// so a warn-only audit is never confused with an audit that could not run. + pub(crate) fn outcome(&self, strict: bool) -> RunOutcome { + if self.summary.fail > 0 || (strict && self.summary.warn > 0) { + RunOutcome::AssertionFailed + } else { + RunOutcome::Success + } + } +} + +/// Builds an [`AuditReport`] from fetched responses. +/// +/// Responses classified as [`ContentTypeGroup::Other`] are ignored — that group +/// is informational and never evaluated. Within a group, per-header verdicts are +/// merged worst-of across every sampled response, so a single bad response +/// downgrades the group. +pub(crate) fn run_analysis(origin: &str, responses: &[FetchedResponse]) -> AuditReport { + // Preserve a stable, deterministic group ordering for output. + let mut grouped: BTreeMap = BTreeMap::new(); + + for response in responses { + let content_type = response.headers.content_type.as_deref().unwrap_or_default(); + let group = ContentTypeGroup::classify(content_type); + if matches!(group, ContentTypeGroup::Other) { + continue; + } + + let accumulator = grouped + .entry(group_order(group)) + .or_insert_with(|| GroupAccumulator::new(group)); + accumulator.add(response.url.as_str(), evaluate(group, &response.headers)); + } + + let groups: Vec = grouped + .into_values() + .map(GroupAccumulator::finish) + .collect(); + + let summary = summarize(&groups); + + AuditReport { + origin: origin.to_owned(), + groups, + summary, + } +} + +/// A stable sort key so groups always render in the same order. +fn group_order(group: ContentTypeGroup) -> usize { + match group { + ContentTypeGroup::Html => 0, + ContentTypeGroup::JavaScript => 1, + ContentTypeGroup::Image => 2, + ContentTypeGroup::StaticAsset => 3, + ContentTypeGroup::RtbJson => 4, + ContentTypeGroup::Other => 5, + } +} + +/// Accumulates sampled URLs and merges per-header verdicts for one group. +struct GroupAccumulator { + content_type: ContentTypeGroup, + urls_sampled: Vec, + // Header name -> merged worst-of verdict, preserving first-seen order. + verdicts: Vec, +} + +impl GroupAccumulator { + fn new(content_type: ContentTypeGroup) -> Self { + Self { + content_type, + urls_sampled: Vec::new(), + verdicts: Vec::new(), + } + } + + fn add(&mut self, url: &str, verdicts: Vec) { + if !self.urls_sampled.iter().any(|sampled| sampled == url) { + self.urls_sampled.push(url.to_owned()); + } + for incoming in verdicts { + match self.verdicts.iter_mut().find(|existing| { + // Key on (header, expected), not header alone: a single + // response can emit two distinct checks under one header + // name (e.g. Surrogate-Control's CDN-TTL comparison and its + // disables-caching check). They carry different `expected` + // strings, so keying on header alone would silently drop the + // second — folding *different* checks instead of the same + // check across sampled URLs. + existing.header == incoming.header && existing.expected == incoming.expected + }) { + // Keep the more severe verdict when the same check is seen on + // multiple sampled responses in this group. + Some(existing) if is_more_severe(incoming.verdict, existing.verdict) => { + *existing = incoming; + } + Some(_) => {} + None => self.verdicts.push(incoming), + } + } + } + + fn finish(self) -> GroupReport { + let verdict = Verdict::rollup(self.verdicts.iter().map(|entry| entry.verdict)); + GroupReport { + content_type: self.content_type, + urls_sampled: self.urls_sampled, + verdict, + verdicts: self.verdicts, + } + } +} + +/// Whether `candidate` is strictly more severe than `current`. +fn is_more_severe(candidate: Verdict, current: Verdict) -> bool { + severity(candidate) > severity(current) +} + +fn severity(verdict: Verdict) -> u8 { + match verdict { + Verdict::Pass => 0, + Verdict::Warn => 1, + Verdict::Fail => 2, + } +} + +fn summarize(groups: &[GroupReport]) -> AuditSummary { + let mut summary = AuditSummary { + total_groups: groups.len(), + pass: 0, + warn: 0, + fail: 0, + }; + for group in groups { + match group.verdict { + Verdict::Pass => summary.pass += 1, + Verdict::Warn => summary.warn += 1, + Verdict::Fail => summary.fail += 1, + } + } + summary +} + +#[cfg(test)] +mod tests { + use super::*; + + fn response(url: &str, content_type: &str, cache_control: &str) -> FetchedResponse { + FetchedResponse { + url: Url::parse(url).expect("should parse test url"), + headers: ResponseHeaders { + content_type: Some(content_type.to_owned()), + cache_control: Some(cache_control.to_owned()), + ..ResponseHeaders::default() + }, + } + } + + #[test] + fn ignores_other_content_types() { + let responses = vec![response( + "https://origin.example/data.bin", + "application/octet-stream", + "public, max-age=60", + )]; + let report = run_analysis("https://origin.example", &responses); + assert_eq!( + report.summary.total_groups, 0, + "Other group should not be counted" + ); + } + + #[test] + fn counts_groups_by_rollup_verdict() { + let responses = vec![ + response("https://origin.example/", "text/html", "no-store"), + response( + "https://origin.example/rtb", + "application/json", + "public, max-age=60", + ), + ]; + let report = run_analysis("https://origin.example", &responses); + assert_eq!(report.summary.total_groups, 2, "should count HTML and RTB"); + assert_eq!( + report.summary.pass, 2, + "HTML no-store and origin JSON (no assumed posture) both pass" + ); + assert_eq!( + report.summary.fail, 0, + "cacheable origin JSON is not failed — it carries no assumed posture" + ); + assert_eq!( + report.outcome(false), + RunOutcome::Success, + "no failures should succeed (exit 0)" + ); + } + + #[test] + fn worst_of_merge_within_group() { + // Two HTML responses: one safe, one cacheable. The group must fail. + let responses = vec![ + response("https://origin.example/a", "text/html", "no-store"), + response( + "https://origin.example/b", + "text/html", + "public, max-age=3600", + ), + ]; + let report = run_analysis("https://origin.example", &responses); + assert_eq!(report.groups.len(), 1, "both HTML pages share one group"); + assert_eq!( + report.groups[0].verdict, + Verdict::Fail, + "a single cacheable HTML response should fail the group" + ); + assert_eq!( + report.groups[0].urls_sampled.len(), + 2, + "both sampled URLs should be recorded" + ); + } + + #[test] + fn warn_only_succeeds_unless_strict() { + let responses = vec![response( + "https://origin.example/app.js", + "application/javascript", + "public, max-age=600", + )]; + let report = run_analysis("https://origin.example", &responses); + assert_eq!(report.summary.warn, 1, "non-immutable JS should warn"); + assert_eq!( + report.outcome(false), + RunOutcome::Success, + "warnings alone exit 0 (spec §59-69); 2 is reserved for a run that could not complete" + ); + assert_eq!( + report.outcome(true), + RunOutcome::AssertionFailed, + "--strict promotes a warning to a failing exit 1" + ); + } + + #[test] + fn distinct_surrogate_control_findings_are_not_collapsed() { + // Two image responses in the same group each emit ONE distinct + // Surrogate-Control verdict: response A the CDN-TTL comparison + // (max-age=30 < browser 86400 -> Warn), response B disables-caching + // (no-store -> Warn). They are legitimately distinct postures now that + // `no-store` no longer also borrows a numeric TTL. Keying the merge on + // header name alone dropped the second; keying on (header, expected) + // must keep both, and keep their verdict values intact. + let response_a = FetchedResponse { + url: Url::parse("https://origin.example/a.png").expect("should parse test url"), + headers: ResponseHeaders { + content_type: Some("image/png".to_owned()), + cache_control: Some("public, max-age=86400".to_owned()), + surrogate_control: Some("max-age=30".to_owned()), + surrogate_key: Some("img".to_owned()), + ..ResponseHeaders::default() + }, + }; + let response_b = FetchedResponse { + url: Url::parse("https://origin.example/b.png").expect("should parse test url"), + headers: ResponseHeaders { + content_type: Some("image/png".to_owned()), + cache_control: Some("public, max-age=86400".to_owned()), + surrogate_control: Some("no-store".to_owned()), + surrogate_key: Some("img".to_owned()), + ..ResponseHeaders::default() + }, + }; + let report = run_analysis("https://origin.example", &[response_a, response_b]); + let image = report + .groups + .iter() + .find(|group| matches!(group.content_type, ContentTypeGroup::Image)) + .expect("should have an image group"); + let surrogate: Vec<(&str, Verdict)> = image + .verdicts + .iter() + .filter(|verdict| verdict.header == "Surrogate-Control") + .map(|verdict| (verdict.expected.as_str(), verdict.verdict)) + .collect(); + assert!( + surrogate.contains(&("CDN TTL >= browser TTL", Verdict::Warn)), + "CDN-TTL finding must survive as a Warn, got {surrogate:?}" + ); + assert!( + surrogate.contains(&("should allow CDN caching", Verdict::Warn)), + "disables-caching finding must survive as a Warn, got {surrogate:?}" + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/origin/audit_headers/fetch.rs b/crates/trusted-server-cli/src/commands/origin/audit_headers/fetch.rs new file mode 100644 index 0000000000..6b4d9cde47 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/origin/audit_headers/fetch.rs @@ -0,0 +1,559 @@ +//! Origin fetching, config-derived URL discovery, and response classification. +//! +//! `ts origin audit-headers` fetches directly from the publisher origin. When no +//! explicit `--url` is given, it fetches the origin root, parses the returned HTML +//! for asset URLs, and audits those. Routes under `/_ts/` are served by the +//! Trusted Server edge — not the origin — so they are never probed here; supply +//! them explicitly if you need to audit them. +//! +//! Every target must use a credential-safe transport (HTTPS, or HTTP only for a +//! loopback development origin), enforced through the shared +//! [`crate::url_guard`] used by `ts origin probe-shareability`. + +use std::io::Read; +use std::path::PathBuf; +use std::time::Duration; + +use scraper::{Html, Selector}; +use url::Url; + +use super::analyze::FetchedResponse; +use super::rules::ResponseHeaders; +use crate::error::{CliResult, cli_error}; + +/// Caps discovery so one crawl of a large page can't fan out unbounded. +const MAX_DISCOVERED_URLS: usize = 50; + +/// Per-request timeout for origin fetches. +const REQUEST_TIMEOUT: Duration = Duration::from_secs(15); + +/// Cap on how many body bytes are read per response. Only the root page's HTML +/// is ever parsed for discovery; every other body is discarded, so this bounds +/// a fast link from pulling an unbounded payload (a large image or video) that +/// the timeout alone would not catch. +const MAX_BODY_BYTES: u64 = 2 * 1024 * 1024; + +/// Arguments for `ts origin audit-headers`. +#[derive(Debug, clap::Args)] +pub struct AuditHeadersArgs { + /// URL to audit. Repeat for several pages. When omitted, URLs are + /// discovered from the origin root page. Every URL must be HTTPS; plain + /// HTTP is accepted only for a loopback development origin. + #[arg(long = "url")] + pub url: Vec, + /// Path to `trusted-server.toml`, used to resolve the origin when no URLs + /// and no `--origin` are given. + #[arg(long, default_value = "trusted-server.toml")] + pub config: PathBuf, + /// Override the origin URL, skipping the config lookup. + #[arg(long)] + pub origin: Option, + /// Follow discovered asset URLs that resolve to a different origin. By + /// default the automatic crawl audits only same-origin assets; explicit + /// URLs are always fetched regardless of this flag. + #[arg(long)] + pub include_cross_origin: bool, + /// Treat WARN verdicts as failures (exit 1), so a CI gate can require a + /// clean posture rather than special-casing an exit code. + #[arg(long)] + pub strict: bool, + /// Emit machine-readable JSON instead of the human table. + #[arg(long)] + pub json: bool, +} + +/// A single origin response: the cache-relevant headers plus the body (needed +/// only to discover assets from the root HTML page). +#[derive(Debug, Clone)] +pub(crate) struct OriginResponse { + /// The cache-relevant response headers. + pub(crate) headers: ResponseHeaders, + /// The response body as text (empty when not decodable as UTF-8). + pub(crate) body: String, +} + +/// Fetches origin responses. Abstracted so the fetch/discovery logic is +/// unit-testable without real network I/O, mirroring the `audit` command's +/// `AuditCollector`. +pub(crate) trait OriginClient { + /// Fetches a single URL from the origin. + fn fetch(&self, url: &Url) -> CliResult; +} + +/// The production [`OriginClient`], backed by a blocking `reqwest` client. +pub(crate) struct ReqwestOriginClient { + client: reqwest::blocking::Client, +} + +impl ReqwestOriginClient { + /// Builds a client with a bounded per-request timeout and no redirect + /// following. Redirects are not followed so the audited URL's own cache + /// headers are classified on their merits — a long-lived cached 3xx is a + /// real misconfiguration this tool should surface, and following would + /// silently attribute the final hop's headers to the requested URL. + pub(crate) fn new() -> CliResult { + // reqwest is built with `rustls-tls-webpki-roots-no-provider`, which + // installs no rustls crypto provider. `ts dev proxy` (macOS-only) + // installs aws-lc-rs, but on other hosts nothing does, so the first + // HTTPS build would panic with "no provider set". Install it here; + // `install_default` is idempotent and returns `Err` if one is already + // installed (e.g. by the proxy on macOS), which is fine to ignore. + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + + let client = reqwest::blocking::Client::builder() + .timeout(REQUEST_TIMEOUT) + .redirect(reqwest::redirect::Policy::none()) + .build() + .map_err(|error| format!("failed to build HTTP client: {error}"))?; + Ok(Self { client }) + } +} + +impl OriginClient for ReqwestOriginClient { + fn fetch(&self, url: &Url) -> CliResult { + let response = self + .client + .get(url.clone()) + .send() + .map_err(|error| format!("failed to fetch {url}: {error}"))?; + + let headers = extract_headers(response.headers()); + + // Read at most MAX_BODY_BYTES: only the root page's HTML is parsed for + // discovery, so a truncated body is harmless for every other response. + let mut buffer = Vec::new(); + let _ = response.take(MAX_BODY_BYTES).read_to_end(&mut buffer); + let body = String::from_utf8_lossy(&buffer).into_owned(); + + Ok(OriginResponse { headers, body }) + } +} + +/// Pulls the cache-relevant headers out of a `reqwest` header map. +fn extract_headers(headers: &reqwest::header::HeaderMap) -> ResponseHeaders { + let value = |name: &str| { + headers + .get(name) + .and_then(|value| value.to_str().ok()) + .map(str::to_owned) + }; + ResponseHeaders { + content_type: value("content-type"), + cache_control: value("cache-control"), + surrogate_control: value("surrogate-control"), + surrogate_key: value("surrogate-key"), + vary: value("vary"), + etag: value("etag"), + } +} + +/// Resolves the target URLs and fetches each, returning classified responses. +pub(crate) fn collect_responses( + args: &AuditHeadersArgs, + client: &dyn OriginClient, +) -> CliResult<(String, Vec)> { + if args.url.is_empty() { + collect_via_discovery(args, client) + } else { + collect_explicit(&args.url, client) + } +} + +/// Fetches an explicit list of URLs. A malformed URL is a hard error (operator +/// typo), but a per-URL fetch failure is tolerated with a warning so one +/// unreachable URL does not discard the verdicts for the others; only an +/// all-fail run errors out. +fn collect_explicit( + urls: &[String], + client: &dyn OriginClient, +) -> CliResult<(String, Vec)> { + let mut responses = Vec::with_capacity(urls.len()); + for raw in urls { + let url = Url::parse(raw).map_err(|error| format!("invalid URL `{raw}`: {error}"))?; + // Transport safety before any fetch: refuse plain HTTP off loopback and + // URL-embedded credentials, shared with `ts origin probe-shareability`. + crate::url_guard::require_credential_safe_transport(&url, "--url")?; + match client.fetch(&url) { + Ok(response) => responses.push(FetchedResponse { + url, + headers: response.headers, + }), + Err(error) => log::warn!("skipping {url}: {error}"), + } + } + if responses.is_empty() { + return cli_error("all requested URLs failed to fetch"); + } + // With explicit URLs there is no single origin; report the first host. + let origin = responses + .first() + .map(|response| response.url.origin().ascii_serialization()) + .unwrap_or_default(); + Ok((origin, responses)) +} + +/// Resolves the origin, fetches its root, and audits the assets discovered in +/// the returned HTML alongside the root itself. +fn collect_via_discovery( + args: &AuditHeadersArgs, + client: &dyn OriginClient, +) -> CliResult<(String, Vec)> { + let origin = resolve_origin(args)?; + let root = + Url::parse(&origin).map_err(|error| format!("invalid origin `{origin}`: {error}"))?; + // The origin root carries no credential, but the same transport rule keeps + // the audit from fetching a plain-HTTP origin off loopback. + crate::url_guard::require_credential_safe_transport(&root, "origin")?; + + let root_response = client.fetch(&root)?; + let discovered = discover_asset_urls(&root, &root_response.body, args.include_cross_origin); + + let mut responses = Vec::with_capacity(discovered.len() + 1); + responses.push(FetchedResponse { + url: root.clone(), + headers: root_response.headers, + }); + + for url in discovered { + // A discovered asset off a safe transport is skipped, not fatal — one + // bad link must not abort the audit of the rest. + if let Err(error) = + crate::url_guard::require_credential_safe_transport(&url, "discovered URL") + { + log::warn!("skipping {url}: {error}"); + continue; + } + // A single asset fetch failing should not abort the whole audit. + match client.fetch(&url) { + Ok(response) => responses.push(FetchedResponse { + url, + headers: response.headers, + }), + Err(error) => log::warn!("skipping {url}: {error}"), + } + } + + Ok((origin, responses)) +} + +/// Resolves the origin from `--origin`, else from the config's +/// `publisher.origin_url`. +fn resolve_origin(args: &AuditHeadersArgs) -> CliResult { + if let Some(origin) = &args.origin { + return Ok(origin.clone()); + } + + let text = std::fs::read_to_string(&args.config).map_err(|error| { + format!( + "failed to read config {}: {error}; pass --origin to audit without a config", + args.config.display() + ) + })?; + let parsed: toml::Value = toml::from_str(&text) + .map_err(|error| format!("failed to parse {}: {error}", args.config.display()))?; + + parsed + .get("publisher") + .and_then(|publisher| publisher.get("origin_url")) + .and_then(toml::Value::as_str) + .map(str::to_owned) + .ok_or(()) + .or_else(|()| { + cli_error(format!( + "publisher.origin_url not found in {}", + args.config.display() + )) + }) +} + +/// Extracts asset URLs from HTML: ` + + + + "#; + let urls = discover_asset_urls(&base, html, false); + assert!( + urls.contains(&Url::parse("https://origin.example/app.js").unwrap()), + "should discover script src" + ); + assert!( + urls.contains(&Url::parse("https://origin.example/style.css").unwrap()), + "should discover stylesheet href" + ); + assert!( + urls.contains(&Url::parse("https://origin.example/logo.png").unwrap()), + "should discover img src" + ); + } + + #[test] + fn discovery_excludes_cross_origin_by_default() { + let base = Url::parse("https://origin.example").expect("should parse base"); + let html = r#" + + + + + "#; + let urls = discover_asset_urls(&base, html, false); + assert!( + urls.contains(&Url::parse("https://origin.example/local.js").unwrap()), + "should keep the same-origin asset" + ); + assert!( + !urls + .iter() + .any(|url| url.host_str() == Some("cdn.other.example")), + "should not follow a cross-origin asset by default" + ); + } + + #[test] + fn discovery_includes_cross_origin_when_opted_in() { + let base = Url::parse("https://origin.example").expect("should parse base"); + let html = r#" + + "#; + let urls = discover_asset_urls(&base, html, true); + assert!( + urls.iter() + .any(|url| url.host_str() == Some("cdn.other.example")), + "should follow a cross-origin asset when opted in" + ); + } + + #[test] + fn discovery_fetches_root_and_assets_but_never_ts_routes() { + let html = r#""#; + let client = FakeClient::new() + .with("https://origin.example/", "text/html", html) + .with("https://origin.example/logo.png", "image/png", "") + .with("https://origin.example/favicon.ico", "image/x-icon", ""); + + let (origin, responses) = + collect_responses(&args("https://origin.example"), &client).expect("should collect"); + + assert_eq!(origin, "https://origin.example"); + let fetched = client.fetched.borrow(); + assert!( + fetched.iter().any(|url| url == "https://origin.example/"), + "should fetch the root" + ); + assert!( + fetched + .iter() + .any(|url| url == "https://origin.example/logo.png"), + "should fetch the discovered image" + ); + assert!( + !fetched.iter().any(|url| url.contains("/_ts/")), + "should never probe edge-only /_ts/ routes against the origin" + ); + assert!( + !responses.is_empty(), + "should return at least the root response" + ); + } + + #[test] + fn explicit_urls_skip_discovery() { + let client = FakeClient::new().with("https://origin.example/rtb", "application/json", ""); + let explicit = AuditHeadersArgs { + url: vec!["https://origin.example/rtb".to_owned()], + config: PathBuf::from("trusted-server.toml"), + origin: None, + include_cross_origin: false, + strict: false, + json: false, + }; + let (_, responses) = collect_responses(&explicit, &client).expect("should collect"); + assert_eq!(responses.len(), 1, "should fetch only the explicit URL"); + assert_eq!( + client.fetched.borrow().len(), + 1, + "should not fetch a root page for explicit URLs" + ); + } + + #[test] + fn invalid_explicit_url_is_rejected() { + let client = FakeClient::new(); + let explicit = AuditHeadersArgs { + url: vec!["not a url".to_owned()], + config: PathBuf::from("trusted-server.toml"), + origin: None, + include_cross_origin: false, + strict: false, + json: false, + }; + let error = collect_responses(&explicit, &client).expect_err("should reject invalid URL"); + assert!(error.contains("invalid URL"), "should explain the failure"); + } + + #[test] + fn explicit_urls_tolerate_partial_failure() { + // One reachable URL, one unreachable: the audit should keep the good + // verdict rather than aborting on the first failure. + let client = FakeClient::new().with("https://origin.example/ok", "text/html", ""); + let explicit = AuditHeadersArgs { + url: vec![ + "https://origin.example/ok".to_owned(), + "https://origin.example/down".to_owned(), + ], + config: PathBuf::from("trusted-server.toml"), + origin: None, + include_cross_origin: false, + strict: false, + json: false, + }; + let (_, responses) = + collect_responses(&explicit, &client).expect("should tolerate one failure"); + assert_eq!( + responses.len(), + 1, + "should keep the reachable URL's response" + ); + } + + #[test] + fn explicit_urls_error_when_all_fail() { + let client = FakeClient::new(); + let explicit = AuditHeadersArgs { + url: vec!["https://origin.example/down".to_owned()], + config: PathBuf::from("trusted-server.toml"), + origin: None, + include_cross_origin: false, + strict: false, + json: false, + }; + let error = + collect_responses(&explicit, &client).expect_err("should error when all URLs fail"); + assert!( + error.contains("all requested URLs failed"), + "should explain the all-fail case" + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/origin/audit_headers/mod.rs b/crates/trusted-server-cli/src/commands/origin/audit_headers/mod.rs new file mode 100644 index 0000000000..91643d2b23 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/origin/audit_headers/mod.rs @@ -0,0 +1,203 @@ +//! `ts origin audit-headers` — audits origin cache-header posture per content type. +//! +//! Fetches origin responses (explicit `--url`, or discovered from the origin +//! root), classifies them by content type, evaluates each against the +//! cacheability rules, and renders a per-type pass/warn/fail report. A `FAIL` +//! verdict — or a `WARN` under `--strict` — is an assertion failure (exit 1); +//! otherwise it succeeds (exit 0). Transport or config errors return `Err`, so +//! `main` exits 2 and a CI gate can tell a warn-only audit from one that never +//! ran. + +mod analyze; +mod fetch; +mod output; +mod rules; + +use std::io::Write; + +pub use fetch::AuditHeadersArgs; + +use crate::error::CliResult; +use crate::run::RunOutcome; +use analyze::AuditReport; +use fetch::{OriginClient, ReqwestOriginClient, collect_responses}; + +/// Runs the header audit against a live origin, writing the report to `out` and +/// mapping the result to a [`RunOutcome`] (`AssertionFailed` on a FAIL, or a +/// WARN under `--strict`; otherwise `Success`). Transport/config failures +/// propagate as `Err`, which `main` turns into exit 2. +pub(crate) fn run(args: &AuditHeadersArgs, out: &mut dyn Write) -> CliResult { + let client = ReqwestOriginClient::new()?; + let report = run_audit_headers(args, &client, out)?; + Ok(report.outcome(args.strict)) +} + +/// Fetches, analyzes, and renders the audit, returning the report. +/// +/// Kept separate from [`run`] — with the [`OriginClient`] and output sink +/// injected — so the full pipeline is unit-testable without network I/O or +/// touching stdout. +fn run_audit_headers( + args: &AuditHeadersArgs, + client: &dyn OriginClient, + out: &mut dyn Write, +) -> CliResult { + let (origin, responses) = collect_responses(args, client)?; + let report = analyze::run_analysis(&origin, &responses); + output::render(&report, args.json, out)?; + Ok(report) +} + +#[cfg(test)] +mod tests { + use std::cell::RefCell; + use std::collections::HashMap; + use std::path::PathBuf; + + use url::Url; + + use super::fetch::OriginResponse; + use super::rules::ResponseHeaders; + use super::*; + + struct FakeClient { + responses: HashMap, + fetched: RefCell>, + } + + impl FakeClient { + fn new() -> Self { + Self { + responses: HashMap::new(), + fetched: RefCell::new(Vec::new()), + } + } + + fn insert(&mut self, url: &str, headers: ResponseHeaders, body: &str) { + self.responses.insert( + url.to_owned(), + OriginResponse { + headers, + body: body.to_owned(), + }, + ); + } + } + + impl OriginClient for FakeClient { + fn fetch(&self, url: &Url) -> CliResult { + self.fetched.borrow_mut().push(url.to_string()); + self.responses + .get(url.as_str()) + .cloned() + .ok_or_else(|| format!("no fake response for {url}")) + } + } + + fn args_for(origin: &str, json: bool) -> AuditHeadersArgs { + AuditHeadersArgs { + url: Vec::new(), + config: PathBuf::from("trusted-server.toml"), + origin: Some(origin.to_owned()), + include_cross_origin: false, + strict: false, + json, + } + } + + #[test] + fn end_to_end_discovery_produces_report() { + let html = r#" + + "#; + + let mut client = FakeClient::new(); + client.insert( + "https://origin.example/", + ResponseHeaders { + content_type: Some("text/html".to_owned()), + cache_control: Some("no-store".to_owned()), + ..ResponseHeaders::default() + }, + html, + ); + client.insert( + "https://origin.example/app.js", + ResponseHeaders { + content_type: Some("application/javascript".to_owned()), + cache_control: Some("public, max-age=31536000, immutable".to_owned()), + surrogate_key: Some("js".to_owned()), + etag: Some("\"abc\"".to_owned()), + ..ResponseHeaders::default() + }, + "", + ); + client.insert( + "https://origin.example/favicon.ico", + ResponseHeaders { + content_type: Some("image/x-icon".to_owned()), + cache_control: Some("public, max-age=86400".to_owned()), + surrogate_key: Some("icon".to_owned()), + ..ResponseHeaders::default() + }, + "", + ); + + let mut buffer = Vec::new(); + let report = run_audit_headers( + &args_for("https://origin.example", false), + &client, + &mut buffer, + ) + .expect("should run end to end"); + + assert!( + report.summary.total_groups >= 2, + "should audit at least HTML and JS groups" + ); + assert_eq!( + report.outcome(false), + RunOutcome::Success, + "clean posture should succeed (exit 0)" + ); + let text = String::from_utf8(buffer).expect("should be utf-8"); + assert!(text.contains("Origin:"), "should render the report"); + } + + #[test] + fn failing_group_sets_exit_code_one() { + let mut client = FakeClient::new(); + client.insert( + "https://origin.example/", + ResponseHeaders { + content_type: Some("text/html".to_owned()), + cache_control: Some("public, max-age=3600".to_owned()), + ..ResponseHeaders::default() + }, + "", + ); + client.insert( + "https://origin.example/favicon.ico", + ResponseHeaders { + content_type: Some("image/x-icon".to_owned()), + cache_control: Some("public, max-age=86400".to_owned()), + surrogate_key: Some("icon".to_owned()), + ..ResponseHeaders::default() + }, + "", + ); + + let mut buffer = Vec::new(); + let report = run_audit_headers( + &args_for("https://origin.example", true), + &client, + &mut buffer, + ) + .expect("should run"); + assert_eq!( + report.outcome(false), + RunOutcome::AssertionFailed, + "cacheable HTML should drive a failing outcome (exit 1)" + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/origin/audit_headers/output.rs b/crates/trusted-server-cli/src/commands/origin/audit_headers/output.rs new file mode 100644 index 0000000000..5a06213f06 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/origin/audit_headers/output.rs @@ -0,0 +1,161 @@ +//! Renders an [`AuditReport`] as a human-readable table or as JSON. +//! +//! Output is written to an injected `Write` sink (rather than `println!`) so the +//! rendering is unit-testable, following the `config init` and `prebid bundle` +//! commands. + +use std::io::Write; + +use super::analyze::AuditReport; +use super::rules::Verdict; +use crate::error::CliResult; + +/// Writes the report to `out` in the format selected by `json`. +pub(crate) fn render(report: &AuditReport, json: bool, out: &mut dyn Write) -> CliResult<()> { + if json { + render_json(report, out) + } else { + render_human(report, out) + } +} + +fn render_json(report: &AuditReport, out: &mut dyn Write) -> CliResult<()> { + let serialized = serde_json::to_string_pretty(report) + .map_err(|error| format!("failed to serialize audit report: {error}"))?; + writeln!(out, "{serialized}").map_err(|error| write_error(&error)) +} + +fn render_human(report: &AuditReport, out: &mut dyn Write) -> CliResult<()> { + writeln!(out, "Origin: {}", report.origin).map_err(|error| write_error(&error))?; + writeln!(out).map_err(|error| write_error(&error))?; + + if report.groups.is_empty() { + writeln!(out, "No auditable content types found.").map_err(|error| write_error(&error))?; + return Ok(()); + } + + for group in &report.groups { + writeln!( + out, + "{} {}", + verdict_glyph(group.verdict), + group.content_type.label() + ) + .map_err(|error| write_error(&error))?; + + for verdict in &group.verdicts { + let recommendation = if verdict.recommendation.is_empty() { + "--" + } else { + &verdict.recommendation + }; + let actual = verdict.actual.as_deref().unwrap_or("(absent)"); + writeln!( + out, + " {} {:<18} {:<40} {}", + verdict_glyph(verdict.verdict), + verdict.header, + actual, + recommendation + ) + .map_err(|error| write_error(&error))?; + } + writeln!(out).map_err(|error| write_error(&error))?; + } + + writeln!( + out, + "Summary (per content type): {} pass, {} warn, {} fail ({} types audited)", + report.summary.pass, report.summary.warn, report.summary.fail, report.summary.total_groups + ) + .map_err(|error| write_error(&error))?; + + Ok(()) +} + +/// A terminal glyph for a verdict. ASCII-safe fallbacks are unnecessary; the +/// glyphs match the spec's sample output. +fn verdict_glyph(verdict: Verdict) -> char { + match verdict { + Verdict::Pass => '✓', + Verdict::Warn => '⚠', + Verdict::Fail => '✗', + } +} + +fn write_error(error: &std::io::Error) -> String { + format!("failed to write command output: {error}") +} + +#[cfg(test)] +mod tests { + use url::Url; + + use super::super::analyze::{FetchedResponse, run_analysis}; + use super::super::rules::ResponseHeaders; + use super::*; + + fn sample_report() -> AuditReport { + let responses = vec![ + FetchedResponse { + url: Url::parse("https://origin.example/").expect("should parse"), + headers: ResponseHeaders { + content_type: Some("text/html".to_owned()), + cache_control: Some("public, max-age=3600".to_owned()), + ..ResponseHeaders::default() + }, + }, + FetchedResponse { + url: Url::parse("https://origin.example/app.js").expect("should parse"), + headers: ResponseHeaders { + content_type: Some("application/javascript".to_owned()), + cache_control: Some("public, max-age=31536000, immutable".to_owned()), + surrogate_key: Some("js".to_owned()), + etag: Some("\"abc\"".to_owned()), + ..ResponseHeaders::default() + }, + }, + ]; + run_analysis("https://origin.example", &responses) + } + + #[test] + fn human_output_includes_origin_and_summary() { + let report = sample_report(); + let mut buffer = Vec::new(); + render(&report, false, &mut buffer).expect("should render"); + let text = String::from_utf8(buffer).expect("should be utf-8"); + assert!( + text.contains("Origin: https://origin.example"), + "should print the origin" + ); + assert!( + text.contains("Summary (per content type):"), + "should print the summary line" + ); + assert!(text.contains("HTML"), "should list the HTML group"); + } + + #[test] + fn json_output_is_valid_and_stable() { + let report = sample_report(); + let mut buffer = Vec::new(); + render(&report, true, &mut buffer).expect("should render"); + let text = String::from_utf8(buffer).expect("should be utf-8"); + let parsed: serde_json::Value = + serde_json::from_str(&text).expect("should emit valid JSON"); + assert_eq!( + parsed["origin"], "https://origin.example", + "should serialize origin" + ); + assert_eq!( + parsed["summary"]["total_groups"], 2, + "should count both groups" + ); + // Verdicts serialize as bare strings, not payload-carrying objects. + assert_eq!( + parsed["groups"][0]["verdict"], "Fail", + "cacheable HTML group should serialize a bare Fail verdict" + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/origin/audit_headers/rules.rs b/crates/trusted-server-cli/src/commands/origin/audit_headers/rules.rs new file mode 100644 index 0000000000..52d8177917 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/origin/audit_headers/rules.rs @@ -0,0 +1,1353 @@ +//! Content-type taxonomy and per-group cacheability rules. +//! +//! The audit classifies each origin response into a [`ContentTypeGroup`] by its +//! `Content-Type`, then evaluates the cache-related response headers against the +//! expected posture for that group. Evaluation is pure — it takes already-parsed +//! header values (see [`ResponseHeaders`]) and returns a list of per-header +//! [`HeaderVerdict`]s, so it is exercised entirely by unit tests without any I/O. + +use std::collections::BTreeSet; + +use serde::Serialize; + +/// One year in seconds — the `max-age` floor for immutable, hashed assets. +const IMMUTABLE_MAX_AGE: u64 = 31_536_000; + +/// One day in seconds — the `max-age` floor for creatives/images. +const IMAGE_MIN_MAX_AGE: u64 = 86_400; + +/// The content-type family a response belongs to. +/// +/// Classification strips `Content-Type` parameters (`text/html; charset=utf-8` +/// becomes `text/html`) and lowercases the value before matching. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +pub(crate) enum ContentTypeGroup { + /// `text/html` — personalized, consent-sensitive. + Html, + /// Hashed, immutable script bundles. + JavaScript, + /// `image/*` creatives. + Image, + /// CSS and fonts. + StaticAsset, + /// `application/json` real-time bidding responses. + RtbJson, + /// Everything else — reported for information, never evaluated. + Other, +} + +impl ContentTypeGroup { + /// Classifies a raw `Content-Type` header value into a group. + /// + /// Parameters after the first `;` are ignored and matching is + /// case-insensitive, so `text/html; charset=utf-8` maps to + /// [`ContentTypeGroup::Html`]. + pub(crate) fn classify(content_type: &str) -> Self { + let mime = content_type + .split(';') + .next() + .unwrap_or(content_type) + .trim() + .to_ascii_lowercase(); + + match mime.as_str() { + "text/html" => Self::Html, + "application/javascript" | "text/javascript" | "application/x-javascript" => { + Self::JavaScript + } + "application/json" => Self::RtbJson, + "text/css" => Self::StaticAsset, + "application/woff" | "application/woff2" => Self::StaticAsset, + other if other.starts_with("image/") => Self::Image, + other if other.starts_with("font/") || other.starts_with("application/font-") => { + Self::StaticAsset + } + _ => Self::Other, + } + } + + /// Whether this group is expected to be CDN-cacheable, and therefore + /// evaluated for `Surrogate-Key` purge coverage. Non-cacheable groups + /// (`Html`, `RtbJson`) never need selective purge, so the check is skipped. + fn is_cacheable(self) -> bool { + matches!(self, Self::JavaScript | Self::Image | Self::StaticAsset) + } + + /// A short, stable display label for terminal output. + pub(crate) fn label(self) -> &'static str { + match self { + Self::Html => "HTML", + Self::JavaScript => "JS Bundle", + Self::Image => "Image", + Self::StaticAsset => "Static Asset", + Self::RtbJson => "RTB/JSON", + Self::Other => "Other", + } + } +} + +/// The outcome of a single header (or group) check. +/// +/// Unit variants only — the human-readable message lives on +/// [`HeaderVerdict::recommendation`], and serializes to a bare string +/// (`"Pass"`, `"Warn"`, `"Fail"`) for the `--json` output. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +pub(crate) enum Verdict { + /// Header posture matches the expected value. + Pass, + /// Suboptimal but not incorrect — hit ratio or purge ergonomics suffer. + Warn, + /// Incorrect — risks stale or cross-user content. + Fail, +} + +impl Verdict { + /// Combines two verdicts into the more severe of the two. + fn worst(self, other: Self) -> Self { + match (self, other) { + (Self::Fail, _) | (_, Self::Fail) => Self::Fail, + (Self::Warn, _) | (_, Self::Warn) => Self::Warn, + _ => Self::Pass, + } + } + + /// Folds an iterator of verdicts into a single worst-of rollup, defaulting + /// to [`Verdict::Pass`] when empty. + pub(crate) fn rollup(verdicts: impl IntoIterator) -> Self { + verdicts.into_iter().fold(Self::Pass, Self::worst) + } +} + +/// A single header's evaluation result within a content-type group. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub(crate) struct HeaderVerdict { + /// The header this verdict is about (e.g. `Cache-Control`). + pub(crate) header: String, + /// Pass / warn / fail for this header. + pub(crate) verdict: Verdict, + /// The observed header value, if the header was present. + pub(crate) actual: Option, + /// The expected posture, described briefly. + pub(crate) expected: String, + /// What to change to satisfy the rule (empty when passing). + pub(crate) recommendation: String, +} + +impl HeaderVerdict { + fn pass(header: &str, actual: Option, expected: &str) -> Self { + Self { + header: header.to_owned(), + verdict: Verdict::Pass, + actual, + expected: expected.to_owned(), + recommendation: String::new(), + } + } + + fn flagged( + header: &str, + verdict: Verdict, + actual: Option, + expected: &str, + recommendation: &str, + ) -> Self { + Self { + header: header.to_owned(), + verdict, + actual, + expected: expected.to_owned(), + recommendation: recommendation.to_owned(), + } + } +} + +/// The parsed, cache-relevant response headers for one origin response. +/// +/// Extracted from the transport layer in `fetch.rs` so this module stays free +/// of any HTTP-client types and is unit-testable in isolation. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub(crate) struct ResponseHeaders { + /// `Content-Type` — drives group classification. + pub(crate) content_type: Option, + /// `Cache-Control`. + pub(crate) cache_control: Option, + /// `Surrogate-Control` (Fastly CDN-only directives). + pub(crate) surrogate_control: Option, + /// `Surrogate-Key` (Fastly purge tags). + pub(crate) surrogate_key: Option, + /// `Vary`. + pub(crate) vary: Option, + /// `ETag`. + pub(crate) etag: Option, +} + +/// A parsed `Cache-Control` (or `Surrogate-Control`) value. +/// +/// Directives are lowercased tokens; `max-age` and `s-maxage` are captured +/// separately because Fastly resolves the shared-cache TTL from +/// `Surrogate-Control` first, then `s-maxage`, then `max-age`. +struct CacheDirectives { + tokens: BTreeSet, + /// Directives that appeared in bare (unqualified) form, i.e. without an + /// `=value`. The qualified form `private="Set-Cookie"` / `no-store="…"` + /// restricts only the *named header fields* (RFC 9111 §5.2.2.7); the + /// response body itself remains storable and shareable by a shared cache. + /// The HTML/RTB safety checks must therefore require the unqualified form. + unqualified: BTreeSet, + max_age: Option, + s_maxage: Option, +} + +/// Folds a parsed TTL into an accumulator, tracking conflicts. The first valid +/// value is kept; a later unparseable (`None`) or duplicate-identical value is +/// ignored (RFC 9111 §4.2.1); a later *different* valid value marks a conflict, +/// after which the caller treats the directive as absent. +fn accumulate(current: &mut Option, conflict: &mut bool, incoming: Option) { + if let Some(value) = incoming { + match *current { + Some(existing) if existing != value => *conflict = true, + None => *current = Some(value), + _ => {} + } + } +} + +impl CacheDirectives { + fn parse(value: &str) -> Self { + let mut tokens = BTreeSet::new(); + let mut unqualified = BTreeSet::new(); + let mut max_age = None; + let mut max_age_conflict = false; + let mut s_maxage = None; + let mut s_maxage_conflict = false; + + for raw in value.split(',') { + let token = raw.trim().to_ascii_lowercase(); + if token.is_empty() { + continue; + } + if let Some((name, seconds)) = token.split_once('=') { + // Trim whitespace on both sides of the quote-stripping so + // `max-age = 3600` and `max-age = "3600"` (legal per HTTP + // parameter syntax) still parse. + let seconds = seconds.trim().trim_matches('"').trim().parse::().ok(); + match name.trim() { + // RFC 9111 §4.2.1: a later unparseable or duplicate-identical + // directive must not discard an already-parsed value, so the + // first valid occurrence wins for those cases. But two + // *different* valid values are a genuine contradiction the + // operator cannot reason about, so the directive is treated + // as absent (invalidated) rather than silently picking one. + "max-age" => accumulate(&mut max_age, &mut max_age_conflict, seconds), + "s-maxage" => accumulate(&mut s_maxage, &mut s_maxage_conflict, seconds), + _ => {} + } + // A `name=value` directive is present but only in qualified + // form — recorded in `tokens` but deliberately NOT in + // `unqualified`. + tokens.insert(name.trim().to_owned()); + } else { + unqualified.insert(token.clone()); + tokens.insert(token); + } + } + + Self { + tokens, + unqualified, + max_age: if max_age_conflict { None } else { max_age }, + s_maxage: if s_maxage_conflict { None } else { s_maxage }, + } + } + + /// Whether the directive is present in any form (bare or qualified). + fn has(&self, directive: &str) -> bool { + self.tokens.contains(directive) + } + + /// Whether the directive appeared in unqualified (bare) form. The qualified + /// form `private="Set-Cookie"` restricts only the named header field + /// (RFC 9111 §5.2.2.7) and leaves the response body storable by a shared + /// cache, so it must not satisfy the HTML/RTB body-safety checks. + fn has_unqualified(&self, directive: &str) -> bool { + self.unqualified.contains(directive) + } + + /// Whether these directives forbid shared-cache storage outright, in + /// unqualified form (`no-store` / `private` / `no-cache`). A single + /// predicate shared by every path that asks "will the CDN cache this?", + /// so the CDN-TTL and disables-caching checks cannot drift apart. Built on + /// [`has_unqualified`] because the qualified form (`private="Set-Cookie"`) + /// restricts only the named header field and leaves the body shareable + /// (RFC 9111 §5.2.2.7), so it does not forbid caching. + fn forbids_caching(&self) -> bool { + self.has_unqualified("no-store") + || self.has_unqualified("private") + || self.has_unqualified("no-cache") + } +} + +/// Whether a `Vary` value contains a hit-ratio-destroying token. +fn vary_has_high_cardinality(vary: &str) -> bool { + vary.split(',').any(|token| { + let token = token.trim().to_ascii_lowercase(); + token == "user-agent" || token == "cookie" + }) +} + +/// Evaluates the cache posture of a single response for its content-type group. +/// +/// Returns one [`HeaderVerdict`] per header the group cares about. +/// [`ContentTypeGroup::Other`] is never evaluated and yields an empty list. +pub(crate) fn evaluate(group: ContentTypeGroup, headers: &ResponseHeaders) -> Vec { + match group { + ContentTypeGroup::Html => evaluate_html(headers), + ContentTypeGroup::JavaScript => evaluate_immutable(group, headers, "JS bundles"), + ContentTypeGroup::Image => evaluate_image(headers), + ContentTypeGroup::StaticAsset => evaluate_immutable(group, headers, "static assets"), + ContentTypeGroup::RtbJson => evaluate_rtb(headers), + ContentTypeGroup::Other => Vec::new(), + } +} + +/// HTML must not be shared across users. `no-store` alone is sufficient +/// (RFC 9111 §5.2.2.5), as is `private` alone (RFC 9111 §5.2.2.7 — a shared +/// cache must not store a `private` response). `no-cache` alone is NOT +/// sufficient: a shared cache may still store the response and, on a 304, +/// serve one user's body to another. +fn evaluate_html(headers: &ResponseHeaders) -> Vec { + let mut verdicts = Vec::new(); + + let expected = "no-store, or private"; + match headers.cache_control.as_deref() { + Some(value) => { + let directives = CacheDirectives::parse(value); + // Only the unqualified forms make the body itself unstorable by a + // shared cache; `private="Set-Cookie"` restricts a header field, not + // the body, and must not satisfy this check (RFC 9111 §5.2.2.7). + let safe = + directives.has_unqualified("no-store") || directives.has_unqualified("private"); + // `public` together with `private` is contradictory. Intermediaries + // resolve the conflict inconsistently and some pick `public`, sharing + // personalized content — flag it rather than trusting `private`. + let contradictory = directives.has("public") && directives.has_unqualified("private"); + if contradictory { + verdicts.push(HeaderVerdict::flagged( + "Cache-Control", + Verdict::Fail, + Some(value.to_owned()), + expected, + "HTML advertises both `public` and `private`; intermediaries resolve this inconsistently and may share personalized content — drop `public`", + )); + } else if safe { + verdicts.push(HeaderVerdict::pass( + "Cache-Control", + Some(value.to_owned()), + expected, + )); + } else { + verdicts.push(HeaderVerdict::flagged( + "Cache-Control", + Verdict::Fail, + Some(value.to_owned()), + expected, + "HTML served without no-store or private risks sharing personalized content across users", + )); + } + } + None => verdicts.push(HeaderVerdict::flagged( + "Cache-Control", + Verdict::Fail, + None, + expected, + "HTML has no Cache-Control; set `no-store` or `private` to prevent shared caching of personalized content", + )), + } + + verdicts.extend(evaluate_vary(headers)); + verdicts.extend(evaluate_surrogate_no_store( + headers, + "CDN may cache personalized HTML; set Surrogate-Control to `no-store` or `private`", + )); + + verdicts +} + +/// JS bundles and static assets share the immutable, long-lived posture. +fn evaluate_immutable( + group: ContentTypeGroup, + headers: &ResponseHeaders, + noun: &str, +) -> Vec { + let mut verdicts = Vec::new(); + + let expected = "public, max-age>=31536000, immutable"; + match headers.cache_control.as_deref() { + Some(value) => { + let directives = CacheDirectives::parse(value); + let immutable = directives.has("public") + && directives.has("immutable") + && directives + .max_age + .is_some_and(|age| age >= IMMUTABLE_MAX_AGE); + if immutable { + verdicts.push(HeaderVerdict::pass( + "Cache-Control", + Some(value.to_owned()), + expected, + )); + } else { + verdicts.push(HeaderVerdict::flagged( + "Cache-Control", + Verdict::Warn, + Some(value.to_owned()), + expected, + &format!("Hashed {noun} should use long-lived immutable caching"), + )); + } + } + None => verdicts.push(HeaderVerdict::flagged( + "Cache-Control", + Verdict::Warn, + None, + expected, + &format!("{noun} have no Cache-Control; set `public, max-age=31536000, immutable`"), + )), + } + + if matches!(group, ContentTypeGroup::JavaScript) { + verdicts.push(evaluate_etag(headers)); + } + verdicts.extend(evaluate_cdn_ttl(headers)); + verdicts.extend(evaluate_surrogate_key(group, headers)); + verdicts.extend(evaluate_surrogate_disables_caching(headers)); + verdicts.extend(evaluate_vary(headers)); + + verdicts +} + +/// Images should be publicly cacheable for at least a day, and the CDN TTL +/// should not undercut the browser TTL. +fn evaluate_image(headers: &ResponseHeaders) -> Vec { + let mut verdicts = Vec::new(); + + let expected = "public, max-age>=86400"; + match headers.cache_control.as_deref() { + Some(value) => { + let directives = CacheDirectives::parse(value); + let cacheable = directives.has("public") + && directives + .max_age + .is_some_and(|age| age >= IMAGE_MIN_MAX_AGE); + if cacheable { + verdicts.push(HeaderVerdict::pass( + "Cache-Control", + Some(value.to_owned()), + expected, + )); + } else { + verdicts.push(HeaderVerdict::flagged( + "Cache-Control", + Verdict::Warn, + Some(value.to_owned()), + expected, + "Images re-fetched too frequently; set `public, max-age=86400` or longer", + )); + } + } + None => verdicts.push(HeaderVerdict::flagged( + "Cache-Control", + Verdict::Warn, + None, + expected, + "Images have no Cache-Control; set `public, max-age=86400` or longer", + )), + } + + verdicts.extend(evaluate_cdn_ttl(headers)); + verdicts.extend(evaluate_surrogate_key(ContentTypeGroup::Image, headers)); + verdicts.extend(evaluate_surrogate_disables_caching(headers)); + verdicts.extend(evaluate_vary(headers)); + + verdicts +} + +/// The CDN (shared-cache) TTL should be at least the browser TTL, so the edge +/// does not re-fetch more often than clients. +/// +/// The CDN TTL is resolved from `Surrogate-Control` (its `max-age`, else its +/// `s-maxage`) when that header is present, otherwise from a +/// `Cache-Control: s-maxage` — matching Fastly's resolution order +/// (`Surrogate-Control` > `s-maxage` > `max-age`). A verdict is emitted only +/// when both a CDN TTL and a browser `max-age` are present. +/// +/// Crucially, when `Surrogate-Control` is present but carries no numeric TTL +/// (e.g. `no-store`), the CDN TTL is `None` and no verdict is emitted here — a +/// `no-store` must never borrow the `Cache-Control: s-maxage` and report a +/// passing CDN TTL for a response the CDN will not cache. That token-only case +/// is handled by [`evaluate_surrogate_disables_caching`]. +fn evaluate_cdn_ttl(headers: &ResponseHeaders) -> Option { + let cache_control = headers.cache_control.as_deref().map(CacheDirectives::parse); + let surrogate_directives = headers + .surrogate_control + .as_deref() + .map(CacheDirectives::parse); + + let browser_ttl = cache_control + .as_ref() + .and_then(|directives| directives.max_age)?; + + // `Surrogate-Control` wins outright over `s-maxage` at the CDN, so the + // `s-maxage` fallback applies only when no `Surrogate-Control` is present. + // A `Surrogate-Control` that forbids caching emits NO CDN-TTL verdict + // whatever number it also carries: `no-store, max-age=86400` means the CDN + // stores nothing, so affirming "the edge TTL is adequate" would be a + // false Pass. The disables-caching Warn covers that case instead. + let cdn_ttl = match surrogate_directives.as_ref() { + Some(directives) if directives.forbids_caching() => None, + Some(directives) => directives.max_age.or(directives.s_maxage), + None => cache_control + .as_ref() + .and_then(|directives| directives.s_maxage), + }?; + + // Label the verdict after whichever header supplied the TTL so it never + // collides with the cacheability `Cache-Control` verdict, and so the + // `--json` schema never carries a phantom header name. + let (header, actual) = match &headers.surrogate_control { + // Reaching here with `Surrogate-Control` present means it supplied the + // numeric TTL (the `None` arm above only runs when it is absent). + Some(surrogate) => ("Surrogate-Control", surrogate.clone()), + None => ( + "Cache-Control (s-maxage)", + headers.cache_control.clone().unwrap_or_default(), + ), + }; + + Some(if cdn_ttl < browser_ttl { + HeaderVerdict::flagged( + header, + Verdict::Warn, + Some(actual), + "CDN TTL >= browser TTL", + "CDN caching shorter than browser caching; raise the shared-cache TTL (Surrogate-Control max-age or s-maxage)", + ) + } else { + HeaderVerdict::pass(header, Some(actual), "CDN TTL >= browser TTL") + }) +} + +/// JSON carries no assumed posture. RTB bid responses come from the TS edge +/// (`POST /auction`, `GET /_ts/page-bids`), not the publisher origin, so the +/// audit never treats origin JSON as bid data and never requires `no-store` +/// (spec §3.2, §6.5). It emits a single informational Pass recording the +/// observed policy, plus an `add_private` note when `no-store` is present +/// without `private` (bare `no-store` forbids storage per RFC 9111 §5.2.2.5, +/// but some CDNs ignore it and TS itself emits `no-store, private`). +fn evaluate_rtb(headers: &ResponseHeaders) -> Vec { + let expected = "no assumed posture (origin JSON is not bid data)"; + let actual = headers.cache_control.clone(); + let recommendation = match headers.cache_control.as_deref() { + Some(value) => { + let directives = CacheDirectives::parse(value); + if directives.has_unqualified("no-store") && !directives.has_unqualified("private") { + "Origin JSON is not graded as bid data; consider adding `private` alongside `no-store` (some CDNs ignore bare `no-store`)" + } else { + "" + } + } + None => "", + }; + + let verdict = if recommendation.is_empty() { + HeaderVerdict::pass("Cache-Control", actual, expected) + } else { + HeaderVerdict::flagged( + "Cache-Control", + Verdict::Pass, + actual, + expected, + recommendation, + ) + }; + vec![verdict] +} + +/// `Surrogate-Control`, when present on HTML, must not permit CDN caching of +/// personalized content. +fn evaluate_surrogate_no_store( + headers: &ResponseHeaders, + recommendation: &str, +) -> Option { + let value = headers.surrogate_control.as_deref()?; + let directives = CacheDirectives::parse(value); + // Require the UNQUALIFIED form: `private="Set-Cookie"` restricts only the + // named header field and leaves the body shareable (RFC 9111 §5.2.2.7), so + // the qualified form must not satisfy the body-safety check for + // personalized HTML/RTB. + let safe = directives.has_unqualified("no-store") || directives.has_unqualified("private"); + Some(if safe { + HeaderVerdict::pass( + "Surrogate-Control", + Some(value.to_owned()), + "no-store or private", + ) + } else { + HeaderVerdict::flagged( + "Surrogate-Control", + Verdict::Fail, + Some(value.to_owned()), + "no-store or private", + recommendation, + ) + }) +} + +/// On a cacheable group, a `Surrogate-Control` that forbids edge caching +/// (`no-store` / `private` / `no-cache`) defeats the point of a cacheable +/// asset. Numeric-TTL cases like `max-age=0` are caught by +/// [`evaluate_cdn_ttl`], which now runs for every cacheable group, so this +/// covers only the token-only directives that carry no numeric value. +fn evaluate_surrogate_disables_caching(headers: &ResponseHeaders) -> Option { + let value = headers.surrogate_control.as_deref()?; + let directives = CacheDirectives::parse(value); + let disables = directives.forbids_caching(); + if disables { + Some(HeaderVerdict::flagged( + "Surrogate-Control", + Verdict::Warn, + Some(value.to_owned()), + "should allow CDN caching", + "Surrogate-Control disables edge caching for a cacheable asset; the CDN will not cache it", + )) + } else { + None + } +} + +/// `Surrogate-Key` enables targeted CDN purge; absence is a warning on +/// cacheable groups only. +fn evaluate_surrogate_key( + group: ContentTypeGroup, + headers: &ResponseHeaders, +) -> Option { + if !group.is_cacheable() { + return None; + } + Some(match headers.surrogate_key.as_deref() { + Some(value) => HeaderVerdict::pass("Surrogate-Key", Some(value.to_owned()), "present"), + None => HeaderVerdict::flagged( + "Surrogate-Key", + Verdict::Warn, + None, + "present", + "Without Surrogate-Key, this content type can't be selectively purged from the CDN — full-cache purges required", + ), + }) +} + +/// `ETag` enables conditional revalidation; absence is a warning. +fn evaluate_etag(headers: &ResponseHeaders) -> HeaderVerdict { + match headers.etag.as_deref() { + Some(value) => HeaderVerdict::pass("ETag", Some(value.to_owned()), "present"), + None => HeaderVerdict::flagged( + "ETag", + Verdict::Warn, + None, + "present", + "Missing ETag means no conditional revalidation", + ), + } +} + +/// `Vary` checks: `*` makes a response uncacheable by every cache (including +/// the CDN), and `User-Agent` / `Cookie` destroy the CDN hit ratio on any +/// group. +fn evaluate_vary(headers: &ResponseHeaders) -> Option { + let value = headers.vary.as_deref()?; + + if value.split(',').any(|token| token.trim() == "*") { + return Some(HeaderVerdict::flagged( + "Vary", + Verdict::Warn, + Some(value.to_owned()), + "no `*`", + "Vary: * disables all caching including the CDN", + )); + } + + if vary_has_high_cardinality(value) { + return Some(HeaderVerdict::flagged( + "Vary", + Verdict::Warn, + Some(value.to_owned()), + "no User-Agent or Cookie", + "Vary on User-Agent/Cookie destroys the CDN hit ratio", + )); + } + + Some(HeaderVerdict::pass( + "Vary", + Some(value.to_owned()), + "no `*`, User-Agent, or Cookie", + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn headers_with(cache_control: Option<&str>) -> ResponseHeaders { + ResponseHeaders { + cache_control: cache_control.map(str::to_owned), + ..ResponseHeaders::default() + } + } + + fn verdict_for(verdicts: &[HeaderVerdict], header: &str) -> Verdict { + verdicts + .iter() + .find(|verdict| verdict.header == header) + .unwrap_or_else(|| panic!("should have a verdict for {header}")) + .verdict + } + + #[test] + fn classify_strips_parameters_and_lowercases() { + assert_eq!( + ContentTypeGroup::classify("text/HTML; charset=utf-8"), + ContentTypeGroup::Html, + "should ignore charset parameter and case" + ); + assert_eq!( + ContentTypeGroup::classify("image/png"), + ContentTypeGroup::Image, + "should match image/* prefix" + ); + assert_eq!( + ContentTypeGroup::classify("font/woff2"), + ContentTypeGroup::StaticAsset, + "should treat fonts as static assets" + ); + assert_eq!( + ContentTypeGroup::classify("application/font-woff"), + ContentTypeGroup::StaticAsset, + "should treat legacy font MIME as static assets" + ); + assert_eq!( + ContentTypeGroup::classify("application/octet-stream"), + ContentTypeGroup::Other, + "should fall through to Other" + ); + } + + #[test] + fn html_passes_on_no_store_alone() { + let verdicts = evaluate(ContentTypeGroup::Html, &headers_with(Some("no-store"))); + assert_eq!( + verdict_for(&verdicts, "Cache-Control"), + Verdict::Pass, + "no-store alone should satisfy RFC 9111 for HTML" + ); + } + + #[test] + fn html_passes_on_private_no_cache() { + let verdicts = evaluate( + ContentTypeGroup::Html, + &headers_with(Some("private, no-cache")), + ); + assert_eq!( + verdict_for(&verdicts, "Cache-Control"), + Verdict::Pass, + "private + no-cache should satisfy HTML" + ); + } + + #[test] + fn html_fails_on_public_cacheable() { + let verdicts = evaluate( + ContentTypeGroup::Html, + &headers_with(Some("public, max-age=3600")), + ); + assert_eq!( + verdict_for(&verdicts, "Cache-Control"), + Verdict::Fail, + "publicly cacheable HTML should fail" + ); + } + + #[test] + fn html_qualified_surrogate_private_is_not_body_safe() { + // Blocker (round 3): a qualified `private="Set-Cookie"` on + // Surrogate-Control restricts only that header field (RFC 9111 + // §5.2.2.7); the HTML body stays storable and shareable, so it must NOT + // satisfy the Surrogate-Control body-safety check for personalized HTML. + let verdicts = evaluate( + ContentTypeGroup::Html, + &ResponseHeaders { + content_type: Some("text/html".to_owned()), + cache_control: Some("no-store".to_owned()), + surrogate_control: Some("max-age=600, private=\"Set-Cookie\"".to_owned()), + ..ResponseHeaders::default() + }, + ); + assert_eq!( + verdict_for(&verdicts, "Surrogate-Control"), + Verdict::Fail, + "qualified private=\"...\" must not pass the Surrogate-Control body-safety check" + ); + } + + #[test] + fn html_unqualified_surrogate_private_is_body_safe() { + let verdicts = evaluate( + ContentTypeGroup::Html, + &ResponseHeaders { + content_type: Some("text/html".to_owned()), + cache_control: Some("no-store".to_owned()), + surrogate_control: Some("private".to_owned()), + ..ResponseHeaders::default() + }, + ); + assert_eq!( + verdict_for(&verdicts, "Surrogate-Control"), + Verdict::Pass, + "unqualified private should satisfy the Surrogate-Control body-safety check" + ); + } + + #[test] + fn rtb_passes_on_no_store_without_private() { + // Origin JSON carries no assumed posture (spec §3.2, §6.5): the audit + // never requires `no-store`, so it passes regardless. + let verdicts = evaluate(ContentTypeGroup::RtbJson, &headers_with(Some("no-store"))); + assert_eq!( + verdict_for(&verdicts, "Cache-Control"), + Verdict::Pass, + "origin JSON carries no assumed posture, so it passes" + ); + } + + #[test] + fn javascript_warns_without_immutable() { + let verdicts = evaluate( + ContentTypeGroup::JavaScript, + &headers_with(Some("public, max-age=31536000")), + ); + assert_eq!( + verdict_for(&verdicts, "Cache-Control"), + Verdict::Warn, + "missing immutable should warn" + ); + } + + #[test] + fn javascript_passes_on_full_immutable_posture() { + let verdicts = evaluate( + ContentTypeGroup::JavaScript, + &ResponseHeaders { + cache_control: Some("public, max-age=31536000, immutable".to_owned()), + surrogate_key: Some("js".to_owned()), + etag: Some("\"abc\"".to_owned()), + ..ResponseHeaders::default() + }, + ); + assert_eq!( + Verdict::rollup(verdicts.iter().map(|verdict| verdict.verdict)), + Verdict::Pass, + "full immutable posture with ETag and Surrogate-Key should pass" + ); + } + + #[test] + fn surrogate_key_skipped_for_non_cacheable_groups() { + let verdicts = evaluate(ContentTypeGroup::Html, &headers_with(Some("no-store"))); + assert!( + !verdicts + .iter() + .any(|verdict| verdict.header == "Surrogate-Key"), + "HTML should not be checked for Surrogate-Key" + ); + } + + #[test] + fn vary_wildcard_warns_on_html() { + let html = evaluate( + ContentTypeGroup::Html, + &ResponseHeaders { + cache_control: Some("no-store".to_owned()), + vary: Some("*".to_owned()), + ..ResponseHeaders::default() + }, + ); + assert_eq!( + verdict_for(&html, "Vary"), + Verdict::Warn, + "Vary: * should warn on HTML" + ); + } + + #[test] + fn vary_high_cardinality_warns_on_assets() { + let verdicts = evaluate( + ContentTypeGroup::JavaScript, + &ResponseHeaders { + cache_control: Some("public, max-age=31536000, immutable".to_owned()), + surrogate_key: Some("js".to_owned()), + etag: Some("\"abc\"".to_owned()), + vary: Some("User-Agent".to_owned()), + ..ResponseHeaders::default() + }, + ); + assert_eq!( + verdict_for(&verdicts, "Vary"), + Verdict::Warn, + "Vary: User-Agent should warn on JS assets" + ); + } + + #[test] + fn image_warns_when_cdn_ttl_below_browser_ttl() { + let verdicts = evaluate( + ContentTypeGroup::Image, + &ResponseHeaders { + cache_control: Some("public, max-age=86400".to_owned()), + surrogate_control: Some("max-age=60".to_owned()), + surrogate_key: Some("img".to_owned()), + ..ResponseHeaders::default() + }, + ); + assert_eq!( + verdict_for(&verdicts, "Surrogate-Control"), + Verdict::Warn, + "CDN TTL below browser TTL should warn" + ); + } + + #[test] + fn other_group_is_never_evaluated() { + let verdicts = evaluate(ContentTypeGroup::Other, &headers_with(Some("whatever"))); + assert!(verdicts.is_empty(), "Other should yield no verdicts"); + } + + #[test] + fn rollup_is_worst_of() { + assert_eq!( + Verdict::rollup([Verdict::Pass, Verdict::Warn, Verdict::Fail]), + Verdict::Fail, + "any fail should dominate" + ); + assert_eq!( + Verdict::rollup([Verdict::Pass, Verdict::Warn]), + Verdict::Warn, + "warn should dominate pass" + ); + assert_eq!( + Verdict::rollup(std::iter::empty()), + Verdict::Pass, + "empty should default to pass" + ); + } + + #[test] + fn html_passes_on_private_alone() { + // RFC 9111 §5.2.2.7: `private` alone forbids a shared cache from storing + // the response, so `private, max-age=0` is a correct personalized-HTML + // posture and must not fail. + let verdicts = evaluate( + ContentTypeGroup::Html, + &headers_with(Some("private, max-age=0")), + ); + assert_eq!( + verdict_for(&verdicts, "Cache-Control"), + Verdict::Pass, + "private alone should satisfy HTML" + ); + } + + #[test] + fn vary_wildcard_warns_on_all_cacheable_groups() { + // Vary: * makes a response uncacheable by any cache, so it must be + // flagged on the immutable/cacheable groups where it is most damaging, + // not only on HTML. + for group in [ + ContentTypeGroup::JavaScript, + ContentTypeGroup::Image, + ContentTypeGroup::StaticAsset, + ] { + let verdicts = evaluate( + group, + &ResponseHeaders { + cache_control: Some("public, max-age=31536000, immutable".to_owned()), + surrogate_key: Some("k".to_owned()), + etag: Some("\"abc\"".to_owned()), + vary: Some("*".to_owned()), + ..ResponseHeaders::default() + }, + ); + assert_eq!( + verdict_for(&verdicts, "Vary"), + Verdict::Warn, + "Vary: * should warn on {}", + group.label() + ); + } + } + + #[test] + fn image_honors_cache_control_s_maxage_as_cdn_ttl() { + // A standard `Cache-Control: s-maxage` shared-cache TTL below the browser + // max-age should warn even without a Surrogate-Control header. + let verdicts = evaluate( + ContentTypeGroup::Image, + &ResponseHeaders { + cache_control: Some("public, max-age=86400, s-maxage=60".to_owned()), + surrogate_key: Some("img".to_owned()), + ..ResponseHeaders::default() + }, + ); + assert_eq!( + verdict_for(&verdicts, "Cache-Control (s-maxage)"), + Verdict::Warn, + "s-maxage below browser TTL should warn, labeled honestly (not a phantom `s-maxage` header)" + ); + } + + #[test] + fn cacheable_group_flags_surrogate_control_no_store() { + // An image with a perfectly cacheable Cache-Control but + // Surrogate-Control: no-store is not cached by the CDN — must not pass. + let verdicts = evaluate( + ContentTypeGroup::Image, + &ResponseHeaders { + cache_control: Some("public, max-age=86400".to_owned()), + surrogate_control: Some("no-store".to_owned()), + surrogate_key: Some("img".to_owned()), + ..ResponseHeaders::default() + }, + ); + assert_eq!( + verdict_for(&verdicts, "Surrogate-Control"), + Verdict::Warn, + "Surrogate-Control: no-store should warn on a cacheable image" + ); + } + + #[test] + fn max_age_with_spaces_around_equals_parses() { + let verdicts = evaluate( + ContentTypeGroup::JavaScript, + &ResponseHeaders { + cache_control: Some("public, max-age = 31536000, immutable".to_owned()), + surrogate_key: Some("js".to_owned()), + etag: Some("\"abc\"".to_owned()), + ..ResponseHeaders::default() + }, + ); + assert_eq!( + verdict_for(&verdicts, "Cache-Control"), + Verdict::Pass, + "whitespace around `=` should not lose the max-age value" + ); + } + + #[test] + fn duplicate_or_malformed_directive_keeps_first_valid_value() { + // RFC 9111 §4.2.1: a later malformed/duplicate directive is treated as + // absent, not allowed to discard an already-parsed value. + let verdicts = evaluate( + ContentTypeGroup::JavaScript, + &ResponseHeaders { + cache_control: Some("public, immutable, max-age=31536000, max-age=foo".to_owned()), + surrogate_key: Some("js".to_owned()), + etag: Some("\"abc\"".to_owned()), + ..ResponseHeaders::default() + }, + ); + assert_eq!( + verdict_for(&verdicts, "Cache-Control"), + Verdict::Pass, + "a trailing malformed max-age must not null the valid one" + ); + } + + #[test] + fn application_woff_classifies_as_static_asset() { + for mime in ["application/woff", "application/woff2"] { + assert_eq!( + ContentTypeGroup::classify(mime), + ContentTypeGroup::StaticAsset, + "{mime} should classify as a static asset (font), not Other" + ); + } + } + + #[test] + fn conflicting_duplicate_max_age_invalidates_the_directive() { + // Two *different* valid max-age values are a contradiction the operator + // cannot reason about, so the directive is treated as absent — unlike a + // duplicate-identical or unparseable value, which keeps the first valid. + // On an immutable JS bundle, invalidating max-age drops it below the + // immutable floor and must warn. + let verdicts = evaluate( + ContentTypeGroup::JavaScript, + &ResponseHeaders { + cache_control: Some("public, immutable, max-age=31536000, max-age=60".to_owned()), + surrogate_key: Some("js".to_owned()), + etag: Some("\"abc\"".to_owned()), + ..ResponseHeaders::default() + }, + ); + assert_eq!( + verdict_for(&verdicts, "Cache-Control"), + Verdict::Warn, + "conflicting max-age values should invalidate the directive" + ); + } + + #[test] + fn duplicate_identical_max_age_is_not_a_conflict() { + // Same valid value twice is not a contradiction — keep it. + let verdicts = evaluate( + ContentTypeGroup::JavaScript, + &ResponseHeaders { + cache_control: Some( + "public, immutable, max-age=31536000, max-age=31536000".to_owned(), + ), + surrogate_key: Some("js".to_owned()), + etag: Some("\"abc\"".to_owned()), + ..ResponseHeaders::default() + }, + ); + assert_eq!( + verdict_for(&verdicts, "Cache-Control"), + Verdict::Pass, + "a repeated identical max-age must not be treated as a conflict" + ); + } + + /// Look up a verdict by both header name and expected string — two distinct + /// checks can share a header name (e.g. Surrogate-Control). + fn verdict_by<'a>( + verdicts: &'a [HeaderVerdict], + header: &str, + expected: &str, + ) -> Option<&'a HeaderVerdict> { + verdicts + .iter() + .find(|verdict| verdict.header == header && verdict.expected == expected) + } + + #[test] + fn surrogate_no_store_does_not_borrow_s_maxage_for_a_passing_cdn_ttl() { + // Blocker 1: `Surrogate-Control: no-store` carries no numeric TTL, so it + // must NOT fall back to `Cache-Control: s-maxage` and report a passing + // CDN TTL for a response the CDN will not cache at all. + let verdicts = evaluate( + ContentTypeGroup::Image, + &ResponseHeaders { + cache_control: Some("public, max-age=86400, s-maxage=60".to_owned()), + surrogate_control: Some("no-store".to_owned()), + surrogate_key: Some("img".to_owned()), + ..ResponseHeaders::default() + }, + ); + assert!( + verdict_by(&verdicts, "Surrogate-Control", "CDN TTL >= browser TTL").is_none() + && verdict_by( + &verdicts, + "Cache-Control (s-maxage)", + "CDN TTL >= browser TTL" + ) + .is_none(), + "no-store must not emit a CDN-TTL verdict by borrowing s-maxage" + ); + assert_eq!( + verdict_by(&verdicts, "Surrogate-Control", "should allow CDN caching") + .map(|verdict| verdict.verdict), + Some(Verdict::Warn), + "no-store on a cacheable asset should warn via disables-caching" + ); + } + + #[test] + fn immutable_flags_surrogate_max_age_zero_on_js() { + // Blocker 3: a hashed JS bundle the edge will never cache + // (`Surrogate-Control: max-age=0`) must be flagged — the CDN-vs-browser + // TTL check now runs for immutable groups, not only images. + let verdicts = evaluate( + ContentTypeGroup::JavaScript, + &ResponseHeaders { + cache_control: Some("public, max-age=31536000, immutable".to_owned()), + surrogate_control: Some("max-age=0".to_owned()), + surrogate_key: Some("js".to_owned()), + etag: Some("\"e\"".to_owned()), + ..ResponseHeaders::default() + }, + ); + assert_eq!( + verdict_by(&verdicts, "Surrogate-Control", "CDN TTL >= browser TTL") + .map(|verdict| verdict.verdict), + Some(Verdict::Warn), + "Surrogate-Control: max-age=0 on an immutable JS bundle should warn" + ); + } + + #[test] + fn html_qualified_private_is_not_accepted_as_unqualified() { + // Blocker 4: `private="Set-Cookie"` restricts a header field, not the + // body (RFC 9111 §5.2.2.7), so publicly cacheable personalized HTML must + // still fail. + let verdicts = evaluate( + ContentTypeGroup::Html, + &headers_with(Some("public, max-age=600, private=\"Set-Cookie\"")), + ); + assert_eq!( + verdict_for(&verdicts, "Cache-Control"), + Verdict::Fail, + "qualified private must not satisfy the HTML body-safety check" + ); + } + + #[test] + fn rtb_json_add_private_note_is_informational_only() { + // `no-store` without `private` on origin JSON yields an informational + // add_private recommendation but stays a Pass (no posture is assumed). + let verdicts = evaluate(ContentTypeGroup::RtbJson, &headers_with(Some("no-store"))); + let cc = verdicts + .iter() + .find(|verdict| verdict.header == "Cache-Control") + .expect("should have a Cache-Control verdict"); + assert_eq!( + cc.verdict, + Verdict::Pass, + "add_private is informational only" + ); + assert!( + !cc.recommendation.is_empty(), + "should carry an add_private recommendation" + ); + } + + #[test] + fn rtb_json_carries_no_assumed_posture() { + // Per revised spec §3.2/§6.5: origin JSON is not treated as bid data, so + // even a publicly cacheable policy must not FAIL — the audit assumes no + // group-specific posture for JSON. + let verdicts = evaluate( + ContentTypeGroup::RtbJson, + &headers_with(Some("public, max-age=600")), + ); + assert_eq!( + verdict_for(&verdicts, "Cache-Control"), + Verdict::Pass, + "origin JSON must not be failed for cache posture" + ); + } + + #[test] + fn html_public_private_contradiction_fails() { + // Blocker 5: `public, private` advertises two contradictory storage + // policies; intermediaries resolve it inconsistently, so it must fail. + for value in ["public, private", "public, max-age=600, private"] { + let verdicts = evaluate(ContentTypeGroup::Html, &headers_with(Some(value))); + assert_eq!( + verdict_for(&verdicts, "Cache-Control"), + Verdict::Fail, + "`{value}` should fail as a public/private contradiction" + ); + } + } + + #[test] + fn surrogate_control_matrix_on_immutable_js() { + // Table over {absent, token-only, numeric} Surrogate-Control against a + // fully-immutable JS bundle. Guards the three regressions that lived in + // the {present, token-only} cell. + let base_cc = "public, max-age=31536000, immutable"; + // Each row: (Surrogate-Control, expected group rollup, expected + // (expected-label, verdict) that the Surrogate-Control header must + // carry -- asserting the verdict VALUE, not just the rollup, so a wrong + // Pass hiding inside a Warn rollup is caught). + type Row<'a> = (Option<&'a str>, Verdict, Option<(&'a str, Verdict)>); + let cases: &[Row] = &[ + // No Surrogate-Control: bundle is fully cacheable -> Pass. + (None, Verdict::Pass, None), + // Token-only that disables edge caching -> Warn (disables-caching). + ( + Some("no-store"), + Verdict::Warn, + Some(("should allow CDN caching", Verdict::Warn)), + ), + ( + Some("private"), + Verdict::Warn, + Some(("should allow CDN caching", Verdict::Warn)), + ), + ( + Some("no-cache"), + Verdict::Warn, + Some(("should allow CDN caching", Verdict::Warn)), + ), + // Numeric zero TTL -> Warn (CDN undercuts browser). + ( + Some("max-age=0"), + Verdict::Warn, + Some(("CDN TTL >= browser TTL", Verdict::Warn)), + ), + // Numeric TTL that meets the browser TTL -> Pass. + ( + Some("max-age=31536000"), + Verdict::Pass, + Some(("CDN TTL >= browser TTL", Verdict::Pass)), + ), + // Token-plus-number: the exact cell the round-3 regression lived in. + // `no-store` forbids caching, so NO passing CDN-TTL verdict may be + // emitted; only the disables-caching Warn. + ( + Some("no-store, max-age=31536000"), + Verdict::Warn, + Some(("should allow CDN caching", Verdict::Warn)), + ), + ]; + for (surrogate, expected, expected_verdict) in cases { + let verdicts = evaluate( + ContentTypeGroup::JavaScript, + &ResponseHeaders { + cache_control: Some(base_cc.to_owned()), + surrogate_control: surrogate.map(str::to_owned), + surrogate_key: Some("js".to_owned()), + etag: Some("\"e\"".to_owned()), + ..ResponseHeaders::default() + }, + ); + assert_eq!( + Verdict::rollup(verdicts.iter().map(|verdict| verdict.verdict)), + *expected, + "Surrogate-Control {surrogate:?} should roll up to {expected:?}" + ); + if let Some((label, verdict)) = expected_verdict { + let found: Vec<(&str, Verdict)> = verdicts + .iter() + .filter(|v| v.header == "Surrogate-Control") + .map(|v| (v.expected.as_str(), v.verdict)) + .collect(); + assert!( + found.contains(&(label, *verdict)), + "Surrogate-Control {surrogate:?} must carry verdict ({label:?}, {verdict:?}), got {found:?}" + ); + // A forbids-caching surrogate (the disables-caching rows, incl. + // the token-plus-number regression cell) must never also emit a + // passing CDN-TTL verdict -- that was the round-3 false Pass. + // Numeric-TTL rows legitimately may, so scope the check. + if *label == "should allow CDN caching" { + assert!( + !found.contains(&("CDN TTL >= browser TTL", Verdict::Pass)), + "forbids-caching Surrogate-Control {surrogate:?} must not emit a passing CDN-TTL verdict, got {found:?}" + ); + } + } + } + } +} diff --git a/crates/trusted-server-cli/src/commands/origin/mod.rs b/crates/trusted-server-cli/src/commands/origin/mod.rs index c385881c26..a6b28989ff 100644 --- a/crates/trusted-server-cli/src/commands/origin/mod.rs +++ b/crates/trusted-server-cli/src/commands/origin/mod.rs @@ -1,11 +1,13 @@ //! `ts origin` — questions about a publisher origin's behaviour. +pub mod audit_headers; pub mod probe; pub mod report; use clap::Subcommand; use crate::error::CliResult; +use crate::run::RunOutcome; /// Subcommands under `ts origin`. #[derive(Debug, Subcommand)] @@ -18,6 +20,16 @@ pub enum OriginCommand { /// `TRUSTED_SERVER_PROBE_ADMISSION_COOKIE` to a cookie that gets past a bot wall. Every --url must be HTTPS; plain HTTP is /// accepted only for a loopback development origin. ProbeShareability(ProbeShareabilityArgs), + + /// Audit origin cache-related response headers per content type. + /// + /// Fetches the publisher origin directly — explicit `--url`, or discovered + /// from the origin root — and grades each response's cache posture for its + /// content type. Sits beside `probe-shareability` and follows its + /// conventions: URLs arrive through a repeatable `--url`, and every target + /// must be HTTPS except on loopback. Exits 1 on a FAIL verdict (or a WARN + /// under `--strict`), 0 otherwise, 2 if the audit could not run. + AuditHeaders(audit_headers::AuditHeadersArgs), } /// Arguments for `ts origin probe-shareability`. @@ -76,12 +88,15 @@ pub struct ProbeShareabilityArgs { /// Returns an error when an origin cannot be reached, or when the probe's verdict is that /// the origin is not shareable — the caller turns that into a non-zero exit so the command /// can gate a deploy. -pub fn run(command: OriginCommand, out: &mut impl std::io::Write) -> CliResult<()> { +pub fn run(command: OriginCommand, out: &mut impl std::io::Write) -> CliResult { match command { OriginCommand::ProbeShareability(mut args) => { load_cookies_from_environment(&mut args); - run_probe(&args, out) + // A failing verdict returns `Err`, which `main` surfaces as exit 2; + // a clean probe is success. + run_probe(&args, out).map(|()| RunOutcome::Success) } + OriginCommand::AuditHeaders(args) => audit_headers::run(&args, out), } } diff --git a/crates/trusted-server-cli/src/run.rs b/crates/trusted-server-cli/src/run.rs index 05b4779f8f..8613a85249 100644 --- a/crates/trusted-server-cli/src/run.rs +++ b/crates/trusted-server-cli/src/run.rs @@ -178,7 +178,7 @@ fn dispatch(args: Args) -> Result { Command::Origin(command) => { let stdout = std::io::stdout(); let mut out = stdout.lock(); - crate::commands::origin::run(command, &mut out).map(|()| RunOutcome::Success) + crate::commands::origin::run(command, &mut out) } } } diff --git a/docs/guide/cli.md b/docs/guide/cli.md index 9466e1f3df..5d7f2d4299 100644 --- a/docs/guide/cli.md +++ b/docs/guide/cli.md @@ -743,6 +743,64 @@ ts prebid client --config publisher-a.toml --out build/prebid `ts prebid client` is local-only. It has no `--adapter` option and does not upload, provision, deploy, or push config. +## Audit origin cache headers + +`ts origin audit-headers` fetches responses from the publisher origin, classifies +each by content type, and grades its cache-related response headers +(`Cache-Control`, `Surrogate-Control`, `s-maxage`, `Surrogate-Key`, `Vary`, +`ETag`) against the expected posture for that type. It reports a per-type +pass/warn/fail verdict naming the responsible header and the recommended value. +It sits beside `ts origin probe-shareability` and follows its conventions: URLs +arrive through a repeatable `--url`, and every target must be HTTPS (plain HTTP +only for a loopback development origin). + +```bash +# Audit the origin from trusted-server.toml (publisher.origin_url), discovering +# assets from the root page's HTML. +ts origin audit-headers + +# Point at an origin explicitly, skipping the config lookup. +ts origin audit-headers --origin https://origin.publisher.com + +# Audit specific URLs instead of crawling. +ts origin audit-headers --url https://origin.publisher.com/app.js --url https://origin.publisher.com/ + +# Fail the command on warnings too (CI gate), and emit machine-readable output. +ts origin audit-headers --strict --json +``` + +When no `--url` is given, discovery fetches the origin root and follows the +`