From bc13f2773ba97e8de07f7ba6ef6f3d0a3e37427f Mon Sep 17 00:00:00 2001 From: Aram Grigoryan <132480+aram356@users.noreply.github.com> Date: Mon, 20 Jul 2026 16:38:03 -0700 Subject: [PATCH 01/16] Upgrade EdgeZero to the deploy-actions branch Point the edgezero-* dependencies at the feature/edgezero-deploy-actions branch (PR #316) and adapt Trusted Server to its API changes: - Wire the new ts CLI subcommands surfaced by edgezero-cli: active-version, healthcheck, and rollback, plus deploy --stage and a --version flag, with argument-parsing coverage. - Migrate TrustedServerAppConfig to the AppConfigMeta::secret_fields() method that replaces the removed SECRET_FIELDS associated constant. --- Cargo.lock | 37 +++-- Cargo.toml | 12 +- crates/trusted-server-cli/src/run.rs | 174 ++++++++++++++++++++++- crates/trusted-server-core/src/config.rs | 4 +- 4 files changed, 205 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 68f14e753..bd92ec5ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1398,7 +1398,7 @@ dependencies = [ [[package]] name = "edgezero-adapter" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#145f1699d8e8f51027804f21faddbf985cff9a1e" dependencies = [ "toml", ] @@ -1406,7 +1406,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-axum" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#145f1699d8e8f51027804f21faddbf985cff9a1e" dependencies = [ "anyhow", "async-trait", @@ -1434,7 +1434,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-cloudflare" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#145f1699d8e8f51027804f21faddbf985cff9a1e" dependencies = [ "anyhow", "async-trait", @@ -1449,7 +1449,7 @@ dependencies = [ "log", "serde_json", "tempfile", - "toml_edit", + "toml_edit 0.25.12+spec-1.1.0", "walkdir", "worker", ] @@ -1457,7 +1457,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-fastly" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#145f1699d8e8f51027804f21faddbf985cff9a1e" dependencies = [ "anyhow", "async-stream", @@ -1479,14 +1479,14 @@ dependencies = [ "serde_json", "sha2 0.10.9", "thiserror 2.0.18", - "toml_edit", + "toml_edit 0.25.12+spec-1.1.0", "walkdir", ] [[package]] name = "edgezero-adapter-spin" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#145f1699d8e8f51027804f21faddbf985cff9a1e" dependencies = [ "anyhow", "async-trait", @@ -1506,14 +1506,14 @@ dependencies = [ "subtle", "thiserror 2.0.18", "toml", - "toml_edit", + "toml_edit 0.25.12+spec-1.1.0", "walkdir", ] [[package]] name = "edgezero-cli" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#145f1699d8e8f51027804f21faddbf985cff9a1e" dependencies = [ "chrono", "clap", @@ -1538,7 +1538,7 @@ dependencies = [ [[package]] name = "edgezero-core" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#145f1699d8e8f51027804f21faddbf985cff9a1e" dependencies = [ "anyhow", "async-compression", @@ -1569,7 +1569,7 @@ dependencies = [ [[package]] name = "edgezero-macros" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#145f1699d8e8f51027804f21faddbf985cff9a1e" dependencies = [ "log", "proc-macro2", @@ -5074,6 +5074,19 @@ dependencies = [ "winnow 0.7.15", ] +[[package]] +name = "toml_edit" +version = "0.25.12+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2153edc6955a6c354fad8f5efd38b6a8769bdccf9fe50f8e1329f81b0baa5d7" +dependencies = [ + "indexmap 2.14.0", + "toml_datetime 1.1.1+spec-1.1.0", + "toml_parser", + "toml_writer", + "winnow 1.0.3", +] + [[package]] name = "toml_parser" version = "1.1.2+spec-1.1.0" @@ -5339,7 +5352,7 @@ dependencies = [ "tokio", "tokio-rustls", "toml", - "toml_edit", + "toml_edit 0.23.10+spec-1.0.0", "trusted-server-core", "url", "webpki-roots", diff --git a/Cargo.toml b/Cargo.toml index 695099d49..54569599b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,12 +53,12 @@ criterion = { version = "0.5", default-features = false, features = ["cargo_benc derive_more = { version = "2.0", features = ["display", "error"] } directories = "5" ed25519-dalek = { version = "2.2", features = ["rand_core"] } -edgezero-adapter-axum = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.4", default-features = false } -edgezero-adapter-cloudflare = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.4", default-features = false } -edgezero-adapter-fastly = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.4", default-features = false } -edgezero-adapter-spin = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.4", default-features = false } -edgezero-cli = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.4" } -edgezero-core = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.4", default-features = false } +edgezero-adapter-axum = { git = "https://github.com/stackpop/edgezero", branch = "feature/edgezero-deploy-actions", default-features = false } +edgezero-adapter-cloudflare = { git = "https://github.com/stackpop/edgezero", branch = "feature/edgezero-deploy-actions", default-features = false } +edgezero-adapter-fastly = { git = "https://github.com/stackpop/edgezero", branch = "feature/edgezero-deploy-actions", default-features = false } +edgezero-adapter-spin = { git = "https://github.com/stackpop/edgezero", branch = "feature/edgezero-deploy-actions", default-features = false } +edgezero-cli = { git = "https://github.com/stackpop/edgezero", branch = "feature/edgezero-deploy-actions" } +edgezero-core = { git = "https://github.com/stackpop/edgezero", branch = "feature/edgezero-deploy-actions", default-features = false } env_logger = "0.11" error-stack = "0.6" fastly = "0.12" diff --git a/crates/trusted-server-cli/src/run.rs b/crates/trusted-server-cli/src/run.rs index 1b0bdfa29..b3be949a5 100644 --- a/crates/trusted-server-cli/src/run.rs +++ b/crates/trusted-server-cli/src/run.rs @@ -2,8 +2,8 @@ use std::process; use clap::{Parser, Subcommand}; use edgezero_cli::args::{ - AuthArgs, BuildArgs, ConfigDiffArgs, ConfigPushArgs, ConfigValidateArgs, DeployArgs, - ProvisionArgs, ServeArgs, + ActiveVersionArgs, AuthArgs, BuildArgs, ConfigDiffArgs, ConfigPushArgs, ConfigValidateArgs, + DeployArgs, HealthcheckArgs, ProvisionArgs, RollbackArgs, ServeArgs, }; use trusted_server_core::config::TrustedServerAppConfig; @@ -13,7 +13,7 @@ use crate::commands::config::init::{ConfigInitArgs, run_config_init}; use crate::prebid_bundle::{NpmPrebidBundleGenerator, PrebidBundleArgs, run_bundle}; #[derive(Debug, Parser)] -#[command(name = "ts", about = "Trusted Server CLI")] +#[command(name = "ts", version, about = "Trusted Server CLI")] struct Args { #[command(subcommand)] command: Command, @@ -21,6 +21,8 @@ struct Args { #[derive(Debug, Subcommand)] enum Command { + /// Print the currently active deployment version for a target adapter. + ActiveVersion(ActiveVersionArgs), /// Audit a public page and write draft Trusted Server artifacts. Audit(AuditArgs), /// Sign in / out / status against an `EdgeZero` adapter. @@ -32,10 +34,14 @@ enum Command { Config(ConfigCommand), /// Deploy the project through a target adapter. Deploy(DeployArgs), + /// Probe a deployed version until it reports healthy. + Healthcheck(HealthcheckArgs), /// Trusted Server Prebid commands. Prebid(PrebidArgs), /// Provision platform resources through a target adapter. Provision(ProvisionArgs), + /// Roll a service back to a previously active deployment version. + Rollback(RollbackArgs), /// Serve the project locally through a target adapter. Serve(ServeArgs), /// Local developer tools (e.g. the macOS-only production-hostname proxy). @@ -79,6 +85,7 @@ pub fn run_from_env() -> Result<(), String> { fn dispatch(args: Args) -> Result<(), String> { match args.command { + Command::ActiveVersion(args) => edgezero_cli::run_active_version(&args), Command::Audit(args) => { let stdout = std::io::stdout(); let mut out = stdout.lock(); @@ -102,6 +109,7 @@ fn dispatch(args: Args) -> Result<(), String> { edgezero_cli::run_config_validate_typed::(&args) } Command::Deploy(args) => edgezero_cli::run_deploy(&args), + Command::Healthcheck(args) => edgezero_cli::run_healthcheck(&args), Command::Prebid(prebid) => { let mut generator = NpmPrebidBundleGenerator; let mut stdout = std::io::stdout(); @@ -113,6 +121,7 @@ fn dispatch(args: Args) -> Result<(), String> { } } Command::Provision(args) => edgezero_cli::run_provision(&args), + Command::Rollback(args) => edgezero_cli::run_rollback(&args), Command::Serve(args) => edgezero_cli::run_serve(&args), Command::Dev(command) => crate::commands::dev::run(command), } @@ -131,6 +140,165 @@ mod tests { Args::try_parse_from(args).expect("should parse args") } + #[test] + fn parses_active_version() { + let args = parse(&[ + "ts", + "active-version", + "--adapter", + "fastly", + "--service-id", + "service-123", + ]); + let Command::ActiveVersion(active_version) = args.command else { + panic!("expected active-version command"); + }; + assert_eq!(active_version.adapter, "fastly"); + assert_eq!(active_version.service_id, "service-123"); + } + + #[test] + fn parses_healthcheck_with_retry_defaults() { + let args = parse(&[ + "ts", + "healthcheck", + "--adapter", + "fastly", + "--service-id", + "service-123", + "--version", + "7", + "--domain", + "edge.example", + ]); + let Command::Healthcheck(healthcheck) = args.command else { + panic!("expected healthcheck command"); + }; + assert_eq!(healthcheck.domain, "edge.example"); + assert_eq!(healthcheck.version, "7"); + assert_eq!(healthcheck.retry, 3, "should default to 3 retries"); + assert_eq!( + healthcheck.retry_delay, 5, + "should default to a 5s retry delay" + ); + assert_eq!(healthcheck.timeout, 10, "should default to a 10s timeout"); + assert!(!healthcheck.staging, "should probe production by default"); + } + + #[test] + fn parses_healthcheck_with_staging_overrides() { + let args = parse(&[ + "ts", + "healthcheck", + "--adapter", + "fastly", + "--service-id", + "service-123", + "--version", + "7", + "--domain", + "edge.example", + "--staging", + "--retry", + "9", + "--retry-delay", + "2", + "--timeout", + "30", + ]); + let Command::Healthcheck(healthcheck) = args.command else { + panic!("expected healthcheck command"); + }; + assert!(healthcheck.staging); + assert_eq!(healthcheck.retry, 9); + assert_eq!(healthcheck.retry_delay, 2); + assert_eq!(healthcheck.timeout, 30); + } + + #[test] + fn healthcheck_requires_domain() { + Args::try_parse_from([ + "ts", + "healthcheck", + "--adapter", + "fastly", + "--service-id", + "service-123", + "--version", + "7", + ]) + .expect_err("should reject healthcheck without a domain"); + } + + #[test] + fn parses_rollback_with_explicit_target() { + let args = parse(&[ + "ts", + "rollback", + "--adapter", + "fastly", + "--service-id", + "service-123", + "--version", + "8", + "--rollback-to", + "7", + ]); + let Command::Rollback(rollback) = args.command else { + panic!("expected rollback command"); + }; + assert_eq!(rollback.version, "8"); + assert_eq!(rollback.rollback_to, Some("7".to_owned())); + assert!(!rollback.staging); + } + + #[test] + fn parses_staging_rollback_without_target() { + let args = parse(&[ + "ts", + "rollback", + "--adapter", + "fastly", + "--service-id", + "service-123", + "--version", + "8", + "--staging", + ]); + let Command::Rollback(rollback) = args.command else { + panic!("expected rollback command"); + }; + assert!(rollback.staging); + assert_eq!( + rollback.rollback_to, None, + "staging rollback should not need an explicit target" + ); + } + + #[test] + fn rollback_requires_service_id() { + Args::try_parse_from(["ts", "rollback", "--adapter", "fastly", "--version", "8"]) + .expect_err("should reject rollback without a service id"); + } + + #[test] + fn parses_deploy_with_staging_flags() { + let args = parse(&[ + "ts", + "deploy", + "--adapter", + "fastly", + "--service-id", + "service-123", + "--stage", + ]); + let Command::Deploy(deploy) = args.command else { + panic!("expected deploy command"); + }; + assert_eq!(deploy.service_id, Some("service-123".to_owned())); + assert!(deploy.stage); + } + #[test] fn parses_audit_with_default_outputs() { let args = parse(&["ts", "audit", "https://publisher.example"]); diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index 7bbecd747..991ed7a2f 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -110,7 +110,9 @@ impl edgezero_core::app_config::AppConfigMeta for TrustedServerAppConfig { // app-config blob. Migrating app-level secrets to `EdgeZero` secret-store // references needs nested/array extraction support and operator migration // work tracked separately. - const SECRET_FIELDS: &'static [edgezero_core::app_config::SecretField] = &[]; + fn secret_fields() -> Vec { + Vec::new() + } } /// Runs Trusted Server deploy-time validation for pushed app config. From b743345fd7d43eaf36e0fce86ba448e27e9e5b6e Mon Sep 17 00:00:00 2001 From: Aram Grigoryan <132480+aram356@users.noreply.github.com> Date: Mon, 3 Aug 2026 17:51:39 -0700 Subject: [PATCH 02/16] Update EdgeZero to latest deploy-actions branch tip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-resolve the six edgezero-* deps from 145f1699 to bb441162 (current tip of feature/edgezero-deploy-actions, PR #316). The deploy staging flag was renamed there from --stage to --staging, standardizing on the same verb healthcheck/rollback/config-push already use; update the deploy CLI parse test to match. No production dispatch change is needed — ts passes the edgezero-cli arg structs through, so the renamed flag is picked up automatically. --- Cargo.lock | 26 +++++++++++++------------- crates/trusted-server-cli/src/run.rs | 4 ++-- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 72fa293ce..e5caa1a7a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -767,7 +767,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] @@ -1398,7 +1398,7 @@ dependencies = [ [[package]] name = "edgezero-adapter" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#145f1699d8e8f51027804f21faddbf985cff9a1e" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#bb4411625856472b1279a3db49aeeac5e8b1507e" dependencies = [ "toml", ] @@ -1406,7 +1406,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-axum" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#145f1699d8e8f51027804f21faddbf985cff9a1e" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#bb4411625856472b1279a3db49aeeac5e8b1507e" dependencies = [ "anyhow", "async-trait", @@ -1434,7 +1434,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-cloudflare" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#145f1699d8e8f51027804f21faddbf985cff9a1e" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#bb4411625856472b1279a3db49aeeac5e8b1507e" dependencies = [ "anyhow", "async-trait", @@ -1457,7 +1457,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-fastly" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#145f1699d8e8f51027804f21faddbf985cff9a1e" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#bb4411625856472b1279a3db49aeeac5e8b1507e" dependencies = [ "anyhow", "async-stream", @@ -1486,7 +1486,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-spin" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#145f1699d8e8f51027804f21faddbf985cff9a1e" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#bb4411625856472b1279a3db49aeeac5e8b1507e" dependencies = [ "anyhow", "async-trait", @@ -1513,7 +1513,7 @@ dependencies = [ [[package]] name = "edgezero-cli" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#145f1699d8e8f51027804f21faddbf985cff9a1e" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#bb4411625856472b1279a3db49aeeac5e8b1507e" dependencies = [ "chrono", "clap", @@ -1538,7 +1538,7 @@ dependencies = [ [[package]] name = "edgezero-core" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#145f1699d8e8f51027804f21faddbf985cff9a1e" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#bb4411625856472b1279a3db49aeeac5e8b1507e" dependencies = [ "anyhow", "async-compression", @@ -1569,7 +1569,7 @@ dependencies = [ [[package]] name = "edgezero-macros" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#145f1699d8e8f51027804f21faddbf985cff9a1e" +source = "git+https://github.com/stackpop/edgezero?branch=feature%2Fedgezero-deploy-actions#bb4411625856472b1279a3db49aeeac5e8b1507e" dependencies = [ "log", "proc-macro2", @@ -3604,7 +3604,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck", - "itertools 0.13.0", + "itertools 0.10.5", "log", "multimap", "once_cell", @@ -3624,7 +3624,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.118", @@ -3637,7 +3637,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.118", @@ -5908,7 +5908,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] diff --git a/crates/trusted-server-cli/src/run.rs b/crates/trusted-server-cli/src/run.rs index b3be949a5..7374c56a7 100644 --- a/crates/trusted-server-cli/src/run.rs +++ b/crates/trusted-server-cli/src/run.rs @@ -290,13 +290,13 @@ mod tests { "fastly", "--service-id", "service-123", - "--stage", + "--staging", ]); let Command::Deploy(deploy) = args.command else { panic!("expected deploy command"); }; assert_eq!(deploy.service_id, Some("service-123".to_owned())); - assert!(deploy.stage); + assert!(deploy.staging); } #[test] From 44f765878c783887a37e35cea2fb756b3949c0a9 Mon Sep 17 00:00:00 2001 From: Christian Date: Mon, 3 Aug 2026 12:30:16 -0500 Subject: [PATCH 03/16] Add DataDome protection decision logs --- .../src/integrations/datadome.rs | 5 +- .../src/integrations/datadome/protection.rs | 81 ++++++++++++++++++- 2 files changed, 82 insertions(+), 4 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/datadome.rs b/crates/trusted-server-core/src/integrations/datadome.rs index a46f626a6..fa5c79992 100644 --- a/crates/trusted-server-core/src/integrations/datadome.rs +++ b/crates/trusted-server-core/src/integrations/datadome.rs @@ -841,9 +841,10 @@ fn build( }; log::info!( - "[datadome] Registering integration (sdk_origin: {}, rewrite_sdk: {})", + "[datadome] Registering integration (sdk_origin: {}, rewrite_sdk: {}, enable_protection: {})", config.sdk_origin, - config.rewrite_sdk + config.rewrite_sdk, + config.enable_protection ); Ok(Some(DataDomeIntegration::try_new(config)?)) diff --git a/crates/trusted-server-core/src/integrations/datadome/protection.rs b/crates/trusted-server-core/src/integrations/datadome/protection.rs index c2759a864..a7c12bbc9 100644 --- a/crates/trusted-server-core/src/integrations/datadome/protection.rs +++ b/crates/trusted-server-core/src/integrations/datadome/protection.rs @@ -98,7 +98,13 @@ impl DataDomeIntegration { .change_context(Self::error("Failed to call DataDome Protection API")) .map_err(ProtectionRequestError::Runtime)?; - Ok(self.classify_protection_response(platform_response.response, input.request.method())) + let status = platform_response.response.status(); + let datadome_status = datadome_response_status(platform_response.response.headers()); + let decision = + self.classify_protection_response(platform_response.response, input.request.method()); + log_protection_result(&input, status, datadome_status, &decision); + + Ok(decision) } fn is_request_protected(&self, input: &RequestFilterInput<'_>) -> bool { @@ -126,7 +132,7 @@ impl DataDomeIntegration { match self.protection_scope.evaluate(&facts, input.services) { ProtectionScopeDecision::Protect => {} ProtectionScopeDecision::Skip { rule_id, reason } => { - log::debug!("[datadome] Skipping Protection API for rule {rule_id} ({reason})"); + log_protection_skip(input, &rule_id, reason); return false; } } @@ -413,6 +419,77 @@ impl DataDomeIntegration { } } +fn log_protection_skip(input: &RequestFilterInput<'_>, rule_id: &str, reason: &str) { + if matches!( + reason, + "client_ip" | "client_ip_source" | "ip_cidr" | "ip_cidr_source" + ) { + log::info!( + "[datadome] protection decision=skipped rule={} reason={} method={} host={} path={} client_ip={}", + rule_id, + reason, + input.request.method(), + request_host(input.request), + input.request.uri().path(), + client_ip_for_log(input), + ); + } else { + log::debug!( + "[datadome] protection decision=skipped rule={} reason={} method={} host={} path={} client_ip={}", + rule_id, + reason, + input.request.method(), + request_host(input.request), + input.request.uri().path(), + client_ip_for_log(input), + ); + } +} + +fn log_protection_result( + input: &RequestFilterInput<'_>, + status: StatusCode, + datadome_status: Option, + decision: &RequestFilterDecision, +) { + let method = input.request.method(); + let host = request_host(input.request); + let path = input.request.uri().path(); + let client_ip = client_ip_for_log(input); + + match decision { + RequestFilterDecision::Respond { .. } => log::info!( + "[datadome] protection decision=blocked status={} method={} host={} path={} client_ip={} route=short_circuit", + status.as_u16(), + method, + host, + path, + client_ip, + ), + RequestFilterDecision::Continue(_) + if status == StatusCode::OK && datadome_status == Some(status.as_u16()) => + { + log::info!( + "[datadome] protection decision=allowed status={} method={} host={} path={} client_ip={} route=continue", + status.as_u16(), + method, + host, + path, + client_ip, + ); + } + RequestFilterDecision::Continue(_) => {} + } +} + +fn client_ip_for_log(input: &RequestFilterInput<'_>) -> String { + input + .services + .client_info() + .client_ip + .map_or_else(|| "unknown".to_string(), |ip| ip.to_string()) +} + struct ProtectionPayload { fields: Vec<(String, String)>, uses_header_client_id: bool, From 302d161e32b3d8e29665d2211f36fb8f85765ebb Mon Sep 17 00:00:00 2001 From: Christian Date: Mon, 3 Aug 2026 14:15:41 -0500 Subject: [PATCH 04/16] Log incoming DataDome client IP --- .../src/integrations/datadome/protection.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/crates/trusted-server-core/src/integrations/datadome/protection.rs b/crates/trusted-server-core/src/integrations/datadome/protection.rs index a7c12bbc9..a86791110 100644 --- a/crates/trusted-server-core/src/integrations/datadome/protection.rs +++ b/crates/trusted-server-core/src/integrations/datadome/protection.rs @@ -36,6 +36,16 @@ impl DataDomeIntegration { &self, input: RequestFilterInput<'_>, ) -> RequestFilterDecision { + if self.config.enable_protection { + log::info!( + "[datadome] protection incoming client_ip={} method={} host={} path={}", + client_ip_for_log(&input), + input.request.method(), + request_host(input.request), + input.request.uri().path(), + ); + } + if !self.config.enable_protection || !self.is_request_protected(&input) { return RequestFilterDecision::Continue(RequestFilterEffects::default()); } From aeb3f4c7e3257c6388c9036607763f528230618f Mon Sep 17 00:00:00 2001 From: Christian Date: Mon, 3 Aug 2026 17:52:25 -0500 Subject: [PATCH 05/16] feat(datadome): suppress client tag for excluded IPs --- .../benches/html_processor_bench.rs | 1 + .../trusted-server-core/src/html_processor.rs | 61 +++ .../src/integrations/datadome.rs | 31 +- .../src/integrations/datadome/protection.rs | 269 ++++++++-- .../src/integrations/registry.rs | 11 +- .../src/platform/test_support.rs | 19 + crates/trusted-server-core/src/publisher.rs | 181 ++++++- docs/guide/integrations/datadome.md | 27 + ...6-08-03-datadome-ip-excluded-client-tag.md | 475 ++++++++++++++++++ ...-datadome-ip-excluded-client-tag-design.md | 339 +++++++++++++ 10 files changed, 1378 insertions(+), 36 deletions(-) create mode 100644 docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md create mode 100644 docs/superpowers/specs/2026-08-03-datadome-ip-excluded-client-tag-design.md diff --git a/crates/trusted-server-core/benches/html_processor_bench.rs b/crates/trusted-server-core/benches/html_processor_bench.rs index 96eec2f1f..7c1303dd4 100644 --- a/crates/trusted-server-core/benches/html_processor_bench.rs +++ b/crates/trusted-server-core/benches/html_processor_bench.rs @@ -13,6 +13,7 @@ fn make_config() -> HtmlProcessorConfig { ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, } } diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index 889234b56..4c827ace0 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -13,6 +13,7 @@ use lol_html::{ text, }; +use crate::integrations::datadome::{DATADOME_INTEGRATION_ID, DataDomeClientTagSuppressed}; use crate::integrations::gpt_diagnostics::GptDiagnosticsRequestDecision; use crate::integrations::{ AttributeRewriteOutcome, IntegrationAttributeContext, IntegrationDocumentState, @@ -175,6 +176,8 @@ pub struct HtmlProcessorConfig { pub max_buffered_body_bytes: usize, /// Request-scoped conditional diagnostics delivery decision. pub gpt_diagnostics: Option, + /// Whether to omit Trusted Server's automatic `DataDome` client-side tag. + pub suppress_datadome_client_side_tag: bool, } impl HtmlProcessorConfig { @@ -196,6 +199,7 @@ impl HtmlProcessorConfig { ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: settings.publisher.max_buffered_body_bytes, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, } } @@ -223,6 +227,13 @@ impl HtmlProcessorConfig { self.gpt_diagnostics = decision; self } + + /// Attach the request-scoped `DataDome` client-tag suppression decision. + #[must_use] + pub fn with_datadome_client_tag_suppression(mut self, suppress: bool) -> Self { + self.suppress_datadome_client_side_tag = suppress; + self + } } /// Create an HTML processor with URL replacement and integration hooks. @@ -235,6 +246,9 @@ impl HtmlProcessorConfig { pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcessor { let post_processors = config.integrations.html_post_processors(); let document_state = IntegrationDocumentState::default(); + if config.suppress_datadome_client_side_tag { + document_state.get_or_insert_with(DATADOME_INTEGRATION_ID, || DataDomeClientTagSuppressed); + } // Simplified URL patterns structure - stores only core data and generates variants on-demand struct UrlPatterns { @@ -692,6 +706,7 @@ mod tests { ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, } } @@ -950,6 +965,46 @@ mod tests { assert_eq!(config.request_scheme, "https"); } + #[test] + fn suppressed_datadome_tag_is_not_injected_into_processed_html() { + let mut settings = create_test_settings(); + settings + .integrations + .insert_config( + "datadome", + &json!({ + "enabled": true, + "client_side_key": "test-client-key", + }), + ) + .expect("should configure DataDome integration"); + let registry = IntegrationRegistry::new(&settings) + .expect("should create integration registry with DataDome"); + let config = HtmlProcessorConfig::from_settings( + &settings, + ®istry, + "origin.example.com", + "test.example.com", + "https", + ) + .with_datadome_client_tag_suppression(true); + let mut processor = create_html_processor(config); + + let output = processor + .process_chunk(b"content", true) + .expect("should process HTML"); + let html = String::from_utf8(output).expect("should produce UTF-8 HTML"); + + assert!( + !html.contains("window.ddjskey"), + "should omit the DataDome client configuration" + ); + assert!( + !html.contains("/integrations/datadome/tags.js"), + "should omit the DataDome client tag URL" + ); + } + #[test] fn test_real_publisher_html() { // Test with publisher HTML from test_publisher.html @@ -1539,6 +1594,7 @@ mod tests { ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let mut processor = create_html_processor(config); let output = processor @@ -1613,6 +1669,7 @@ mod tests { ad_bids_state: state, max_buffered_body_bytes: 16 * 1024 * 1024, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let mut processor = create_html_processor(config); let output = processor @@ -1649,6 +1706,7 @@ mod tests { ad_bids_state: state, max_buffered_body_bytes: 16 * 1024 * 1024, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let mut processor = create_html_processor(config); // Malformed HTML with two elements (common in CMS template pages) @@ -1684,6 +1742,7 @@ mod tests { ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let mut processor = create_html_processor(config); let output = processor @@ -1737,6 +1796,7 @@ mod tests { ad_bids_state: state, max_buffered_body_bytes: 16 * 1024 * 1024, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let mut processor = create_html_processor(config); let output = processor @@ -1764,6 +1824,7 @@ mod tests { ad_bids_state: state, max_buffered_body_bytes: 16 * 1024 * 1024, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let mut processor = create_html_processor(config); let output = processor diff --git a/crates/trusted-server-core/src/integrations/datadome.rs b/crates/trusted-server-core/src/integrations/datadome.rs index fa5c79992..db5932fb2 100644 --- a/crates/trusted-server-core/src/integrations/datadome.rs +++ b/crates/trusted-server-core/src/integrations/datadome.rs @@ -88,7 +88,12 @@ pub use protection_scope::{ use protection_scope::ProtectionScope; -pub(super) const DATADOME_INTEGRATION_ID: &str = "datadome"; +pub(crate) const DATADOME_INTEGRATION_ID: &str = "datadome"; + +/// Request marker indicating that Trusted Server should omit its automatic +/// `DataDome` client-side tag for the current response. +#[derive(Debug, Clone, Copy)] +pub(crate) struct DataDomeClientTagSuppressed; /// Regex pattern for matching and rewriting `DataDome` URLs in script content. /// @@ -765,7 +770,15 @@ impl IntegrationHeadInjector for DataDomeIntegration { DATADOME_INTEGRATION_ID } - fn head_inserts(&self, _ctx: &IntegrationHtmlContext<'_>) -> Vec { + fn head_inserts(&self, ctx: &IntegrationHtmlContext<'_>) -> Vec { + if ctx + .document_state + .get::(DATADOME_INTEGRATION_ID) + .is_some() + { + return Vec::new(); + } + if !self.config.inject_client_side_tag || self.config.client_side_key.trim().is_empty() { return Vec::new(); } @@ -1249,6 +1262,20 @@ mod tests { #[test] fn head_injector_omits_client_side_tag_when_disabled_or_blank() { + let mut suppressed = test_config(); + suppressed.client_side_key = "test-client-key".to_string(); + let suppressed_integration = DataDomeIntegration::new(suppressed); + let suppressed_state = crate::integrations::IntegrationDocumentState::default(); + suppressed_state + .get_or_insert_with(DATADOME_INTEGRATION_ID, || DataDomeClientTagSuppressed); + let suppressed_ctx = html_context_for_tests(&suppressed_state); + assert!( + suppressed_integration + .head_inserts(&suppressed_ctx) + .is_empty(), + "should omit the tag when the request is IP-excluded" + ); + let mut blank_key = test_config(); blank_key.client_side_key = " ".to_string(); let integration = DataDomeIntegration::new(blank_key); diff --git a/crates/trusted-server-core/src/integrations/datadome/protection.rs b/crates/trusted-server-core/src/integrations/datadome/protection.rs index a86791110..4c74c71e8 100644 --- a/crates/trusted-server-core/src/integrations/datadome/protection.rs +++ b/crates/trusted-server-core/src/integrations/datadome/protection.rs @@ -34,19 +34,18 @@ enum ProtectionRequestError { impl DataDomeIntegration { pub(super) async fn filter_protection_request( &self, - input: RequestFilterInput<'_>, + mut input: RequestFilterInput<'_>, ) -> RequestFilterDecision { if self.config.enable_protection { log::info!( - "[datadome] protection incoming client_ip={} method={} host={} path={}", - client_ip_for_log(&input), + "[datadome] protection incoming method={} host={} path={}", input.request.method(), request_host(input.request), input.request.uri().path(), ); } - if !self.config.enable_protection || !self.is_request_protected(&input) { + if !self.config.enable_protection || !self.is_request_protected(&mut input) { return RequestFilterDecision::Continue(RequestFilterEffects::default()); } @@ -117,8 +116,8 @@ impl DataDomeIntegration { Ok(decision) } - fn is_request_protected(&self, input: &RequestFilterInput<'_>) -> bool { - let req = input.request; + fn is_request_protected(&self, input: &mut RequestFilterInput<'_>) -> bool { + let req = &*input.request; if req.method() == Method::OPTIONS { return false; } @@ -142,6 +141,13 @@ impl DataDomeIntegration { match self.protection_scope.evaluate(&facts, input.services) { ProtectionScopeDecision::Protect => {} ProtectionScopeDecision::Skip { rule_id, reason } => { + let client_tag_omitted = is_ip_exclusion_reason(reason); + if client_tag_omitted { + input + .request + .extensions_mut() + .insert(super::DataDomeClientTagSuppressed); + } log_protection_skip(input, &rule_id, reason); return false; } @@ -210,7 +216,7 @@ impl DataDomeIntegration { input: &RequestFilterInput<'_>, server_side_key: &Redacted, ) -> ProtectionPayload { - let req = input.request; + let req = &*input.request; let client_info = input.services.client_info(); let mut fields = Vec::new(); let header_client_id = header_value(req, HEADER_DATADOME_CLIENT_ID); @@ -429,29 +435,31 @@ impl DataDomeIntegration { } } -fn log_protection_skip(input: &RequestFilterInput<'_>, rule_id: &str, reason: &str) { - if matches!( +fn is_ip_exclusion_reason(reason: &str) -> bool { + matches!( reason, "client_ip" | "client_ip_source" | "ip_cidr" | "ip_cidr_source" - ) { + ) +} + +fn log_protection_skip(input: &RequestFilterInput<'_>, rule_id: &str, reason: &str) { + if is_ip_exclusion_reason(reason) { log::info!( - "[datadome] protection decision=skipped rule={} reason={} method={} host={} path={} client_ip={}", + "[datadome] protection decision=skipped rule={} reason={} client_tag=omitted method={} host={} path={}", rule_id, reason, input.request.method(), request_host(input.request), input.request.uri().path(), - client_ip_for_log(input), ); } else { log::debug!( - "[datadome] protection decision=skipped rule={} reason={} method={} host={} path={} client_ip={}", + "[datadome] protection decision=skipped rule={} reason={} method={} host={} path={}", rule_id, reason, input.request.method(), request_host(input.request), input.request.uri().path(), - client_ip_for_log(input), ); } } @@ -465,41 +473,30 @@ fn log_protection_result( let method = input.request.method(); let host = request_host(input.request); let path = input.request.uri().path(); - let client_ip = client_ip_for_log(input); match decision { RequestFilterDecision::Respond { .. } => log::info!( - "[datadome] protection decision=blocked status={} method={} host={} path={} client_ip={} route=short_circuit", + "[datadome] protection decision=blocked status={} method={} host={} path={} route=short_circuit", status.as_u16(), method, host, path, - client_ip, ), RequestFilterDecision::Continue(_) if status == StatusCode::OK && datadome_status == Some(status.as_u16()) => { log::info!( - "[datadome] protection decision=allowed status={} method={} host={} path={} client_ip={} route=continue", + "[datadome] protection decision=allowed status={} method={} host={} path={} route=continue", status.as_u16(), method, host, path, - client_ip, ); } RequestFilterDecision::Continue(_) => {} } } -fn client_ip_for_log(input: &RequestFilterInput<'_>) -> String { - input - .services - .client_info() - .client_ip - .map_or_else(|| "unknown".to_string(), |ip| ip.to_string()) -} - struct ProtectionPayload { fields: Vec<(String, String)>, uses_header_client_id: bool, @@ -732,11 +729,17 @@ fn truncate_utf8(value: &str, limit: i32) -> String { #[cfg(test)] mod tests { use std::collections::HashMap; + use std::net::{IpAddr, Ipv4Addr}; use std::sync::Arc; - use crate::integrations::datadome::DataDomeConfig; + use crate::integrations::datadome::{ + DataDomeConfig, ProtectionExclusionRuleConfig, ProtectionMatcherConfig, + }; + use crate::platform::GeoInfo; use crate::platform::test_support::{ - HashMapSecretStore, NoopConfigStore, NoopSecretStore, build_services_with_config_and_secret, + HashMapConfigStore, HashMapSecretStore, NoopConfigStore, NoopSecretStore, + build_services_with_config_and_secret, build_services_with_config_and_secret_and_client_ip, + noop_services_with_client_ip, }; use crate::settings::Settings; @@ -751,6 +754,210 @@ mod tests { DataDomeIntegration::try_new(config).expect("should create integration") } + fn request_for_filter() -> Request { + request_builder() + .method(Method::GET.as_str()) + .uri("https://publisher.example/page") + .body(EdgeBody::empty()) + .expect("should build filter request") + } + + fn filter_marks_request( + config: DataDomeConfig, + services: &RuntimeServices, + ) -> Request { + filter_marks_request_with_geo(config, services, None) + } + + fn filter_marks_request_with_geo( + config: DataDomeConfig, + services: &RuntimeServices, + geo_info: Option<&GeoInfo>, + ) -> Request { + let integration = + DataDomeIntegration::try_new(config).expect("should create DataDome integration"); + let settings = Settings::default(); + let mut request = request_for_filter(); + let decision = futures::executor::block_on(integration.filter_protection_request( + RequestFilterInput { + settings: &settings, + services, + request: &mut request, + geo_info, + is_integration_route: false, + }, + )); + assert!( + matches!(decision, RequestFilterDecision::Continue(_)), + "an excluded request should continue without a Protection API response" + ); + request + } + + fn has_client_tag_suppression_marker(request: &Request) -> bool { + request + .extensions() + .get::() + .is_some() + } + + #[test] + fn ip_exclusions_mark_requests_for_client_tag_suppression() { + let ip = IpAddr::V4(Ipv4Addr::new(192, 0, 2, 10)); + let mut inline = DataDomeConfig { + enabled: true, + enable_protection: true, + protection_excluded_ip_cidrs: vec!["192.0.2.0/24".to_string()], + ..DataDomeConfig::default() + }; + let inline_request = + filter_marks_request(inline.clone(), &noop_services_with_client_ip(ip)); + assert!( + has_client_tag_suppression_marker(&inline_request), + "inline IP exclusions should mark the request" + ); + + inline.protection_excluded_ip_cidrs.clear(); + inline.protection_excluded_ip_cidr_sources = + vec![super::super::ProtectionIpCidrSourceConfig { + config_store: "datadome-test-source".to_string(), + key: "inline-source".to_string(), + }]; + let mut source_values = HashMap::new(); + source_values.insert("inline-source".to_string(), "192.0.2.0/24".to_string()); + let source_services = build_services_with_config_and_secret_and_client_ip( + HashMapConfigStore::new(source_values), + NoopSecretStore, + ip, + ); + let source_request = filter_marks_request(inline, &source_services); + assert!( + has_client_tag_suppression_marker(&source_request), + "Config Store IP exclusions should mark the request" + ); + + let structured_ip = DataDomeConfig { + enabled: true, + enable_protection: true, + protection_exclusion_rules: vec![ProtectionExclusionRuleConfig { + id: "structured-ip".to_string(), + enabled: true, + methods: Vec::new(), + matcher: ProtectionMatcherConfig::IpCidr { + cidrs: vec!["192.0.2.0/24".to_string()], + }, + }], + ..DataDomeConfig::default() + }; + let structured_request = + filter_marks_request(structured_ip, &noop_services_with_client_ip(ip)); + assert!( + has_client_tag_suppression_marker(&structured_request), + "structured IP exclusions should mark the request" + ); + + let structured_source = DataDomeConfig { + enabled: true, + enable_protection: true, + protection_exclusion_rules: vec![ProtectionExclusionRuleConfig { + id: "structured-ip-source".to_string(), + enabled: true, + methods: Vec::new(), + matcher: ProtectionMatcherConfig::IpCidrSource { + config_store: "datadome-test-source".to_string(), + key: "structured-source".to_string(), + }, + }], + ..DataDomeConfig::default() + }; + let mut structured_values = HashMap::new(); + structured_values.insert("structured-source".to_string(), "192.0.2.0/24".to_string()); + let structured_services = build_services_with_config_and_secret_and_client_ip( + HashMapConfigStore::new(structured_values), + NoopSecretStore, + ip, + ); + let structured_source_request = + filter_marks_request(structured_source, &structured_services); + assert!( + has_client_tag_suppression_marker(&structured_source_request), + "structured Config Store IP exclusions should mark the request" + ); + } + + #[test] + fn non_ip_exclusions_do_not_mark_requests_for_client_tag_suppression() { + let ip = IpAddr::V4(Ipv4Addr::new(192, 0, 2, 10)); + let cases = [DataDomeConfig { + enabled: true, + enable_protection: true, + protection_exclusion_rules: vec![ProtectionExclusionRuleConfig { + id: "path".to_string(), + enabled: true, + methods: Vec::new(), + matcher: ProtectionMatcherConfig::PathExact { + paths: vec!["/page".to_string()], + }, + }], + ..DataDomeConfig::default() + }]; + + for config in cases { + let request = filter_marks_request(config, &noop_services_with_client_ip(ip)); + assert!( + !has_client_tag_suppression_marker(&request), + "non-IP exclusions should not mark the request" + ); + } + } + + #[test] + fn asn_exclusions_do_not_mark_requests_for_client_tag_suppression() { + let config = DataDomeConfig { + enabled: true, + enable_protection: true, + protection_excluded_asns: vec![64500], + ..DataDomeConfig::default() + }; + let geo_info = GeoInfo { + city: String::new(), + country: String::new(), + continent: String::new(), + latitude: 0.0, + longitude: 0.0, + metro_code: 0, + region: None, + asn: Some(64500), + }; + let request = filter_marks_request_with_geo( + config, + &noop_services_with_client_ip(IpAddr::V4(Ipv4Addr::new(192, 0, 2, 10))), + Some(&geo_info), + ); + assert!( + !has_client_tag_suppression_marker(&request), + "ASN exclusions should not mark the request" + ); + } + + #[test] + fn non_matching_ip_does_not_mark_request_for_client_tag_suppression() { + let config = DataDomeConfig { + enabled: true, + enable_protection: true, + protection_excluded_ip_cidrs: vec!["192.0.2.0/24".to_string()], + ..DataDomeConfig::default() + }; + let request = filter_marks_request( + config, + &noop_services_with_client_ip(IpAddr::V4(Ipv4Addr::new(198, 51, 100, 10))), + ); + assert!( + !has_client_tag_suppression_marker(&request), + "a non-matching IP should not mark the request" + ); + } + #[test] fn load_server_side_key_reads_secret_store() { let mut secrets = HashMap::new(); @@ -835,7 +1042,7 @@ mod tests { // the Protection API. let services = build_services_with_config_and_secret(NoopConfigStore, NoopSecretStore); let settings = Settings::default(); - let request = request_builder() + let mut request = request_builder() .method(Method::OPTIONS.as_str()) .uri("https://publisher.example/_ts/api/v1/identify") .body(EdgeBody::empty()) @@ -846,7 +1053,7 @@ mod tests { RequestFilterInput { settings: &settings, services: &services, - request: &request, + request: &mut request, geo_info: None, is_integration_route: false, }, diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index 0644fb522..291a54242 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -327,7 +327,7 @@ pub trait IntegrationProxy: Send + Sync { pub struct RequestFilterInput<'a> { pub settings: &'a Settings, pub services: &'a RuntimeServices, - pub request: &'a Request, + pub request: &'a mut Request, pub geo_info: Option<&'a GeoInfo>, /// Whether the request matches a registered integration proxy route. pub is_integration_route: bool, @@ -1345,6 +1345,8 @@ mod tests { } struct EnrichingRequestFilter; + #[derive(Clone, Copy)] + struct RequestAnnotation; #[async_trait(?Send)] impl IntegrationRequestFilter for EnrichingRequestFilter { @@ -1354,8 +1356,9 @@ mod tests { async fn filter_request( &self, - _input: RequestFilterInput<'_>, + input: RequestFilterInput<'_>, ) -> Result> { + input.request.extensions_mut().insert(RequestAnnotation); Ok(RequestFilterDecision::Continue(RequestFilterEffects { request_headers: vec![HeaderMutation::set("x-datadome-isbot", "1")], response_headers: vec![HeaderMutation::set("x-dd-b", "allowed")], @@ -1487,6 +1490,10 @@ mod tests { Some("1"), "should apply DataDome-style request enrichment before routing" ); + assert!( + req.extensions().get::().is_some(), + "should preserve private request annotations for downstream routing" + ); match outcome { RequestFilterRegistryOutcome::Continue(effects) => { assert_eq!( diff --git a/crates/trusted-server-core/src/platform/test_support.rs b/crates/trusted-server-core/src/platform/test_support.rs index 94b42161c..13b6d70cd 100644 --- a/crates/trusted-server-core/src/platform/test_support.rs +++ b/crates/trusted-server-core/src/platform/test_support.rs @@ -631,6 +631,25 @@ pub(crate) fn build_services_with_config_and_secret( .build() } +pub(crate) fn build_services_with_config_and_secret_and_client_ip( + config_store: impl PlatformConfigStore + 'static, + secret_store: impl PlatformSecretStore + 'static, + client_ip: IpAddr, +) -> RuntimeServices { + RuntimeServices::builder() + .config_store(Arc::new(config_store)) + .secret_store(Arc::new(secret_store)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::new(NoopBackend)) + .http_client(Arc::new(NoopHttpClient)) + .geo(Arc::new(NoopGeo)) + .client_info(ClientInfo { + client_ip: Some(client_ip), + ..ClientInfo::default() + }) + .build() +} + pub(crate) fn build_request_signing_services() -> RuntimeServices { let signing_key = SigningKey::generate(&mut OsRng); let key_b64 = general_purpose::STANDARD.encode(signing_key.as_bytes()); diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index d3410b4ed..48e0eaacf 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -357,6 +357,7 @@ struct ProcessResponseParams<'a> { integration_registry: &'a IntegrationRegistry, ad_slots_script: Option<&'a str>, ad_bids_state: &'a Arc>>, + suppress_datadome_client_side_tag: bool, gpt_diagnostics: Option<&'a crate::integrations::gpt_diagnostics::GptDiagnosticsRequestDecision>, } @@ -383,6 +384,7 @@ impl PublisherBodyProcessor { integration_registry, ad_slots_script: params.ad_slots_script.as_deref().map(str::to_string), ad_bids_state: Arc::clone(¶ms.ad_bids_state), + suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, gpt_diagnostics: params.gpt_diagnostics.clone(), })?) } else if is_rsc_flight { @@ -460,6 +462,7 @@ fn process_response_streaming( integration_registry: params.integration_registry, ad_slots_script: params.ad_slots_script.map(str::to_string), ad_bids_state: params.ad_bids_state.clone(), + suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, gpt_diagnostics: params.gpt_diagnostics.cloned(), })?; StreamingPipeline::new(config, processor) @@ -944,6 +947,7 @@ struct HtmlStreamProcessorParams<'a> { integration_registry: &'a IntegrationRegistry, ad_slots_script: Option, ad_bids_state: Arc>>, + suppress_datadome_client_side_tag: bool, gpt_diagnostics: Option, } @@ -960,7 +964,8 @@ fn create_html_stream_processor( params.request_scheme, ) .with_ad_state(params.ad_slots_script, params.ad_bids_state) - .with_gpt_diagnostics(params.gpt_diagnostics); + .with_gpt_diagnostics(params.gpt_diagnostics) + .with_datadome_client_tag_suppression(params.suppress_datadome_client_side_tag); Ok(create_html_processor(config)) } @@ -1081,6 +1086,8 @@ pub struct OwnedProcessResponseParams { pub(crate) dispatched_auction: Option, /// Price granularity used to bucket bids when building `tsjs.bids`. pub(crate) price_granularity: PriceGranularity, + /// Whether to omit Trusted Server's automatic `DataDome` client-side tag. + pub(crate) suppress_datadome_client_side_tag: bool, /// Request-scoped conditional diagnostics delivery decision. pub(crate) gpt_diagnostics: Option, @@ -1431,6 +1438,28 @@ fn response_carries_body(method: &Method, status: StatusCode) -> bool { && status != StatusCode::NOT_MODIFIED } +/// Prevent shared caches from replaying tag-suppressed HTML to other clients. +fn apply_datadome_client_tag_cache_privacy( + response: &mut Response, + method: &Method, + suppress_datadome_client_side_tag: bool, + content_type: &str, +) { + if !suppress_datadome_client_side_tag + || !response_carries_body(method, response.status()) + || !is_html_content_type(content_type) + { + return; + } + + response.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static("private, max-age=0"), + ); + response.headers_mut().remove("surrogate-control"); + response.headers_mut().remove("fastly-surrogate-control"); +} + /// Drop a bodiless response's body and correct its framing headers. /// /// The response keeps no body, and its `Content-Length` is corrected where the @@ -1547,6 +1576,7 @@ pub fn stream_publisher_body( integration_registry, ad_slots_script: params.ad_slots_script.as_deref(), ad_bids_state: ¶ms.ad_bids_state, + suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, gpt_diagnostics: params.gpt_diagnostics.as_ref(), }; process_response_streaming(body, output, &borrowed) @@ -1640,6 +1670,7 @@ pub async fn stream_publisher_body_async( integration_registry, ad_slots_script: params.ad_slots_script.as_deref().map(str::to_string), ad_bids_state: params.ad_bids_state.clone(), + suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, gpt_diagnostics: params.gpt_diagnostics.clone(), }) { Ok(processor) => processor, @@ -2859,6 +2890,11 @@ pub async fn handle_publisher_request( // sets the flag unconditionally and tolerates buffered fallback): adapters // without streaming support may reject the flag outright rather than // silently buffering, which would fail every publisher fetch. + let request_method = req.method().clone(); + let suppress_datadome_client_side_tag = req + .extensions() + .get::() + .is_some(); let mut platform_request = PlatformHttpRequest::new(req, backend_name); if services.http_client().supports_streaming_responses() { platform_request = platform_request.with_stream_response(); @@ -3051,6 +3087,12 @@ pub async fn handle_publisher_request( content_encoding ); + apply_datadome_client_tag_cache_privacy( + &mut response, + &request_method, + suppress_datadome_client_side_tag, + &content_type, + ); let body = std::mem::replace(response.body_mut(), EdgeBody::empty()); response.headers_mut().remove(header::CONTENT_LENGTH); @@ -3066,6 +3108,7 @@ pub async fn handle_publisher_request( content_type, ad_slots_script: ad_slots_script.clone(), ad_bids_state: ad_bids_state.clone(), + suppress_datadome_client_side_tag, auction_observation, auction_request: auction_request_for_telemetry, dispatched_auction, @@ -4282,6 +4325,7 @@ mod tests { dispatched_auction: None, price_granularity: Default::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, } } @@ -5274,6 +5318,121 @@ mod tests { ); } + #[test] + fn suppressed_datadome_tag_reaches_publisher_html_pipeline() { + let mut settings = create_test_settings(); + settings + .integrations + .insert_config( + "datadome", + &serde_json::json!({ + "enabled": true, + "client_side_key": "test-client-key", + }), + ) + .expect("should configure DataDome integration"); + let registry = IntegrationRegistry::new(&settings) + .expect("should create integration registry with DataDome"); + let mut params = make_stream_params(&settings, "identity"); + params.content_type = "text/html; charset=utf-8".to_string(); + params.suppress_datadome_client_side_tag = true; + let mut output = Vec::new(); + + stream_publisher_body( + EdgeBody::from(b"content".to_vec()), + &mut output, + ¶ms, + &settings, + ®istry, + ) + .expect("should process suppressed HTML"); + + let html = String::from_utf8(output).expect("should produce UTF-8 HTML"); + assert!( + !html.contains("window.ddjskey"), + "publisher processing should omit the DataDome client configuration" + ); + assert!( + !html.contains("/integrations/datadome/tags.js"), + "publisher processing should omit the DataDome client tag URL" + ); + } + + #[test] + fn suppressed_datadome_html_is_private_and_not_shared_cached() { + let mut response = Response::builder() + .status(StatusCode::OK) + .header(header::CACHE_CONTROL, "public, max-age=600") + .header("surrogate-control", "max-age=600") + .header("fastly-surrogate-control", "max-age=600") + .body(EdgeBody::empty()) + .expect("should build cacheable HTML response"); + + super::apply_datadome_client_tag_cache_privacy( + &mut response, + &Method::GET, + true, + "text/html; charset=utf-8", + ); + + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("private, max-age=0"), + "suppressed HTML should be private" + ); + assert!( + response.headers().get("surrogate-control").is_none(), + "suppressed HTML should not retain Surrogate-Control" + ); + assert!( + response.headers().get("fastly-surrogate-control").is_none(), + "suppressed HTML should not retain Fastly-Surrogate-Control" + ); + } + + #[test] + fn datadome_cache_privacy_does_not_change_non_html_or_unsuppressed_responses() { + let mut response = Response::builder() + .status(StatusCode::OK) + .header(header::CACHE_CONTROL, "public, max-age=600") + .header("surrogate-control", "max-age=600") + .body(EdgeBody::empty()) + .expect("should build cacheable response"); + + super::apply_datadome_client_tag_cache_privacy( + &mut response, + &Method::GET, + false, + "text/html; charset=utf-8", + ); + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("public, max-age=600"), + "unsuppressed HTML should retain its existing cache policy" + ); + + super::apply_datadome_client_tag_cache_privacy( + &mut response, + &Method::GET, + true, + "text/css", + ); + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("public, max-age=600"), + "non-HTML should retain its existing cache policy" + ); + } + #[test] fn response_carries_body_preserves_bodiless_metadata() { // A processable GET 200 buffers a body and recomputes Content-Length. @@ -6229,6 +6388,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let mut output = Vec::new(); @@ -6277,6 +6437,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let mut output = Vec::new(); @@ -6314,6 +6475,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let body = EdgeBody::from_stream(futures::stream::iter(vec![Ok::<_, io::Error>( bytes::Bytes::from_static(b"live"), @@ -6429,6 +6591,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let body = EdgeBody::stream(futures::stream::iter(vec![ bytes::Bytes::from_static(b"body{background:url('https://origin.example.com/"), @@ -6482,6 +6645,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let compressed = gzip_encode(b"body{background:url('https://origin.example.com/asset.png')}"); @@ -6538,6 +6702,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let compressed = deflate_encode(b"body{background:url('https://origin.example.com/asset.png')}"); @@ -6594,6 +6759,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let compressed = brotli_encode(b"body{background:url('https://origin.example.com/asset.png')}"); @@ -6650,6 +6816,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let compressed = brotli_encode(b"body{background:url('https://origin.example.com/asset.png')}"); @@ -6694,6 +6861,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, } } @@ -6888,6 +7056,7 @@ mod tests { )), price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let body = EdgeBody::stream(futures::stream::iter(vec![ bytes::Bytes::from_static(b"hello"), @@ -6952,6 +7121,7 @@ mod tests { )), price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; // The `` that triggers bid injection lives in the SECOND gzip // member. `flate2::read::GzDecoder` decodes only the first member, so @@ -7015,6 +7185,7 @@ mod tests { )), price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let body = EdgeBody::stream(futures::stream::iter(vec![bytes::Bytes::from_static( b"body{background:url('https://origin.example.com/asset.png')}", @@ -7071,6 +7242,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let publisher_response = PublisherResponse::Stream { response, @@ -7207,6 +7379,7 @@ mod tests { dispatched_auction, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, } } @@ -7558,6 +7731,7 @@ mod tests { )), price_granularity: PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, } }; let make_stream_response = || PublisherResponse::Stream { @@ -7737,6 +7911,7 @@ mod tests { )), price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let publisher_response = PublisherResponse::Stream { response, @@ -7804,6 +7979,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let mut output = Vec::new(); @@ -7854,6 +8030,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let bogus_body = EdgeBody::from(b"not gzip".to_vec()); @@ -7962,6 +8139,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let mut output = Vec::new(); stream_publisher_body(body, &mut output, ¶ms, &settings, ®istry) @@ -8019,6 +8197,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let mut output = Vec::new(); diff --git a/docs/guide/integrations/datadome.md b/docs/guide/integrations/datadome.md index 9c342679b..1743a1b75 100644 --- a/docs/guide/integrations/datadome.md +++ b/docs/guide/integrations/datadome.md @@ -173,6 +173,33 @@ Static assets are excluded by default using a case-insensitive file-extension re Auction traffic at `/auction` is protected by default. +### IP-excluded client-side tag behavior + +On the Fastly adapter, a request that matches an IP-based DataDome exclusion +also omits Trusted Server's automatically injected client-side DataDome tag +from processed HTML. This keeps the client-side layer consistent with the +server-side Protection API skip. + +This behavior applies to: + +- `protection_excluded_ip_cidrs`; +- `protection_excluded_ip_cidr_sources`; +- structured `ip_cidr` rules; and +- structured `ip_cidr_source` rules. + +ASN, method, path, query-parameter, static-asset, and internal-route +exclusions do not automatically suppress the client-side tag. DataDome tags +already present in publisher HTML are not removed or changed by this behavior, +and `/integrations/datadome/tags.js` remains available when requested directly. + +Because the processed HTML differs by client IP, tag-suppressed HTML is marked +`private, max-age=0` and removed from shared surrogate caches. The decision is +reported in the existing protection log, for example: + +```text +[datadome] protection decision=skipped rule=excluded-ip-cidrs reason=client_ip client_tag=omitted method=GET host=example.com path=/page +``` + ### Structured exclusion rules Use structured rules for all DataDome protection exclusions. Each rule has an `id`, optional `methods`, and a typed matcher. The default configuration includes a `path_regex` rule for common static assets. diff --git a/docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md b/docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md new file mode 100644 index 000000000..5ea5f6ff3 --- /dev/null +++ b/docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md @@ -0,0 +1,475 @@ +# DataDome IP-excluded client tag suppression — Implementation Plan + +> **Status:** Approved for implementation +> +> **For implementers:** Work task by task and keep the workspace buildable. +> Follow `CLAUDE.md`: use target-matched Cargo aliases, do not use bare +> workspace tests, and do not add an internal HTTP header for request state. + +**Goal:** When Fastly's authoritative client IP matches a DataDome IP exclusion, +skip the Protection API call and omit only Trusted Server's automatically +injected DataDome client tag from every processed HTML response. + +**Issue:** #994 +**Design:** +`docs/superpowers/specs/2026-08-03-datadome-ip-excluded-client-tag-design.md` + +## Approved behavior + +| Request condition | Protection API | Trusted Server auto-injected tag | Publisher-originated tag | +| ------------------------------------------------------------- | --------------- | -------------------------------- | ------------------------ | +| Inline IP CIDR match | Skipped | Omitted | Unchanged | +| Config Store IP CIDR-source match | Skipped | Omitted | Unchanged | +| Structured `ip_cidr` match | Skipped | Omitted | Unchanged | +| Structured `ip_cidr_source` match | Skipped | Omitted | Unchanged | +| ASN, method, path, query, static, or internal-route exclusion | Skipped | Preserved | Unchanged | +| No exclusion match | Called normally | Preserved | Unchanged | +| Protection API fail-open | Continued | Preserved | Unchanged | + +The Fastly-only scope means that other adapters receive the default +non-suppressed value. Do not add a configuration option and do not modify their +request-filter wiring. + +## Runtime contracts + +1. **Trusted identity source:** determine exclusion from + `RuntimeServices::client_info().client_ip`, never a caller-provided header. +2. **Single evaluation:** use the existing `ProtectionScope` decision. Do not + evaluate CIDRs a second time while injecting HTML; this avoids diverging + Config Store/cache behavior. +3. **Private marker:** communicate the decision with a typed request extension, + never a request/response header. The marker cannot leak to the origin or + client. +4. **Precise scope:** tag suppression is keyed only on decision reasons + `client_ip`, `client_ip_source`, `ip_cidr`, and `ip_cidr_source`. +5. **Cache safety:** an HTML response with the tag omitted differs by client IP. + A suppressed processed HTML response must be `private, max-age=0` and have + `Surrogate-Control` and `Fastly-Surrogate-Control` removed. Do not alter + cache headers when the response is not processed HTML, because this feature + does not alter that body. +6. **No behavior drift:** DataDome proxy endpoints, response-header effects, + `rewrite_sdk`, and DataDome tags that were already in origin HTML retain + their current behavior. + +## File map + +| File | Change | +| ----------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `crates/trusted-server-core/src/integrations/registry.rs` | Permit filters to attach private typed request extensions while retaining header-effect semantics. Extend the HTML context with the propagated boolean. | +| `crates/trusted-server-core/src/integrations/datadome.rs` | Define the crate-private marker and have the head injector honor the HTML-context flag. | +| `crates/trusted-server-core/src/integrations/datadome/protection.rs` | Recognize IP scope skips, attach the marker, and add `client_tag=omitted` to the existing info log. | +| `crates/trusted-server-core/src/html_processor.rs` | Carry the per-response suppression boolean from config to all integration HTML contexts. | +| `crates/trusted-server-core/src/publisher.rs` | Snapshot the marker before origin dispatch, propagate it through every HTML streaming path, and apply cache privacy to suppressed processed HTML. | +| `docs/guide/integrations/datadome.md` | Document the Fastly IP-exclusion behavior and its limits. | +| `docs/superpowers/specs/2026-08-03-datadome-ip-excluded-client-tag-design.md` | Already updated with the cache-variance safeguard. | + +No changes are expected in `trusted-server.example.toml`, JavaScript bundles, +or non-Fastly adapters. + +--- + +## Task 1: Make the request-filter input capable of private annotations + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/registry.rs` +- Test: its existing `#[cfg(test)]` module + +The current `RequestFilterInput` holds `&Request`. Change it to hold +`&mut Request` so a request filter can add a typed extension. This is +the narrowest safe transport because the registry already has exclusive mutable +access to the request while it invokes each filter. + +- [ ] **Step 1: Add a regression test for an extension-producing filter.** Create + a test-only zero-sized marker and filter that writes it to + `input.request.extensions_mut()`. Run `IntegrationRegistry::filter_request` + and assert the original mutable request has the marker afterward. In the + same test, verify normal `RequestFilterEffects` still apply their request + header mutation and return their response header mutation. +- [ ] **Step 2: Change `RequestFilterInput::request` to a mutable borrow.** Keep + the `IntegrationRequestFilter` method signature and `RequestFilterEffects` + unchanged. +- [ ] **Step 3: Update `IntegrationRegistry::filter_request`.** Pass its existing + `&mut Request` directly to each `RequestFilterInput`. Keep the ordering: + filter mutation first, then registry-applied request-header effects, then + the next filter. +- [ ] **Step 4: Update all direct filter tests and test filters.** Calls that build + `RequestFilterInput` must construct a mutable request and pass + `request: &mut request`. Read-only filters should continue to compile by + simply not mutating the request. +- [ ] **Step 5: Run focused tests.** + +```bash +cargo test-fastly integrations::registry +``` + +**Acceptance:** a filter can retain a typed marker for downstream route handling +without emitting a synthetic `x-*` header, and existing header effects retain +their behavior. + +--- + +## Task 2: Mark IP-based DataDome exclusions and log the outcome + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/datadome.rs` +- Modify: `crates/trusted-server-core/src/integrations/datadome/protection.rs` +- Test: `crates/trusted-server-core/src/integrations/datadome/protection.rs` +- Reuse: `crates/trusted-server-core/src/integrations/datadome/protection_scope.rs` + +- [ ] **Step 1: Add a crate-private marker in `datadome.rs`.** Define a + zero-sized type with a behavior-oriented name, such as + `DataDomeClientTagSuppressed`. It must be visible to `publisher.rs` and + `protection.rs` through `pub(crate)`, but must not be exported as public + integration configuration or API. +- [ ] **Step 2: Add an IP-reason predicate beside protection logging.** Centralize + the exact four eligible scope reasons in one helper: + +```rust +matches!(reason, "client_ip" | "client_ip_source" | "ip_cidr" | "ip_cidr_source") +``` + + Do not infer eligibility from rule ID: Config Store source rule IDs are + operator-configured strings. + +- [ ] **Step 3: Make `filter_protection_request` own a mutable input and pass it + mutably to `is_request_protected`.** In the existing + `ProtectionScopeDecision::Skip` arm: + + 1. determine whether the reason is IP-based; + 2. if so, insert the typed marker into `input.request.extensions_mut()`; + 3. call the updated skip logger with `client_tag_omitted = true`; and + 4. return `false` exactly as today so the Protection API is not called. + + Do not set the marker for the early method/integration/internal-route + returns. Do not set it when the API call returns a fail-open error. + +- [ ] **Step 4: Update `log_protection_skip`.** Keep IP exclusions at `info` and + non-IP exclusions at `debug`. For the IP branch, extend the existing + structured text after the reason with `client_tag=omitted`; retain rule, + reason, method, host, and path, but do not include the client IP. The + desired shape is: + +```text +[datadome] protection decision=skipped rule=excluded-ip-cidrs reason=client_ip client_tag=omitted method=GET host=example.com path=/page +``` + +- [ ] **Step 5: Add filter-level marker tests.** Add small helpers in the + protection test module to build `RuntimeServices` with a fixed client IP, + optional Config Store data, and a mutable request. For each case, call + `filter_protection_request`, assert it returns `Continue`, and inspect the + request extension: + + - inline `protection_excluded_ip_cidrs` match → marker present; + - `protection_excluded_ip_cidr_sources` match → marker present; + - structured `ProtectionMatcherConfig::IpCidr` match → marker present; + - structured `ProtectionMatcherConfig::IpCidrSource` match → marker present. + + Clear the process-global CIDR-source test cache before and after source + tests so cached values cannot affect another case. + +- [ ] **Step 6: Add negative filter-level tests.** Assert the marker is absent + for a non-matching IP, a configured ASN match, a structured path match, + a structured query match, an excluded method, and an internal/integration + route. Reuse the existing `ProtectionScope` unit tests for matching + semantics; these new tests verify only the new side effect. +- [ ] **Step 7: Preserve API-call behavior.** For an IP marker test, use an HTTP + client double that records calls or errors if called. Assert no Protection + API request is sent. This protects against accidentally marking a request + while still invoking DataDome. +- [ ] **Step 8: Run focused tests.** + +```bash +cargo test-fastly datadome::protection +cargo test-fastly datadome::protection_scope +``` + +**Acceptance:** only the four IP decision reasons add the private marker and +produce the augmented informational skip log; all other exclusion and fail-open +paths keep their current tag behavior. + +--- + +## Task 3: Thread suppression through publisher response processing + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-core/src/html_processor.rs` +- Modify: `crates/trusted-server-core/src/integrations/registry.rs` +- Test: `publisher.rs` and `html_processor.rs` test modules + +### Data flow to implement + +```text +DataDomeClientTagSuppressed request extension + -> bool captured by handle_publisher_request before origin dispatch + -> OwnedProcessResponseParams + -> ProcessResponseParams / HtmlStreamProcessorParams + -> HtmlProcessorConfig + -> IntegrationHtmlContext + -> DataDomeIntegration::head_inserts +``` + +- [ ] **Step 1: Capture the marker once in `handle_publisher_request`.** Read + `req.extensions().get::().is_some()` before + `req` is rewritten and moved into `PlatformHttpRequest`. Store the boolean + only in the `PublisherResponse::Stream` parameters, because that is the + only response route that passes through HTML injection. +- [ ] **Step 2: Add a boolean to the owned and borrowed publisher-processing + parameter structs.** Add a clearly named field such as + `suppress_datadome_client_side_tag` to: + + - `OwnedProcessResponseParams`; + - `ProcessResponseParams`; and + - `HtmlStreamProcessorParams`. + + Pass it through all three existing HTML construction sites: + + - `PublisherBodyProcessor::new` for async buffered processing; + - `process_response_streaming` for synchronous processing; and + - `stream_publisher_body_async` for the Fastly streaming auction-hold path. + + Every test fixture that constructs `OwnedProcessResponseParams` directly + must set `false` unless it explicitly exercises suppression. + +- [ ] **Step 3: Extend `HtmlProcessorConfig`.** Add the same boolean, default it + to `false` in `from_settings`, and add a narrow builder method used by + `create_html_stream_processor`. Update direct `HtmlProcessorConfig` + fixtures and the benchmark fixture to set `false` explicitly. +- [ ] **Step 4: Extend `IntegrationHtmlContext`.** Add the boolean as immutable + request-scoped context. Populate it at both construction sites in + `html_processor.rs`: + + - the streaming `` element handler; and + - `HtmlWithPostProcessing::process_chunk` for full-document post-processors. + + Update every test helper that constructs `IntegrationHtmlContext` to set + `false` by default. + +- [ ] **Step 5: Add plumbing tests.** + + - `HtmlProcessorConfig::from_settings` defaults to non-suppressed. + - A test head injector records the context flag and sees `true` when a config + is built with suppression. + - A `publisher.rs` route test inserts the DataDome marker into a request, + receives a processable HTML `PublisherResponse::Stream`, and verifies the + owned parameters carry `true`. + - A buffered and a streaming-body path both preserve `true` to head injection. + +- [ ] **Step 6: Run focused tests.** + +```bash +cargo test-fastly html_processor +cargo test-fastly publisher +``` + +**Acceptance:** the decision is read once from a private request extension and +is available to every head injector for every processed HTML response, including +Fastly's streaming path. + +--- + +## Task 4: Omit only Trusted Server's injected DataDome tag + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/datadome.rs` +- Test: `crates/trusted-server-core/src/integrations/datadome.rs` +- Test: `crates/trusted-server-core/src/html_processor.rs` or `publisher.rs` + +- [ ] **Step 1: Add a direct head-injector regression test.** With a client-side + key configured and `ctx.suppress_datadome_client_side_tag = true`, assert + `head_inserts()` returns an empty vector. The same config with `false` + must still return exactly one snippet containing both `window.ddjskey` and + the configured tag URL. +- [ ] **Step 2: Implement the guard as the first condition in + `DataDomeIntegration::head_inserts`.** Return an empty vector when the + context flag is true; otherwise retain all current serialization, + escaping, blank-key, and `inject_client_side_tag` behavior unchanged. +- [ ] **Step 3: Add an end-to-end HTML pipeline test.** Configure the DataDome + integration with a client-side key, process representative HTML with + suppression enabled, and assert the result contains neither: + +```text +window.ddjskey= +/integrations/datadome/tags.js +``` + + Repeat with suppression disabled and assert both appear. + +- [ ] **Step 4: Pin publisher-originated-tag behavior.** Feed origin HTML that + contains a DataDome `tags.js` element. With suppression enabled, assert + that element remains in output and is rewritten by `rewrite_sdk` exactly + as before. This distinguishes automatic injection from origin markup. +- [ ] **Step 5: Pin direct route behavior.** Retain or add a DataDome proxy test + showing that `GET /integrations/datadome/tags.js` remains registered and + fetches/proxies the SDK normally; suppression affects only HTML injection. +- [ ] **Step 6: Run focused tests.** + +```bash +cargo test-fastly datadome +``` + +**Acceptance:** suppression removes only the generated configuration/script +pair; nothing removes publisher markup or disables DataDome endpoints. + +--- + +## Task 5: Make tag-suppressed processed HTML private + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Test: `crates/trusted-server-core/src/publisher.rs` + +The automatic tag makes the processed HTML vary by client IP. Cache privacy is +therefore a correctness and protection requirement, not an optional +optimization. + +- [ ] **Step 1: Add a failing cache-privacy test.** Build a `PublisherResponse` + with a processable HTML content type, suppression `true`, and cacheable + origin headers (`Cache-Control`, `Surrogate-Control`, and + `Fastly-Surrogate-Control`). Assert the stream response is: + + - `Cache-Control: private, max-age=0`; and + - missing both surrogate cache headers. + +- [ ] **Step 2: Apply privacy only in the `ResponseRoute::Stream` HTML arm.** + After response classification confirms a processable HTML stream, use the + existing per-user ad-stack policy as the model. Do not alter cache headers + for CSS, RSC, non-processable pass-through, unsupported encodings, HEAD, + 204/205/304, or responses without suppression: none has a body variation + created by this feature. +- [ ] **Step 3: Add non-regression cache tests.** Verify that: + + - non-suppressed processed HTML keeps its existing cache headers unless + another existing policy changes them; + - a suppressed CSS/non-HTML stream is not made private by this feature; and + - existing ad-stack privacy behavior remains unchanged when both features are + active. + +- [ ] **Step 4: Run focused tests.** + +```bash +cargo test-fastly publisher +``` + +**Acceptance:** a shared cache cannot replay an IP-excluded client's tagless +HTML to a non-excluded visitor, while unchanged responses retain their existing +cacheability. + +--- + +## Task 6: Add Fastly-path regression coverage + +**Files:** + +- Modify: `crates/trusted-server-adapter-fastly/src/app.rs` tests only if the + existing dispatch helpers can exercise the DataDome registry with a stubbed + publisher response. +- Otherwise, document the existing core filter + publisher pipeline tests as + the executable behavioral coverage; do not refactor Fastly production code + merely to enable a duplicate test. + +- [ ] **Step 1: Extend the existing Fastly request-filter dispatch regression + test or add a focused equivalent.** Configure a DataDome request filter, + insert trusted `ClientInfo` into the request extensions with a matching + IP, and confirm the filter runs before publisher routing. +- [ ] **Step 2: Assert that the routed request retains the private DataDome + marker.** The assertion must inspect request extensions or the processed + HTML result, not an HTTP header. +- [ ] **Step 3: Ensure no actual DataDome API call occurs for the matching IP.** + Use a recording/failing HTTP client or the existing Fastly test seam. +- [ ] **Step 4: Add the non-matching counterpart.** It must not receive the + marker and must continue to inject the configured tag when HTML is + processed. +- [ ] **Step 5: Run Fastly adapter tests.** + +```bash +cargo test-fastly +``` + +**Acceptance:** the production adapter's actual filter ordering preserves the +marker from authoritative Fastly client metadata through publisher HTML +processing. If the current test seam cannot stub a full origin response, retain +this as focused request-filter-order coverage and rely on Task 3's core +pipeline tests for body output rather than expanding adapter production code. + +--- + +## Task 7: Document operator-visible behavior + +**Files:** + +- Modify: `docs/guide/integrations/datadome.md` +- Do not modify: `trusted-server.example.toml` + +- [ ] **Step 1: Add a subsection adjacent to “Protected traffic” or “Client-side + setup.”** State that, on Fastly, an IP exclusion skips the Protection API + and suppresses only Trusted Server's automatic DataDome tag injection on + processed HTML. +- [ ] **Step 2: List the four covered IP sources.** Use the exact configuration + names and structured rule types. +- [ ] **Step 3: State the exclusions that do not suppress the client tag.** ASN, + method, path, query, static-asset, and internal-route exclusions retain + normal auto-injection. +- [ ] **Step 4: State the limits.** Publisher-originated/manual tags are not + removed; `/integrations/datadome/tags.js` remains available; no new + configuration is required; and tag-suppressed processed HTML is private + to prevent shared-cache replay. +- [ ] **Step 5: Add the diagnostic example.** Use an example-only host/IP and + include `client_tag=omitted` with rule and reason. +- [ ] **Step 6: Format-check the changed documentation.** + +```bash +cd docs +npx prettier --check guide/integrations/datadome.md \ + superpowers/specs/2026-08-03-datadome-ip-excluded-client-tag-design.md \ + superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md +``` + +**Acceptance:** operators can predict exactly when the tag will be omitted and +understand that this is an IP-based Fastly behavior, not a general exclusion +side effect. + +--- + +## Final verification + +- [ ] Confirm the working tree contains only the intended core, Fastly-test, + guide, spec, and plan changes. +- [ ] Run formatting. + +```bash +cargo fmt --all -- --check +``` + +- [ ] Run the relevant target-matched test suites. + +```bash +cargo test-fastly +cargo test-axum +cargo test-cloudflare +``` + +- [ ] Run required lint suites. + +```bash +cargo clippy-fastly +cargo clippy-axum +cargo clippy-cloudflare +``` + +- [ ] Run the docs check from Task 7. +- [ ] Review the diff for accidental exposure of the marker as a request or + response header, duplicate CIDR evaluation, unintended publisher-tag + removal, or shared-cacheable tag-suppressed HTML. + +## Deferred acceptance + +Do **not** perform live production/browser verification in this change. After +deployment, the separate testing workflow should verify that a matching +whitelisted IP receives processed HTML without Trusted Server's +`/integrations/datadome/tags.js` injection, while an unlisted IP retains it. diff --git a/docs/superpowers/specs/2026-08-03-datadome-ip-excluded-client-tag-design.md b/docs/superpowers/specs/2026-08-03-datadome-ip-excluded-client-tag-design.md new file mode 100644 index 000000000..c48ad2fda --- /dev/null +++ b/docs/superpowers/specs/2026-08-03-datadome-ip-excluded-client-tag-design.md @@ -0,0 +1,339 @@ +# DataDome IP-excluded client tag suppression + +**Issue:** #994 +**Date:** 2026-08-03 +**Status:** Proposed + +## Problem + +Trusted Server has two DataDome protection layers: + +1. Server-side Protection API validation, which can be skipped for configured + client IP CIDRs. +2. Client-side tag auto-injection, which adds `window.ddjskey`, + `window.ddoptions`, and the configured `tags.js` script to processed HTML. + +When a request matches an IP-based server-side exclusion, the Protection API is +skipped, but the client-side tag is currently still injected. The browser can +therefore continue running client-side DataDome protection for a request that +was explicitly whitelisted at Trusted Server. + +The desired behavior is that Fastly requests skipped by an IP-based DataDome +exclusion also omit Trusted Server's automatically injected client-side tag. + +## Goals + +- Suppress Trusted Server's automatically injected DataDome client-side tag for + Fastly requests skipped by an IP-based protection exclusion. +- Reuse the existing authoritative protection-scope decision. +- Cover all supported IP-based exclusion mechanisms: + - `protection_excluded_ip_cidrs` + - `protection_excluded_ip_cidr_sources` + - structured `ip_cidr` rules + - structured `ip_cidr_source` rules +- Preserve current behavior for non-IP exclusions. +- Leave publisher-originated or manually configured DataDome tags untouched. +- Add an informational diagnostic indicating that the client tag is omitted. +- Keep the implementation independent of caller-supplied IP headers. +- Prevent a shared cache from replaying IP-specific, tag-suppressed HTML to + non-excluded visitors. + +## Non-goals + +- Do not add a configuration flag or make this behavior opt-in. +- Do not change Axum, Cloudflare, or Spin request-filter wiring. This behavior + is intentionally scoped to the Fastly adapter, where the DataDome server-side + request filter is currently run. +- Do not suppress DataDome tags that originate in publisher HTML. +- Do not remove or disable the `/integrations/datadome/tags.js` route. +- Do not change DataDome signal-collection proxy behavior. +- Do not change ASN, path, query-parameter, method, static-asset, or internal + route exclusions. +- Do not perform live production verification as part of implementation. + +## Confirmed decisions + +1. **Adapter scope:** Fastly only. +2. **IP scope:** all four IP-based exclusion mechanisms listed above. +3. **Tag scope:** Trusted Server's auto-injected tag only. +4. **HTML scope:** every HTML response that enters the existing HTML processing + pipeline. +5. **Logging:** enrich the existing IP-exclusion skip log with + `client_tag=omitted`, including the matched rule and reason. +6. **Live testing:** deferred until after implementation and deployment/testing + workflow review. + +## Current architecture + +### Server-side protection + +`DataDomeIntegration::is_request_protected()` in +`crates/trusted-server-core/src/integrations/datadome/protection.rs` evaluates +method, internal-route, ASN, IP, and structured exclusion conditions. It uses +the client IP from `RuntimeServices::client_info()`, which is populated from +trusted Fastly request metadata. It does not use a caller-supplied IP header. + +The current function reduces the protection-scope result to a boolean. For an +IP exclusion it logs the skip and returns `false`, causing the request filter to +continue without calling the Protection API. + +The Fastly EdgeZero fallback path runs this request filter before route +selection and publisher proxying. The request continues into +`handle_publisher_request()` after the filter returns a continue decision. + +### Client-side injection + +`DataDomeIntegration::head_inserts()` in +`crates/trusted-server-core/src/integrations/datadome.rs` emits the client-side +snippet when: + +- `inject_client_side_tag` is true; and +- `client_side_key` is non-empty. + +The injector currently receives `IntegrationHtmlContext`, which contains HTML +host/scheme and document state but no request IP or protection decision. + +The publisher response path carries request-specific values through: + +```text +Request + -> OwnedProcessResponseParams + -> HtmlStreamProcessorParams + -> HtmlProcessorConfig + -> IntegrationHtmlContext + -> IntegrationHeadInjector +``` + +The existing DataDome attribute rewriter separately rewrites DataDome URLs +found in publisher HTML. That behavior must remain unchanged. + +## Design + +### 1. Capture an IP-exclusion marker at the request filter + +The request filter must attach a typed, internal request-scoped marker when the +existing protection-scope evaluation returns a skip for one of these reasons: + +- `client_ip` +- `client_ip_source` +- `ip_cidr` +- `ip_cidr_source` + +The marker must be attached only after the existing scope decision confirms the +IP exclusion. It must not be inferred from request headers or recomputed later +in the HTML pipeline. + +The request-filter API currently exposes an immutable request view. Add the +smallest internal mechanism needed for a filter to attach a typed request +extension without introducing a caller-visible header. Header mutations should +continue to use `RequestFilterEffects` as they do today. + +The marker should be a zero-sized or otherwise minimal internal type. It only +needs to answer whether Trusted Server's DataDome client tag should be +suppressed; the existing skip log supplies the rule ID and reason. + +The marker must not be attached for: + +- `OPTIONS` or other excluded methods before scope evaluation; +- internal or integration routes; +- ASN exclusions; +- path, query, or other non-IP structured exclusions; +- unmatched IP rules; +- Protection API fail-open behavior; or +- requests where `enable_protection` is false and the request filter does not + run. + +### 2. Enrich the existing skip log + +For IP-based skips, extend the existing informational log with +`client_tag=omitted`: + +```text +[datadome] protection decision=skipped rule=excluded-ip-cidrs reason=client_ip client_tag=omitted method=GET host=example.com path=/page +``` + +The existing rule ID, reason, and request metadata remain part of the log. +Client IP values are not included. Non-IP skip logs retain their current +behavior and level. + +This log represents the request policy decision. It may also apply to a +non-HTML response, for which no HTML tag would have been injected anyway. + +### 3. Propagate the marker into HTML processing + +Before the publisher request is moved into the platform HTTP client, snapshot +whether the request carries the marker. Carry that request-scoped boolean +through `OwnedProcessResponseParams`, `HtmlStreamProcessorParams`, and +`HtmlProcessorConfig`. + +The value should default to `false` in all existing constructors and direct +unit-test fixtures. Non-Fastly adapters will naturally retain the default +because they do not currently produce the Fastly request-filter marker. + +Expose the value to head injectors through the existing HTML processing context +or equivalent request-scoped integration context. The propagation must work for +both: + +- the normal buffered HTML path; and +- the streaming HTML path, including the auction-hold path. + +The value is irrelevant for non-HTML, RSC, pass-through, and unmodified +responses, which should retain their current processing. + +### 4. Keep IP-specific HTML out of shared cache + +A processed HTML response differs by client IP when the generated tag is +suppressed. In the `PublisherResponse::Stream` path, when suppression is active +and the response is HTML, set `Cache-Control: private, max-age=0` and remove +`Surrogate-Control` and `Fastly-Surrogate-Control` before the body is streamed. + +This matches the existing per-user ad-stack cache policy. It prevents Fastly or +another shared cache from replaying a tag-suppressed response to a visitor whose +IP does not match an exclusion. Do not change cache headers for non-HTML, +pass-through, or unmodified responses because their output does not vary by this +feature. + +### 5. Suppress only the generated DataDome snippet + +At the start of `DataDomeIntegration::head_inserts()`: + +1. Check the request-scoped suppression marker. +2. If present, return no DataDome head inserts. +3. Otherwise preserve the current `inject_client_side_tag` and + `client_side_key` checks and emit the existing snippet unchanged. + +When suppression is active, omit both: + +```html + + +``` + +Do not alter: + +- publisher-originated DataDome script tags; +- `rewrite_sdk` behavior; +- the DataDome SDK proxy route; +- the signal collection API proxy; +- DataDome configuration serialization for non-suppressed requests; or +- injection behavior for requests without the marker. + +## Testing plan + +### Protection-filter tests + +Add or extend tests in +`crates/trusted-server-core/src/integrations/datadome/protection.rs` to verify +that the marker is attached for: + +- a matching inline IPv4 CIDR; +- a matching Config Store-backed CIDR source; +- a matching structured `ip_cidr` rule; and +- a matching structured `ip_cidr_source` rule. + +Verify that the marker is absent for: + +- a non-matching IP; +- an ASN exclusion; +- a path exclusion; +- a query-parameter exclusion; +- an excluded method; and +- an internal or integration route. + +Verify the existing protection behavior remains unchanged: IP-matched requests +continue without a Protection API call. + +### Head-injector tests + +Add tests in +`crates/trusted-server-core/src/integrations/datadome.rs` verifying that: + +- a configured client tag is omitted when suppression is active; +- a configured client tag is emitted when suppression is inactive; +- a blank client-side key remains a no-op; and +- `inject_client_side_tag = false` remains a no-op. + +### HTML pipeline tests + +Add coverage for the request-scoped value flowing through the HTML processor, +including the streaming path. Confirm that a suppressed processed HTML response +contains neither the injected `window.ddjskey` configuration nor the configured +DataDome `tags.js` script. For a suppressed HTML stream, assert the response is +private and has no surrogate cache headers. Confirm a non-suppressed HTML stream +retains its origin cache behavior. + +Confirm that publisher-originated DataDome tags remain in the output and are +still rewritten according to the existing `rewrite_sdk` behavior. + +### Fastly dispatch tests + +Add a Fastly adapter dispatch test with: + +- DataDome protection enabled; +- a client IP matching an inline exclusion; +- a configured client-side key; and +- an HTML publisher response. + +The test should verify that the request continues without a Protection API +call, the response includes the `client_tag=omitted` decision log through the +existing test logging seam where available, and the generated tag is absent. + +Also cover a non-excluded request to confirm the generated tag remains present. + +## Documentation changes + +Update `docs/guide/integrations/datadome.md` to state that IP-excluded Fastly +requests skip both: + +- server-side Protection API validation; and +- Trusted Server's automatic client-side tag injection. + +Document that this does not remove or disable publisher-originated DataDome +tags, and that non-IP exclusions do not automatically suppress the client-side +tag. + +No configuration template changes are required because this behavior has no +new setting. + +## Files expected to change + +- `crates/trusted-server-core/src/integrations/registry.rs` + - Support the internal request-scoped annotation mechanism. +- `crates/trusted-server-core/src/integrations/datadome.rs` + - Define the marker and conditionally suppress head injection. +- `crates/trusted-server-core/src/integrations/datadome/protection.rs` + - Attach the marker for IP-based scope skips and enrich the skip log. +- `crates/trusted-server-core/src/integrations/registry.rs` or the relevant + HTML context definition + - Carry the suppression decision into head injection. +- `crates/trusted-server-core/src/html_processor.rs` + - Carry the request-scoped value into HTML integration context. +- `crates/trusted-server-core/src/publisher.rs` + - Snapshot and propagate the request marker through response processing. +- `docs/guide/integrations/datadome.md` + - Document the behavior. +- Relevant unit and Fastly adapter test modules. + +The exact split between registry request annotations and HTML context plumbing +should remain minimal and should not introduce a new public configuration API. + +## Verification + +Implementation verification should use the repository's target-matched +commands: + +```bash +cargo fmt --all -- --check +cargo test-fastly +cargo test-axum +cargo test-cloudflare +cargo clippy-fastly +cargo clippy-axum +cargo clippy-cloudflare +``` + +No live production validation is required for this implementation task. Live +browser verification will be performed later through the deployment/testing +workflow. From fc963341a085da137212e5e2ee83ec07c462a082 Mon Sep 17 00:00:00 2001 From: Christian Date: Wed, 5 Aug 2026 19:49:24 -0500 Subject: [PATCH 06/16] Add DataDome staging test bypass --- .../src/integrations/datadome.rs | 128 +++++++++++++- .../src/integrations/datadome/protection.rs | 161 +++++++++++++++++- docs/guide/integrations/datadome.md | 52 ++++-- 3 files changed, 328 insertions(+), 13 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/datadome.rs b/crates/trusted-server-core/src/integrations/datadome.rs index db5932fb2..4dd55b28f 100644 --- a/crates/trusted-server-core/src/integrations/datadome.rs +++ b/crates/trusted-server-core/src/integrations/datadome.rs @@ -61,7 +61,7 @@ use async_trait::async_trait; use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt}; use http::header; -use http::{Method, StatusCode}; +use http::{HeaderName, Method, StatusCode}; use regex::Regex; use serde::Deserialize; use serde_json::Value as JsonValue; @@ -77,6 +77,7 @@ use crate::integrations::{ collect_body_bounded, collect_response_bounded, ensure_integration_backend, }; use crate::platform::{PlatformHttpRequest, RuntimeServices}; +use crate::redacted::Redacted; use crate::settings::{IntegrationConfig, Settings}; mod protection; @@ -117,6 +118,27 @@ static DATADOME_URL_PATTERN: LazyLock = LazyLock::new(|| { .expect("DataDome URL rewrite regex should compile") }); +/// Temporary static-header bypass for server-side `DataDome` protection. +/// +/// This is intended only for an access-controlled staging environment. A +/// matching header bypasses the server-side Protection API and is removed +/// before the publisher origin receives the request. +#[derive(Debug, Default, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ProtectionTestBypassConfig { + /// Enables the bypass. Defaults to disabled when the section is present. + #[serde(default)] + pub enabled: bool, + + /// Header name carrying the temporary bypass credential. + #[serde(default)] + pub header_name: String, + + /// Static credential expected in [`Self::header_name`]. + #[serde(default)] + pub credential: Redacted, +} + /// Configuration for `DataDome` integration. #[derive(Debug, Clone, Deserialize, Validate)] #[serde(deny_unknown_fields)] @@ -199,6 +221,10 @@ pub struct DataDomeConfig { )] pub protection_exclusion_rules: Vec, + /// Temporary static-header bypass for access-controlled staging tests. + #[serde(default)] + pub protection_test_bypass: Option, + /// Reserved flag for future GraphQL payload extraction. #[serde(default)] pub enable_graphql_support: bool, @@ -329,6 +355,7 @@ impl Default for DataDomeConfig { protection_excluded_ip_cidr_sources: Vec::new(), protection_ip_list_cache_ttl_seconds: default_protection_ip_list_cache_ttl_seconds(), protection_exclusion_rules: default_protection_exclusion_rules(), + protection_test_bypass: None, enable_graphql_support: false, client_side_key: String::new(), inject_client_side_tag: default_inject_client_side_tag(), @@ -362,6 +389,9 @@ impl DataDomeIntegration { config.server_side_key_secret_name = config.server_side_key_secret_name.trim().to_string(); config.protection_api_origin = config.protection_api_origin.trim().to_string(); config.client_side_tag_url = config.client_side_tag_url.trim().to_string(); + if let Some(bypass) = &mut config.protection_test_bypass { + bypass.header_name = bypass.header_name.trim().to_string(); + } if config.enable_protection { if config.server_side_key_secret_store.is_empty() @@ -373,6 +403,7 @@ impl DataDomeIntegration { } Self::validate_protection_api_origin(&config.protection_api_origin)?; } + Self::validate_protection_test_bypass(&config)?; if config.inject_client_side_tag { Self::validate_client_side_tag_url(&config.client_side_tag_url)?; @@ -422,6 +453,36 @@ impl DataDomeIntegration { Ok(()) } + fn validate_protection_test_bypass( + config: &DataDomeConfig, + ) -> Result<(), Report> { + let Some(bypass) = config + .protection_test_bypass + .as_ref() + .filter(|bypass| bypass.enabled) + else { + return Ok(()); + }; + + if !config.enable_protection { + return Err(Report::new(Self::error( + "protection_test_bypass requires enable_protection to be true", + ))); + } + if HeaderName::from_bytes(bypass.header_name.as_bytes()).is_err() { + return Err(Report::new(Self::error( + "protection_test_bypass.header_name must be a valid HTTP header name", + ))); + } + if bypass.credential.expose().is_empty() { + return Err(Report::new(Self::error( + "protection_test_bypass.credential must not be empty when enabled", + ))); + } + + Ok(()) + } + fn validate_client_side_tag_url(tag_url: &str) -> Result<(), Report> { if tag_url.starts_with('/') && !tag_url.starts_with("//") { if tag_url.chars().any(is_unsafe_client_side_tag_path_char) { @@ -1098,6 +1159,71 @@ mod tests { config.server_side_key_secret_name, "datadome_server_side_key" ); + assert!( + config.protection_test_bypass.is_none(), + "the temporary test bypass should be disabled by default" + ); + } + + #[test] + fn protection_test_bypass_deserializes_nested_configuration() { + let config: DataDomeConfig = toml::from_str( + r#" + enabled = true + enable_protection = true + + [protection_test_bypass] + enabled = true + header_name = "x-ts-datadome-test-bypass" + credential = "temporary-test-credential" + "#, + ) + .expect("should deserialize DataDome test bypass configuration"); + let bypass = config + .protection_test_bypass + .expect("should deserialize the nested test bypass configuration"); + + assert!(bypass.enabled, "should retain the enabled flag"); + assert_eq!( + bypass.header_name, "x-ts-datadome-test-bypass", + "should retain the configured header name" + ); + assert_eq!( + bypass.credential.expose(), + "temporary-test-credential", + "should retain the configured credential" + ); + } + + #[test] + fn protection_test_bypass_requires_protection_header_and_credential() { + for (enable_protection, header_name, credential, expected_message) in [ + ( + false, + "x-ts-datadome-test-bypass", + "temporary-test-credential", + "requires enable_protection", + ), + (true, "", "temporary-test-credential", "header_name"), + (true, "x-ts-datadome-test-bypass", "", "credential"), + ] { + let mut config = test_config(); + config.enable_protection = enable_protection; + config.protection_test_bypass = Some(ProtectionTestBypassConfig { + enabled: true, + header_name: header_name.to_string(), + credential: Redacted::new(credential.to_string()), + }); + + let err = match DataDomeIntegration::try_new(config) { + Ok(_) => panic!("should reject invalid protection test bypass configuration"), + Err(err) => err, + }; + assert!( + format!("{err:?}").contains(expected_message), + "should explain the invalid protection test bypass configuration" + ); + } } #[test] diff --git a/crates/trusted-server-core/src/integrations/datadome/protection.rs b/crates/trusted-server-core/src/integrations/datadome/protection.rs index 4c74c71e8..e67a83e36 100644 --- a/crates/trusted-server-core/src/integrations/datadome/protection.rs +++ b/crates/trusted-server-core/src/integrations/datadome/protection.rs @@ -4,6 +4,8 @@ use edgezero_core::body::Body as EdgeBody; use edgezero_core::http::{HeaderMap, HeaderName, request_builder}; use error_stack::{Report, ResultExt}; use http::{Method, Request, Response, StatusCode, header}; +use sha2::{Digest as _, Sha256}; +use subtle::ConstantTimeEq as _; use url::Url; use crate::error::TrustedServerError; @@ -45,10 +47,20 @@ impl DataDomeIntegration { ); } + let test_bypass_matched = self.take_protection_test_bypass_header(input.request); if !self.config.enable_protection || !self.is_request_protected(&mut input) { return RequestFilterDecision::Continue(RequestFilterEffects::default()); } + if test_bypass_matched { + input + .request + .extensions_mut() + .insert(super::DataDomeClientTagSuppressed); + log_protection_test_bypass(&input); + return RequestFilterDecision::Continue(RequestFilterEffects::default()); + } + match self.filter_protection_request_inner(input).await { Ok(decision) => decision, Err(ProtectionRequestError::Setup(err)) => { @@ -156,6 +168,26 @@ impl DataDomeIntegration { true } + fn take_protection_test_bypass_header(&self, req: &mut Request) -> bool { + let Some(bypass) = self + .config + .protection_test_bypass + .as_ref() + .filter(|bypass| bypass.enabled) + else { + return false; + }; + let header_name = HeaderName::from_bytes(bypass.header_name.as_bytes()) + .expect("should validate protection test bypass header name during setup"); + let Some(value) = req.headers_mut().remove(&header_name) else { + return false; + }; + + let actual = Sha256::digest(value.as_bytes()); + let expected = Sha256::digest(bypass.credential.expose().as_bytes()); + bool::from(actual.ct_eq(&expected)) + } + fn protection_validate_url(&self) -> String { format!( "{}{}", @@ -442,6 +474,15 @@ fn is_ip_exclusion_reason(reason: &str) -> bool { ) } +fn log_protection_test_bypass(input: &RequestFilterInput<'_>) { + log::info!( + "[datadome] protection decision=skipped rule=protection-test-bypass reason=test_bypass client_tag=omitted method={} host={} path={}", + input.request.method(), + request_host(input.request), + input.request.uri().path(), + ); +} + fn log_protection_skip(input: &RequestFilterInput<'_>, rule_id: &str, reason: &str) { if is_ip_exclusion_reason(reason) { log::info!( @@ -734,12 +775,13 @@ mod tests { use crate::integrations::datadome::{ DataDomeConfig, ProtectionExclusionRuleConfig, ProtectionMatcherConfig, + ProtectionTestBypassConfig, }; use crate::platform::GeoInfo; use crate::platform::test_support::{ - HashMapConfigStore, HashMapSecretStore, NoopConfigStore, NoopSecretStore, + HashMapConfigStore, HashMapSecretStore, NoopConfigStore, NoopSecretStore, StubHttpClient, build_services_with_config_and_secret, build_services_with_config_and_secret_and_client_ip, - noop_services_with_client_ip, + build_services_with_secret_and_http_client, noop_services_with_client_ip, }; use crate::settings::Settings; @@ -801,6 +843,121 @@ mod tests { .is_some() } + #[test] + fn protection_test_bypass_skips_api_suppresses_tag_and_strips_header() { + let config = DataDomeConfig { + enabled: true, + enable_protection: true, + protection_test_bypass: Some(ProtectionTestBypassConfig { + enabled: true, + header_name: "x-ts-datadome-test-bypass".to_string(), + credential: Redacted::new("temporary-test-credential".to_string()), + }), + ..DataDomeConfig::default() + }; + let integration = DataDomeIntegration::try_new(config).expect("should create integration"); + let http_client = Arc::new(StubHttpClient::new()); + let services = + build_services_with_secret_and_http_client(NoopSecretStore, http_client.clone()); + let settings = Settings::default(); + let mut request = request_for_filter(); + request.headers_mut().insert( + "x-ts-datadome-test-bypass", + edgezero_core::http::HeaderValue::from_static("temporary-test-credential"), + ); + + let decision = futures::executor::block_on(integration.filter_protection_request( + RequestFilterInput { + settings: &settings, + services: &services, + request: &mut request, + geo_info: None, + is_integration_route: false, + }, + )); + + assert!( + matches!(decision, RequestFilterDecision::Continue(_)), + "a matching test credential should continue without a challenge" + ); + assert!( + has_client_tag_suppression_marker(&request), + "the bypass should suppress the automatic DataDome client tag" + ); + assert!( + request.headers().get("x-ts-datadome-test-bypass").is_none(), + "the bypass credential must not reach the publisher origin" + ); + assert!( + http_client.recorded_backend_names().is_empty(), + "a matching test credential must not call the Protection API" + ); + } + + #[test] + fn protection_test_bypass_strips_invalid_credential_without_bypassing() { + let config = DataDomeConfig { + enabled: true, + enable_protection: true, + protection_test_bypass: Some(ProtectionTestBypassConfig { + enabled: true, + header_name: "x-ts-datadome-test-bypass".to_string(), + credential: Redacted::new("temporary-test-credential".to_string()), + }), + ..DataDomeConfig::default() + }; + let integration = DataDomeIntegration::try_new(config).expect("should create integration"); + let mut secrets = HashMap::new(); + secrets.insert( + "datadome_server_side_key".to_string(), + b"server-side-key".to_vec(), + ); + let http_client = Arc::new(StubHttpClient::new()); + http_client.push_response_with_headers( + 200, + Vec::new(), + vec![(HEADER_DATADOME_RESPONSE, "200")], + ); + let services = build_services_with_secret_and_http_client( + HashMapSecretStore::new(secrets), + http_client.clone(), + ); + let settings = Settings::default(); + let mut request = request_for_filter(); + request.headers_mut().insert( + "x-ts-datadome-test-bypass", + edgezero_core::http::HeaderValue::from_static("wrong-credential"), + ); + + let decision = futures::executor::block_on(integration.filter_protection_request( + RequestFilterInput { + settings: &settings, + services: &services, + request: &mut request, + geo_info: None, + is_integration_route: false, + }, + )); + + assert!( + matches!(decision, RequestFilterDecision::Continue(_)), + "an allowed Protection API response should continue" + ); + assert!( + !has_client_tag_suppression_marker(&request), + "a non-matching credential must not suppress the DataDome client tag" + ); + assert!( + request.headers().get("x-ts-datadome-test-bypass").is_none(), + "an invalid bypass credential must not reach the publisher origin" + ); + assert_eq!( + http_client.recorded_backend_names().len(), + 1, + "a non-matching credential must still call the Protection API" + ); + } + #[test] fn ip_exclusions_mark_requests_for_client_tag_suppression() { let ip = IpAddr::V4(Ipv4Addr::new(192, 0, 2, 10)); diff --git a/docs/guide/integrations/datadome.md b/docs/guide/integrations/datadome.md index 1743a1b75..ba3081f2a 100644 --- a/docs/guide/integrations/datadome.md +++ b/docs/guide/integrations/datadome.md @@ -86,6 +86,7 @@ patterns = ["(?i)\\.(avi|flv|mka|mkv|mov|mp4|mpeg|mpg|mp3|flac|ogg|ogm|opus|wav| | `protection_excluded_ip_cidr_sources` | array | `[]` | Config Store sources containing dynamic client IP CIDR bypass lists | | `protection_ip_list_cache_ttl_seconds` | integer | `300` | Process-local cache TTL for Config Store-backed IP CIDR bypass lists | | `protection_exclusion_rules` | array | Static asset path regex | Structured method/path/query/IP/ASN exclusion rules | +| `protection_test_bypass` | object | omitted | Temporary static-header bypass for access-controlled staging tests | | `enable_graphql_support` | boolean | `false` | Reserved for future GraphQL body inspection; ignored in v1 | | `client_side_key` | string | `""` | DataDome client-side JavaScript key used for tag injection | | `inject_client_side_tag` | boolean | `true` | Auto-inject the browser tag when `client_side_key` is non-empty | @@ -168,36 +169,67 @@ A request is protected when all of the following are true: 5. The client IP does not match `protection_excluded_ip_cidrs` or any Config Store-backed CIDR source. 6. The client ASN is not listed in `protection_excluded_asns`. 7. No `protection_exclusion_rules` match. +8. The request does not contain a matching enabled `protection_test_bypass` credential. Static assets are excluded by default using a case-insensitive file-extension regex. Trusted Server internal routes such as `/static/tsjs=`, `/integrations/`, `/first-party/`, admin routes, discovery routes, and signature-verification routes are also excluded by default. Auction traffic at `/auction` is protected by default. -### IP-excluded client-side tag behavior +### Staging test bypass + +For short-lived browser automation on an access-controlled staging site, you +can configure a static header credential that skips only the server-side +Protection API: + +```toml +[integrations.datadome.protection_test_bypass] +enabled = true +header_name = "x-ts-datadome-test-bypass" +credential = "temporary-test-credential" +``` + +`protection_test_bypass` requires `enable_protection = true`; it is disabled +when omitted. Treat the credential as a temporary secret: configure it only +while needed, protect the site with an outer access control such as Basic Auth, +and remove the section when testing finishes. Do not enable it in production. + +A matching header is compared in constant time, removed before the request can +reach DataDome or the publisher origin, and never logged. With Playwright, +apply it to the browser context: + +```ts +await context.setExtraHTTPHeaders({ + "X-TS-DataDome-Test-Bypass": process.env.DATADOME_TEST_BYPASS!, +}); +``` + +### Client-side tag suppression behavior On the Fastly adapter, a request that matches an IP-based DataDome exclusion -also omits Trusted Server's automatically injected client-side DataDome tag -from processed HTML. This keeps the client-side layer consistent with the -server-side Protection API skip. +or the configured test-bypass credential also omits Trusted Server's +automatically injected client-side DataDome tag from processed HTML. This keeps +the client-side layer consistent with the server-side Protection API skip. This behavior applies to: - `protection_excluded_ip_cidrs`; - `protection_excluded_ip_cidr_sources`; -- structured `ip_cidr` rules; and -- structured `ip_cidr_source` rules. +- structured `ip_cidr` rules; +- structured `ip_cidr_source` rules; and +- a matching enabled `protection_test_bypass` credential. ASN, method, path, query-parameter, static-asset, and internal-route exclusions do not automatically suppress the client-side tag. DataDome tags already present in publisher HTML are not removed or changed by this behavior, and `/integrations/datadome/tags.js` remains available when requested directly. -Because the processed HTML differs by client IP, tag-suppressed HTML is marked -`private, max-age=0` and removed from shared surrogate caches. The decision is -reported in the existing protection log, for example: +Because the processed HTML differs by client IP or test credential, +tag-suppressed HTML is marked `private, max-age=0` and removed from shared +surrogate caches. The decision is reported in the existing protection log, for +example: ```text -[datadome] protection decision=skipped rule=excluded-ip-cidrs reason=client_ip client_tag=omitted method=GET host=example.com path=/page +[datadome] protection decision=skipped rule=protection-test-bypass reason=test_bypass client_tag=omitted method=GET host=example.com path=/page ``` ### Structured exclusion rules From 1ce7892b145d3643a0e916311291035759d3e7ea Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 6 Aug 2026 10:49:17 -0500 Subject: [PATCH 07/16] Log DataDome test-bypass registration state Log whether protection_test_bypass is enabled when registering the DataDome integration and include configured header name when enabled. Keep credential secret out of logs., --- .../src/integrations/datadome.rs | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/datadome.rs b/crates/trusted-server-core/src/integrations/datadome.rs index 4dd55b28f..112d41da1 100644 --- a/crates/trusted-server-core/src/integrations/datadome.rs +++ b/crates/trusted-server-core/src/integrations/datadome.rs @@ -914,12 +914,26 @@ fn build( return Ok(None); }; - log::info!( - "[datadome] Registering integration (sdk_origin: {}, rewrite_sdk: {}, enable_protection: {})", - config.sdk_origin, - config.rewrite_sdk, - config.enable_protection - ); + if let Some(bypass) = config + .protection_test_bypass + .as_ref() + .filter(|bypass| bypass.enabled) + { + log::info!( + "[datadome] Registering integration (sdk_origin: {}, rewrite_sdk: {}, enable_protection: {}, protection_test_bypass: enabled, protection_test_bypass_header: {})", + config.sdk_origin, + config.rewrite_sdk, + config.enable_protection, + bypass.header_name, + ); + } else { + log::info!( + "[datadome] Registering integration (sdk_origin: {}, rewrite_sdk: {}, enable_protection: {}, protection_test_bypass: disabled)", + config.sdk_origin, + config.rewrite_sdk, + config.enable_protection, + ); + } Ok(Some(DataDomeIntegration::try_new(config)?)) } From 7e6f365ad04b329dfa138cac0736c12220ebb8be Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 6 Aug 2026 12:49:49 -0500 Subject: [PATCH 08/16] fix datadome staging bypass privacy --- crates/trusted-server-core/src/config.rs | 42 ++++- .../src/integrations/datadome.rs | 111 +++++------ .../src/integrations/datadome/protection.rs | 173 +++++++++++++----- crates/trusted-server-core/src/publisher.rs | 102 ++++++++++- .../src/response_privacy.rs | 3 +- docs/guide/integrations/datadome.md | 31 ++-- ...6-08-03-datadome-ip-excluded-client-tag.md | 2 +- ...-datadome-ip-excluded-client-tag-design.md | 4 +- 8 files changed, 347 insertions(+), 121 deletions(-) diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index cdee3b222..e74ef4150 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -155,7 +155,9 @@ fn validate_enabled_integrations( validate_integration::(settings, "sourcepoint")?; validate_integration::(settings, "osano")?; validate_integration::(settings, "google_tag_manager")?; - validate_integration::(settings, "datadome")?; + if let Some(config) = settings.integration_config::("datadome")? { + crate::integrations::datadome::DataDomeIntegration::validate_config_for_startup(config)?; + } validate_integration::(settings, "gpt")?; validate_integration::(settings, "gpt_diagnostics")?; @@ -404,6 +406,44 @@ password = "production-admin-password-32-bytes" ); } + #[test] + fn deploy_validation_rejects_invalid_datadome_test_bypass() { + for (enable_protection, store, name, expected_message) in [ + ( + false, + "ts_secrets", + "datadome_test_bypass", + "requires enable_protection", + ), + (true, "", "datadome_test_bypass", "credential_secret_store"), + (true, "ts_secrets", "", "credential_secret_name"), + ] { + let mut settings = valid_settings(); + settings + .integrations + .insert_config( + "datadome", + &serde_json::json!({ + "enabled": true, + "enable_protection": enable_protection, + "protection_test_bypass": { + "enabled": true, + "credential_secret_store": store, + "credential_secret_name": name, + }, + }), + ) + .expect("should insert DataDome config"); + + let err = validate_settings_for_deploy(&settings) + .expect_err("should reject invalid DataDome test bypass"); + assert!( + format!("{err:?}").contains(expected_message), + "error should mention the invalid bypass setting: {err:?}" + ); + } + } + #[test] fn validate_trait_reports_deploy_errors() { let mut settings = valid_settings(); diff --git a/crates/trusted-server-core/src/integrations/datadome.rs b/crates/trusted-server-core/src/integrations/datadome.rs index 112d41da1..2486c16be 100644 --- a/crates/trusted-server-core/src/integrations/datadome.rs +++ b/crates/trusted-server-core/src/integrations/datadome.rs @@ -61,7 +61,7 @@ use async_trait::async_trait; use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt}; use http::header; -use http::{HeaderName, Method, StatusCode}; +use http::{Method, StatusCode}; use regex::Regex; use serde::Deserialize; use serde_json::Value as JsonValue; @@ -77,7 +77,6 @@ use crate::integrations::{ collect_body_bounded, collect_response_bounded, ensure_integration_backend, }; use crate::platform::{PlatformHttpRequest, RuntimeServices}; -use crate::redacted::Redacted; use crate::settings::{IntegrationConfig, Settings}; mod protection; @@ -90,6 +89,7 @@ pub use protection_scope::{ use protection_scope::ProtectionScope; pub(crate) const DATADOME_INTEGRATION_ID: &str = "datadome"; +pub(crate) const HEADER_DATADOME_TEST_BYPASS: &str = "x-ts-datadome-bypass"; /// Request marker indicating that Trusted Server should omit its automatic /// `DataDome` client-side tag for the current response. @@ -121,8 +121,9 @@ static DATADOME_URL_PATTERN: LazyLock = LazyLock::new(|| { /// Temporary static-header bypass for server-side `DataDome` protection. /// /// This is intended only for an access-controlled staging environment. A -/// matching header bypasses the server-side Protection API and is removed -/// before the publisher origin receives the request. +/// matching `x-ts-datadome-bypass` header bypasses the server-side Protection +/// API and is removed before the publisher origin receives the request. The +/// credential itself is loaded from the Secret Store at runtime. #[derive(Debug, Default, Clone, Deserialize)] #[serde(deny_unknown_fields)] pub struct ProtectionTestBypassConfig { @@ -130,13 +131,13 @@ pub struct ProtectionTestBypassConfig { #[serde(default)] pub enabled: bool, - /// Header name carrying the temporary bypass credential. - #[serde(default)] - pub header_name: String, + /// Secret Store containing the temporary bypass credential. + #[serde(default = "default_protection_test_bypass_secret_store")] + pub credential_secret_store: String, - /// Static credential expected in [`Self::header_name`]. - #[serde(default)] - pub credential: Redacted, + /// Secret name containing the temporary bypass credential. + #[serde(default = "default_protection_test_bypass_secret_name")] + pub credential_secret_name: String, } /// Configuration for `DataDome` integration. @@ -278,6 +279,14 @@ fn default_server_side_key_secret_name() -> String { "datadome_server_side_key".to_string() } +fn default_protection_test_bypass_secret_store() -> String { + "ts_secrets".to_string() +} + +fn default_protection_test_bypass_secret_name() -> String { + "datadome_test_bypass".to_string() +} + fn default_timeout_ms() -> u32 { 1500 } @@ -390,7 +399,8 @@ impl DataDomeIntegration { config.protection_api_origin = config.protection_api_origin.trim().to_string(); config.client_side_tag_url = config.client_side_tag_url.trim().to_string(); if let Some(bypass) = &mut config.protection_test_bypass { - bypass.header_name = bypass.header_name.trim().to_string(); + bypass.credential_secret_store = bypass.credential_secret_store.trim().to_string(); + bypass.credential_secret_name = bypass.credential_secret_name.trim().to_string(); } if config.enable_protection { @@ -453,6 +463,12 @@ impl DataDomeIntegration { Ok(()) } + pub(crate) fn validate_config_for_startup( + config: DataDomeConfig, + ) -> Result<(), Report> { + Self::try_new(config).map(|_| ()) + } + fn validate_protection_test_bypass( config: &DataDomeConfig, ) -> Result<(), Report> { @@ -469,14 +485,9 @@ impl DataDomeIntegration { "protection_test_bypass requires enable_protection to be true", ))); } - if HeaderName::from_bytes(bypass.header_name.as_bytes()).is_err() { + if bypass.credential_secret_store.is_empty() || bypass.credential_secret_name.is_empty() { return Err(Report::new(Self::error( - "protection_test_bypass.header_name must be a valid HTTP header name", - ))); - } - if bypass.credential.expose().is_empty() { - return Err(Report::new(Self::error( - "protection_test_bypass.credential must not be empty when enabled", + "protection_test_bypass credential_secret_store and credential_secret_name must not be empty when enabled", ))); } @@ -914,28 +925,25 @@ fn build( return Ok(None); }; - if let Some(bypass) = config + let integration = DataDomeIntegration::try_new(config)?; + let protection_test_bypass = integration + .config .protection_test_bypass .as_ref() - .filter(|bypass| bypass.enabled) - { - log::info!( - "[datadome] Registering integration (sdk_origin: {}, rewrite_sdk: {}, enable_protection: {}, protection_test_bypass: enabled, protection_test_bypass_header: {})", - config.sdk_origin, - config.rewrite_sdk, - config.enable_protection, - bypass.header_name, - ); - } else { - log::info!( - "[datadome] Registering integration (sdk_origin: {}, rewrite_sdk: {}, enable_protection: {}, protection_test_bypass: disabled)", - config.sdk_origin, - config.rewrite_sdk, - config.enable_protection, - ); - } + .is_some_and(|bypass| bypass.enabled); + log::info!( + "[datadome] Registering integration (sdk_origin: {}, rewrite_sdk: {}, enable_protection: {}, protection_test_bypass: {})", + integration.config.sdk_origin, + integration.config.rewrite_sdk, + integration.config.enable_protection, + if protection_test_bypass { + "enabled" + } else { + "disabled" + }, + ); - Ok(Some(DataDomeIntegration::try_new(config)?)) + Ok(Some(integration)) } /// Register the `DataDome` integration with Trusted Server. @@ -1188,8 +1196,8 @@ mod tests { [protection_test_bypass] enabled = true - header_name = "x-ts-datadome-test-bypass" - credential = "temporary-test-credential" + credential_secret_store = "ts_secrets" + credential_secret_name = "datadome_test_bypass" "#, ) .expect("should deserialize DataDome test bypass configuration"); @@ -1199,34 +1207,33 @@ mod tests { assert!(bypass.enabled, "should retain the enabled flag"); assert_eq!( - bypass.header_name, "x-ts-datadome-test-bypass", - "should retain the configured header name" + bypass.credential_secret_store, "ts_secrets", + "should retain the configured credential Secret Store" ); assert_eq!( - bypass.credential.expose(), - "temporary-test-credential", - "should retain the configured credential" + bypass.credential_secret_name, "datadome_test_bypass", + "should retain the configured credential secret name" ); } #[test] - fn protection_test_bypass_requires_protection_header_and_credential() { - for (enable_protection, header_name, credential, expected_message) in [ + fn protection_test_bypass_requires_protection_and_secret_references() { + for (enable_protection, store, name, expected_message) in [ ( false, - "x-ts-datadome-test-bypass", - "temporary-test-credential", + "ts_secrets", + "datadome_test_bypass", "requires enable_protection", ), - (true, "", "temporary-test-credential", "header_name"), - (true, "x-ts-datadome-test-bypass", "", "credential"), + (true, "", "datadome_test_bypass", "credential_secret_store"), + (true, "ts_secrets", "", "credential_secret_name"), ] { let mut config = test_config(); config.enable_protection = enable_protection; config.protection_test_bypass = Some(ProtectionTestBypassConfig { enabled: true, - header_name: header_name.to_string(), - credential: Redacted::new(credential.to_string()), + credential_secret_store: store.to_string(), + credential_secret_name: name.to_string(), }); let err = match DataDomeIntegration::try_new(config) { diff --git a/crates/trusted-server-core/src/integrations/datadome/protection.rs b/crates/trusted-server-core/src/integrations/datadome/protection.rs index e67a83e36..bc06214ff 100644 --- a/crates/trusted-server-core/src/integrations/datadome/protection.rs +++ b/crates/trusted-server-core/src/integrations/datadome/protection.rs @@ -38,20 +38,8 @@ impl DataDomeIntegration { &self, mut input: RequestFilterInput<'_>, ) -> RequestFilterDecision { - if self.config.enable_protection { - log::info!( - "[datadome] protection incoming method={} host={} path={}", - input.request.method(), - request_host(input.request), - input.request.uri().path(), - ); - } - - let test_bypass_matched = self.take_protection_test_bypass_header(input.request); - if !self.config.enable_protection || !self.is_request_protected(&mut input) { - return RequestFilterDecision::Continue(RequestFilterEffects::default()); - } - + let test_bypass_matched = + self.take_protection_test_bypass_header(input.request, input.services); if test_bypass_matched { input .request @@ -61,6 +49,10 @@ impl DataDomeIntegration { return RequestFilterDecision::Continue(RequestFilterEffects::default()); } + if !self.config.enable_protection || !self.is_request_protected(&mut input) { + return RequestFilterDecision::Continue(RequestFilterEffects::default()); + } + match self.filter_protection_request_inner(input).await { Ok(decision) => decision, Err(ProtectionRequestError::Setup(err)) => { @@ -168,7 +160,11 @@ impl DataDomeIntegration { true } - fn take_protection_test_bypass_header(&self, req: &mut Request) -> bool { + fn take_protection_test_bypass_header( + &self, + req: &mut Request, + services: &RuntimeServices, + ) -> bool { let Some(bypass) = self .config .protection_test_bypass @@ -177,14 +173,32 @@ impl DataDomeIntegration { else { return false; }; - let header_name = HeaderName::from_bytes(bypass.header_name.as_bytes()) - .expect("should validate protection test bypass header name during setup"); - let Some(value) = req.headers_mut().remove(&header_name) else { + let Some(value) = req.headers_mut().remove(super::HEADER_DATADOME_TEST_BYPASS) else { return false; }; + let store_name = StoreName::from(bypass.credential_secret_store.as_str()); + let credential = match services + .secret_store() + .get_string(&store_name, &bypass.credential_secret_name) + { + Ok(credential) if !credential.is_empty() => credential, + Ok(_) => { + log::warn!( + "[datadome] DataDome test bypass credential is empty; ignoring bypass header" + ); + return false; + } + Err(err) => { + log::warn!( + "[datadome] Failed to load DataDome test bypass credential; ignoring bypass header: {err:?}" + ); + return false; + } + }; + let actual = Sha256::digest(value.as_bytes()); - let expected = Sha256::digest(bypass.credential.expose().as_bytes()); + let expected = Sha256::digest(credential.as_bytes()); bool::from(actual.ct_eq(&expected)) } @@ -476,31 +490,25 @@ fn is_ip_exclusion_reason(reason: &str) -> bool { fn log_protection_test_bypass(input: &RequestFilterInput<'_>) { log::info!( - "[datadome] protection decision=skipped rule=protection-test-bypass reason=test_bypass client_tag=omitted method={} host={} path={}", + "[datadome] protection decision=skipped rule=protection-test-bypass reason=test_bypass client_tag=omitted method={}", input.request.method(), - request_host(input.request), - input.request.uri().path(), ); } fn log_protection_skip(input: &RequestFilterInput<'_>, rule_id: &str, reason: &str) { if is_ip_exclusion_reason(reason) { log::info!( - "[datadome] protection decision=skipped rule={} reason={} client_tag=omitted method={} host={} path={}", + "[datadome] protection decision=skipped rule={} reason={} client_tag=omitted method={}", rule_id, reason, input.request.method(), - request_host(input.request), - input.request.uri().path(), ); } else { log::debug!( - "[datadome] protection decision=skipped rule={} reason={} method={} host={} path={}", + "[datadome] protection decision=skipped rule={} reason={} method={}", rule_id, reason, input.request.method(), - request_host(input.request), - input.request.uri().path(), ); } } @@ -512,26 +520,20 @@ fn log_protection_result( decision: &RequestFilterDecision, ) { let method = input.request.method(); - let host = request_host(input.request); - let path = input.request.uri().path(); match decision { RequestFilterDecision::Respond { .. } => log::info!( - "[datadome] protection decision=blocked status={} method={} host={} path={} route=short_circuit", + "[datadome] protection decision=blocked status={} method={} route=short_circuit", status.as_u16(), method, - host, - path, ), RequestFilterDecision::Continue(_) if status == StatusCode::OK && datadome_status == Some(status.as_u16()) => { log::info!( - "[datadome] protection decision=allowed status={} method={} host={} path={} route=continue", + "[datadome] protection decision=allowed status={} method={} route=continue", status.as_u16(), method, - host, - path, ); } RequestFilterDecision::Continue(_) => {} @@ -850,19 +852,26 @@ mod tests { enable_protection: true, protection_test_bypass: Some(ProtectionTestBypassConfig { enabled: true, - header_name: "x-ts-datadome-test-bypass".to_string(), - credential: Redacted::new("temporary-test-credential".to_string()), + credential_secret_store: "ts_secrets".to_string(), + credential_secret_name: "datadome_test_bypass".to_string(), }), ..DataDomeConfig::default() }; let integration = DataDomeIntegration::try_new(config).expect("should create integration"); + let mut secrets = HashMap::new(); + secrets.insert( + "datadome_test_bypass".to_string(), + b"temporary-test-credential".to_vec(), + ); let http_client = Arc::new(StubHttpClient::new()); - let services = - build_services_with_secret_and_http_client(NoopSecretStore, http_client.clone()); + let services = build_services_with_secret_and_http_client( + HashMapSecretStore::new(secrets), + http_client.clone(), + ); let settings = Settings::default(); let mut request = request_for_filter(); request.headers_mut().insert( - "x-ts-datadome-test-bypass", + super::super::HEADER_DATADOME_TEST_BYPASS, edgezero_core::http::HeaderValue::from_static("temporary-test-credential"), ); @@ -885,7 +894,10 @@ mod tests { "the bypass should suppress the automatic DataDome client tag" ); assert!( - request.headers().get("x-ts-datadome-test-bypass").is_none(), + request + .headers() + .get(super::super::HEADER_DATADOME_TEST_BYPASS) + .is_none(), "the bypass credential must not reach the publisher origin" ); assert!( @@ -894,6 +906,68 @@ mod tests { ); } + #[test] + fn protection_test_bypass_wins_over_other_exclusions() { + let config = DataDomeConfig { + enabled: true, + enable_protection: true, + protection_exclusion_rules: vec![ProtectionExclusionRuleConfig { + id: "staging-page-exclusion".to_string(), + enabled: true, + methods: Vec::new(), + matcher: ProtectionMatcherConfig::PathExact { + paths: vec!["/page".to_string()], + }, + }], + protection_test_bypass: Some(ProtectionTestBypassConfig { + enabled: true, + credential_secret_store: "ts_secrets".to_string(), + credential_secret_name: "datadome_test_bypass".to_string(), + }), + ..DataDomeConfig::default() + }; + let integration = DataDomeIntegration::try_new(config).expect("should create integration"); + let mut secrets = HashMap::new(); + secrets.insert( + "datadome_test_bypass".to_string(), + b"temporary-test-credential".to_vec(), + ); + let http_client = Arc::new(StubHttpClient::new()); + let services = build_services_with_secret_and_http_client( + HashMapSecretStore::new(secrets), + http_client.clone(), + ); + let settings = Settings::default(); + let mut request = request_for_filter(); + request.headers_mut().insert( + super::super::HEADER_DATADOME_TEST_BYPASS, + edgezero_core::http::HeaderValue::from_static("temporary-test-credential"), + ); + + let decision = futures::executor::block_on(integration.filter_protection_request( + RequestFilterInput { + settings: &settings, + services: &services, + request: &mut request, + geo_info: None, + is_integration_route: false, + }, + )); + + assert!( + matches!(decision, RequestFilterDecision::Continue(_)), + "a matching test credential should continue" + ); + assert!( + has_client_tag_suppression_marker(&request), + "a matching test credential should suppress the tag even on an excluded path" + ); + assert!( + http_client.recorded_backend_names().is_empty(), + "a matching test credential must not call the Protection API" + ); + } + #[test] fn protection_test_bypass_strips_invalid_credential_without_bypassing() { let config = DataDomeConfig { @@ -901,8 +975,8 @@ mod tests { enable_protection: true, protection_test_bypass: Some(ProtectionTestBypassConfig { enabled: true, - header_name: "x-ts-datadome-test-bypass".to_string(), - credential: Redacted::new("temporary-test-credential".to_string()), + credential_secret_store: "ts_secrets".to_string(), + credential_secret_name: "datadome_test_bypass".to_string(), }), ..DataDomeConfig::default() }; @@ -912,6 +986,10 @@ mod tests { "datadome_server_side_key".to_string(), b"server-side-key".to_vec(), ); + secrets.insert( + "datadome_test_bypass".to_string(), + b"temporary-test-credential".to_vec(), + ); let http_client = Arc::new(StubHttpClient::new()); http_client.push_response_with_headers( 200, @@ -925,7 +1003,7 @@ mod tests { let settings = Settings::default(); let mut request = request_for_filter(); request.headers_mut().insert( - "x-ts-datadome-test-bypass", + super::super::HEADER_DATADOME_TEST_BYPASS, edgezero_core::http::HeaderValue::from_static("wrong-credential"), ); @@ -948,7 +1026,10 @@ mod tests { "a non-matching credential must not suppress the DataDome client tag" ); assert!( - request.headers().get("x-ts-datadome-test-bypass").is_none(), + request + .headers() + .get(super::super::HEADER_DATADOME_TEST_BYPASS) + .is_none(), "an invalid bypass credential must not reach the publisher origin" ); assert_eq!( diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 48e0eaacf..5fc8dac0e 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1452,12 +1452,21 @@ fn apply_datadome_client_tag_cache_privacy( return; } - response.headers_mut().insert( - header::CACHE_CONTROL, - HeaderValue::from_static("private, max-age=0"), - ); - response.headers_mut().remove("surrogate-control"); - response.headers_mut().remove("fastly-surrogate-control"); + let already_uncacheable = response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()) + .map(str::to_ascii_lowercase) + .is_some_and(|value| value.contains("private") || value.contains("no-store")); + if !already_uncacheable { + response.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static("private, max-age=0"), + ); + } + for header_name in CDN_CACHE_HEADERS { + response.headers_mut().remove(*header_name); + } } /// Drop a bodiless response's body and correct its framing headers. @@ -2895,6 +2904,10 @@ pub async fn handle_publisher_request( .extensions() .get::() .is_some(); + if suppress_datadome_client_side_tag { + req.headers_mut().remove(header::IF_NONE_MATCH); + req.headers_mut().remove(header::IF_MODIFIED_SINCE); + } let mut platform_request = PlatformHttpRequest::new(req, backend_name); if services.http_client().supports_streaming_responses() { platform_request = platform_request.with_stream_response(); @@ -5203,6 +5216,50 @@ mod tests { ); } + #[tokio::test] + async fn suppressed_publisher_request_removes_conditional_validators() { + let settings = create_test_settings(); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + b"origin".to_vec(), + vec![("content-type", "text/html; charset=utf-8")], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let mut req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/page") + .header(header::HOST, "publisher.example") + .header(header::IF_NONE_MATCH, "\"cached-page\"") + .header(header::IF_MODIFIED_SINCE, "Wed, 21 Oct 2015 07:28:00 GMT") + .body(EdgeBody::empty()) + .expect("should build conditional request"); + req.extensions_mut() + .insert(crate::integrations::datadome::DataDomeClientTagSuppressed); + + let _response = run_publisher_proxy(&settings, &services, req).await; + + let headers = stub + .recorded_request_headers() + .into_iter() + .next() + .expect("should record one outbound request"); + assert!( + headers + .iter() + .all(|(name, _)| !name.eq_ignore_ascii_case(header::IF_NONE_MATCH.as_str())), + "suppressed requests must not forward If-None-Match" + ); + assert!( + headers + .iter() + .all(|(name, _)| !name.eq_ignore_ascii_case(header::IF_MODIFIED_SINCE.as_str())), + "suppressed requests must not forward If-Modified-Since" + ); + } + #[tokio::test] async fn publisher_origin_fetch_leaves_stream_response_disabled_when_unsupported() { let settings = create_test_settings(); @@ -5365,6 +5422,8 @@ mod tests { .header(header::CACHE_CONTROL, "public, max-age=600") .header("surrogate-control", "max-age=600") .header("fastly-surrogate-control", "max-age=600") + .header("cloudflare-cdn-cache-control", "max-age=600") + .header("cdn-cache-control", "max-age=600") .body(EdgeBody::empty()) .expect("should build cacheable HTML response"); @@ -5391,6 +5450,37 @@ mod tests { response.headers().get("fastly-surrogate-control").is_none(), "suppressed HTML should not retain Fastly-Surrogate-Control" ); + assert!( + response + .headers() + .get("cloudflare-cdn-cache-control") + .is_none(), + "suppressed HTML should not retain Cloudflare-CDN-Cache-Control" + ); + assert!( + response.headers().get("cdn-cache-control").is_none(), + "suppressed HTML should not retain CDN-Cache-Control" + ); + + let mut no_store_response = Response::builder() + .status(StatusCode::OK) + .header(header::CACHE_CONTROL, "no-store") + .body(EdgeBody::empty()) + .expect("should build no-store HTML response"); + super::apply_datadome_client_tag_cache_privacy( + &mut no_store_response, + &Method::GET, + true, + "text/html; charset=utf-8", + ); + assert_eq!( + no_store_response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("no-store"), + "suppressed HTML should preserve an existing no-store policy" + ); } #[test] diff --git a/crates/trusted-server-core/src/response_privacy.rs b/crates/trusted-server-core/src/response_privacy.rs index e23348211..2205ae892 100644 --- a/crates/trusted-server-core/src/response_privacy.rs +++ b/crates/trusted-server-core/src/response_privacy.rs @@ -22,6 +22,7 @@ pub const CDN_CACHE_HEADERS: &[&str] = &[ "fastly-surrogate-control", "cdn-cache-control", "cloudflare-cdn-cache-control", + "cdn-cache-control", ]; /// Forces cookie-bearing responses to stay private to shared caches. @@ -37,7 +38,7 @@ pub fn enforce_set_cookie_cache_privacy(response: &mut Response) { if !response.headers().contains_key(header::SET_COOKIE) { return; } - // Surrogate cache headers must come off every cookie-bearing response, even + // Shared-cache control headers must come off every cookie-bearing response, even // one already carrying a stricter `no-store`/`private` directive — they are // independent of Cache-Control and would otherwise let a shared cache store // and replay one visitor's Set-Cookie. diff --git a/docs/guide/integrations/datadome.md b/docs/guide/integrations/datadome.md index ba3081f2a..35eda5d3e 100644 --- a/docs/guide/integrations/datadome.md +++ b/docs/guide/integrations/datadome.md @@ -184,23 +184,30 @@ Protection API: ```toml [integrations.datadome.protection_test_bypass] enabled = true -header_name = "x-ts-datadome-test-bypass" -credential = "temporary-test-credential" +credential_secret_store = "ts_secrets" +credential_secret_name = "datadome_test_bypass" ``` `protection_test_bypass` requires `enable_protection = true`; it is disabled -when omitted. Treat the credential as a temporary secret: configure it only -while needed, protect the site with an outer access control such as Basic Auth, -and remove the section when testing finishes. Do not enable it in production. +when omitted. Store the temporary credential in the configured Secret Store, +configure this section only while needed, protect the site with an outer access +control such as Basic Auth, and remove the section when testing finishes. Do not +enable it in production. -A matching header is compared in constant time, removed before the request can -reach DataDome or the publisher origin, and never logged. With Playwright, -apply it to the browser context: +The fixed `x-ts-datadome-bypass` header is compared in constant time, removed +before the request can reach DataDome or the publisher origin, and never +logged. Scope the header to the staging origin; do not attach it to every +request in a browser context because that can disclose the credential to +third-party origins. With Playwright: ```ts -await context.setExtraHTTPHeaders({ - "X-TS-DataDome-Test-Bypass": process.env.DATADOME_TEST_BYPASS!, -}); +await context.route('https://staging.example.com/**', async (route) => { + const headers = { + ...route.request().headers(), + 'x-ts-datadome-bypass': process.env.DATADOME_TEST_BYPASS!, + } + await route.continue({ headers }) +}) ``` ### Client-side tag suppression behavior @@ -229,7 +236,7 @@ surrogate caches. The decision is reported in the existing protection log, for example: ```text -[datadome] protection decision=skipped rule=protection-test-bypass reason=test_bypass client_tag=omitted method=GET host=example.com path=/page +[datadome] protection decision=skipped rule=protection-test-bypass reason=test_bypass client_tag=omitted method=GET ``` ### Structured exclusion rules diff --git a/docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md b/docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md index 5ea5f6ff3..0dfc923cf 100644 --- a/docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md +++ b/docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md @@ -152,7 +152,7 @@ matches!(reason, "client_ip" | "client_ip_source" | "ip_cidr" | "ip_cidr_source" desired shape is: ```text -[datadome] protection decision=skipped rule=excluded-ip-cidrs reason=client_ip client_tag=omitted method=GET host=example.com path=/page +[datadome] protection decision=skipped rule=excluded-ip-cidrs reason=client_ip client_tag=omitted method=GET ``` - [ ] **Step 5: Add filter-level marker tests.** Add small helpers in the diff --git a/docs/superpowers/specs/2026-08-03-datadome-ip-excluded-client-tag-design.md b/docs/superpowers/specs/2026-08-03-datadome-ip-excluded-client-tag-design.md index c48ad2fda..d6d811780 100644 --- a/docs/superpowers/specs/2026-08-03-datadome-ip-excluded-client-tag-design.md +++ b/docs/superpowers/specs/2026-08-03-datadome-ip-excluded-client-tag-design.md @@ -149,10 +149,10 @@ For IP-based skips, extend the existing informational log with `client_tag=omitted`: ```text -[datadome] protection decision=skipped rule=excluded-ip-cidrs reason=client_ip client_tag=omitted method=GET host=example.com path=/page +[datadome] protection decision=skipped rule=excluded-ip-cidrs reason=client_ip client_tag=omitted method=GET ``` -The existing rule ID, reason, and request metadata remain part of the log. +The existing rule ID, reason, and method remain part of the log. Host and path are omitted to avoid placing dynamic request data in the protection logs. Client IP values are not included. Non-IP skip logs retain their current behavior and level. From 126c17fc73437adc7869d1e512901678eadec223 Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 6 Aug 2026 14:56:44 -0500 Subject: [PATCH 09/16] Improve publisher HTML cache policy when SSAT is inactive --- crates/trusted-server-core/src/publisher.rs | 105 ++++++++++++++------ 1 file changed, 77 insertions(+), 28 deletions(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index d3410b4ed..8919f09b3 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -2932,33 +2932,49 @@ pub async fn handle_publisher_request( // // Gate on `should_run_ad_stack` rather than content-type alone: when no slot // matched, the feature is disabled, or this is not an ad-eligible navigation, - // no per-user `tsjs.adSlots`/`tsjs.bids` are injected, so forcing private - // here would needlessly strip shared cacheability from ordinary publisher - // HTML. Applies regardless of the auction *outcome* (empty bids still inject - // per-user slot state). The separate EC-cookie cache net in the adapter's - // `finalize_response` keeps first-visit identity responses private. + // no per-user `tsjs.adSlots`/`tsjs.bids` are injected. Applies regardless of + // the auction *outcome* (empty bids still inject per-user slot state). The + // separate EC-cookie cache net in the adapter's `finalize_response` keeps + // first-visit identity responses private. let origin_content_type = response .headers() .get(header::CONTENT_TYPE) .and_then(|h| h.to_str().ok()) .unwrap_or_default(); - if should_run_ad_stack && is_html_content_type(origin_content_type) { - response.headers_mut().insert( - header::CACHE_CONTROL, - HeaderValue::from_static("private, no-store"), - ); - response.headers_mut().remove(header::ETAG); - response.headers_mut().remove(header::LAST_MODIFIED); - // Every CDN-targeted cache directive, not just the browser-facing - // `Cache-Control` above: an origin emitting any of these would otherwise - // instruct an intermediary to store a synthesized per-navigation - // document. `Surrogate-Control` and `Fastly-Surrogate-Control` cover - // Fastly; `CDN-Cache-Control` is the standard targeted field (RFC 9213) - // and `Cloudflare-CDN-Cache-Control` is the Cloudflare-specific field - // that overrides it there, so both are needed to close the gap on the - // Cloudflare adapter. - for directive in CDN_CACHE_HEADERS { - response.headers_mut().remove(*directive); + if is_html_content_type(origin_content_type) { + if should_run_ad_stack { + response.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static("private, no-store"), + ); + response.headers_mut().remove(header::ETAG); + response.headers_mut().remove(header::LAST_MODIFIED); + // Every CDN-targeted cache directive, not just the browser-facing + // `Cache-Control` above: an origin emitting any of these would otherwise + // instruct an intermediary to store a synthesized per-navigation + // document. `Surrogate-Control` and `Fastly-Surrogate-Control` cover + // Fastly; `CDN-Cache-Control` is the standard targeted field (RFC 9213) + // and `Cloudflare-CDN-Cache-Control` is the Cloudflare-specific field + // that overrides it there, so both are needed to close the gap on the + // Cloudflare adapter. + for directive in CDN_CACHE_HEADERS { + response.headers_mut().remove(*directive); + } + } else { + let origin_cache_control = response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()) + .map(str::to_ascii_lowercase); + if !origin_cache_control + .as_deref() + .is_some_and(|value| value.contains("private") || value.contains("no-store")) + { + response.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static("max-age=60"), + ); + } } } @@ -4697,13 +4713,16 @@ mod tests { .expect("should build conditional navigation request") } - fn queue_cacheable_html_response(stub: &StubHttpClient) { + fn queue_html_response_with_cache_control( + stub: &StubHttpClient, + cache_control: &'static str, + ) { stub.push_response_with_headers( 200, b"origin".to_vec(), vec![ ("content-type", "text/html; charset=utf-8"), - ("cache-control", "public, max-age=300"), + ("cache-control", cache_control), ("etag", ORIGIN_ETAG), ("last-modified", ORIGIN_LAST_MODIFIED), ("surrogate-control", "max-age=300"), @@ -4773,7 +4792,7 @@ mod tests { // Arrange let settings = settings_with_enabled_auction_and_creative_opportunities(); let stub = Arc::new(StubHttpClient::new()); - queue_cacheable_html_response(&stub); + queue_html_response_with_cache_control(&stub, "public, max-age=300"); let services = build_services_with_http_client( Arc::clone(&stub) as Arc ); @@ -4866,11 +4885,11 @@ mod tests { } #[tokio::test] - async fn navigation_without_matched_slots_preserves_origin_cache_policy() { + async fn navigation_without_matched_slots_uses_short_browser_cache_policy() { // Arrange let settings = settings_with_enabled_auction_and_creative_opportunities(); let stub = Arc::new(StubHttpClient::new()); - queue_cacheable_html_response(&stub); + queue_html_response_with_cache_control(&stub, "public, max-age=300"); let services = build_services_with_http_client( Arc::clone(&stub) as Arc ); @@ -4916,7 +4935,7 @@ mod tests { ); for (header_name, expected) in [ - (header::CACHE_CONTROL, "public, max-age=300"), + (header::CACHE_CONTROL, "max-age=60"), (header::ETAG, ORIGIN_ETAG), (header::LAST_MODIFIED, ORIGIN_LAST_MODIFIED), ( @@ -4947,6 +4966,36 @@ mod tests { } } + #[tokio::test] + async fn navigation_without_matched_slots_preserves_private_origin_cache_policy() { + let settings = settings_with_enabled_auction_and_creative_opportunities(); + + for cache_control in ["private, max-age=0", "No-Store"] { + // Arrange + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, cache_control); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + + // Act + let response = + run_with_slots(&settings, &services, &[], conditional_navigation_request()) + .await; + let response_head = response_head(response); + + // Assert + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some(cache_control), + "origin {cache_control} policy should not be weakened" + ); + } + } + #[tokio::test] async fn eligible_navigation_rejects_unexpected_origin_304() { for content_type in [None, Some("text/html; charset=utf-8")] { From 92e2a78062dc1b57194cbd8873a73f8ef7e5bdec Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 6 Aug 2026 17:17:40 -0500 Subject: [PATCH 10/16] Document dedicated server-side ad template switch --- ...-server-side-ad-templates-cache-control.md | 238 ++++++++++++++++++ 1 file changed, 238 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-06-server-side-ad-templates-cache-control.md diff --git a/docs/superpowers/plans/2026-08-06-server-side-ad-templates-cache-control.md b/docs/superpowers/plans/2026-08-06-server-side-ad-templates-cache-control.md new file mode 100644 index 000000000..f96bdcb28 --- /dev/null +++ b/docs/superpowers/plans/2026-08-06-server-side-ad-templates-cache-control.md @@ -0,0 +1,238 @@ +# Dedicated Server-Side Ad Templates Switch and Cache Policy Plan + +> **For agentic workers:** Implement this plan task-by-task, keeping the dedicated +> template switch separate from the global auction configuration. + +**Goal:** Add an explicit on/off switch for server-side ad templates, while +retaining the browser-facing cache policy from issue #1007: + +- Server-side ad templates active: `Cache-Control: private, no-store`. +- Server-side ad templates inactive: `Cache-Control: max-age=60`, unless the + origin already sends `private` or `no-store`. +- CDN-specific cache headers must not change when templates are inactive. + +**Issue context:** The current cache-policy change uses the runtime +`should_run_ad_stack` gate. That gate is also affected by `[auction].enabled`, +which is not the right configuration boundary for publisher templates. A +browser can call `POST /auction`, and that endpoint is a separate server-run +auction API. The new switch must disable publisher HTML/page-bids template +delivery without disabling that API. + +## Configuration decision + +Add this field to the existing `[creative_opportunities]` section: + +```toml +[creative_opportunities] +enabled = true +``` + +Use `enabled = false` to turn off server-side ad templates while retaining the +slot definitions and keeping direct `POST /auction` behavior available. + +### Compatibility rules + +- The field defaults to `true` when omitted, preserving existing behavior for + deployments that already have `[creative_opportunities]` configured. +- The section remains optional. An absent section continues to mean that the + feature is unavailable. +- Serialize the default `true` value as omitted, matching the existing + rollback-compatibility pattern for newer creative-opportunity fields. An + explicit `false` must remain serialized so the setting is not silently lost. +- `auction.enabled` remains a separate auction/orchestrator setting. Do not use + it as the dedicated template switch and do not thread the new template flag + into `POST /auction`. + +## Current cache behavior to retain + +The existing HTML policy block in `publisher.rs` must remain structurally +consistent with the current issue #952 behavior: + +1. For an eligible request that runs the server-side ad stack and receives HTML: + - Set `Cache-Control: private, no-store`. + - Remove `ETag` and `Last-Modified`. + - Remove `Surrogate-Control`, `Fastly-Surrogate-Control`, `CDN-Cache-Control`, + and `Cloudflare-CDN-Cache-Control`. +2. For HTML where the server-side ad stack does not run, including an explicit + template disable: + - Read the browser-facing `Cache-Control` header. + - If its value contains `private` or `no-store`, case-insensitively, preserve + the origin value exactly. + - Otherwise set exactly `Cache-Control: max-age=60`. + - Leave validators and all CDN-specific cache headers untouched. +3. Preserve the later adapter response-privacy finalization for cookie-bearing + responses; this plan does not refactor that behavior. + +## File map + +### Configuration and compatibility + +- `crates/trusted-server-core/src/creative_opportunities.rs` + - Add `CreativeOpportunitiesConfig::enabled` with a default-true serde + implementation and documentation. + - Add a small accessor if it improves readability, but keep the source of + truth in this config type. + - Update config constructors and serialization tests. +- `crates/trusted-server-core/src/settings.rs` + - Keep `creative_opportunities` parsing and runtime preparation compatible with + the new field. + - Make `creative_opportunity_slots()` return an empty slice when the section + is absent or explicitly disabled, so all adapters receive one consistent + runtime view. + - Add TOML and environment-override coverage for `enabled = false`. +- `crates/trusted-server-core/src/config.rs` + - Extend legacy-schema tests to prove default `enabled = true` is omitted from + serialized blobs and remains readable by older binaries. + - Prove an explicit `enabled = false` is serialized, making rollback failure + loud rather than silently re-enabling templates. +- `trusted-server.example.toml` + - Document `creative_opportunities.enabled` and show how to turn templates off + without deleting slot definitions. +- `docs/guide/configuration.md` + - Add the field to the creative-opportunities reference and document the + environment override: + `TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__ENABLED=false`. + - Clarify that this switch controls publisher HTML/page-bids template + delivery, not direct `POST /auction` callers. +- `CHANGELOG.md` + - Add an entry describing the dedicated template switch and cache behavior. + +### Publisher execution and cache policy + +- `crates/trusted-server-core/src/publisher.rs` + - Include the dedicated flag in the initial publisher eligibility decision. + - Do not match, dispatch, or inject server-side ad templates when the flag is + false, even if slots are configured and `[auction].enabled` is true. + - Apply the issue #1007 inactive-HTML cache policy in this state. + - Update skip-reason diagnostics/telemetry so `ad_templates_disabled` is + distinguishable from `auction_disabled`, consent denial, bots, prefetch, and + no matching slots. + - Update `handle_page_bids` so an explicit template disable returns the normal + empty JSON shape (`slots: []`, `bids: {}`) rather than slot definitions. Keep + the current `404` behavior for an absent `[creative_opportunities]` section. + - Extend the existing SSAT cache-policy and eligibility tests. +- `crates/trusted-server-core/src/auction/endpoints.rs` + - Do not gate `POST /auction` on the new template flag. + - Add a regression test or test fixture proving that disabling + `creative_opportunities.enabled` does not suppress a direct auction request + when providers are configured. + - Separately document/verify the existing behavior of `[auction].enabled` for + this endpoint; do not conflate that global setting with the new template + switch. + +### Adapter propagation and browser behavior + +The adapters already pass `Settings::creative_opportunity_slots()` into the +publisher/page-bids handlers. Update and verify these call sites so the central +empty-slice behavior is honored; avoid adding four divergent config checks: + +- `crates/trusted-server-adapter-fastly/src/app.rs` +- `crates/trusted-server-adapter-axum/src/app.rs` +- `crates/trusted-server-adapter-cloudflare/src/app.rs` +- `crates/trusted-server-adapter-spin/src/app.rs` + +No route-level flag is needed if the core `Settings` accessor and handlers are +correct. Add adapter route assertions only where existing fixtures make them +useful. + +The browser runtime already defaults `window.tsjs.adSlots` and +`window.tsjs.bids` to empty values when the edge does not inject templates. If +terminology is updated, adjust these comments/tests without changing runtime +semantics: + +- `crates/trusted-server-js/lib/src/core/index.ts` +- `crates/trusted-server-js/lib/src/integrations/gpt/index.ts` +- Relevant page-bids tests under `crates/trusted-server-js/lib/test/integrations/gpt/` + +## Implementation tasks + +### Task 1: Add and serialize the dedicated setting + +- [ ] Add `enabled: bool` to `CreativeOpportunitiesConfig` with default `true`. +- [ ] Use `skip_serializing_if` so the default value does not appear in stored + config blobs; explicit `false` must serialize. +- [ ] Update all Rust struct literals in `creative_opportunities.rs` and + `publisher.rs` tests. +- [ ] Add parsing, default, false-value, and environment-override tests. +- [ ] Update the legacy compatibility tests in `config.rs`. + +### Task 2: Thread the setting through publisher eligibility + +- [ ] Update `should_run_server_side_ad_stack` to accept the dedicated template + flag as an explicit gate, with a descriptive parameter/doc comment. +- [ ] Ensure initial publisher slot matching and `Settings::creative_opportunity_slots` + do not expose slots when templates are disabled. +- [ ] Preserve the existing `[auction].enabled` and consent gates as separate + conditions. +- [ ] Add an `ad_templates_disabled` diagnostic/telemetry skip reason where the + current branch records a skipped auction. + +### Task 3: Apply the cache policy to the dedicated-off state + +- [ ] Keep the active-SSAT `private, no-store` behavior and validator/CDN header + removal unchanged. +- [ ] Keep the inactive-HTML `max-age=60` behavior from issue #1007. +- [ ] Verify that explicit template disable changes only browser-facing + `Cache-Control` for cacheable HTML; preserve `ETag`, `Last-Modified`, and + every CDN-specific header. +- [ ] Verify that origin `private`, `PRIVATE`, `no-store`, and `No-Store` values + remain unchanged. + +### Task 4: Gate SPA page-bids/template delivery + +- [ ] Include `co_config.enabled` in the `ad_stack_enabled` decision in + `handle_page_bids`. +- [ ] Return empty slots and bids for an explicit disable while retaining the + endpoint and its existing response privacy headers. +- [ ] Keep the absent-section `404` behavior unchanged. +- [ ] Add tests for enabled, disabled, absent, consent-denied, bot, and prefetch + cases as appropriate; preserve existing tests for `[auction].enabled=false`. + +### Task 5: Protect direct `POST /auction` from accidental coupling + +- [ ] Add a focused endpoint test with `creative_opportunities.enabled=false` + and a recording provider. +- [ ] Assert that the provider still sees the direct auction request and that + the response remains a normal OpenRTB response. +- [ ] If the test reveals that `[auction].enabled=false` also needs a separate + product decision for `/auction`, record that as a follow-up rather than + changing it as part of the template-switch work. + +### Task 6: Update docs, examples, comments, and adapter coverage + +- [ ] Update the example config, configuration guide, and changelog. +- [ ] Update stale comments that call `[auction].enabled` the universal template + kill switch. +- [ ] Verify all four adapter call sites use the centralized disabled-slot view. +- [ ] Run JS tests if comments or tests are touched; no JS behavior change is + expected. + +## Test plan + +Use target-matched commands; do not run bare workspace tests because the +workspace contains multiple runtime targets. + +- [ ] `cargo test-axum -p trusted-server-core publisher` +- [ ] `cargo test-fastly` +- [ ] `cargo test-axum` +- [ ] `cargo test-cloudflare` +- [ ] `cargo test-spin` +- [ ] `cargo fmt --all -- --check` +- [ ] `cargo clippy-fastly` +- [ ] `cargo clippy-axum` +- [ ] `cargo clippy-cloudflare` +- [ ] `cargo clippy-cloudflare-wasm` +- [ ] `cargo clippy-spin-native` +- [ ] `cargo clippy-spin-wasm` +- [ ] `cd crates/trusted-server-js/lib && npx vitest run` if JS tests/comments change +- [ ] `cd docs && npm run format` if documentation formatting is required + +## Non-goals + +- Do not change CDN-specific cache policy for inactive templates. +- Do not change adapter response privacy or cookie handling. +- Do not use `auction.rewrite_creatives` as the template switch; it controls + creative URL rewriting, not whether the server-side template stack runs. +- Do not gate or disable direct `POST /auction` as part of this feature. +- Do not remove slot definitions when the switch is off; the point of the switch + is to provide a reversible runtime control. From 58054463a16ce801198f877b687b579a33a9d9a3 Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 6 Aug 2026 18:03:25 -0500 Subject: [PATCH 11/16] Add dedicated server-side ad template switch --- CHANGELOG.md | 1 + .../src/auction/endpoints.rs | 127 +++++++++- crates/trusted-server-core/src/config.rs | 27 ++ .../src/creative_opportunities.rs | 47 +++- crates/trusted-server-core/src/publisher.rs | 237 +++++++++++++++--- crates/trusted-server-core/src/settings.rs | 46 +++- .../trusted-server-js/lib/src/core/index.ts | 8 +- .../lib/src/integrations/gpt/index.ts | 12 +- .../test/integrations/gpt/spa_hook.test.ts | 6 +- docs/guide/configuration.md | 20 +- trusted-server.example.toml | 3 + 11 files changed, 480 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 36655763c..7c43a6fa7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Publisher HTML now uses `Cache-Control: max-age=60` when server-side ad templates are inactive, while preserving origin `private`/`no-store` policies and CDN-specific cache headers. Set `[creative_opportunities].enabled = false` to disable publisher HTML and SPA template delivery without disabling direct `POST /auction` callers. - **Breaking** — `bid_param_zone_overrides` inner values must now be JSON objects; previously non-object or empty values (`"header" = "x"`, `"header" = {}`) were accepted and silently produced a dead rule at runtime. They now fail at startup with a configuration error. Operators upgrading should audit their `bid_param_zone_overrides` config for non-object zone entries. - **Breaking** — Integration configuration strings are no longer globally reinterpreted as JSON scalars. Operators upgrading should audit `[integrations.*]` settings and use native TOML/typed-config booleans and numbers (for example, `enabled = true`, not `enabled = "true"`); quoted numeric and boolean scalars now fail validation instead of silently converting. - **Breaking** — Sourcepoint browser module inclusion now requires explicit `[integrations.sourcepoint].enabled = true`; operators relying on the previous unconditional Sourcepoint module should enable the integration before upgrading. diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index 36e083948..b80b9339c 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -554,10 +554,14 @@ mod tests { use crate::consent::types::ConsentContext; use crate::openrtb::Uid; use crate::platform::test_support::{ - NoopBackend, NoopConfigStore, NoopGeo, NoopHttpClient, NoopSecretStore, noop_services, + NoopBackend, NoopConfigStore, NoopGeo, NoopHttpClient, NoopSecretStore, StubHttpClient, + noop_services, }; - use crate::platform::{ClientInfo, PlatformPendingRequest, PlatformResponse}; - use crate::test_support::tests::create_test_settings; + use crate::platform::{ + ClientInfo, PlatformHttpClient, PlatformHttpRequest, PlatformPendingRequest, + PlatformResponse, + }; + use crate::test_support::tests::{crate_test_settings_str, create_test_settings}; use base64::Engine as _; use base64::engine::general_purpose::STANDARD as BASE64; use serde_json::json; @@ -642,6 +646,123 @@ mod tests { } } + /// Provider used to prove that direct `/auction` remains available when + /// publisher server-side ad templates are disabled. + struct TemplateSwitchProbeProvider { + calls: Arc>, + } + + #[async_trait::async_trait(?Send)] + impl AuctionProvider for TemplateSwitchProbeProvider { + fn provider_name(&self) -> &'static str { + "template_switch_probe" + } + + async fn request_bids( + &self, + _request: &AuctionRequest, + context: &AuctionContext<'_>, + ) -> Result> { + *self.calls.lock().expect("should lock provider call count") += 1; + let request = Request::builder() + .method("POST") + .uri("https://bidder.example/auction") + .body(EdgeBody::empty()) + .expect("should build probe provider request"); + context + .services + .http_client() + .send_async(PlatformHttpRequest::new( + request, + "template-switch-probe-backend", + )) + .await + .change_context(TrustedServerError::Auction { + message: "probe provider launch failed".to_string(), + }) + } + + async fn parse_response( + &self, + _response: PlatformResponse, + _response_time_ms: u64, + ) -> Result> { + Ok(AuctionResponse::success( + self.provider_name(), + Vec::new(), + 0, + )) + } + + fn timeout_ms(&self) -> u32 { + 100 + } + + fn backend_name(&self, _services: &RuntimeServices, _timeout_ms: u32) -> Option { + Some("template-switch-probe-backend".to_string()) + } + } + + #[tokio::test] + async fn direct_auction_remains_available_when_templates_are_disabled() { + let settings_toml = format!( + "{}\n[auction]\nenabled = true\nproviders = [\"template_switch_probe\"]\n\n[creative_opportunities]\nenabled = false\ngam_network_id = \"12345\"\n", + crate_test_settings_str() + ); + let settings = Settings::from_toml(&settings_toml) + .expect("should parse settings with disabled templates"); + let calls = Arc::new(Mutex::new(0)); + let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + orchestrator.register_provider(Arc::new(TemplateSwitchProbeProvider { + calls: Arc::clone(&calls), + })); + + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"probe response".to_vec()); + let services = RuntimeServices::builder() + .config_store(Arc::new(NoopConfigStore)) + .secret_store(Arc::new(NoopSecretStore)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::new(NoopBackend)) + .http_client(Arc::clone(&stub) as Arc) + .geo(Arc::new(NoopGeo)) + .client_info(ClientInfo::default()) + .build(); + let ec_context = make_ec_context(Jurisdiction::NonRegulated, None); + let body = json!({ + "adUnits": [{ + "code": "div-gpt-ad-1", + "mediaTypes": { "banner": { "sizes": [[300, 250]] } } + }] + }); + let req = Request::builder() + .method("POST") + .uri("https://test-publisher.com/auction") + .body(EdgeBody::from( + serde_json::to_vec(&body).expect("should serialize body"), + )) + .expect("should build auction request"); + + let response = handle_auction( + &settings, + &orchestrator, + None, + None, + &ec_context, + &services, + req, + ) + .await + .expect("direct auction should remain available"); + + assert_eq!( + *calls.lock().expect("should lock provider call count"), + 1, + "disabling publisher templates must not disable direct /auction" + ); + assert_eq!(response.status(), StatusCode::OK); + } + #[tokio::test] async fn auction_endpoint_consent_gate_returns_no_bid_without_contacting_providers() { // GDPR/unknown jurisdiction lacking effective TCF Purpose 1 must not run diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index cdee3b222..033e6c908 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -321,10 +321,37 @@ formats = [{ width = 300, height = 250 }] fn absent_gam_unit_template_is_accepted_by_legacy_schema() { let creative_opportunities = serialized_creative_opportunities(None); + assert!( + creative_opportunities.get("enabled").is_none(), + "default template switch should be omitted for legacy binaries" + ); serde_json::from_value::(creative_opportunities) .expect("should accept absent GAM unit template"); } + #[test] + fn disabled_creative_opportunities_flag_is_visible_to_legacy_schema() { + let mut toml = crate_test_settings_str(); + toml.push_str( + r#" + +[creative_opportunities] +enabled = false +gam_network_id = "99999" +"#, + ); + let app_config: TrustedServerAppConfig = + toml::from_str(&toml).expect("should deserialize app config wrapper"); + let creative_opportunities = serde_json::to_value(app_config) + .expect("should serialize app config wrapper") + .get("creative_opportunities") + .cloned() + .expect("should contain creative opportunities"); + + serde_json::from_value::(creative_opportunities) + .expect_err("legacy binaries should reject an explicit disabled switch"); + } + #[test] fn deploy_validation_rejects_placeholders() { let settings = Settings::from_toml( diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index 10a85b3e8..a6b70e1d6 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -183,10 +183,27 @@ fn derive_section(path: &str, section_root: &str, section_segment: usize) -> Str } } +const fn default_enabled() -> bool { + true +} + +const fn is_default_enabled(value: &bool) -> bool { + *value == default_enabled() +} + /// Top-level configuration for the creative opportunities system. #[derive(Debug, Clone, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct CreativeOpportunitiesConfig { + /// Enables server-side ad template delivery on publisher HTML and page-bids requests. + /// + /// This does not disable the direct `POST /auction` endpoint. The default is + /// `true` so existing creative-opportunity configurations retain their behavior. + #[serde( + default = "default_enabled", + skip_serializing_if = "is_default_enabled" + )] + pub enabled: bool, /// GAM network ID used to build default unit paths. pub gam_network_id: String, /// Maximum time in milliseconds to wait for the server-side auction before @@ -244,7 +261,7 @@ pub struct CreativeOpportunitiesConfig { /// [`section_root`](Self::section_root) are omitted. #[serde(default, skip_serializing_if = "Option::is_none")] pub section_segment: Option, - /// Slot templates. Empty vec = feature disabled (no auction fired, no globals injected). + /// Slot templates. An empty vec or `enabled = false` disables template delivery. #[serde(default, deserialize_with = "vec_from_seq_or_map")] pub slot: Vec, } @@ -1139,12 +1156,39 @@ mod tests { assert_eq!(derive_section("/%%%/x", "home", 0), "_"); } + #[test] + fn enabled_defaults_true_and_is_omitted_from_serialized_config() { + let config = make_config_with_section_template(None); + assert!( + config.enabled, + "template delivery should default to enabled" + ); + let value = serde_json::to_value(&config).expect("should serialize config"); + assert!( + value.get("enabled").is_none(), + "default enabled value should be omitted for rollback compatibility" + ); + } + + #[test] + fn disabled_template_switch_is_serialized() { + let mut config = make_config_with_section_template(None); + config.enabled = false; + let value = serde_json::to_value(&config).expect("should serialize config"); + assert_eq!( + value.get("enabled"), + Some(&serde_json::Value::Bool(false)), + "explicitly disabled template delivery must remain in config blobs" + ); + } + fn make_config_with_section_template( section_root: Option<&str>, ) -> CreativeOpportunitiesConfig { let mut slot = make_slot("ad-header-0", vec!["/news/*"]); slot.gam_unit_path = Some("/{network_id}/example/{section}".to_string()); CreativeOpportunitiesConfig { + enabled: true, gam_network_id: "99999".to_string(), auction_timeout_ms: None, price_granularity: PriceGranularity::default(), @@ -1542,6 +1586,7 @@ mod tests { // Older binaries deserialize this struct with `deny_unknown_fields`, so // a pushed config blob must not carry `"section_root": null`. let config = CreativeOpportunitiesConfig { + enabled: true, gam_network_id: "99999".to_string(), auction_timeout_ms: None, price_granularity: PriceGranularity::default(), diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 8919f09b3..87ab8317b 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1751,27 +1751,34 @@ pub(crate) fn is_prefetch_request(req: &Request) -> bool { header("sec-purpose") || header("purpose") } -/// Returns true only when the publisher request should run the full -/// server-side ad stack: auction dispatch plus initial ad-slot injection. +#[derive(Debug, Clone, Copy)] +struct ServerSideAdStackConfig { + /// Dedicated `[creative_opportunities].enabled` switch. + ad_templates_enabled: bool, + /// Global `[auction].enabled` gate used by publisher/page-bids flows. + auction_enabled: bool, +} + +/// Returns true only when the publisher should inject and run server-side ad templates. /// -/// `auction_enabled` is the global `[auction].enabled` kill switch — when -/// false, no automatic server-side auction or ad-slot injection runs. -pub(crate) fn should_run_server_side_ad_stack( +/// This includes auction dispatch plus initial ad-slot injection. +fn should_run_server_side_ad_stack( is_get: bool, is_navigation: bool, is_prefetch: bool, is_bot: bool, has_matched_slots: bool, consent_allows_auction: bool, - auction_enabled: bool, + config: ServerSideAdStackConfig, ) -> bool { is_get && is_navigation && !is_prefetch && !is_bot + && config.ad_templates_enabled && has_matched_slots && consent_allows_auction - && auction_enabled + && config.auction_enabled } /// Write winning bids from an auction result into the shared `ad_bids_state` lock. @@ -2632,7 +2639,10 @@ pub async fn handle_publisher_request( let is_prefetch = is_prefetch_request(&req); let is_bot = is_bot_user_agent(&req); - let matched_slots = if is_get { + let creative_opportunities = settings.creative_opportunities.as_ref(); + let ad_templates_enabled = creative_opportunities.is_some_and(|co_config| co_config.enabled); + let ad_templates_disabled = creative_opportunities.is_some_and(|co_config| !co_config.enabled); + let matched_slots = if is_get && ad_templates_enabled { settings .creative_opportunities .as_ref() @@ -2655,7 +2665,10 @@ pub async fn handle_publisher_request( is_bot, !matched_slots.is_empty(), consent_allows_auction, - auction.orchestrator.is_enabled(), + ServerSideAdStackConfig { + ad_templates_enabled, + auction_enabled: auction.orchestrator.is_enabled(), + }, ); let should_run_auction = should_run_ad_stack; // Diagnostic: shows which gate suppresses the server-side auction. Pair with @@ -2663,14 +2676,16 @@ pub async fn handle_publisher_request( // when `consent_allows_auction=false`. log::debug!( "server-side ad-stack gate: is_get={is_get} is_navigation={is_navigation} \ - is_prefetch={is_prefetch} is_bot={is_bot} matched_slots={} \ - consent_allows_auction={consent_allows_auction} orchestrator_enabled={} \ - -> should_run_auction={should_run_auction}", + is_prefetch={is_prefetch} is_bot={is_bot} ad_templates_enabled={ad_templates_enabled} \ + matched_slots={} consent_allows_auction={consent_allows_auction} \ + orchestrator_enabled={} -> should_run_auction={should_run_auction}", matched_slots.len(), auction.orchestrator.is_enabled(), ); - if matched_slots.is_empty() && settings.creative_opportunities.is_some() { + if ad_templates_disabled { + log::debug!("Server-side ad templates are disabled by configuration"); + } else if matched_slots.is_empty() && settings.creative_opportunities.is_some() { log::debug!( "No creative opportunity slots matched path '{}' — skipping auction and injection", request_path @@ -2795,7 +2810,9 @@ pub async fn handle_publisher_request( } } } else { - let skip_reason = if !auction.orchestrator.is_enabled() { + let skip_reason = if ad_templates_disabled { + "ad_templates_disabled" + } else if !auction.orchestrator.is_enabled() { "auction_disabled" } else if !consent_allows_auction { "consent_denied" @@ -3771,7 +3788,11 @@ pub async fn handle_page_bids( }) .unwrap_or_else(|| "/".to_string()); - let matched_slots = match_renderable_slots(auction.slots, co_config, &path_param); + let matched_slots = if co_config.enabled { + match_renderable_slots(auction.slots, co_config, &path_param) + } else { + Vec::new() + }; let request_info = crate::http_util::RequestInfo::from_request(&req, services.client_info()); let ec_id = ec_context.ec_value().filter(|_| ec_context.ec_allowed()); @@ -3790,7 +3811,10 @@ pub async fn handle_page_bids( let is_bot = is_bot_user_agent(&req); let auction_enabled = auction.orchestrator.is_enabled(); - if !auction_enabled { + let ad_templates_enabled = co_config.enabled; + if !ad_templates_enabled { + log::debug!("page-bids: [creative_opportunities].enabled is false — skipping templates"); + } else if !auction_enabled { log::debug!("page-bids: [auction].enabled is false — skipping auction"); } else if matched_slots.is_empty() { log::debug!( @@ -3806,14 +3830,14 @@ pub async fn handle_page_bids( ); } - // The [auction].enabled kill switch and a consent denial disable the entire - // server-side ad stack. In those states the endpoint must return no slots, - // so the SPA hook does not assign `ts.adSlots` and call `adInit()` — - // otherwise the kill switch/consent gate would stop SSP calls but still let - // the client create/refresh GPT slots. Bot/prefetch requests, by contrast, - // keep their slot definitions (the placement structure is unchanged) but - // skip the live auction, matching the existing bot/prefetch behaviour. - let ad_stack_enabled = auction_enabled && consent_allows_auction; + // The dedicated template switch, [auction].enabled, and a consent denial + // disable the entire server-side ad stack. In those states the endpoint must + // return no slots, so the SPA hook does not assign `ts.adSlots` and call + // `adInit()` — otherwise the gate would stop SSP calls but still let the + // client create/refresh GPT slots client-side. Bot/prefetch requests, by + // contrast, keep their slot definitions (the placement structure is + // unchanged) but skip the live auction, matching the existing behavior. + let ad_stack_enabled = ad_templates_enabled && auction_enabled && consent_allows_auction; let winning_bids = if matched_slots.is_empty() { std::collections::HashMap::new() @@ -3903,7 +3927,9 @@ pub async fn handle_page_bids( } } } else { - let skip_reason = if !auction_enabled { + let skip_reason = if !ad_templates_enabled { + "ad_templates_disabled" + } else if !auction_enabled { "auction_disabled" } else if !consent_allows_auction { "consent_denied" @@ -4655,6 +4681,15 @@ mod tests { .expect("should parse settings with auction and creative opportunities enabled") } + fn settings_with_disabled_ad_templates() -> Settings { + let toml = format!( + "{}\n[auction]\nenabled = true\n\n\ + [creative_opportunities]\nenabled = false\ngam_network_id = \"12345\"\n", + crate_test_settings_str() + ); + Settings::from_toml(&toml).expect("should parse settings with disabled ad templates") + } + fn settings_with_dispatching_provider() -> Settings { let toml = format!( "{}\n[auction]\nenabled = true\nproviders = [\"{UNEXPECTED_304_PROVIDER}\"]\n\n\ @@ -4966,6 +5001,72 @@ mod tests { } } + #[tokio::test] + async fn disabled_ad_templates_use_short_browser_cache_policy() { + // Arrange + let settings = settings_with_disabled_ad_templates(); + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, "public, max-age=300"); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let slots = [article_slot()]; + + // Act + let response = run_with_slots( + &settings, + &services, + &slots, + conditional_navigation_request(), + ) + .await; + let response_head = response_head(response); + + // Assert + assert_eq!( + stub.recorded_cache_bypass_flags(), + vec![false], + "disabled server-side ad templates should not bypass the origin cache" + ); + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("max-age=60"), + "disabled server-side ad templates should use the short browser cache policy" + ); + for (header_name, expected) in [ + (header::ETAG, ORIGIN_ETAG), + (header::LAST_MODIFIED, ORIGIN_LAST_MODIFIED), + ( + header::HeaderName::from_static("surrogate-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("fastly-surrogate-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("cdn-cache-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("cloudflare-cdn-cache-control"), + "max-age=300", + ), + ] { + assert_eq!( + response_head + .headers + .get(&header_name) + .and_then(|value| value.to_str().ok()), + Some(expected), + "disabled server-side ad templates should preserve {header_name}" + ); + } + } + #[tokio::test] async fn navigation_without_matched_slots_preserves_private_origin_cache_policy() { let settings = settings_with_enabled_auction_and_creative_opportunities(); @@ -5450,39 +5551,69 @@ mod tests { #[test] fn server_side_ad_stack_runs_only_when_all_auction_gates_pass() { + let enabled_config = ServerSideAdStackConfig { + ad_templates_enabled: true, + auction_enabled: true, + }; assert!( - should_run_server_side_ad_stack(true, true, false, false, true, true, true), - "GET, real navigation, matched slots, and consent should run TS ad stack" + should_run_server_side_ad_stack(true, true, false, false, true, true, enabled_config,), + "GET, real navigation, enabled templates, matched slots, and consent should run TS ad stack" ); assert!( - !should_run_server_side_ad_stack(false, true, false, false, true, true, true), + !should_run_server_side_ad_stack(false, true, false, false, true, true, enabled_config,), "non-GET requests should skip TS ad stack" ); assert!( - !should_run_server_side_ad_stack(true, false, false, false, true, true, true), + !should_run_server_side_ad_stack(true, false, false, false, true, true, enabled_config,), "non-document requests should skip TS ad stack" ); assert!( - !should_run_server_side_ad_stack(true, true, true, false, true, true, true), + !should_run_server_side_ad_stack(true, true, true, false, true, true, enabled_config,), "prefetch requests should skip TS ad stack and injection" ); assert!( - !should_run_server_side_ad_stack(true, true, false, true, true, true, true), + !should_run_server_side_ad_stack(true, true, false, true, true, true, enabled_config,), "bot requests should skip TS ad stack and injection" ); assert!( - !should_run_server_side_ad_stack(true, true, false, false, false, true, true), + !should_run_server_side_ad_stack(true, true, false, false, false, true, enabled_config,), "requests with no matching slots should skip TS ad stack" ); assert!( - !should_run_server_side_ad_stack(true, true, false, false, true, false, true), + !should_run_server_side_ad_stack(true, true, false, false, true, false, enabled_config,), "requests without required consent should skip TS ad stack and injection" ); assert!( - !should_run_server_side_ad_stack(true, true, false, false, true, true, false), + !should_run_server_side_ad_stack( + true, + true, + false, + false, + true, + true, + ServerSideAdStackConfig { + ad_templates_enabled: true, + auction_enabled: false, + }, + ), "disabled [auction].enabled kill switch should skip TS ad stack and injection" ); + assert!( + !should_run_server_side_ad_stack( + true, + true, + false, + false, + true, + true, + ServerSideAdStackConfig { + ad_templates_enabled: false, + auction_enabled: true, + }, + ), + "disabled [creative_opportunities].enabled switch should skip TS ad stack and injection" + ); } #[tokio::test] @@ -8120,6 +8251,7 @@ mod tests { fn make_config() -> CreativeOpportunitiesConfig { CreativeOpportunitiesConfig { + enabled: true, gam_network_id: "21765378893".to_string(), auction_timeout_ms: Some(500), price_granularity: PriceGranularity::Dense, @@ -9337,6 +9469,14 @@ mod tests { Settings::from_toml(&toml).expect("should parse settings with creative_opportunities") } + fn settings_with_co_templates_disabled() -> Settings { + let toml = format!( + "{}\n[auction]\nenabled = true\n\n[creative_opportunities]\nenabled = false\ngam_network_id = \"12345\"\n", + crate_test_settings_str() + ); + Settings::from_toml(&toml).expect("should parse settings with disabled templates") + } + async fn run_page_bids( settings: &Settings, orchestrator: &AuctionOrchestrator, @@ -9886,6 +10026,35 @@ mod tests { ); } + #[tokio::test] + async fn disabled_server_side_ad_templates_return_no_slots_or_bids() { + // The dedicated template switch must suppress publisher/page-bids + // delivery without using the global auction switch. + let settings = settings_with_co_templates_disabled(); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let slots = article_slot(); + let req = make_page_bids_request("/2024/01/my-article/"); + + let body = run_page_bids_consent_allowed(&settings, &orchestrator, &slots, req).await; + + assert_eq!( + body["slots"] + .as_array() + .expect("slots should be array") + .len(), + 0, + "disabled server-side ad templates must not return slot definitions" + ); + assert_eq!( + body["bids"] + .as_object() + .expect("bids should be object") + .len(), + 0, + "disabled server-side ad templates must not produce bids" + ); + } + #[tokio::test] async fn consent_denied_returns_no_slots_or_bids() { // When consent denies the server-side auction (here: Jurisdiction diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 1021ee9ec..06152659c 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -2095,13 +2095,14 @@ impl Settings { Ok(()) } - /// Returns compiled creative opportunity slots, or empty slice if feature is disabled. + /// Returns compiled creative opportunity slots when template delivery is enabled. #[must_use] pub fn creative_opportunity_slots( &self, ) -> &[crate::creative_opportunities::CreativeOpportunitySlot] { self.creative_opportunities .as_ref() + .filter(|co| co.enabled) .map(|co| co.slot.as_slice()) .unwrap_or(&[]) } @@ -5010,6 +5011,10 @@ formats = [{ width = 300, height = 250 }] let co = settings .creative_opportunities .expect("should have creative_opportunities"); + assert!( + co.enabled, + "creative-opportunity templates should default to enabled" + ); assert_eq!(co.gam_network_id, "21765378893"); assert_eq!(co.auction_timeout_ms, Some(500)); assert_eq!( @@ -5019,6 +5024,45 @@ formats = [{ width = 300, height = 250 }] ); } + #[test] + fn settings_disables_creative_opportunity_slots_when_configured_off() { + let toml = format!( + "{}\n[creative_opportunities]\nenabled = false\ngam_network_id = \"21765378893\"\n\n[[creative_opportunities.slot]]\nid = \"atf\"\npage_patterns = [\"/\"]\nformats = [{{ width = 300, height = 250 }}]\n", + crate_test_settings_str() + ); + let settings = Settings::from_toml(&toml).expect("should parse disabled templates"); + assert!( + settings.creative_opportunity_slots().is_empty(), + "disabled template delivery should expose no runtime slots" + ); + } + + #[test] + fn settings_creative_opportunity_enabled_flag_supports_environment_override() { + let toml = format!( + "{}\n[creative_opportunities]\nenabled = true\ngam_network_id = \"21765378893\"\n", + crate_test_settings_str() + ); + let env_key = format!( + "{}{}CREATIVE_OPPORTUNITIES{}ENABLED", + ENVIRONMENT_VARIABLE_PREFIX, + ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR + ); + + temp_env::with_var(env_key, Some("false"), || { + let settings = Settings::from_toml_and_env(&toml) + .expect("should parse template enabled environment override"); + assert!( + !settings + .creative_opportunities + .expect("should have creative opportunities") + .enabled, + "environment override should disable template delivery" + ); + }); + } + #[test] fn settings_rejects_invalid_creative_opportunity_slot_id() { let toml = r#" diff --git a/crates/trusted-server-js/lib/src/core/index.ts b/crates/trusted-server-js/lib/src/core/index.ts index 5d8e41971..b2c4e41e1 100644 --- a/crates/trusted-server-js/lib/src/core/index.ts +++ b/crates/trusted-server-js/lib/src/core/index.ts @@ -36,10 +36,10 @@ api.getConfig = getConfig; // Provide core requestAds API api.requestAds = requestAds; // Defensive defaults: the edge injects adSlots (head-open) and bids (before -// ) only when the server-side ad stack runs for the request. When it -// is gated off (kill switch, consent fail-closed, bots, prefetch), page code -// reading window.tsjs.bids / window.tsjs.adSlots must still see defined -// values instead of throwing. Injected scripts overwrite these wholesale. +// ) only when server-side ad templates run for the request. When template +// delivery is disabled or gated off (auction/consent, bots, prefetch), page code +// reading window.tsjs.bids / window.tsjs.adSlots must still see defined values +// instead of throwing. Injected scripts overwrite these wholesale. api.adSlots ??= []; api.bids ??= {}; // Point global tsjs diff --git a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts index 30cfbbbdd..acea49fa6 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts @@ -689,8 +689,8 @@ export function installTsAdInit(): void { ts.prevSlotTargetingKeys = nextSlotTargetingKeys; // Whether this call produced any TS slot to render. A gated page-bids - // response (auction kill switch or consent denial) returns no slots, so - // the loops above leave these empty. + // response (template switch, auction gate, or consent denial) returns no + // slots, so the loops above leave these empty. const hasRenderableWork = slotsToDisplay.length > 0 || slotsToRefresh.length > 0; // enableSingleRequest and enableServices must only be called once per page @@ -971,10 +971,10 @@ export function installSpaAuctionHook(): void { // This route is now the committed, loaded state — a later failed // navigation rolls back here, and a return trip no-ops correctly. lastAppliedPath = path; - // An empty page-bids response (auction kill switch or consent gate) carries - // no TS slots. Only run adInit() when there are slots to apply or prior TS - // state to sweep — otherwise a consent-denied or kill-switched navigation - // must not enter the GPT command queue and risk activating services. + // An empty page-bids response (template switch, auction, or consent gate) + // carries no TS slots. Only run adInit() when there are slots to apply or + // prior TS state to sweep — otherwise a gated navigation must not enter + // the GPT command queue and risk activating services. const hasPriorTsState = (ts.prevGptSlots?.length ?? 0) > 0 || Object.keys(ts.prevSlotTargetingKeys ?? {}).length > 0 || diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts index 19739e3dd..36c41aa71 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts @@ -115,9 +115,9 @@ describe('installSpaAuctionHook', () => { }); it('skips adInit on an empty page-bids response with no prior TS state', async () => { - // A gated page-bids response (auction kill switch or consent denial) returns - // no slots. With no prior TS state to sweep, the hook must not call adInit() - // so a consent-denied navigation cannot activate the publisher's GPT setup. + // A gated page-bids response (template switch, auction gate, or consent + // denial) returns no slots. With no prior TS state to sweep, the hook must + // not call adInit() so a gated navigation cannot activate publisher GPT. fetchStub.mockResolvedValue({ ok: true, json: async () => ({ slots: [], bids: {} }), diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index aa2b0264b..ef991eea0 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -1312,8 +1312,16 @@ Defines the ad slots the trusted server offers on a page: which pages each slot appears on (`page_patterns`), its supported sizes (`formats`), and the GAM ad unit it maps to (`gam_unit_path`). +`enabled` is the dedicated server-side ad-template switch. It defaults to `true` +for compatibility with existing configurations. Set it to `false` to stop +publisher HTML and SPA page-bids template delivery while retaining the slot +configuration and direct `POST /auction` endpoint. The browser-facing cache +policy for a disabled template stack is `Cache-Control: max-age=60`, unless the +origin already sends `private` or `no-store`. + ```toml [creative_opportunities] +enabled = true # set to false to disable server-side ad templates gam_network_id = "123456789" price_granularity = "dense" @@ -1332,6 +1340,13 @@ page_patterns = ["/", "/news", "/news/*", "/reviews", "/reviews/*"] formats = [{ width = 728, height = 90 }] ``` +The same switch can be overridden through the legacy environment-variable +loader: + +```bash +TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__ENABLED=false +``` + ### `gam_unit_path` templating `gam_unit_path` is a template. A publisher whose ad unit varies by site section @@ -1385,8 +1400,9 @@ publisher-specific. Startup fails if `{section}` is used without a valid `section_root`. Startup rejects a blank `gam_network_id` only when an absent path/default or a `{network_id}` template consumes it; static paths and templates without `{network_id}` do not consume it. A -`[creative_opportunities]` block with no slots is disabled, so its -`gam_network_id` is not checked. +`[creative_opportunities]` block with `enabled = false` or no slots is +inactive, so no publisher templates are delivered and its `gam_network_id` is +not checked when no slot uses it. Both knobs are config-driven, so the URL→section convention stays with the publisher: `section_segment` selects which segment names the section, and diff --git a/trusted-server.example.toml b/trusted-server.example.toml index e78d2b255..ef5bf5487 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -155,6 +155,9 @@ ja4_endpoint_enabled = false auction_html_comment = false [creative_opportunities] +# Set to false to disable server-side ad templates while retaining slot definitions +# and direct POST /auction callers. +enabled = true gam_network_id = "123456789" # FCP is not affected by this value — body content above has already # streamed and painted before the hold begins. What this caps is the slip on From d0fffb9d4e886681b92a50e826cfa877f89d1a51 Mon Sep 17 00:00:00 2001 From: Christian Date: Tue, 11 Aug 2026 08:04:10 -0500 Subject: [PATCH 12/16] Address DataDome review feedback --- .../trusted-server-core/src/html_processor.rs | 24 ++- .../src/integrations/datadome.rs | 26 ++- .../src/integrations/datadome/protection.rs | 199 +++++++++++++++--- .../src/integrations/registry.rs | 49 ++++- crates/trusted-server-core/src/publisher.rs | 67 +++--- .../src/response_privacy.rs | 1 - docs/guide/integrations/datadome.md | 24 ++- ...6-08-03-datadome-ip-excluded-client-tag.md | 7 - 8 files changed, 304 insertions(+), 93 deletions(-) diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index 4c827ace0..3bff588fe 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -966,7 +966,7 @@ mod tests { } #[test] - fn suppressed_datadome_tag_is_not_injected_into_processed_html() { + fn suppressed_datadome_tag_preserves_and_rewrites_publisher_tag() { let mut settings = create_test_settings(); settings .integrations @@ -991,7 +991,10 @@ mod tests { let mut processor = create_html_processor(config); let output = processor - .process_chunk(b"content", true) + .process_chunk( + br#"content"#, + true, + ) .expect("should process HTML"); let html = String::from_utf8(output).expect("should produce UTF-8 HTML"); @@ -1000,8 +1003,21 @@ mod tests { "should omit the DataDome client configuration" ); assert!( - !html.contains("/integrations/datadome/tags.js"), - "should omit the DataDome client tag URL" + html.contains("id=\"publisher-datadome\""), + "should preserve the publisher-originated DataDome tag" + ); + assert!( + html.contains("src=\"/integrations/datadome/tags.js\""), + "should rewrite the publisher-originated DataDome tag" + ); + assert!( + !html.contains("https://js.datadome.co/tags.js"), + "should remove the original third-party DataDome URL" + ); + assert_eq!( + html.matches("/integrations/datadome/tags.js").count(), + 1, + "should leave exactly one publisher-originated DataDome tag" ); } diff --git a/crates/trusted-server-core/src/integrations/datadome.rs b/crates/trusted-server-core/src/integrations/datadome.rs index 2486c16be..75e6c3fdc 100644 --- a/crates/trusted-server-core/src/integrations/datadome.rs +++ b/crates/trusted-server-core/src/integrations/datadome.rs @@ -68,6 +68,7 @@ use serde_json::Value as JsonValue; use url::Url; use validator::Validate; +use crate::constants::ENV_FASTLY_IS_STAGING; use crate::error::TrustedServerError; use crate::integrations::{ AttributeRewriteAction, INTEGRATION_MAX_BODY_BYTES, IntegrationAttributeContext, @@ -469,6 +470,17 @@ impl DataDomeIntegration { Self::try_new(config).map(|_| ()) } + fn active_protection_test_bypass(&self) -> Option<&ProtectionTestBypassConfig> { + if std::env::var(ENV_FASTLY_IS_STAGING).as_deref() != Ok("1") { + return None; + } + + self.config + .protection_test_bypass + .as_ref() + .filter(|bypass| bypass.enabled) + } + fn validate_protection_test_bypass( config: &DataDomeConfig, ) -> Result<(), Report> { @@ -926,18 +938,26 @@ fn build( }; let integration = DataDomeIntegration::try_new(config)?; - let protection_test_bypass = integration + let protection_test_bypass_configured = integration .config .protection_test_bypass .as_ref() .is_some_and(|bypass| bypass.enabled); + let protection_test_bypass_active = integration.active_protection_test_bypass().is_some(); + if protection_test_bypass_configured && !protection_test_bypass_active { + log::warn!( + "[datadome] DataDome test bypass is configured but inactive because FASTLY_IS_STAGING is not 1" + ); + } log::info!( "[datadome] Registering integration (sdk_origin: {}, rewrite_sdk: {}, enable_protection: {}, protection_test_bypass: {})", integration.config.sdk_origin, integration.config.rewrite_sdk, integration.config.enable_protection, - if protection_test_bypass { - "enabled" + if protection_test_bypass_active { + "active" + } else if protection_test_bypass_configured { + "configured-inactive" } else { "disabled" }, diff --git a/crates/trusted-server-core/src/integrations/datadome/protection.rs b/crates/trusted-server-core/src/integrations/datadome/protection.rs index bc06214ff..f2803b38f 100644 --- a/crates/trusted-server-core/src/integrations/datadome/protection.rs +++ b/crates/trusted-server-core/src/integrations/datadome/protection.rs @@ -165,15 +165,11 @@ impl DataDomeIntegration { req: &mut Request, services: &RuntimeServices, ) -> bool { - let Some(bypass) = self - .config - .protection_test_bypass - .as_ref() - .filter(|bypass| bypass.enabled) - else { + let value = req.headers_mut().remove(super::HEADER_DATADOME_TEST_BYPASS); + let Some(bypass) = self.active_protection_test_bypass() else { return false; }; - let Some(value) = req.headers_mut().remove(super::HEADER_DATADOME_TEST_BYPASS) else { + let Some(value) = value else { return false; }; @@ -806,6 +802,23 @@ mod tests { .expect("should build filter request") } + fn filter_with_staging( + integration: &DataDomeIntegration, + settings: &Settings, + services: &RuntimeServices, + request: &mut Request, + ) -> RequestFilterDecision { + temp_env::with_var(crate::constants::ENV_FASTLY_IS_STAGING, Some("1"), || { + futures::executor::block_on(integration.filter_protection_request(RequestFilterInput { + settings, + services, + request, + geo_info: None, + is_integration_route: false, + })) + }) + } + fn filter_marks_request( config: DataDomeConfig, services: &RuntimeServices, @@ -875,15 +888,7 @@ mod tests { edgezero_core::http::HeaderValue::from_static("temporary-test-credential"), ); - let decision = futures::executor::block_on(integration.filter_protection_request( - RequestFilterInput { - settings: &settings, - services: &services, - request: &mut request, - geo_info: None, - is_integration_route: false, - }, - )); + let decision = filter_with_staging(&integration, &settings, &services, &mut request); assert!( matches!(decision, RequestFilterDecision::Continue(_)), @@ -906,6 +911,148 @@ mod tests { ); } + #[test] + fn protection_test_bypass_header_is_stripped_when_unconfigured_or_disabled() { + for protection_test_bypass in [ + None, + Some(ProtectionTestBypassConfig { + enabled: false, + credential_secret_store: "ts_secrets".to_string(), + credential_secret_name: "datadome_test_bypass".to_string(), + }), + ] { + let config = DataDomeConfig { + enabled: true, + enable_protection: true, + protection_test_bypass, + ..DataDomeConfig::default() + }; + let integration = + DataDomeIntegration::try_new(config).expect("should create integration"); + let mut secrets = HashMap::new(); + secrets.insert( + "datadome_server_side_key".to_string(), + b"server-side-key".to_vec(), + ); + let http_client = Arc::new(StubHttpClient::new()); + http_client.push_response_with_headers( + 200, + Vec::new(), + vec![(HEADER_DATADOME_RESPONSE, "200")], + ); + let services = build_services_with_secret_and_http_client( + HashMapSecretStore::new(secrets), + http_client.clone(), + ); + let settings = Settings::default(); + let mut request = request_for_filter(); + request.headers_mut().insert( + super::super::HEADER_DATADOME_TEST_BYPASS, + edgezero_core::http::HeaderValue::from_static("stale-test-credential"), + ); + + let decision = filter_with_staging(&integration, &settings, &services, &mut request); + + assert!( + matches!(decision, RequestFilterDecision::Continue(_)), + "an allowed Protection API response should continue" + ); + assert!( + request + .headers() + .get(super::super::HEADER_DATADOME_TEST_BYPASS) + .is_none(), + "the bypass header must be stripped when the bypass is unconfigured or disabled" + ); + assert!( + !has_client_tag_suppression_marker(&request), + "an inactive bypass must not suppress the DataDome client tag" + ); + assert_eq!( + http_client.recorded_backend_names().len(), + 1, + "an inactive bypass must still call the Protection API" + ); + } + } + + #[test] + fn protection_test_bypass_is_inactive_outside_staging() { + let config = DataDomeConfig { + enabled: true, + enable_protection: true, + protection_test_bypass: Some(ProtectionTestBypassConfig { + enabled: true, + credential_secret_store: "ts_secrets".to_string(), + credential_secret_name: "datadome_test_bypass".to_string(), + }), + ..DataDomeConfig::default() + }; + let integration = DataDomeIntegration::try_new(config).expect("should create integration"); + let mut secrets = HashMap::new(); + secrets.insert( + "datadome_server_side_key".to_string(), + b"server-side-key".to_vec(), + ); + secrets.insert( + "datadome_test_bypass".to_string(), + b"temporary-test-credential".to_vec(), + ); + let http_client = Arc::new(StubHttpClient::new()); + http_client.push_response_with_headers( + 200, + Vec::new(), + vec![(HEADER_DATADOME_RESPONSE, "200")], + ); + let services = build_services_with_secret_and_http_client( + HashMapSecretStore::new(secrets), + http_client.clone(), + ); + let settings = Settings::default(); + let mut request = request_for_filter(); + request.headers_mut().insert( + super::super::HEADER_DATADOME_TEST_BYPASS, + edgezero_core::http::HeaderValue::from_static("temporary-test-credential"), + ); + + let decision = temp_env::with_var( + crate::constants::ENV_FASTLY_IS_STAGING, + None::<&str>, + || { + futures::executor::block_on(integration.filter_protection_request( + RequestFilterInput { + settings: &settings, + services: &services, + request: &mut request, + geo_info: None, + is_integration_route: false, + }, + )) + }, + ); + + assert!( + matches!(decision, RequestFilterDecision::Continue(_)), + "an allowed Protection API response should continue" + ); + assert!( + request + .headers() + .get(super::super::HEADER_DATADOME_TEST_BYPASS) + .is_none(), + "the bypass credential must be stripped outside staging" + ); + assert!( + !has_client_tag_suppression_marker(&request), + "the bypass must not suppress the DataDome client tag outside staging" + ); + assert_eq!( + http_client.recorded_backend_names().len(), + 1, + "the bypass must still call the Protection API outside staging" + ); + } + #[test] fn protection_test_bypass_wins_over_other_exclusions() { let config = DataDomeConfig { @@ -944,15 +1091,7 @@ mod tests { edgezero_core::http::HeaderValue::from_static("temporary-test-credential"), ); - let decision = futures::executor::block_on(integration.filter_protection_request( - RequestFilterInput { - settings: &settings, - services: &services, - request: &mut request, - geo_info: None, - is_integration_route: false, - }, - )); + let decision = filter_with_staging(&integration, &settings, &services, &mut request); assert!( matches!(decision, RequestFilterDecision::Continue(_)), @@ -1007,15 +1146,7 @@ mod tests { edgezero_core::http::HeaderValue::from_static("wrong-credential"), ); - let decision = futures::executor::block_on(integration.filter_protection_request( - RequestFilterInput { - settings: &settings, - services: &services, - request: &mut request, - geo_info: None, - is_integration_route: false, - }, - )); + let decision = filter_with_staging(&integration, &settings, &services, &mut request); assert!( matches!(decision, RequestFilterDecision::Continue(_)), diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index 291a54242..16cbac868 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -1,4 +1,4 @@ -use std::any::Any; +use std::any::{Any, TypeId}; use std::collections::BTreeMap; use std::sync::{Arc, Mutex}; @@ -98,17 +98,19 @@ pub struct IntegrationScriptContext<'a> { pub document_state: &'a IntegrationDocumentState, } +type IntegrationDocumentStateMap = BTreeMap<(&'static str, TypeId), Arc>; + /// Per-document state shared between HTML/script rewriters and post-processors. /// /// This exists to support multi-phase HTML processing without requiring a second HTML parse. #[derive(Clone, Default)] pub struct IntegrationDocumentState { - inner: Arc>>>, + inner: Arc>, } impl std::fmt::Debug for IntegrationDocumentState { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let keys: Vec<&'static str> = { + let keys: Vec<(&'static str, TypeId)> = { let guard = self .inner .lock() @@ -136,7 +138,7 @@ impl IntegrationDocumentState { .inner .lock() .expect("should lock integration document state"); - let value = guard.get(integration_id)?; + let value = guard.get(&(integration_id, TypeId::of::()))?; let cloned: Arc = Arc::clone(value); cloned.downcast::().ok() } @@ -159,17 +161,15 @@ impl IntegrationDocumentState { .lock() .expect("should lock integration document state"); - if let Some(existing) = guard.get(integration_id) + let key = (integration_id, TypeId::of::()); + if let Some(existing) = guard.get(&key) && let Ok(downcast) = Arc::clone(existing).downcast::() { return downcast; } let value: Arc = Arc::new(init()); - guard.insert( - integration_id, - Arc::clone(&value) as Arc, - ); + guard.insert(key, Arc::clone(&value) as Arc); value } @@ -1405,6 +1405,37 @@ mod tests { } } + #[test] + fn document_state_keeps_multiple_types_for_one_integration() { + let state = IntegrationDocumentState::default(); + let number = state.get_or_insert_with("test", || 7_u32); + let label = state.get_or_insert_with("test", || "first".to_string()); + let repeated_number = state.get_or_insert_with("test", || 99_u32); + + assert!( + Arc::ptr_eq(&number, &repeated_number), + "repeated insertion should preserve the original typed state" + ); + assert_eq!( + *state.get::("test").expect("should retrieve number"), + 7, + "should retain numeric state" + ); + assert_eq!( + state + .get::("test") + .expect("should retrieve label") + .as_str(), + "first", + "should retain string state under the same integration ID" + ); + assert_eq!( + label.as_str(), + "first", + "should return inserted string state" + ); + } + #[test] fn default_html_post_processor_should_process_is_false() { let processor = NoopHtmlPostProcessor; diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 57c4be3db..79ebf4436 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1423,6 +1423,14 @@ pub async fn publisher_response_into_streaming_response( } } +/// Removes request headers that can produce a bodyless or partial origin response. +fn strip_conditional_and_range_headers(req: &mut Request) { + req.headers_mut().remove(header::IF_NONE_MATCH); + req.headers_mut().remove(header::IF_MODIFIED_SINCE); + req.headers_mut().remove(header::RANGE); + req.headers_mut().remove(header::IF_RANGE); +} + /// Returns `true` when a buffered publisher response should carry a body and a /// recomputed `Content-Length`. /// @@ -1464,6 +1472,8 @@ fn apply_datadome_client_tag_cache_privacy( HeaderValue::from_static("private, max-age=0"), ); } + response.headers_mut().remove(header::ETAG); + response.headers_mut().remove(header::LAST_MODIFIED); for header_name in CDN_CACHE_HEADERS { response.headers_mut().remove(*header_name); } @@ -2881,11 +2891,15 @@ pub async fn handle_publisher_request( } ); - if should_run_ad_stack { - req.headers_mut().remove(header::IF_NONE_MATCH); - req.headers_mut().remove(header::IF_MODIFIED_SINCE); - req.headers_mut().remove(header::RANGE); - req.headers_mut().remove(header::IF_RANGE); + let suppress_datadome_client_side_tag = req + .extensions() + .get::() + .is_some(); + if should_run_ad_stack || suppress_datadome_client_side_tag { + // The origin content type is not known yet, so request hints cannot safely + // narrow this to HTML without allowing 304 or 206 responses to bypass a + // response mutation that becomes necessary after the fetch. + strip_conditional_and_range_headers(&mut req); } // Only advertise encodings the rewrite pipeline can decode and re-encode. @@ -2912,14 +2926,6 @@ pub async fn handle_publisher_request( // without streaming support may reject the flag outright rather than // silently buffering, which would fail every publisher fetch. let request_method = req.method().clone(); - let suppress_datadome_client_side_tag = req - .extensions() - .get::() - .is_some(); - if suppress_datadome_client_side_tag { - req.headers_mut().remove(header::IF_NONE_MATCH); - req.headers_mut().remove(header::IF_MODIFIED_SINCE); - } let mut platform_request = PlatformHttpRequest::new(req, backend_name); if services.http_client().supports_streaming_responses() { platform_request = platform_request.with_stream_response(); @@ -5339,6 +5345,8 @@ mod tests { .header(header::HOST, "publisher.example") .header(header::IF_NONE_MATCH, "\"cached-page\"") .header(header::IF_MODIFIED_SINCE, "Wed, 21 Oct 2015 07:28:00 GMT") + .header(header::RANGE, "bytes=0-18") + .header(header::IF_RANGE, "\"cached-page\"") .body(EdgeBody::empty()) .expect("should build conditional request"); req.extensions_mut() @@ -5351,18 +5359,19 @@ mod tests { .into_iter() .next() .expect("should record one outbound request"); - assert!( - headers - .iter() - .all(|(name, _)| !name.eq_ignore_ascii_case(header::IF_NONE_MATCH.as_str())), - "suppressed requests must not forward If-None-Match" - ); - assert!( - headers - .iter() - .all(|(name, _)| !name.eq_ignore_ascii_case(header::IF_MODIFIED_SINCE.as_str())), - "suppressed requests must not forward If-Modified-Since" - ); + for header_name in [ + header::IF_NONE_MATCH, + header::IF_MODIFIED_SINCE, + header::RANGE, + header::IF_RANGE, + ] { + assert!( + headers + .iter() + .all(|(name, _)| !name.eq_ignore_ascii_case(header_name.as_str())), + "suppressed requests must not forward {header_name}" + ); + } } #[tokio::test] @@ -5529,6 +5538,8 @@ mod tests { .header("fastly-surrogate-control", "max-age=600") .header("cloudflare-cdn-cache-control", "max-age=600") .header("cdn-cache-control", "max-age=600") + .header(header::ETAG, "\"origin-tag\"") + .header(header::LAST_MODIFIED, "Wed, 21 Oct 2015 07:28:00 GMT") .body(EdgeBody::empty()) .expect("should build cacheable HTML response"); @@ -5566,6 +5577,12 @@ mod tests { response.headers().get("cdn-cache-control").is_none(), "suppressed HTML should not retain CDN-Cache-Control" ); + for header_name in [header::ETAG, header::LAST_MODIFIED] { + assert!( + !response.headers().contains_key(&header_name), + "suppressed HTML should not retain {header_name}" + ); + } let mut no_store_response = Response::builder() .status(StatusCode::OK) diff --git a/crates/trusted-server-core/src/response_privacy.rs b/crates/trusted-server-core/src/response_privacy.rs index 2205ae892..c2b1447e3 100644 --- a/crates/trusted-server-core/src/response_privacy.rs +++ b/crates/trusted-server-core/src/response_privacy.rs @@ -22,7 +22,6 @@ pub const CDN_CACHE_HEADERS: &[&str] = &[ "fastly-surrogate-control", "cdn-cache-control", "cloudflare-cdn-cache-control", - "cdn-cache-control", ]; /// Forces cookie-bearing responses to stay private to shared caches. diff --git a/docs/guide/integrations/datadome.md b/docs/guide/integrations/datadome.md index 35eda5d3e..ff2eb6aaa 100644 --- a/docs/guide/integrations/datadome.md +++ b/docs/guide/integrations/datadome.md @@ -169,7 +169,7 @@ A request is protected when all of the following are true: 5. The client IP does not match `protection_excluded_ip_cidrs` or any Config Store-backed CIDR source. 6. The client ASN is not listed in `protection_excluded_asns`. 7. No `protection_exclusion_rules` match. -8. The request does not contain a matching enabled `protection_test_bypass` credential. +8. The request does not contain a matching enabled `protection_test_bypass` credential while `FASTLY_IS_STAGING=1`. Static assets are excluded by default using a case-insensitive file-extension regex. Trusted Server internal routes such as `/static/tsjs=`, `/integrations/`, `/first-party/`, admin routes, discovery routes, and signature-verification routes are also excluded by default. @@ -182,6 +182,7 @@ can configure a static header credential that skips only the server-side Protection API: ```toml +# Runtime activation also requires FASTLY_IS_STAGING=1. [integrations.datadome.protection_test_bypass] enabled = true credential_secret_store = "ts_secrets" @@ -189,14 +190,17 @@ credential_secret_name = "datadome_test_bypass" ``` `protection_test_bypass` requires `enable_protection = true`; it is disabled -when omitted. Store the temporary credential in the configured Secret Store, -configure this section only while needed, protect the site with an outer access -control such as Basic Auth, and remove the section when testing finishes. Do not -enable it in production. - -The fixed `x-ts-datadome-bypass` header is compared in constant time, removed -before the request can reach DataDome or the publisher origin, and never -logged. Scope the header to the staging origin; do not attach it to every +when omitted and is runtime-active only when `FASTLY_IS_STAGING=1`. A retained +section cannot bypass protection in a production or other non-staging runtime. +Store the temporary credential in the configured Secret Store, configure this +section only while needed, protect the site with an outer access control such +as Basic Auth, and remove the section when testing finishes. + +Whenever the enabled DataDome request filter runs, the fixed +`x-ts-datadome-bypass` header is removed before configuration or credential +checks. It therefore cannot reach DataDome or the publisher origin when the +bypass is absent, disabled, inactive, or invalid. Active credentials are +compared in constant time and never logged. Scope the header to the staging origin; do not attach it to every request in a browser context because that can disclose the credential to third-party origins. With Playwright: @@ -223,7 +227,7 @@ This behavior applies to: - `protection_excluded_ip_cidr_sources`; - structured `ip_cidr` rules; - structured `ip_cidr_source` rules; and -- a matching enabled `protection_test_bypass` credential. +- a matching enabled `protection_test_bypass` credential in a staging runtime. ASN, method, path, query-parameter, static-asset, and internal-route exclusions do not automatically suppress the client-side tag. DataDome tags diff --git a/docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md b/docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md index 0dfc923cf..2d9b0eae2 100644 --- a/docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md +++ b/docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md @@ -136,7 +136,6 @@ matches!(reason, "client_ip" | "client_ip_source" | "ip_cidr" | "ip_cidr_source" - [ ] **Step 3: Make `filter_protection_request` own a mutable input and pass it mutably to `is_request_protected`.** In the existing `ProtectionScopeDecision::Skip` arm: - 1. determine whether the reason is IP-based; 2. if so, insert the typed marker into `input.request.extensions_mut()`; 3. call the updated skip logger with `client_tag_omitted = true`; and @@ -160,7 +159,6 @@ matches!(reason, "client_ip" | "client_ip_source" | "ip_cidr" | "ip_cidr_source" optional Config Store data, and a mutable request. For each case, call `filter_protection_request`, assert it returns `Continue`, and inspect the request extension: - - inline `protection_excluded_ip_cidrs` match → marker present; - `protection_excluded_ip_cidr_sources` match → marker present; - structured `ProtectionMatcherConfig::IpCidr` match → marker present; @@ -220,7 +218,6 @@ DataDomeClientTagSuppressed request extension - [ ] **Step 2: Add a boolean to the owned and borrowed publisher-processing parameter structs.** Add a clearly named field such as `suppress_datadome_client_side_tag` to: - - `OwnedProcessResponseParams`; - `ProcessResponseParams`; and - `HtmlStreamProcessorParams`. @@ -241,7 +238,6 @@ DataDomeClientTagSuppressed request extension - [ ] **Step 4: Extend `IntegrationHtmlContext`.** Add the boolean as immutable request-scoped context. Populate it at both construction sites in `html_processor.rs`: - - the streaming `` element handler; and - `HtmlWithPostProcessing::process_chunk` for full-document post-processors. @@ -249,7 +245,6 @@ DataDomeClientTagSuppressed request extension `false` by default. - [ ] **Step 5: Add plumbing tests.** - - `HtmlProcessorConfig::from_settings` defaults to non-suppressed. - A test head injector records the context flag and sees `true` when a config is built with suppression. @@ -332,7 +327,6 @@ optimization. with a processable HTML content type, suppression `true`, and cacheable origin headers (`Cache-Control`, `Surrogate-Control`, and `Fastly-Surrogate-Control`). Assert the stream response is: - - `Cache-Control: private, max-age=0`; and - missing both surrogate cache headers. @@ -343,7 +337,6 @@ optimization. 204/205/304, or responses without suppression: none has a body variation created by this feature. - [ ] **Step 3: Add non-regression cache tests.** Verify that: - - non-suppressed processed HTML keeps its existing cache headers unless another existing policy changes them; - a suppressed CSS/non-HTML stream is not made private by this feature; and From 73b700bf50afae07bfe89dbe0cd539550c18c4fb Mon Sep 17 00:00:00 2001 From: Christian Date: Wed, 12 Aug 2026 11:07:03 -0500 Subject: [PATCH 13/16] Harden DataDome exclusion and bypass handling --- .../src/integrations/datadome.rs | 9 +- .../src/integrations/datadome/protection.rs | 419 +++++++++++++++--- .../integrations/datadome/protection_scope.rs | 260 +++++++++-- .../src/platform/test_support.rs | 10 +- crates/trusted-server-core/src/publisher.rs | 256 ++++++++--- .../src/response_privacy.rs | 56 ++- docs/guide/integrations/datadome.md | 56 ++- ...6-08-03-datadome-ip-excluded-client-tag.md | 22 +- ...-datadome-ip-excluded-client-tag-design.md | 32 +- 9 files changed, 917 insertions(+), 203 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/datadome.rs b/crates/trusted-server-core/src/integrations/datadome.rs index 75e6c3fdc..d95ee35ee 100644 --- a/crates/trusted-server-core/src/integrations/datadome.rs +++ b/crates/trusted-server-core/src/integrations/datadome.rs @@ -90,6 +90,7 @@ pub use protection_scope::{ use protection_scope::ProtectionScope; pub(crate) const DATADOME_INTEGRATION_ID: &str = "datadome"; +/// Fixed request header used by the staging-only protection test bypass. pub(crate) const HEADER_DATADOME_TEST_BYPASS: &str = "x-ts-datadome-bypass"; /// Request marker indicating that Trusted Server should omit its automatic @@ -136,7 +137,7 @@ pub struct ProtectionTestBypassConfig { #[serde(default = "default_protection_test_bypass_secret_store")] pub credential_secret_store: String, - /// Secret name containing the temporary bypass credential. + /// Secret name containing at least 32 bytes of high-entropy bypass material. #[serde(default = "default_protection_test_bypass_secret_name")] pub credential_secret_name: String, } @@ -464,6 +465,12 @@ impl DataDomeIntegration { Ok(()) } + /// Validates `DataDome` configuration before runtime registration. + /// + /// # Errors + /// + /// Returns an error when protection, bypass, or client-tag configuration is + /// invalid. pub(crate) fn validate_config_for_startup( config: DataDomeConfig, ) -> Result<(), Report> { diff --git a/crates/trusted-server-core/src/integrations/datadome/protection.rs b/crates/trusted-server-core/src/integrations/datadome/protection.rs index f2803b38f..665060014 100644 --- a/crates/trusted-server-core/src/integrations/datadome/protection.rs +++ b/crates/trusted-server-core/src/integrations/datadome/protection.rs @@ -9,6 +9,7 @@ use subtle::ConstantTimeEq as _; use url::Url; use crate::error::TrustedServerError; +use crate::http_util::is_navigation_request; use crate::integrations::{ HeaderMutation, RequestFilterDecision, RequestFilterEffects, RequestFilterInput, }; @@ -16,7 +17,11 @@ use crate::platform::{PlatformBackendSpec, PlatformHttpRequest, RuntimeServices, use crate::redacted::Redacted; use super::DataDomeIntegration; -use super::protection_scope::{ProtectionRequestFacts, ProtectionScopeDecision}; +use super::protection_scope::{ + ProtectionRequestFacts, ProtectionScopeDecision, ProtectionSkipReason, +}; + +const MIN_TEST_BYPASS_CREDENTIAL_BYTES: usize = 32; const VALIDATE_REQUEST_PATH: &str = "/validate-request"; const REQUEST_MODULE_NAME: &str = "Trusted-Server-Rust"; @@ -144,15 +149,18 @@ impl DataDomeIntegration { }; match self.protection_scope.evaluate(&facts, input.services) { ProtectionScopeDecision::Protect => {} - ProtectionScopeDecision::Skip { rule_id, reason } => { - let client_tag_omitted = is_ip_exclusion_reason(reason); - if client_tag_omitted { + ProtectionScopeDecision::Skip { + rule_id, + reason, + suppress_client_tag, + } => { + if suppress_client_tag { input .request .extensions_mut() .insert(super::DataDomeClientTagSuppressed); } - log_protection_skip(input, &rule_id, reason); + log_protection_skip(input, &rule_id, reason, suppress_client_tag); return false; } } @@ -165,11 +173,23 @@ impl DataDomeIntegration { req: &mut Request, services: &RuntimeServices, ) -> bool { - let value = req.headers_mut().remove(super::HEADER_DATADOME_TEST_BYPASS); - let Some(bypass) = self.active_protection_test_bypass() else { + let supplied_values = req + .headers() + .get_all(super::HEADER_DATADOME_TEST_BYPASS) + .iter() + .cloned() + .collect::>(); + req.headers_mut().remove(super::HEADER_DATADOME_TEST_BYPASS); + if supplied_values.is_empty() { return false; - }; - let Some(value) = value else { + } + if supplied_values.len() != 1 { + log::warn!( + "[datadome] Multiple DataDome test bypass headers supplied; ignoring bypass" + ); + return false; + } + let Some(bypass) = self.active_protection_test_bypass() else { return false; }; @@ -178,10 +198,10 @@ impl DataDomeIntegration { .secret_store() .get_string(&store_name, &bypass.credential_secret_name) { - Ok(credential) if !credential.is_empty() => credential, + Ok(credential) if credential.len() >= MIN_TEST_BYPASS_CREDENTIAL_BYTES => credential, Ok(_) => { log::warn!( - "[datadome] DataDome test bypass credential is empty; ignoring bypass header" + "[datadome] DataDome test bypass credential does not meet security requirements; ignoring bypass header" ); return false; } @@ -193,7 +213,7 @@ impl DataDomeIntegration { } }; - let actual = Sha256::digest(value.as_bytes()); + let actual = Sha256::digest(supplied_values[0].as_bytes()); let expected = Sha256::digest(credential.as_bytes()); bool::from(actual.ct_eq(&expected)) } @@ -421,13 +441,16 @@ impl DataDomeIntegration { let (parts, body) = response.into_parts(); let status = parts.status; let Some(datadome_status) = datadome_response_status(&parts.headers) else { - log::warn!("[datadome] Protection API response missing X-DataDomeResponse"); + log::warn!( + "[datadome] Protection API response has missing or non-numeric verdict: api_status={} datadome_status=missing_or_invalid", + status.as_u16() + ); return RequestFilterDecision::Continue(RequestFilterEffects::default()); }; if datadome_status != status.as_u16() { log::warn!( - "[datadome] Protection API status/header mismatch: status={} header={}", + "[datadome] Protection API status/verdict mismatch: api_status={} datadome_status={}", status.as_u16(), datadome_status ); @@ -470,20 +493,14 @@ impl DataDomeIntegration { } log::warn!( - "[datadome] Protection API returned fail-open status {}", - status.as_u16() + "[datadome] Protection API returned unexpected fail-open status: api_status={} datadome_status={}", + status.as_u16(), + datadome_status ); RequestFilterDecision::Continue(RequestFilterEffects::default()) } } -fn is_ip_exclusion_reason(reason: &str) -> bool { - matches!( - reason, - "client_ip" | "client_ip_source" | "ip_cidr" | "ip_cidr_source" - ) -} - fn log_protection_test_bypass(input: &RequestFilterInput<'_>) { log::info!( "[datadome] protection decision=skipped rule=protection-test-bypass reason=test_bypass client_tag=omitted method={}", @@ -491,14 +508,37 @@ fn log_protection_test_bypass(input: &RequestFilterInput<'_>) { ); } -fn log_protection_skip(input: &RequestFilterInput<'_>, rule_id: &str, reason: &str) { - if is_ip_exclusion_reason(reason) { +fn suppression_skip_log_level(suppress_client_tag: bool, is_navigation: bool) -> log::Level { + if suppress_client_tag && is_navigation { + log::Level::Info + } else { + log::Level::Debug + } +} + +fn log_protection_skip( + input: &RequestFilterInput<'_>, + rule_id: &str, + reason: ProtectionSkipReason, + suppress_client_tag: bool, +) { + let reason = reason.as_str(); + if suppression_skip_log_level(suppress_client_tag, is_navigation_request(input.request)) + == log::Level::Info + { log::info!( "[datadome] protection decision=skipped rule={} reason={} client_tag=omitted method={}", rule_id, reason, input.request.method(), ); + } else if suppress_client_tag { + log::debug!( + "[datadome] protection decision=skipped rule={} reason={} client_tag=omitted method={}", + rule_id, + reason, + input.request.method(), + ); } else { log::debug!( "[datadome] protection decision=skipped rule={} reason={} method={}", @@ -509,6 +549,29 @@ fn log_protection_skip(input: &RequestFilterInput<'_>, rule_id: &str, reason: &s } } +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +enum ProtectionResultKind { + Allowed, + Blocked, + FailedOpen, +} + +fn classify_logged_protection_result( + status: StatusCode, + datadome_status: Option, + decision: &RequestFilterDecision, +) -> ProtectionResultKind { + match decision { + RequestFilterDecision::Respond { .. } => ProtectionResultKind::Blocked, + RequestFilterDecision::Continue(_) + if status == StatusCode::OK && datadome_status == Some(status.as_u16()) => + { + ProtectionResultKind::Allowed + } + RequestFilterDecision::Continue(_) => ProtectionResultKind::FailedOpen, + } +} + fn log_protection_result( input: &RequestFilterInput<'_>, status: StatusCode, @@ -516,23 +579,30 @@ fn log_protection_result( decision: &RequestFilterDecision, ) { let method = input.request.method(); - - match decision { - RequestFilterDecision::Respond { .. } => log::info!( - "[datadome] protection decision=blocked status={} method={} route=short_circuit", + let result_kind = classify_logged_protection_result(status, datadome_status, decision); + let datadome_status = datadome_status + .map(|value| value.to_string()) + .unwrap_or_else(|| "missing_or_invalid".to_string()); + + match result_kind { + ProtectionResultKind::Blocked => log::info!( + "[datadome] protection decision=blocked api_status={} datadome_status={} method={} route=short_circuit", status.as_u16(), + datadome_status, + method, + ), + ProtectionResultKind::Allowed => log::info!( + "[datadome] protection decision=allowed api_status={} datadome_status={} method={} route=continue", + status.as_u16(), + datadome_status, + method, + ), + ProtectionResultKind::FailedOpen => log::warn!( + "[datadome] protection decision=failed_open api_status={} datadome_status={} method={} route=continue", + status.as_u16(), + datadome_status, method, ), - RequestFilterDecision::Continue(_) - if status == StatusCode::OK && datadome_status == Some(status.as_u16()) => - { - log::info!( - "[datadome] protection decision=allowed status={} method={} route=continue", - status.as_u16(), - method, - ); - } - RequestFilterDecision::Continue(_) => {} } } @@ -830,11 +900,24 @@ mod tests { config: DataDomeConfig, services: &RuntimeServices, geo_info: Option<&GeoInfo>, + ) -> Request { + filter_marks_request_for_uri(config, services, geo_info, "https://publisher.example/page") + } + + fn filter_marks_request_for_uri( + config: DataDomeConfig, + services: &RuntimeServices, + geo_info: Option<&GeoInfo>, + uri: &str, ) -> Request { let integration = DataDomeIntegration::try_new(config).expect("should create DataDome integration"); let settings = Settings::default(); - let mut request = request_for_filter(); + let mut request = request_builder() + .method(Method::GET.as_str()) + .uri(uri) + .body(EdgeBody::empty()) + .expect("should build filter request"); let decision = futures::executor::block_on(integration.filter_protection_request( RequestFilterInput { settings: &settings, @@ -874,7 +957,7 @@ mod tests { let mut secrets = HashMap::new(); secrets.insert( "datadome_test_bypass".to_string(), - b"temporary-test-credential".to_vec(), + b"temporary-test-credential-32-bytes!".to_vec(), ); let http_client = Arc::new(StubHttpClient::new()); let services = build_services_with_secret_and_http_client( @@ -885,7 +968,7 @@ mod tests { let mut request = request_for_filter(); request.headers_mut().insert( super::super::HEADER_DATADOME_TEST_BYPASS, - edgezero_core::http::HeaderValue::from_static("temporary-test-credential"), + edgezero_core::http::HeaderValue::from_static("temporary-test-credential-32-bytes!"), ); let decision = filter_with_staging(&integration, &settings, &services, &mut request); @@ -996,7 +1079,7 @@ mod tests { ); secrets.insert( "datadome_test_bypass".to_string(), - b"temporary-test-credential".to_vec(), + b"temporary-test-credential-32-bytes!".to_vec(), ); let http_client = Arc::new(StubHttpClient::new()); http_client.push_response_with_headers( @@ -1012,7 +1095,7 @@ mod tests { let mut request = request_for_filter(); request.headers_mut().insert( super::super::HEADER_DATADOME_TEST_BYPASS, - edgezero_core::http::HeaderValue::from_static("temporary-test-credential"), + edgezero_core::http::HeaderValue::from_static("temporary-test-credential-32-bytes!"), ); let decision = temp_env::with_var( @@ -1077,7 +1160,7 @@ mod tests { let mut secrets = HashMap::new(); secrets.insert( "datadome_test_bypass".to_string(), - b"temporary-test-credential".to_vec(), + b"temporary-test-credential-32-bytes!".to_vec(), ); let http_client = Arc::new(StubHttpClient::new()); let services = build_services_with_secret_and_http_client( @@ -1088,7 +1171,7 @@ mod tests { let mut request = request_for_filter(); request.headers_mut().insert( super::super::HEADER_DATADOME_TEST_BYPASS, - edgezero_core::http::HeaderValue::from_static("temporary-test-credential"), + edgezero_core::http::HeaderValue::from_static("temporary-test-credential-32-bytes!"), ); let decision = filter_with_staging(&integration, &settings, &services, &mut request); @@ -1127,7 +1210,7 @@ mod tests { ); secrets.insert( "datadome_test_bypass".to_string(), - b"temporary-test-credential".to_vec(), + b"temporary-test-credential-32-bytes!".to_vec(), ); let http_client = Arc::new(StubHttpClient::new()); http_client.push_response_with_headers( @@ -1170,6 +1253,158 @@ mod tests { ); } + #[test] + fn duplicate_test_bypass_headers_fail_closed_and_are_all_stripped() { + let config = DataDomeConfig { + enabled: true, + enable_protection: true, + protection_test_bypass: Some(ProtectionTestBypassConfig { + enabled: true, + credential_secret_store: "ts_secrets".to_string(), + credential_secret_name: "datadome_test_bypass".to_string(), + }), + ..DataDomeConfig::default() + }; + let integration = DataDomeIntegration::try_new(config).expect("should create integration"); + let mut secrets = HashMap::new(); + secrets.insert( + "datadome_server_side_key".to_string(), + b"server-side-key".to_vec(), + ); + secrets.insert( + "datadome_test_bypass".to_string(), + b"temporary-test-credential-32-bytes!".to_vec(), + ); + let http_client = Arc::new(StubHttpClient::new()); + http_client.push_response_with_headers( + 200, + Vec::new(), + vec![(HEADER_DATADOME_RESPONSE, "200")], + ); + let services = build_services_with_secret_and_http_client( + HashMapSecretStore::new(secrets), + http_client.clone(), + ); + let settings = Settings::default(); + let mut request = request_for_filter(); + for value in [ + "temporary-test-credential-32-bytes!", + "temporary-test-credential-32-bytes!", + ] { + request.headers_mut().append( + super::super::HEADER_DATADOME_TEST_BYPASS, + edgezero_core::http::HeaderValue::from_static(value), + ); + } + + let decision = filter_with_staging(&integration, &settings, &services, &mut request); + + assert!(matches!(decision, RequestFilterDecision::Continue(_))); + assert!( + request + .headers() + .get(super::super::HEADER_DATADOME_TEST_BYPASS) + .is_none(), + "all duplicate bypass values should be stripped" + ); + assert!(!has_client_tag_suppression_marker(&request)); + assert_eq!(http_client.recorded_backend_names().len(), 1); + } + + #[test] + fn test_bypass_credential_requires_at_least_32_bytes() { + for (credential, should_match) in [ + (Some("1234567890123456789012345678901"), false), + (Some("12345678901234567890123456789012"), true), + (Some(""), false), + (None, false), + ] { + let config = DataDomeConfig { + enabled: true, + enable_protection: true, + protection_test_bypass: Some(ProtectionTestBypassConfig { + enabled: true, + credential_secret_store: "ts_secrets".to_string(), + credential_secret_name: "datadome_test_bypass".to_string(), + }), + ..DataDomeConfig::default() + }; + let integration = + DataDomeIntegration::try_new(config).expect("should create integration"); + let mut secrets = HashMap::new(); + secrets.insert( + "datadome_server_side_key".to_string(), + b"server-side-key".to_vec(), + ); + if let Some(credential) = credential { + secrets.insert( + "datadome_test_bypass".to_string(), + credential.as_bytes().to_vec(), + ); + } + let http_client = Arc::new(StubHttpClient::new()); + if !should_match { + http_client.push_response_with_headers( + 200, + Vec::new(), + vec![(HEADER_DATADOME_RESPONSE, "200")], + ); + } + let services = build_services_with_secret_and_http_client( + HashMapSecretStore::new(secrets), + http_client.clone(), + ); + let settings = Settings::default(); + let mut request = request_for_filter(); + let supplied = credential.unwrap_or("12345678901234567890123456789012"); + request.headers_mut().insert( + super::super::HEADER_DATADOME_TEST_BYPASS, + edgezero_core::http::HeaderValue::from_str(supplied) + .expect("should build bypass header"), + ); + + let decision = filter_with_staging(&integration, &settings, &services, &mut request); + + assert!(matches!(decision, RequestFilterDecision::Continue(_))); + assert_eq!(has_client_tag_suppression_marker(&request), should_match); + assert_eq!( + http_client.recorded_backend_names().is_empty(), + should_match, + "only a credential meeting the minimum should skip the API" + ); + } + } + + #[test] + fn protection_result_classifier_and_suppression_log_level_cover_outcomes() { + let continue_decision = RequestFilterDecision::Continue(RequestFilterEffects::default()); + let blocked_decision = RequestFilterDecision::Respond { + response: Box::new(Response::new(EdgeBody::empty())), + effects: RequestFilterEffects::default(), + }; + + assert_eq!( + classify_logged_protection_result(StatusCode::OK, Some(200), &continue_decision), + ProtectionResultKind::Allowed + ); + assert_eq!( + classify_logged_protection_result(StatusCode::FORBIDDEN, Some(403), &blocked_decision), + ProtectionResultKind::Blocked + ); + for (status, datadome_status) in [ + (StatusCode::OK, None), + (StatusCode::OK, Some(403)), + (StatusCode::CREATED, Some(201)), + ] { + assert_eq!( + classify_logged_protection_result(status, datadome_status, &continue_decision), + ProtectionResultKind::FailedOpen + ); + } + assert_eq!(suppression_skip_log_level(true, true), log::Level::Info); + assert_eq!(suppression_skip_log_level(true, false), log::Level::Debug); + } + #[test] fn ip_exclusions_mark_requests_for_client_tag_suppression() { let ip = IpAddr::V4(Ipv4Addr::new(192, 0, 2, 10)); @@ -1257,29 +1492,89 @@ mod tests { #[test] fn non_ip_exclusions_do_not_mark_requests_for_client_tag_suppression() { let ip = IpAddr::V4(Ipv4Addr::new(192, 0, 2, 10)); - let cases = [DataDomeConfig { - enabled: true, - enable_protection: true, - protection_exclusion_rules: vec![ProtectionExclusionRuleConfig { - id: "path".to_string(), - enabled: true, - methods: Vec::new(), - matcher: ProtectionMatcherConfig::PathExact { - paths: vec!["/page".to_string()], + let cases = [ + ( + ProtectionMatcherConfig::PathExact { + paths: vec!["/exact".to_string()], }, - }], - ..DataDomeConfig::default() - }]; + "https://publisher.example/exact", + ), + ( + ProtectionMatcherConfig::PathPrefix { + prefixes: vec!["/prefix/".to_string()], + }, + "https://publisher.example/prefix/page", + ), + ( + ProtectionMatcherConfig::PathRegex { + patterns: vec![r"^/regex/[0-9]+$".to_string()], + }, + "https://publisher.example/regex/42", + ), + ( + ProtectionMatcherConfig::QueryParamNonEmpty { + names: vec!["skip".to_string()], + }, + "https://publisher.example/page?skip=yes", + ), + ]; - for config in cases { - let request = filter_marks_request(config, &noop_services_with_client_ip(ip)); + for (matcher, uri) in cases { + let config = DataDomeConfig { + enabled: true, + enable_protection: true, + protection_exclusion_rules: vec![ProtectionExclusionRuleConfig { + id: "non-ip".to_string(), + enabled: true, + methods: Vec::new(), + matcher, + }], + ..DataDomeConfig::default() + }; + let request = + filter_marks_request_for_uri(config, &noop_services_with_client_ip(ip), None, uri); assert!( !has_client_tag_suppression_marker(&request), - "non-IP exclusions should not mark the request" + "matching non-IP exclusion should not mark {uri}" ); } } + #[test] + fn overlapping_path_and_ip_exclusions_still_mark_request() { + let ip = IpAddr::V4(Ipv4Addr::new(192, 0, 2, 10)); + let config = DataDomeConfig { + enabled: true, + enable_protection: true, + protection_exclusion_rules: vec![ + ProtectionExclusionRuleConfig { + id: "path-first".to_string(), + enabled: true, + methods: Vec::new(), + matcher: ProtectionMatcherConfig::PathExact { + paths: vec!["/page".to_string()], + }, + }, + ProtectionExclusionRuleConfig { + id: "ip-second".to_string(), + enabled: true, + methods: Vec::new(), + matcher: ProtectionMatcherConfig::IpCidr { + cidrs: vec!["192.0.2.0/24".to_string()], + }, + }, + ], + ..DataDomeConfig::default() + }; + + let request = filter_marks_request(config, &noop_services_with_client_ip(ip)); + + assert!( + has_client_tag_suppression_marker(&request), + "overlapping IP exclusion should suppress even when path remains the primary reason" + ); + } + #[test] fn asn_exclusions_do_not_mark_requests_for_client_tag_suppression() { let config = DataDomeConfig { diff --git a/crates/trusted-server-core/src/integrations/datadome/protection_scope.rs b/crates/trusted-server-core/src/integrations/datadome/protection_scope.rs index cce56f2fa..a83f73217 100644 --- a/crates/trusted-server-core/src/integrations/datadome/protection_scope.rs +++ b/crates/trusted-server-core/src/integrations/datadome/protection_scope.rs @@ -91,12 +91,52 @@ pub(super) struct ProtectionRequestFacts<'a> { pub(super) asn: Option, } +/// Typed reason why `DataDome` protection was skipped. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(super) enum ProtectionSkipReason { + Method, + ClientIp, + ClientIpSource, + Asn, + PathExact, + PathPrefix, + PathRegex, + QueryParamNonEmpty, + IpCidr, + IpCidrSource, +} + +impl ProtectionSkipReason { + pub(super) const fn as_str(self) -> &'static str { + match self { + Self::Method => "method", + Self::ClientIp => "client_ip", + Self::ClientIpSource => "client_ip_source", + Self::Asn => "asn", + Self::PathExact => "path_exact", + Self::PathPrefix => "path_prefix", + Self::PathRegex => "path_regex", + Self::QueryParamNonEmpty => "query_param_non_empty", + Self::IpCidr => "ip_cidr", + Self::IpCidrSource => "ip_cidr_source", + } + } + + pub(super) const fn is_ip_based(self) -> bool { + matches!( + self, + Self::ClientIp | Self::ClientIpSource | Self::IpCidr | Self::IpCidrSource + ) + } +} + /// Result of evaluating whether `DataDome` protection should run. pub(super) enum ProtectionScopeDecision { Protect, Skip { rule_id: String, - reason: &'static str, + reason: ProtectionSkipReason, + suppress_client_tag: bool, }, } @@ -210,51 +250,75 @@ impl ProtectionScope { facts: &ProtectionRequestFacts<'_>, services: &RuntimeServices, ) -> ProtectionScopeDecision { - if self.excluded_methods.matches(facts.method) { - return ProtectionScopeDecision::Skip { - rule_id: "excluded-methods".to_string(), - reason: "method", - }; - } + let mut primary = self + .excluded_methods + .matches(facts.method) + .then(|| ("excluded-methods".to_string(), ProtectionSkipReason::Method)); if let Some(client_ip) = facts.client_ip { if cidrs_match(&self.excluded_ip_cidrs, client_ip) { + let (rule_id, reason) = primary.unwrap_or_else(|| { + ( + "excluded-ip-cidrs".to_string(), + ProtectionSkipReason::ClientIp, + ) + }); return ProtectionScopeDecision::Skip { - rule_id: "excluded-ip-cidrs".to_string(), - reason: "client_ip", + rule_id, + reason, + suppress_client_tag: true, }; } for source in &self.excluded_ip_cidr_sources { if source.matches(client_ip, services, self.ip_list_cache_ttl) { + let (rule_id, reason) = primary.unwrap_or_else(|| { + (source.rule_id(), ProtectionSkipReason::ClientIpSource) + }); return ProtectionScopeDecision::Skip { - rule_id: source.rule_id(), - reason: "client_ip_source", + rule_id, + reason, + suppress_client_tag: true, }; } } } - if facts - .asn - .is_some_and(|asn| self.excluded_asns.contains(&asn)) + if primary.is_none() + && facts + .asn + .is_some_and(|asn| self.excluded_asns.contains(&asn)) { - return ProtectionScopeDecision::Skip { - rule_id: "excluded-asns".to_string(), - reason: "asn", - }; + primary = Some(("excluded-asns".to_string(), ProtectionSkipReason::Asn)); } for rule in &self.exclusion_rules { + let reason = rule.matcher.reason(); + if primary.is_some() && !reason.is_ip_based() { + continue; + } if rule.matches(facts, services, self.ip_list_cache_ttl) { - return ProtectionScopeDecision::Skip { - rule_id: rule.id.clone(), - reason: rule.matcher.reason(), - }; + let suppress_client_tag = reason.is_ip_based(); + let (rule_id, reason) = primary.unwrap_or_else(|| (rule.id.clone(), reason)); + if suppress_client_tag { + return ProtectionScopeDecision::Skip { + rule_id, + reason, + suppress_client_tag, + }; + } + primary = Some((rule_id, reason)); } } - ProtectionScopeDecision::Protect + match primary { + Some((rule_id, reason)) => ProtectionScopeDecision::Skip { + rule_id, + reason, + suppress_client_tag: false, + }, + None => ProtectionScopeDecision::Protect, + } } } @@ -489,15 +553,15 @@ impl ProtectionMatcher { } } - fn reason(&self) -> &'static str { + fn reason(&self) -> ProtectionSkipReason { match self { - ProtectionMatcher::PathExact(_) => "path_exact", - ProtectionMatcher::PathPrefix(_) => "path_prefix", - ProtectionMatcher::PathRegex(_) => "path_regex", - ProtectionMatcher::QueryParamNonEmpty(_) => "query_param_non_empty", - ProtectionMatcher::Asn(_) => "asn", - ProtectionMatcher::IpCidr(_) => "ip_cidr", - ProtectionMatcher::IpCidrSource(_) => "ip_cidr_source", + ProtectionMatcher::PathExact(_) => ProtectionSkipReason::PathExact, + ProtectionMatcher::PathPrefix(_) => ProtectionSkipReason::PathPrefix, + ProtectionMatcher::PathRegex(_) => ProtectionSkipReason::PathRegex, + ProtectionMatcher::QueryParamNonEmpty(_) => ProtectionSkipReason::QueryParamNonEmpty, + ProtectionMatcher::Asn(_) => ProtectionSkipReason::Asn, + ProtectionMatcher::IpCidr(_) => ProtectionSkipReason::IpCidr, + ProtectionMatcher::IpCidrSource(_) => ProtectionSkipReason::IpCidrSource, } } } @@ -741,7 +805,8 @@ mod tests { assert!(matches!( decision, ProtectionScopeDecision::Skip { - reason: "method", + reason: ProtectionSkipReason::Method, + suppress_client_tag: false, .. } )); @@ -758,7 +823,11 @@ mod tests { assert!(matches!( decision, - ProtectionScopeDecision::Skip { reason: "asn", .. } + ProtectionScopeDecision::Skip { + reason: ProtectionSkipReason::Asn, + suppress_client_tag: false, + .. + } )); } @@ -783,7 +852,8 @@ mod tests { assert!(matches!( decision, ProtectionScopeDecision::Skip { - reason: "client_ip", + reason: ProtectionSkipReason::ClientIp, + suppress_client_tag: true, .. } )); @@ -817,7 +887,8 @@ mod tests { assert!(matches!( decision, ProtectionScopeDecision::Skip { - reason: "client_ip_source", + reason: ProtectionSkipReason::ClientIpSource, + suppress_client_tag: true, .. } )); @@ -840,7 +911,8 @@ mod tests { assert!(matches!( scope.evaluate(&facts("GET", "/app.JSON", None, None, None), &services), ProtectionScopeDecision::Skip { - reason: "path_regex", + reason: ProtectionSkipReason::PathRegex, + suppress_client_tag: false, .. } )); @@ -850,6 +922,119 @@ mod tests { )); } + #[test] + fn method_skip_detects_overlapping_inline_ip_exclusion() { + let mut config = config_with_protection(); + config.protection_excluded_methods = vec!["GET".to_string()]; + config.protection_excluded_ip_cidrs = vec!["192.0.2.0/24".to_string()]; + let scope = ProtectionScope::compile(&config).expect("should compile scope"); + let services = crate::platform::test_support::noop_services(); + + let decision = scope.evaluate( + &facts( + "GET", + "/page", + None, + Some(IpAddr::V4(Ipv4Addr::new(192, 0, 2, 10))), + None, + ), + &services, + ); + + assert!(matches!( + decision, + ProtectionScopeDecision::Skip { + reason: ProtectionSkipReason::Method, + suppress_client_tag: true, + .. + } + )); + } + + #[test] + fn earlier_non_ip_skip_detects_later_structured_ip_exclusion() { + for primary_asn in [None, Some(64500)] { + let mut config = config_with_protection(); + config.protection_excluded_asns = primary_asn.into_iter().collect(); + config.protection_exclusion_rules = vec![ + ProtectionExclusionRuleConfig { + id: "path".to_string(), + enabled: primary_asn.is_none(), + methods: Vec::new(), + matcher: ProtectionMatcherConfig::PathExact { + paths: vec!["/page".to_string()], + }, + }, + ProtectionExclusionRuleConfig { + id: "ip".to_string(), + enabled: true, + methods: Vec::new(), + matcher: ProtectionMatcherConfig::IpCidr { + cidrs: vec!["192.0.2.0/24".to_string()], + }, + }, + ]; + let scope = ProtectionScope::compile(&config).expect("should compile scope"); + let services = crate::platform::test_support::noop_services(); + + let decision = scope.evaluate( + &facts( + "GET", + "/page", + None, + Some(IpAddr::V4(Ipv4Addr::new(192, 0, 2, 10))), + primary_asn, + ), + &services, + ); + + assert!(matches!( + decision, + ProtectionScopeDecision::Skip { + reason: ProtectionSkipReason::Asn | ProtectionSkipReason::PathExact, + suppress_client_tag: true, + .. + } + )); + } + } + + #[test] + fn method_scoped_structured_ip_does_not_suppress_when_scope_does_not_apply() { + let mut config = config_with_protection(); + config.protection_excluded_methods = vec!["POST".to_string()]; + config.protection_exclusion_rules = vec![ProtectionExclusionRuleConfig { + id: "get-ip".to_string(), + enabled: true, + methods: vec!["GET".to_string()], + matcher: ProtectionMatcherConfig::IpCidr { + cidrs: vec!["192.0.2.0/24".to_string()], + }, + }]; + let scope = ProtectionScope::compile(&config).expect("should compile scope"); + let services = crate::platform::test_support::noop_services(); + + let decision = scope.evaluate( + &facts( + "POST", + "/page", + None, + Some(IpAddr::V4(Ipv4Addr::new(192, 0, 2, 10))), + None, + ), + &services, + ); + + assert!(matches!( + decision, + ProtectionScopeDecision::Skip { + reason: ProtectionSkipReason::Method, + suppress_client_tag: false, + .. + } + )); + } + #[test] fn rule_query_param_non_empty_matches_rsc() { let mut config = config_with_protection(); @@ -870,7 +1055,8 @@ mod tests { &services ), ProtectionScopeDecision::Skip { - reason: "query_param_non_empty", + reason: ProtectionSkipReason::QueryParamNonEmpty, + suppress_client_tag: false, .. } )); diff --git a/crates/trusted-server-core/src/platform/test_support.rs b/crates/trusted-server-core/src/platform/test_support.rs index 10389c787..917f1bf50 100644 --- a/crates/trusted-server-core/src/platform/test_support.rs +++ b/crates/trusted-server-core/src/platform/test_support.rs @@ -769,6 +769,14 @@ pub(crate) fn build_services_with_backend_and_http_client( pub(crate) fn build_services_with_secret_and_http_client( secret_store: impl PlatformSecretStore + 'static, http_client: Arc, +) -> RuntimeServices { + build_services_with_secret_http_client_and_client_ip(secret_store, http_client, None) +} + +pub(crate) fn build_services_with_secret_http_client_and_client_ip( + secret_store: impl PlatformSecretStore + 'static, + http_client: Arc, + client_ip: Option, ) -> RuntimeServices { RuntimeServices::builder() .config_store(Arc::new(NoopConfigStore)) @@ -778,7 +786,7 @@ pub(crate) fn build_services_with_secret_and_http_client( .http_client(http_client) .geo(Arc::new(NoopGeo)) .client_info(ClientInfo { - client_ip: None, + client_ip, tls_protocol: None, tls_cipher: None, ..ClientInfo::default() diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 66b8a9553..4bed98327 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -59,7 +59,7 @@ use crate::http_util::{RequestInfo, is_navigation_request, serve_static_with_eta use crate::integrations::IntegrationRegistry; use crate::platform::{GeoInfo, PlatformBackendSpec, PlatformHttpRequest, RuntimeServices}; use crate::price_bucket::{PriceGranularity, price_bucket}; -use crate::response_privacy::CDN_CACHE_HEADERS; +use crate::response_privacy::enforce_synthesized_html_cache_privacy; use crate::rsc_flight::RscFlightUrlRewriter; use crate::settings::Settings; use crate::streaming_processor::{ @@ -1424,6 +1424,22 @@ pub async fn publisher_response_into_streaming_response( } } +/// Returns whether a request can render an HTML document context. +fn is_html_document_request(req: &Request) -> bool { + if let Some(destination) = req + .headers() + .get("sec-fetch-dest") + .and_then(|value| value.to_str().ok()) + { + return matches!( + destination.trim().to_ascii_lowercase().as_str(), + "document" | "embed" | "fencedframe" | "frame" | "iframe" | "object" + ); + } + + is_navigation_request(req) +} + /// Removes request headers that can produce a bodyless or partial origin response. fn strip_conditional_and_range_headers(req: &mut Request) { req.headers_mut().remove(header::IF_NONE_MATCH); @@ -1454,29 +1470,11 @@ fn apply_datadome_client_tag_cache_privacy( suppress_datadome_client_side_tag: bool, content_type: &str, ) { - if !suppress_datadome_client_side_tag - || !response_carries_body(method, response.status()) - || !is_html_content_type(content_type) + if suppress_datadome_client_side_tag + && response_carries_body(method, response.status()) + && is_html_content_type(content_type) { - return; - } - - let already_uncacheable = response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()) - .map(str::to_ascii_lowercase) - .is_some_and(|value| value.contains("private") || value.contains("no-store")); - if !already_uncacheable { - response.headers_mut().insert( - header::CACHE_CONTROL, - HeaderValue::from_static("private, max-age=0"), - ); - } - response.headers_mut().remove(header::ETAG); - response.headers_mut().remove(header::LAST_MODIFIED); - for header_name in CDN_CACHE_HEADERS { - response.headers_mut().remove(*header_name); + enforce_synthesized_html_cache_privacy(response); } } @@ -2900,10 +2898,11 @@ pub async fn handle_publisher_request( .extensions() .get::() .is_some(); - if should_run_ad_stack || suppress_datadome_client_side_tag { - // The origin content type is not known yet, so request hints cannot safely - // narrow this to HTML without allowing 304 or 206 responses to bypass a - // response mutation that becomes necessary after the fetch. + if should_run_ad_stack || (suppress_datadome_client_side_tag && is_html_document_request(&req)) + { + // HTML document contexts whose output may be synthesized must not + // receive a cached 304 or partial 206. Non-document subresources contain + // no executable injected tag, so retain their validators and ranges. strip_conditional_and_range_headers(&mut req); } @@ -3015,23 +3014,7 @@ pub async fn handle_publisher_request( .and_then(|h| h.to_str().ok()) .unwrap_or_default(); if should_run_ad_stack && is_html_content_type(origin_content_type) { - response.headers_mut().insert( - header::CACHE_CONTROL, - HeaderValue::from_static("private, no-store"), - ); - response.headers_mut().remove(header::ETAG); - response.headers_mut().remove(header::LAST_MODIFIED); - // Every CDN-targeted cache directive, not just the browser-facing - // `Cache-Control` above: an origin emitting any of these would otherwise - // instruct an intermediary to store a synthesized per-navigation - // document. `Surrogate-Control` and `Fastly-Surrogate-Control` cover - // Fastly; `CDN-Cache-Control` is the standard targeted field (RFC 9213) - // and `Cloudflare-CDN-Cache-Control` is the Cloudflare-specific field - // that overrides it there, so both are needed to close the gap on the - // Cloudflare adapter. - for directive in CDN_CACHE_HEADERS { - response.headers_mut().remove(*directive); - } + enforce_synthesized_html_cache_privacy(&mut response); } let content_type = response @@ -4218,7 +4201,8 @@ mod tests { use crate::auction::types::{AdFormat, AdSlot, MediaType}; use crate::integrations::IntegrationRegistry; use crate::platform::test_support::{ - StubHttpClient, build_services_with_http_client, noop_services, + NoopSecretStore, StubHttpClient, build_services_with_http_client, + build_services_with_secret_http_client_and_client_ip, noop_services, noop_services_with_telemetry_sink, }; use crate::test_support::tests::create_test_settings; @@ -5386,7 +5370,7 @@ mod tests { } #[tokio::test] - async fn suppressed_publisher_request_removes_conditional_validators() { + async fn suppressed_navigation_removes_conditional_and_range_headers() { let settings = create_test_settings(); let stub = Arc::new(StubHttpClient::new()); stub.push_response_with_headers( @@ -5401,6 +5385,7 @@ mod tests { .method(Method::GET) .uri("https://publisher.example/page") .header(header::HOST, "publisher.example") + .header("sec-fetch-dest", "document") .header(header::IF_NONE_MATCH, "\"cached-page\"") .header(header::IF_MODIFIED_SINCE, "Wed, 21 Oct 2015 07:28:00 GMT") .header(header::RANGE, "bytes=0-18") @@ -5427,7 +5412,105 @@ mod tests { headers .iter() .all(|(name, _)| !name.eq_ignore_ascii_case(header_name.as_str())), - "suppressed requests must not forward {header_name}" + "suppressed navigations must not forward {header_name}" + ); + } + } + + #[tokio::test] + async fn suppressed_iframe_removes_conditional_and_range_headers() { + let settings = create_test_settings(); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + b"frame".to_vec(), + vec![("content-type", "text/html; charset=utf-8")], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let mut req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/frame") + .header(header::HOST, "publisher.example") + .header("sec-fetch-dest", "iframe") + .header(header::IF_NONE_MATCH, "\"cached-frame\"") + .header(header::IF_MODIFIED_SINCE, "Wed, 21 Oct 2015 07:28:00 GMT") + .header(header::RANGE, "bytes=0-18") + .header(header::IF_RANGE, "\"cached-frame\"") + .body(EdgeBody::empty()) + .expect("should build conditional iframe request"); + req.extensions_mut() + .insert(crate::integrations::datadome::DataDomeClientTagSuppressed); + + let _response = run_publisher_proxy(&settings, &services, req).await; + + let headers = stub + .recorded_request_headers() + .into_iter() + .next() + .expect("should record one outbound request"); + for header_name in [ + header::IF_NONE_MATCH, + header::IF_MODIFIED_SINCE, + header::RANGE, + header::IF_RANGE, + ] { + assert!( + headers + .iter() + .all(|(name, _)| !name.eq_ignore_ascii_case(header_name.as_str())), + "suppressed iframe documents must not forward {header_name}" + ); + } + } + + #[tokio::test] + async fn suppressed_subresource_preserves_conditional_and_range_headers() { + let settings = create_test_settings(); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + b"video".to_vec(), + vec![("content-type", "video/mp4")], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let mut req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/video.mp4") + .header(header::HOST, "publisher.example") + .header("sec-fetch-dest", "video") + .header(header::IF_NONE_MATCH, "\"cached-video\"") + .header(header::IF_MODIFIED_SINCE, "Wed, 21 Oct 2015 07:28:00 GMT") + .header(header::RANGE, "bytes=0-18") + .header(header::IF_RANGE, "\"cached-video\"") + .body(EdgeBody::empty()) + .expect("should build conditional subresource request"); + req.extensions_mut() + .insert(crate::integrations::datadome::DataDomeClientTagSuppressed); + + let _response = run_publisher_proxy(&settings, &services, req).await; + + let headers = stub + .recorded_request_headers() + .into_iter() + .next() + .expect("should record one outbound request"); + for (header_name, expected) in [ + (header::IF_NONE_MATCH, "\"cached-video\""), + (header::IF_MODIFIED_SINCE, "Wed, 21 Oct 2015 07:28:00 GMT"), + (header::RANGE, "bytes=0-18"), + (header::IF_RANGE, "\"cached-video\""), + ] { + assert_eq!( + headers + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case(header_name.as_str())) + .map(|(_, value)| value.as_str()), + Some(expected), + "suppressed subresources should preserve {header_name}" ); } } @@ -5547,6 +5630,77 @@ mod tests { ); } + #[tokio::test] + async fn datadome_filter_marker_survives_into_publisher_html_pipeline() { + let mut settings = create_test_settings(); + settings + .integrations + .insert_config( + "datadome", + &serde_json::json!({ + "enabled": true, + "enable_protection": true, + "protection_excluded_ip_cidrs": ["192.0.2.0/24"], + "client_side_key": "test-client-key", + }), + ) + .expect("should configure DataDome integration"); + let registry = IntegrationRegistry::new(&settings) + .expect("should create integration registry with DataDome"); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + b"content".to_vec(), + vec![("content-type", "text/html; charset=utf-8")], + ); + let services = build_services_with_secret_http_client_and_client_ip( + NoopSecretStore, + Arc::clone(&stub) as Arc, + Some("192.0.2.10".parse().expect("should parse client IP")), + ); + let mut req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/page") + .header(header::HOST, "publisher.example") + .header("sec-fetch-dest", "document") + .body(EdgeBody::empty()) + .expect("should build request"); + + let filter_outcome = registry + .filter_request(crate::integrations::RequestFilterRegistryInput { + settings: &settings, + services: &services, + req: &mut req, + geo_info: None, + }) + .await + .expect("should run DataDome filter"); + assert!(matches!( + filter_outcome, + crate::integrations::RequestFilterRegistryOutcome::Continue(_) + )); + let publisher_response = run_publisher_proxy(&settings, &services, req).await; + let response = buffer_publisher_response_async( + publisher_response, + &Method::GET, + &settings, + ®istry, + &AuctionOrchestrator::new(settings.auction.clone()), + &services, + ) + .await + .expect("should buffer publisher response"); + let html = response_body_string(response); + + assert!(!html.contains("window.ddjskey")); + assert!(!html.contains("/integrations/datadome/tags.js")); + assert_eq!( + stub.recorded_backend_names().len(), + 1, + "only the publisher origin should be called" + ); + } + #[test] fn suppressed_datadome_tag_reaches_publisher_html_pipeline() { let mut settings = create_test_settings(); @@ -5613,8 +5767,8 @@ mod tests { .headers() .get(header::CACHE_CONTROL) .and_then(|value| value.to_str().ok()), - Some("private, max-age=0"), - "suppressed HTML should be private" + Some("private, no-store"), + "suppressed HTML should be private and non-storable" ); assert!( response.headers().get("surrogate-control").is_none(), @@ -5658,8 +5812,8 @@ mod tests { .headers() .get(header::CACHE_CONTROL) .and_then(|value| value.to_str().ok()), - Some("no-store"), - "suppressed HTML should preserve an existing no-store policy" + Some("private, no-store"), + "suppressed HTML should use the exact synthesized-HTML policy" ); } diff --git a/crates/trusted-server-core/src/response_privacy.rs b/crates/trusted-server-core/src/response_privacy.rs index c2b1447e3..21ba9f20b 100644 --- a/crates/trusted-server-core/src/response_privacy.rs +++ b/crates/trusted-server-core/src/response_privacy.rs @@ -24,6 +24,27 @@ pub const CDN_CACHE_HEADERS: &[&str] = &[ "cloudflare-cdn-cache-control", ]; +fn strip_cdn_cache_headers(response: &mut Response) { + for name in CDN_CACHE_HEADERS { + response.headers_mut().remove(*name); + } +} + +/// Forces synthesized HTML to be private and non-storable. +/// +/// Use this exact policy whenever Trusted Server changes an origin HTML +/// representation with request-specific content: force `private, no-store`, +/// remove origin validators, and remove all CDN-targeted cache directives. +pub(crate) fn enforce_synthesized_html_cache_privacy(response: &mut Response) { + response.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static("private, no-store"), + ); + response.headers_mut().remove(header::ETAG); + response.headers_mut().remove(header::LAST_MODIFIED); + strip_cdn_cache_headers(response); +} + /// Forces cookie-bearing responses to stay private to shared caches. /// /// Any response that sets a per-user cookie (notably the EC identity cookie) @@ -41,9 +62,7 @@ pub fn enforce_set_cookie_cache_privacy(response: &mut Response) { // one already carrying a stricter `no-store`/`private` directive — they are // independent of Cache-Control and would otherwise let a shared cache store // and replay one visitor's Set-Cookie. - for name in CDN_CACHE_HEADERS { - response.headers_mut().remove(*name); - } + strip_cdn_cache_headers(response); // Cache-Control directives are case-insensitive (RFC 9111 §5.2), so match // against a lowercased copy — `No-Store` / `Private` must count. let already_uncacheable = response @@ -149,6 +168,37 @@ mod tests { s } + #[test] + fn synthesized_html_is_forced_no_store_without_validators_or_cdn_headers() { + let mut response = response_builder() + .header(header::CACHE_CONTROL, "private, max-age=600") + .header(header::ETAG, "\"origin\"") + .header(header::LAST_MODIFIED, "Wed, 21 Oct 2015 07:28:00 GMT") + .header("surrogate-control", "max-age=600") + .header("fastly-surrogate-control", "max-age=600") + .header("cdn-cache-control", "max-age=600") + .header("cloudflare-cdn-cache-control", "max-age=600") + .body(edgezero_core::body::Body::empty()) + .expect("should build response"); + + enforce_synthesized_html_cache_privacy(&mut response); + + assert_eq!( + response.headers()[header::CACHE_CONTROL], + "private, no-store", + "synthesized HTML should always be non-storable" + ); + for header_name in [header::ETAG.as_str(), header::LAST_MODIFIED.as_str()] + .into_iter() + .chain(CDN_CACHE_HEADERS.iter().copied()) + { + assert!( + !response.headers().contains_key(header_name), + "synthesized HTML should remove {header_name}" + ); + } + } + #[test] fn downgrades_public_cache_control_on_cookie_response() { let settings = settings_with_response_headers(&[("cache-control", "public, max-age=600")]); diff --git a/docs/guide/integrations/datadome.md b/docs/guide/integrations/datadome.md index ff2eb6aaa..0c2d8ae6c 100644 --- a/docs/guide/integrations/datadome.md +++ b/docs/guide/integrations/datadome.md @@ -86,7 +86,7 @@ patterns = ["(?i)\\.(avi|flv|mka|mkv|mov|mp4|mpeg|mpg|mp3|flac|ogg|ogm|opus|wav| | `protection_excluded_ip_cidr_sources` | array | `[]` | Config Store sources containing dynamic client IP CIDR bypass lists | | `protection_ip_list_cache_ttl_seconds` | integer | `300` | Process-local cache TTL for Config Store-backed IP CIDR bypass lists | | `protection_exclusion_rules` | array | Static asset path regex | Structured method/path/query/IP/ASN exclusion rules | -| `protection_test_bypass` | object | omitted | Temporary static-header bypass for access-controlled staging tests | +| `protection_test_bypass` | object | omitted | Staging-only fixed-header bypass; secret must contain at least 32 bytes | | `enable_graphql_support` | boolean | `false` | Reserved for future GraphQL body inspection; ignored in v1 | | `client_side_key` | string | `""` | DataDome client-side JavaScript key used for tag injection | | `inject_client_side_tag` | boolean | `true` | Auto-inject the browser tag when `client_side_key` is non-empty | @@ -190,19 +190,25 @@ credential_secret_name = "datadome_test_bypass" ``` `protection_test_bypass` requires `enable_protection = true`; it is disabled -when omitted and is runtime-active only when `FASTLY_IS_STAGING=1`. A retained +when omitted and is runtime-active only when `FASTLY_IS_STAGING=1`. +`FASTLY_IS_STAGING` is supplied at runtime by Fastly (`1` in staging and `0` in +production); it is not compiled into or promoted with the Wasm artifact. Verify +staging through the `X-TS-ENV: staging` response signal and the integration +activation log, and verify production omits that response signal. A retained section cannot bypass protection in a production or other non-staging runtime. -Store the temporary credential in the configured Secret Store, configure this -section only while needed, protect the site with an outer access control such -as Basic Auth, and remove the section when testing finishes. - -Whenever the enabled DataDome request filter runs, the fixed -`x-ts-datadome-bypass` header is removed before configuration or credential -checks. It therefore cannot reach DataDome or the publisher origin when the -bypass is absent, disabled, inactive, or invalid. Active credentials are -compared in constant time and never logged. Scope the header to the staging origin; do not attach it to every -request in a browser context because that can disclose the credential to -third-party origins. With Playwright: +Store a randomly generated credential containing at least 32 bytes of +high-entropy material in the configured Secret Store, configure this section +only while needed, protect the site with an outer access control such as Basic +Auth, and remove the section when testing finishes. + +Whenever the enabled DataDome request filter runs on the Fastly adapter, the +fixed `x-ts-datadome-bypass` header is removed before configuration or +credential checks. It therefore cannot reach DataDome or the publisher origin +through that path when the bypass is absent, disabled, inactive, or invalid. +Active credentials are compared in constant time and never logged. Duplicate +header values fail closed. Scope the header to the staging origin; do not attach +it to every request in a browser context because that can disclose the +credential to third-party origins. With Playwright: ```ts await context.route('https://staging.example.com/**', async (route) => { @@ -229,15 +235,25 @@ This behavior applies to: - structured `ip_cidr_source` rules; and - a matching enabled `protection_test_bypass` credential in a staging runtime. -ASN, method, path, query-parameter, static-asset, and internal-route -exclusions do not automatically suppress the client-side tag. DataDome tags -already present in publisher HTML are not removed or changed by this behavior, -and `/integrations/datadome/tags.js` remains available when requested directly. +Method, ASN, path, query-parameter, static-asset, and internal-route exclusions +alone do not suppress the client-side tag. However, a simultaneous matching IP +exclusion suppresses it regardless of which first-match rule and reason are +logged. DataDome tags already present in publisher HTML are not removed or +changed by this behavior, and `/integrations/datadome/tags.js` remains available +when requested directly. Because the processed HTML differs by client IP or test credential, -tag-suppressed HTML is marked `private, max-age=0` and removed from shared -surrogate caches. The decision is reported in the existing protection log, for -example: +tag-suppressed HTML is marked `private, no-store`, has origin validators +removed, and has shared-surrogate cache directives removed. This response-time +policy cannot invalidate tag-bearing HTML already held by a shared cache in +front of Trusted Server. Guaranteed suppression requires bypassing or purging +that cache, or avoiding shared caching ahead of Trusted Server. + +IP-exclusion suppression skips are logged at `info` for navigations and `debug` +for subresources; matching test-bypass events remain at `info` as security audit +events. Protection API result logs classify `allowed`, `blocked`, and +`failed_open` outcomes and use distinct `api_status` and `datadome_status` +fields. For example: ```text [datadome] protection decision=skipped rule=protection-test-bypass reason=test_bypass client_tag=omitted method=GET diff --git a/docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md b/docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md index 2d9b0eae2..9f2c1674f 100644 --- a/docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md +++ b/docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md @@ -40,10 +40,9 @@ request-filter wiring. 3. **Private marker:** communicate the decision with a typed request extension, never a request/response header. The marker cannot leak to the origin or client. -4. **Precise scope:** tag suppression is keyed only on decision reasons - `client_ip`, `client_ip_source`, `ip_cidr`, and `ip_cidr_source`. +4. **Precise scope:** tag suppression uses typed scope metadata and applies whenever an IP exclusion overlaps the primary first-match skip reason. 5. **Cache safety:** an HTML response with the tag omitted differs by client IP. - A suppressed processed HTML response must be `private, max-age=0` and have + A suppressed processed HTML response must be `private, no-store` and have `Surrogate-Control` and `Fastly-Surrogate-Control` removed. Do not alter cache headers when the response is not processed HTML, because this feature does not alter that body. @@ -183,9 +182,11 @@ cargo test-fastly datadome::protection cargo test-fastly datadome::protection_scope ``` -**Acceptance:** only the four IP decision reasons add the private marker and -produce the augmented informational skip log; all other exclusion and fail-open -paths keep their current tag behavior. +**Acceptance:** typed scope metadata adds the private marker whenever an +applicable IP exclusion matches, including overlap with an earlier primary skip +reason. Navigation suppression uses the augmented informational log, +subresources use debug, and non-overlapping/fail-open paths keep their current +tag behavior. --- @@ -327,7 +328,7 @@ optimization. with a processable HTML content type, suppression `true`, and cacheable origin headers (`Cache-Control`, `Surrogate-Control`, and `Fastly-Surrogate-Control`). Assert the stream response is: - - `Cache-Control: private, max-age=0`; and + - `Cache-Control: private, no-store`; and - missing both surrogate cache headers. - [ ] **Step 2: Apply privacy only in the `ResponseRoute::Stream` HTML arm.** @@ -349,9 +350,10 @@ optimization. cargo test-fastly publisher ``` -**Acceptance:** a shared cache cannot replay an IP-excluded client's tagless -HTML to a non-excluded visitor, while unchanged responses retain their existing -cacheability. +**Acceptance:** newly synthesized tagless HTML is private and non-storable, +while unchanged responses retain their existing cacheability. A shared cache in +front of Trusted Server that already holds tag-bearing HTML must be bypassed or +purged for guaranteed suppression. --- diff --git a/docs/superpowers/specs/2026-08-03-datadome-ip-excluded-client-tag-design.md b/docs/superpowers/specs/2026-08-03-datadome-ip-excluded-client-tag-design.md index d6d811780..0bc7a856d 100644 --- a/docs/superpowers/specs/2026-08-03-datadome-ip-excluded-client-tag-design.md +++ b/docs/superpowers/specs/2026-08-03-datadome-ip-excluded-client-tag-design.md @@ -112,16 +112,11 @@ found in publisher HTML. That behavior must remain unchanged. ### 1. Capture an IP-exclusion marker at the request filter The request filter must attach a typed, internal request-scoped marker when the -existing protection-scope evaluation returns a skip for one of these reasons: - -- `client_ip` -- `client_ip_source` -- `ip_cidr` -- `ip_cidr_source` - -The marker must be attached only after the existing scope decision confirms the -IP exclusion. It must not be inferred from request headers or recomputed later -in the HTML pipeline. +existing protection-scope evaluation reports `suppress_client_tag` metadata. +The scope preserves the current first-match rule ID and typed reason for +logging, while independently checking applicable IP exclusions when an earlier +method, ASN, path, or query rule already matched. The marker must not be +inferred from request headers or recomputed later in the HTML pipeline. The request-filter API currently exposes an immutable request view. Add the smallest internal mechanism needed for a filter to attach a typed request @@ -134,19 +129,17 @@ suppressed; the existing skip log supplies the rule ID and reason. The marker must not be attached for: -- `OPTIONS` or other excluded methods before scope evaluation; - internal or integration routes; -- ASN exclusions; -- path, query, or other non-IP structured exclusions; -- unmatched IP rules; +- method, ASN, path, or query exclusions without an overlapping IP match; +- unmatched or method-inapplicable IP rules; - Protection API fail-open behavior; or - requests where `enable_protection` is false and the request filter does not run. ### 2. Enrich the existing skip log -For IP-based skips, extend the existing informational log with -`client_tag=omitted`: +For navigation skips that suppress the tag, extend the existing informational +log with `client_tag=omitted` (subresource suppression skips use `debug`): ```text [datadome] protection decision=skipped rule=excluded-ip-cidrs reason=client_ip client_tag=omitted method=GET @@ -184,8 +177,11 @@ responses, which should retain their current processing. A processed HTML response differs by client IP when the generated tag is suppressed. In the `PublisherResponse::Stream` path, when suppression is active -and the response is HTML, set `Cache-Control: private, max-age=0` and remove -`Surrogate-Control` and `Fastly-Surrogate-Control` before the body is streamed. +and the response is HTML, use the shared synthesized-HTML policy: +`Cache-Control: private, no-store`, no origin validators, and no CDN-targeted +cache headers. This response-time policy cannot invalidate tag-bearing HTML +already held by a fronting shared cache; guaranteed suppression requires +bypassing or purging that cache. This matches the existing per-user ad-stack cache policy. It prevents Fastly or another shared cache from replaying a tag-suppressed response to a visitor whose From 2650aa27294fa726456cc47f75a124aab5db7254 Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 13 Aug 2026 17:03:58 -0500 Subject: [PATCH 14/16] Show observed slot size in GPT diagnostics --- .../trusted-server-js/lib/src/core/types.ts | 6 + .../src/integrations/gpt_diagnostics/api.ts | 2 + .../integrations/gpt_diagnostics/badges.ts | 5 +- .../src/integrations/gpt_diagnostics/index.ts | 5 + .../integrations/gpt_diagnostics/overlay.ts | 5 +- .../gpt_diagnostics/slot_size_observer.ts | 146 ++++++++++++++++ .../src/integrations/gpt_diagnostics/store.ts | 40 +++++ .../integrations/gpt_diagnostics/api.test.ts | 1 + .../gpt_diagnostics/badges.test.ts | 2 +- .../gpt_diagnostics/overlay.test.ts | 2 +- .../slot_size_observer.test.ts | 158 ++++++++++++++++++ .../gpt_diagnostics/store.test.ts | 32 ++++ docs/guide/integrations/gpt-diagnostics.md | 18 +- 13 files changed, 417 insertions(+), 5 deletions(-) create mode 100644 crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/slot_size_observer.ts create mode 100644 crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/slot_size_observer.test.ts diff --git a/crates/trusted-server-js/lib/src/core/types.ts b/crates/trusted-server-js/lib/src/core/types.ts index 0c68d43fe..464203046 100644 --- a/crates/trusted-server-js/lib/src/core/types.ts +++ b/crates/trusted-server-js/lib/src/core/types.ts @@ -207,7 +207,13 @@ export interface GptDiagnosticsRequestCycle { viewableAtMs?: number; durations: GptDiagnosticsDurations; isEmpty?: boolean; + /** Exact size fact GPT reported in its `slotRenderEnded` callback. */ size?: Size; + /** + * Outer CSS box observed on the uniquely bound, connected slot element after + * a filled GPT render. This is not an assertion about internal creative pixels. + */ + observedSlotSize?: Size; isBackfill?: boolean; slotContentChanged?: boolean; incompleteSequence: boolean; diff --git a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/api.ts b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/api.ts index 99f876b3f..8871165dd 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/api.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/api.ts @@ -64,6 +64,7 @@ function cloneExportSnapshot(snapshot: GptDiagnosticsExportV1): GptDiagnosticsEx ...cycle, durations: { ...cycle.durations }, size: cycle.size ? [...cycle.size] : undefined, + observedSlotSize: cycle.observedSlotSize ? [...cycle.observedSlotSize] : undefined, adManager: cycle.adManager ? { ...cycle.adManager, @@ -192,6 +193,7 @@ export class GptDiagnosticsApiController { ...cycle, durations: { ...cycle.durations }, size: cycle.size ? [...cycle.size] : undefined, + observedSlotSize: cycle.observedSlotSize ? [...cycle.observedSlotSize] : undefined, adManager: cycle.adManager ? { ...cycle.adManager, diff --git a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/badges.ts b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/badges.ts index 57fce3d85..ab39f1f09 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/badges.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/badges.ts @@ -115,7 +115,10 @@ function badgeText(cycle: GptDiagnosticsRequestCycle): string { const delivery = deliveryLabel(cycle); if (delivery) firstLine.push(delivery); if (cycle.requestPath === 'competing') firstLine.push('Competing paths'); - if (cycle.size) firstLine.push(`${cycle.size[0]}×${cycle.size[1]}`); + if (cycle.size) firstLine.push(`GPT ${cycle.size[0]}×${cycle.size[1]}`); + if (cycle.observedSlotSize) { + firstLine.push(`Box ${cycle.observedSlotSize[0]}×${cycle.observedSlotSize[1]}`); + } const timingLine: string[] = []; const response = formatMilliseconds(cycle.durations.requestToResponseMs); diff --git a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/index.ts index 75bf97823..d7271710c 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/index.ts @@ -7,6 +7,7 @@ import { GptDiagnosticsBindingManager } from './binding'; import { GptDiagnosticsObserver } from './observer'; import type { GptObserverWindow } from './observer'; import { GptDiagnosticsOverlay } from './overlay'; +import { GptDiagnosticsSlotSizeObserver } from './slot_size_observer'; import { GptDiagnosticsStore } from './store'; interface GptDiagnosticsRuntime { @@ -44,6 +45,7 @@ export function installGptDiagnosticsRuntime( let bindings: GptDiagnosticsBindingManager | undefined; let badges: GptDiagnosticsBadgeManager | undefined; let overlay: GptDiagnosticsOverlay | undefined; + let slotSizeObserver: GptDiagnosticsSlotSizeObserver | undefined; let apiController: GptDiagnosticsApiController | undefined; try { @@ -59,6 +61,7 @@ export function installGptDiagnosticsRuntime( window: target, document: target.document, }); + slotSizeObserver = new GptDiagnosticsSlotSizeObserver(store, bindings, { window: target }); overlay = new GptDiagnosticsOverlay(store, bindings, { window: target, document: target.document, @@ -83,6 +86,7 @@ export function installGptDiagnosticsRuntime( apiController?.destroy(); overlay?.destroy(); badges?.destroy(); + slotSizeObserver?.destroy(); bindings?.destroy(); delete target.__tsjs_gpt_diagnostics_runtime; }, @@ -95,6 +99,7 @@ export function installGptDiagnosticsRuntime( apiController?.destroy(); overlay?.destroy(); badges?.destroy(); + slotSizeObserver?.destroy(); bindings?.destroy(); log.warn('gpt diagnostics: runtime installation failed', error); return undefined; diff --git a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/overlay.ts b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/overlay.ts index e99f1345b..ca5d8dd6d 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/overlay.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/overlay.ts @@ -277,7 +277,10 @@ function cycleFacts(cycle: GptDiagnosticsRequestCycle): string[] { if (cycle.loadAtMs !== undefined) facts.push('GPT slot onload observed'); if (cycle.viewableAtMs !== undefined) facts.push('GPT impressionViewable observed'); if (cycle.incompleteSequence) facts.push('Incomplete sequence'); - if (cycle.size) facts.push(`Rendered size ${cycle.size[0]}×${cycle.size[1]}`); + if (cycle.size) facts.push(`GPT reported size ${cycle.size[0]}×${cycle.size[1]}`); + if (cycle.observedSlotSize) { + facts.push(`Observed slot box ${cycle.observedSlotSize[0]}×${cycle.observedSlotSize[1]}`); + } if (cycle.isBackfill !== undefined) facts.push(`Backfill ${cycle.isBackfill ? 'yes' : 'no'}`); if (cycle.slotContentChanged !== undefined) { facts.push(`Slot content changed ${cycle.slotContentChanged ? 'yes' : 'no'}`); diff --git a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/slot_size_observer.ts b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/slot_size_observer.ts new file mode 100644 index 000000000..ab49a451a --- /dev/null +++ b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/slot_size_observer.ts @@ -0,0 +1,146 @@ +import type { Size } from '../../core/types'; + +import type { GptDiagnosticsBindingManager } from './binding'; +import type { GptDiagnosticsStoreSnapshot } from './store'; + +interface SlotSizeStore { + snapshot(): GptDiagnosticsStoreSnapshot; + recordObservedSlotSize(runtimeSlotNumber: number, requestNumber: number, size: Size): void; + subscribe(listener: () => void): () => void; +} + +interface SlotSizeBindings { + get: GptDiagnosticsBindingManager['get']; + subscribe(listener: () => void): () => void; +} + +type SlotSizeWindow = Window & { + ResizeObserver?: typeof ResizeObserver; +}; + +interface SlotSizeObserverOptions { + window?: SlotSizeWindow; + scheduleFrame?: (callback: () => void) => void; +} + +interface ObservedCycle { + runtimeSlotNumber: number; + requestNumber: number; +} + +function defaultScheduleFrame(callback: () => void): void { + if (typeof requestAnimationFrame === 'function') { + requestAnimationFrame(() => callback()); + } else { + queueMicrotask(callback); + } +} + +function latestFilledCycle( + slot: GptDiagnosticsStoreSnapshot['slots'][number] +): ObservedCycle | undefined { + const cycle = slot.requests[slot.requests.length - 1]; + if (!cycle || cycle.isEmpty !== false || cycle.renderAtMs === undefined) return undefined; + return { runtimeSlotNumber: slot.runtimeSlotNumber, requestNumber: cycle.requestNumber }; +} + +/** + * Observes the outer CSS boxes of uniquely bound elements after filled GPT renders. + * + * Measurements remain separately labelled from GPT's reported creative size and + * are conditionally written with the runtime-slot and request-cycle identity that + * was current when the measurement was scheduled. + */ +export class GptDiagnosticsSlotSizeObserver { + private readonly store: SlotSizeStore; + private readonly bindings: SlotSizeBindings; + private readonly window: SlotSizeWindow; + private readonly scheduleFrame: (callback: () => void) => void; + private readonly unsubscribeStore: () => void; + private readonly unsubscribeBindings: () => void; + private resizeObserver?: ResizeObserver; + private refreshScheduled = false; + private destroyed = false; + + constructor( + store: SlotSizeStore, + bindings: SlotSizeBindings, + options: SlotSizeObserverOptions = {} + ) { + this.store = store; + this.bindings = bindings; + this.window = options.window ?? (window as unknown as SlotSizeWindow); + this.scheduleFrame = options.scheduleFrame ?? defaultScheduleFrame; + this.unsubscribeStore = this.store.subscribe(this.scheduleRefresh); + this.unsubscribeBindings = this.bindings.subscribe(this.scheduleRefresh); + this.refresh(); + } + + destroy(): void { + if (this.destroyed) return; + this.destroyed = true; + this.unsubscribeStore(); + this.unsubscribeBindings(); + this.resizeObserver?.disconnect(); + } + + private readonly scheduleRefresh = (): void => { + if (this.destroyed || this.refreshScheduled) return; + this.refreshScheduled = true; + this.scheduleFrame(() => { + this.refreshScheduled = false; + this.refresh(); + }); + }; + + private refresh(): void { + if (this.destroyed) return; + this.resizeObserver?.disconnect(); + const observations = new Map(); + const ResizeObserverConstructor = this.window.ResizeObserver; + if (typeof ResizeObserverConstructor === 'function') { + this.resizeObserver = new ResizeObserverConstructor((entries) => { + for (const entry of entries) { + const element = entry.target; + if (!(element instanceof this.window.HTMLElement)) continue; + const cycle = observations.get(element); + if (cycle) this.scheduleMeasure(element, cycle); + } + }); + } + + for (const slot of this.store.snapshot().slots) { + const cycle = latestFilledCycle(slot); + const binding = this.bindings.get(slot.runtimeSlotNumber); + if (!cycle || binding.binding.status !== 'bound' || !binding.element?.isConnected) continue; + observations.set(binding.element, cycle); + this.resizeObserver?.observe(binding.element); + this.scheduleMeasure(binding.element, cycle); + } + } + + private scheduleMeasure(element: HTMLElement, cycle: ObservedCycle): void { + this.scheduleFrame(() => this.measure(element, cycle)); + } + + private measure(element: HTMLElement, cycle: ObservedCycle): void { + const binding = this.bindings.get(cycle.runtimeSlotNumber); + if (binding.binding.status !== 'bound' || binding.element !== element || !element.isConnected) { + return; + } + + const rectangle = element.getBoundingClientRect(); + if ( + !Number.isFinite(rectangle.width) || + !Number.isFinite(rectangle.height) || + rectangle.width < 0 || + rectangle.height < 0 + ) { + return; + } + this.store.recordObservedSlotSize(cycle.runtimeSlotNumber, cycle.requestNumber, [ + rectangle.width, + rectangle.height, + ]); + } +} diff --git a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/store.ts b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/store.ts index 0324a56cc..2917dc206 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/store.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/store.ts @@ -249,6 +249,7 @@ function copyCycle(cycle: MutableRequestCycle, nowMs: number): GptDiagnosticsReq ...cycle, durations: derivedDurations(cycle), size: cycle.size ? ([...cycle.size] as Size) : undefined, + observedSlotSize: cycle.observedSlotSize ? ([...cycle.observedSlotSize] as Size) : undefined, adManager: cycle.adManager ? { ...cycle.adManager, @@ -666,6 +667,45 @@ export class GptDiagnosticsStore { ); } + /** + * Retain an outer CSS box only when this exact slot and request cycle still + * identify a filled render. Async DOM measurements use this guard so a prior + * render cannot alter a later refresh cycle. + */ + recordObservedSlotSize(runtimeSlotNumber: number, requestNumber: number, size: Size): void { + if ( + !Number.isSafeInteger(requestNumber) || + requestNumber <= 0 || + !Number.isFinite(size[0]) || + !Number.isFinite(size[1]) || + size[0] < 0 || + size[1] < 0 + ) { + return; + } + + const record = this.slots.get(runtimeSlotNumber); + const cycle = record?.requests.find((candidate) => candidate.requestNumber === requestNumber); + if ( + !cycle || + record.requests[record.requests.length - 1] !== cycle || + cycle.isEmpty !== false || + cycle.renderAtMs === undefined + ) { + return; + } + + const observedSlotSize: Size = [size[0], size[1]]; + if ( + cycle.observedSlotSize?.[0] === observedSlotSize[0] && + cycle.observedSlotSize[1] === observedSlotSize[1] + ) { + return; + } + cycle.observedSlotSize = observedSlotSize; + this.notify(); + } + recordSlotOnload(slot: GptDiagnosticsSlotLike): void { const timestampMs = this.timestamp(); this.matchCycle( diff --git a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/api.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/api.test.ts index 2e2ae2d2b..6ac993df7 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/api.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/api.test.ts @@ -348,6 +348,7 @@ describe('GptDiagnosticsApiController', () => { 'incompleteSequence', 'isBackfill', 'isEmpty', + 'observedSlotSize', 'renderAtMs', 'requestNumber', 'requestPath', diff --git a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/badges.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/badges.test.ts index 675e4f442..56de45842 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/badges.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/badges.test.ts @@ -260,7 +260,7 @@ describe('GptDiagnosticsBadgeManager', () => { renderToViewableMs: 1000, }, }) - ).toBe('Filled · 728×90\nResponse 276 ms · Render 42 ms\nViewable after 1 s'); + ).toBe('Filled · GPT 728×90\nResponse 276 ms · Render 42 ms\nViewable after 1 s'); expect( gptDiagnosticsBadgeTextForTest({ requestNumber: 1, diff --git a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/overlay.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/overlay.test.ts index b89a8f507..e3d80dae0 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/overlay.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/overlay.test.ts @@ -457,7 +457,7 @@ describe('GptDiagnosticsOverlay', () => { expect(root!.textContent).toContain('/example/site/filled-slot'); expect(root!.textContent).toContain('Empty'); expect(root!.textContent).toContain('Previous requests (1)'); - expect(root!.textContent).toContain('Rendered size 300×250'); + expect(root!.textContent).toContain('GPT reported size 300×250'); expect(root!.textContent).toContain('Backfill yes'); expect(root!.textContent).toContain('GPT slot onload observed'); expect(root!.textContent).toContain('GPT impressionViewable observed'); diff --git a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/slot_size_observer.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/slot_size_observer.test.ts new file mode 100644 index 000000000..86ad532c7 --- /dev/null +++ b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/slot_size_observer.test.ts @@ -0,0 +1,158 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import type { GptDiagnosticsRequestCycle } from '../../../src/core/types'; +import { GptDiagnosticsSlotSizeObserver } from '../../../src/integrations/gpt_diagnostics/slot_size_observer'; +import type { GptDiagnosticsStoreSnapshot } from '../../../src/integrations/gpt_diagnostics/store'; + +class ResizeObserverMock { + static instances: ResizeObserverMock[] = []; + readonly observe = vi.fn(); + readonly disconnect = vi.fn(); + + constructor(readonly callback: ResizeObserverCallback) { + ResizeObserverMock.instances.push(this); + } + + emit(element: Element): void { + this.callback([{ target: element } as ResizeObserverEntry], this as unknown as ResizeObserver); + } +} + +function cycle(requestNumber: number, isEmpty: boolean | undefined): GptDiagnosticsRequestCycle { + return { + requestNumber, + isEmpty, + renderAtMs: 1, + durations: {}, + incompleteSequence: false, + }; +} + +function snapshot(requests: GptDiagnosticsRequestCycle[]): GptDiagnosticsStoreSnapshot { + return { + gptObserved: true, + slots: [ + { + runtimeSlotNumber: 1, + slotElementId: 'ad-slot-example', + requests, + }, + ], + callbackIssues: [], + attributionIssues: [], + coverage: { + slotRequested: { observed: 0, matched: 0, unmatched: 0, ambiguous: 0 }, + slotResponseReceived: { observed: 0, matched: 0, unmatched: 0, ambiguous: 0 }, + slotRenderEnded: { observed: 0, matched: 0, unmatched: 0, ambiguous: 0 }, + slotOnload: { observed: 0, matched: 0, unmatched: 0, ambiguous: 0 }, + impressionViewable: { observed: 0, matched: 0, unmatched: 0, ambiguous: 0 }, + slotVisibilityChanged: { observed: 0, matched: 0, unmatched: 0, ambiguous: 0 }, + }, + metadata: { + droppedCallbacks: 0, + droppedAttributionIssues: 0, + evictedSlots: 0, + evictedRequestCycles: 0, + }, + }; +} + +describe('GptDiagnosticsSlotSizeObserver', () => { + afterEach(() => { + ResizeObserverMock.instances = []; + document.body.replaceChildren(); + }); + + it('keeps GPT 1×1 distinct from the observed outer box and updates it on resize', () => { + const element = document.createElement('div'); + document.body.append(element); + const getBoundingClientRect = vi.spyOn(element, 'getBoundingClientRect'); + getBoundingClientRect.mockReturnValue({ width: 728, height: 90 } as DOMRect); + const requests = [cycle(1, false)]; + requests[0].size = [1, 1]; + const store = { + snapshot: () => snapshot(requests), + recordObservedSlotSize: vi.fn(), + subscribe: () => () => undefined, + }; + const bindings = { + get: () => ({ binding: { status: 'bound' as const }, element, visible: true }), + subscribe: () => () => undefined, + }; + + const observer = new GptDiagnosticsSlotSizeObserver(store, bindings, { + window: { HTMLElement, ResizeObserver: ResizeObserverMock } as unknown as Window, + scheduleFrame: (callback) => callback(), + }); + + expect(store.recordObservedSlotSize).toHaveBeenCalledWith(1, 1, [728, 90]); + expect(requests[0].size).toEqual([1, 1]); + + getBoundingClientRect.mockReturnValue({ width: 970, height: 250 } as DOMRect); + ResizeObserverMock.instances.at(-1)!.emit(element); + expect(store.recordObservedSlotSize).toHaveBeenLastCalledWith(1, 1, [970, 250]); + observer.destroy(); + }); + + it.each(['unbound', 'ambiguous'] as const)('does not observe %s slots', (status) => { + const element = document.createElement('div'); + document.body.append(element); + const store = { + snapshot: () => snapshot([cycle(1, false)]), + recordObservedSlotSize: vi.fn(), + subscribe: () => () => undefined, + }; + const bindings = { + get: () => ({ binding: { status }, element, visible: false }), + subscribe: () => () => undefined, + }; + + const observer = new GptDiagnosticsSlotSizeObserver(store, bindings, { + window: { HTMLElement, ResizeObserver: ResizeObserverMock } as unknown as Window, + scheduleFrame: (callback) => callback(), + }); + + expect(store.recordObservedSlotSize).not.toHaveBeenCalled(); + expect(ResizeObserverMock.instances.at(-1)!.observe).not.toHaveBeenCalled(); + observer.destroy(); + }); + + it('cannot apply a delayed prior-cycle measurement to a later refresh', () => { + const element = document.createElement('div'); + document.body.append(element); + vi.spyOn(element, 'getBoundingClientRect').mockReturnValue({ + width: 300, + height: 250, + } as DOMRect); + const requests = [cycle(1, false)]; + const listeners: Array<() => void> = []; + const store = { + snapshot: () => snapshot(requests), + recordObservedSlotSize: vi.fn(), + subscribe: (listener: () => void) => { + listeners.push(listener); + return () => undefined; + }, + }; + const bindings = { + get: () => ({ binding: { status: 'bound' as const }, element, visible: true }), + subscribe: () => () => undefined, + }; + const frames: Array<() => void> = []; + const observer = new GptDiagnosticsSlotSizeObserver(store, bindings, { + window: { HTMLElement, ResizeObserver: ResizeObserverMock } as unknown as Window, + scheduleFrame: (callback) => frames.push(callback), + }); + const firstObserver = ResizeObserverMock.instances[0]; + + requests.push(cycle(2, false)); + listeners[0](); + frames.shift()!(); + firstObserver.emit(element); + while (frames.length > 0) frames.shift()!(); + + expect(store.recordObservedSlotSize).toHaveBeenCalledWith(1, 1, [300, 250]); + expect(store.recordObservedSlotSize).toHaveBeenCalledWith(1, 2, [300, 250]); + observer.destroy(); + }); +}); diff --git a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/store.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/store.test.ts index 4c6721f3a..2d29d8b18 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/store.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/store.test.ts @@ -519,6 +519,38 @@ describe('GptDiagnosticsStore', () => { expect(cycle.responseClass).toBe('reservation'); }); + it('retains an observed outer slot box separately from GPT reported size', () => { + const store = new GptDiagnosticsStore({ now: () => 10 }); + const slot = fakeSlot('ad-slot-outer-box'); + + store.recordSlotRequested(slot); + store.recordSlotResponseReceived(slot); + store.recordSlotRenderEnded(slot, { isEmpty: false, size: [1, 1] }); + store.recordObservedSlotSize(1, 1, [728, 90]); + + const cycle = store.snapshot().slots[0].requests[0]; + expect(cycle.size).toEqual([1, 1]); + expect(cycle.observedSlotSize).toEqual([728, 90]); + }); + + it('rejects a stale prior-cycle outer-box measurement after a refresh', () => { + const store = new GptDiagnosticsStore({ now: () => 10 }); + const slot = fakeSlot('ad-slot-stale-outer-box'); + + store.recordSlotRequested(slot); + store.recordSlotResponseReceived(slot); + store.recordSlotRenderEnded(slot, { isEmpty: false }); + store.recordSlotRequested(slot); + store.recordSlotResponseReceived(slot); + store.recordSlotRenderEnded(slot, { isEmpty: false }); + store.recordObservedSlotSize(1, 1, [300, 250]); + store.recordObservedSlotSize(1, 2, [970, 250]); + + const requests = store.snapshot().slots[0].requests; + expect(requests[0].observedSlotSize).toBeUndefined(); + expect(requests[1].observedSlotSize).toEqual([970, 250]); + }); + it('separates a fill without Ad Manager identifiers from a reservation', () => { const store = new GptDiagnosticsStore({ now: () => 10 }); const slot = fakeSlot('ad-slot-default'); diff --git a/docs/guide/integrations/gpt-diagnostics.md b/docs/guide/integrations/gpt-diagnostics.md index 78c657bf6..d19376b4b 100644 --- a/docs/guide/integrations/gpt-diagnostics.md +++ b/docs/guide/integrations/gpt-diagnostics.md @@ -89,7 +89,7 @@ Each request cycle can show: - GPT slot-onload, impression-viewable, and visibility observations. - Non-negative request-to-response, response-to-render, render-to-load, and render-to-viewable durations. -- Rendered size, backfill, and slot-content-change facts exposed by GPT. +- GPT-reported rendered size, a separately labelled observed outer slot box when safely bound, backfill, and slot-content-change facts. - Current DOM binding status and viewport intersection. Elapsed time alone never changes a pending GPT request to Incomplete. Incomplete @@ -276,6 +276,22 @@ because selector support is unavailable or throws, the export reports `dom_uniqueness_unverifiable`. Framework replacement of an element with a new unique element using the same exact ID is rebound automatically. +For an explicitly filled render, diagnostics can also retain `observedSlotSize`: the +current outer CSS box of the uniquely bound, connected slot element. This is measured +after `slotRenderEnded`. When `ResizeObserver` is available, it remains current +while that same request cycle is latest for the GPT slot; otherwise it is the most +recently sampled box. It is displayed separately from `size`, which remains the exact +`slotRenderEnded.size` fact GPT reported. The observed box may differ from GPT's +reported size (for example, a flexible APS creative can report `1×1` while its +allocated outer slot box is larger). It is a publisher-page layout measurement, not a +claim about universal internal creative-pixel dimensions. Empty, unbound, missing, or +ambiguous slots do not report an observed box; delayed measurements from an older +cycle are rejected after a refresh. + +Cross-origin and SafeFrame boundaries prevent diagnostics from inspecting iframe +content. It does not inspect iframe content or alter the APS sandbox, so it cannot +use this field to prove the inner creative's pixels. + Badges and the panel live in a closed Shadow DOM. Diagnostics do not add attributes, classes, or inline styles to publisher slot elements. From fa9326900bb43621fdb4202bda5b10de3beb590a Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Fri, 14 Aug 2026 16:22:38 +0530 Subject: [PATCH 15/16] Merge ESI #1013 into rc/202608 --- .github/workflows/test.yml | 16 + Cargo.lock | 113 +- Cargo.toml | 1 + .../trusted-server-adapter-fastly/Cargo.toml | 2 + .../trusted-server-adapter-fastly/src/app.rs | 40 +- .../src/esi_assembly.rs | 287 + .../trusted-server-adapter-fastly/src/main.rs | 62 +- .../src/template_cache.rs | 505 ++ crates/trusted-server-core/Cargo.toml | 1 + .../benches/html_processor_bench.rs | 8 +- .../src/creative_opportunities.rs | 315 +- .../trusted-server-core/src/html_processor.rs | 147 +- .../src/integrations/datadome/protection.rs | 65 +- .../src/integrations/gpt_bootstrap.js | 6 +- .../src/integrations/gpt_diagnostics.rs | 77 + .../trusted-server-core/src/platform/mod.rs | 13 + .../src/platform/template_assembly.rs | 77 + .../src/platform/template_cache.rs | 1096 +++ .../trusted-server-core/src/platform/types.rs | 74 + crates/trusted-server-core/src/publisher.rs | 7449 ++++++++++++++++- .../src/response_privacy.rs | 112 +- .../trusted-server-js/lib/src/core/types.ts | 12 +- .../lib/src/integrations/gpt/index.ts | 16 +- .../integrations/gpt/gpt_bootstrap.test.ts | 30 + .../gpt/schedule_initial_ad_init.test.ts | 63 + docs/guide/configuration.md | 116 + ...2026-08-08-1009-measurement-and-stage-0.md | 1044 +++ .../2026-08-08-1009-measurement-findings.md | 503 ++ .../2026-08-10-1009-esi-validation-spike.md | 967 +++ .../2026-08-12-1009-esi-merge-hardening.md | 294 + .../2026-08-14-1009-esi-parser-assembly.md | 104 + ...08-esi-cacheable-root-validation-design.md | 864 ++ ...11-1009-streaming-assembly-architecture.md | 260 + ...6-08-12-1009-esi-merge-hardening-design.md | 195 + ...6-08-14-1009-esi-parser-assembly-design.md | 153 + scripts/c2-local-test.sh | 607 ++ trusted-server.example.toml | 25 + 37 files changed, 15276 insertions(+), 443 deletions(-) create mode 100644 crates/trusted-server-adapter-fastly/src/esi_assembly.rs create mode 100644 crates/trusted-server-adapter-fastly/src/template_cache.rs create mode 100644 crates/trusted-server-core/src/platform/template_assembly.rs create mode 100644 crates/trusted-server-core/src/platform/template_cache.rs create mode 100644 docs/superpowers/plans/2026-08-08-1009-measurement-and-stage-0.md create mode 100644 docs/superpowers/plans/2026-08-08-1009-measurement-findings.md create mode 100644 docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md create mode 100644 docs/superpowers/plans/2026-08-12-1009-esi-merge-hardening.md create mode 100644 docs/superpowers/plans/2026-08-14-1009-esi-parser-assembly.md create mode 100644 docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md create mode 100644 docs/superpowers/specs/2026-08-11-1009-streaming-assembly-architecture.md create mode 100644 docs/superpowers/specs/2026-08-12-1009-esi-merge-hardening-design.md create mode 100644 docs/superpowers/specs/2026-08-14-1009-esi-parser-assembly-design.md create mode 100755 scripts/c2-local-test.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fb0233b03..4f0bb6cfb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,6 +27,11 @@ jobs: run: echo "viceroy-version=$(grep '^viceroy ' .tool-versions | awk '{print $2}')" >> $GITHUB_OUTPUT shell: bash + - name: Retrieve Node.js version + id: node-version + run: echo "node-version=$(grep '^nodejs ' .tool-versions | awk '{print $2}')" >> $GITHUB_OUTPUT + shell: bash + - name: Set up Rust toolchain uses: actions-rust-lang/setup-rust-toolchain@v1 with: @@ -45,9 +50,20 @@ jobs: if: steps.cache-viceroy.outputs.cache-hit != 'true' run: cargo install viceroy --version "${{ steps.viceroy-version.outputs.viceroy-version }}" --locked --force + - name: Use Node.js for the served-seam contract + uses: actions/setup-node@v4 + with: + node-version: ${{ steps.node-version.outputs.node-version }} + - name: Run tests run: cargo test-fastly + - name: Run C2 ESI local harness + run: ./scripts/c2-local-test.sh esi + + - name: Run inline control harness + run: ./scripts/c2-local-test.sh inline + test-axum: name: cargo test (axum native) runs-on: ubuntu-latest diff --git a/Cargo.lock b/Cargo.lock index ed1b9c7a3..5c4748b41 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -146,7 +146,7 @@ dependencies = [ "asn1-rs-derive", "asn1-rs-impl", "displaydoc", - "nom", + "nom 7.1.3", "num-traits", "rusticata-macros", "thiserror 1.0.69", @@ -254,6 +254,15 @@ dependencies = [ "tungstenite", ] +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -573,7 +582,18 @@ checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" dependencies = [ "cfg-if", "cipher", - "cpufeatures", + "cpufeatures 0.2.17", +] + +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", ] [[package]] @@ -583,7 +603,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" dependencies = [ "aead", - "chacha20", + "chacha20 0.9.1", "cipher", "poly1305", "zeroize", @@ -916,6 +936,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -1041,7 +1070,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "curve25519-dalek-derive", "digest 0.10.7", "fiat-crypto", @@ -1186,7 +1215,7 @@ checksum = "5cd0a5c643689626bec213c4d8bd4d96acc8ffdb4ad4bb6bc16abf27d5f4b553" dependencies = [ "asn1-rs", "displaydoc", - "nom", + "nom 7.1.3", "num-bigint", "num-traits", "rusticata-macros", @@ -1690,6 +1719,26 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "esi" +version = "0.7.1" +source = "git+https://github.com/stackpop/esi.git?rev=4c53feab4d22ad9a84641b4c46f3f63bc6d197e2#4c53feab4d22ad9a84641b4c46f3f63bc6d197e2" +dependencies = [ + "atoi", + "base64", + "bytes", + "chrono", + "fastly", + "html-escape", + "log", + "md5", + "nom 8.0.0", + "percent-encoding", + "rand 0.10.2", + "regex", + "thiserror 2.0.18", +] + [[package]] name = "etcetera" version = "0.10.0" @@ -2034,6 +2083,7 @@ dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", + "rand_core 0.10.1", ] [[package]] @@ -2188,6 +2238,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "html-escape" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9356095b4b41197bba32173600e1582792cda618f65d12f68e2e77d273413c5" + [[package]] name = "html5ever" version = "0.35.0" @@ -2914,6 +2970,12 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8863b587001c1b9a8a4e36008cebc6b3612cb1226fe2de94858e06092687b608" +[[package]] +name = "md5" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ebb8d8732c6a6df3d8f032a82911cfc747e00efb95cc46e8d0acd5b5b88570c" + [[package]] name = "memchr" version = "2.8.2" @@ -2984,6 +3046,15 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + [[package]] name = "num" version = "0.4.3" @@ -3477,7 +3548,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" dependencies = [ - "cpufeatures", + "cpufeatures 0.2.17", "opaque-debug", "universal-hash", ] @@ -3775,6 +3846,17 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20 0.10.1", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + [[package]] name = "rand_chacha" version = "0.3.1" @@ -3813,6 +3895,12 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "rcgen" version = "0.13.2" @@ -4079,7 +4167,7 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" dependencies = [ - "nom", + "nom 7.1.3", ] [[package]] @@ -4494,7 +4582,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest 0.10.7", ] @@ -4506,7 +4594,7 @@ checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" dependencies = [ "block-buffer 0.9.0", "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest 0.9.0", "opaque-debug", ] @@ -4518,7 +4606,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest 0.10.7", ] @@ -5286,9 +5374,11 @@ dependencies = [ "base64", "bytes", "chrono", + "derive_more", "edgezero-adapter-fastly", "edgezero-core", "error-stack", + "esi", "fastly", "fern", "futures", @@ -5386,6 +5476,7 @@ dependencies = [ "hex", "hmac", "http", + "httpdate", "iab_gpp", "jose-jwk", "log", @@ -6338,7 +6429,7 @@ dependencies = [ "data-encoding", "der-parser", "lazy_static", - "nom", + "nom 7.1.3", "oid-registry", "ring", "rusticata-macros", diff --git a/Cargo.toml b/Cargo.toml index a5a63ca3a..ab5638f5b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -71,6 +71,7 @@ getrandom = "0.2" hex = "0.4.3" hmac = "0.12.1" http = "1.4.0" +httpdate = "1.0.3" http-body-util = "0.1" hyper = "1" hyper-util = "0.1" diff --git a/crates/trusted-server-adapter-fastly/Cargo.toml b/crates/trusted-server-adapter-fastly/Cargo.toml index b6bc0f1a1..3d42ae388 100644 --- a/crates/trusted-server-adapter-fastly/Cargo.toml +++ b/crates/trusted-server-adapter-fastly/Cargo.toml @@ -15,9 +15,11 @@ async-trait = { workspace = true } base64 = { workspace = true } bytes = { workspace = true } chrono = { workspace = true } +derive_more = { workspace = true } edgezero-adapter-fastly = { workspace = true, features = ["fastly"] } edgezero-core = { workspace = true } error-stack = { workspace = true } +esi = { git = "https://github.com/stackpop/esi.git", rev = "4c53feab4d22ad9a84641b4c46f3f63bc6d197e2" } fastly = { workspace = true } fern = { workspace = true } futures = { workspace = true } diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index d6090c983..6be93b4b3 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -257,6 +257,11 @@ fn build_per_request_services(state: &AppState, ctx: &RequestContext) -> Runtime .config_store(Arc::new(FastlyPlatformConfigStore)) .secret_store(Arc::new(FastlyPlatformSecretStore)) .kv_store(Arc::clone(&state.default_kv_store)) + // Spike-only (#1009). Constructed unconditionally, but only read when the + // assembly mode is a shared-template one — which defaults to Inline, so this + // is inert until an operator opts in. + .template_cache(Arc::new(crate::template_cache::FastlyTemplateCache::new())) + .template_assembler(Arc::new(crate::esi_assembly::FastlyTemplateAssembler)) .backend(Arc::new(FastlyPlatformBackend)) .http_client(Arc::new(FastlyPlatformHttpClient)) .geo(Arc::new(FastlyPlatformGeo)) @@ -1239,12 +1244,15 @@ mod tests { use super::{ AppState, NAMED_ROUTES, NamedRouteHandler, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, - TrustedServerApp, build_state_from_settings, startup_error_router, + TrustedServerApp, build_per_request_services, build_state_from_settings, + startup_error_router, }; use bytes::Bytes; use edgezero_core::body::Body; + use edgezero_core::context::RequestContext; use edgezero_core::http::{Method, Response, StatusCode, header, request_builder}; use edgezero_core::key_value_store::NoopKvStore; + use edgezero_core::params::PathParams; use edgezero_core::router::RouterService; use std::net::{IpAddr, Ipv4Addr}; use std::sync::Mutex; @@ -1379,6 +1387,36 @@ mod tests { TrustedServerApp::routes_for_state(&state) } + #[test] + fn per_request_services_register_the_fastly_template_assembler() { + let state = build_state_from_settings(test_settings()).expect("should build test state"); + let context = RequestContext::new( + empty_request(Method::GET, "/article"), + PathParams::default(), + ); + + let services = build_per_request_services(&state, &context); + let template = format!( + "article{}", + trusted_server_core::publisher::AD_ASSEMBLY_SEAM + ); + let fragment = b""; + let assembled = services + .template_assembler() + .assemble(template.as_bytes(), fragment) + .expect("Fastly services should provide ESI assembly"); + + assert_eq!( + assembled, + template + .replace( + trusted_server_core::publisher::AD_ASSEMBLY_SEAM, + std::str::from_utf8(fragment).expect("fragment should be UTF-8") + ) + .into_bytes() + ); + } + /// Builds a router whose `AppState` uses a registry containing the given /// request filters (and no routes), so dispatch-level request-filter /// behavior can be exercised without a real integration. diff --git a/crates/trusted-server-adapter-fastly/src/esi_assembly.rs b/crates/trusted-server-adapter-fastly/src/esi_assembly.rs new file mode 100644 index 000000000..4a92c6f78 --- /dev/null +++ b/crates/trusted-server-adapter-fastly/src/esi_assembly.rs @@ -0,0 +1,287 @@ +//! Fastly cold-response assembly backed by the repaired `stackpop/esi` parser. +//! +//! C2 stores an inert marker. This module creates one synthetic ESI include only in a +//! request-private working copy, resolves it from an already-built fragment, and never +//! performs an HTTP request. + +use std::io::Cursor; + +use esi::{CacheConfig, Configuration, DcaMode, PendingFragmentContent, Processor}; +use fastly::http::StatusCode; +use fastly::{Request, Response}; +use trusted_server_core::platform::{PlatformTemplateAssembler, TemplateAssemblyError}; +use trusted_server_core::publisher::AD_ASSEMBLY_SEAM; + +const INTERNAL_FRAGMENT_PATH: &str = "/_ts/internal/reader-ad-state"; +const SYNTHETIC_ESI_INCLUDE: &[u8] = b""; + +/// Why the Fastly ESI adapter refused or failed to assemble a document. +#[derive(Debug, derive_more::Display)] +enum EsiAssemblyError { + /// The inert seam marker was missing or repeated. + #[display("expected exactly one inert seam marker, found {count}")] + InvalidMarkerCount { count: usize }, + /// Publisher bytes contained ESI instructions outside TS's synthetic seam. + #[display("publisher-authored ESI directives are not allowed")] + PublisherEsiDirective, + /// The parser dispatched a URL other than TS's one synthetic fragment. + #[display("unexpected fragment request path `{path}` (query present: {has_query})")] + UnexpectedFragmentRequest { path: String, has_query: bool }, + /// The pinned parser could not process the document. + #[display("ESI processing failed: {message}")] + Processing { message: String }, + /// The parser changed bytes outside the one synthetic include. + #[display("ESI output was not an exact seam substitution")] + OutputMismatch, +} + +impl core::error::Error for EsiAssemblyError {} + +/// ESI configuration with every cache- and recursion-sensitive option explicit. +fn assembly_configuration() -> Configuration { + Configuration::default() + .with_escaped(false) + .with_default_dca(DcaMode::None) + .with_inherit_parent_dca(false) + .with_max_include_depth(1) + .with_edge_control(false) + .with_caching(CacheConfig { + is_includes_cacheable: false, + includes_default_ttl: None, + includes_force_ttl: None, + is_rendered_cacheable: false, + rendered_cache_control: false, + rendered_ttl: None, + }) +} + +fn contains_esi_directive(bytes: &[u8]) -> bool { + bytes + .windows(b" Result<(Vec, usize), EsiAssemblyError> { + let marker = AD_ASSEMBLY_SEAM.as_bytes(); + let positions = template + .windows(marker.len()) + .enumerate() + .filter_map(|(at, window)| (window == marker).then_some(at)) + .collect::>(); + if positions.len() != 1 { + return Err(EsiAssemblyError::InvalidMarkerCount { + count: positions.len(), + }); + } + if contains_esi_directive(template) { + return Err(EsiAssemblyError::PublisherEsiDirective); + } + + let at = positions[0]; + let mut working = + Vec::with_capacity(template.len() - marker.len() + SYNTHETIC_ESI_INCLUDE.len()); + working.extend_from_slice(&template[..at]); + working.extend_from_slice(SYNTHETIC_ESI_INCLUDE); + working.extend_from_slice(&template[at + marker.len()..]); + Ok((working, at)) +} + +fn completed_fragment_response( + request: &Request, + fragment: &[u8], +) -> Result { + let path = request.get_path().to_string(); + let has_query = request.get_url().query().is_some(); + if path != INTERNAL_FRAGMENT_PATH || has_query { + return Err(EsiAssemblyError::UnexpectedFragmentRequest { path, has_query }); + } + + Ok(PendingFragmentContent::CompletedRequest(Box::new( + Response::from_status(StatusCode::OK) + .with_header( + fastly::http::header::CONTENT_TYPE, + "text/html; charset=utf-8", + ) + .with_body(fragment.to_vec()), + ))) +} + +fn assemble_with_observer( + template: &[u8], + fragment: &[u8], + on_dispatch: F, +) -> Result, EsiAssemblyError> +where + F: Fn() + 'static, +{ + let (working, seam_at) = template_with_synthetic_include(template)?; + let fragment_len = fragment.len(); + let fragment_response = fragment.to_vec(); + let dispatcher = move |request, _index| { + on_dispatch(); + completed_fragment_response(&request, &fragment_response) + .map_err(|error| esi::ESIError::FragmentRequestError(error.to_string())) + }; + let mut processor = Processor::new(None, assembly_configuration()); + let mut output = Vec::with_capacity(template.len() + fragment_len); + processor + .process_stream(Cursor::new(working), &mut output, Some(&dispatcher), None) + .map_err(|error| EsiAssemblyError::Processing { + message: error.to_string(), + })?; + let expected_len = template.len() - AD_ASSEMBLY_SEAM.len() + fragment_len; + let output_tail_at = seam_at + fragment_len; + let template_tail_at = seam_at + AD_ASSEMBLY_SEAM.len(); + if output.len() != expected_len + || output[..seam_at] != template[..seam_at] + || &output[seam_at..output_tail_at] != fragment + || output[output_tail_at..] != template[template_tail_at..] + { + return Err(EsiAssemblyError::OutputMismatch); + } + Ok(output) +} + +fn assemble(template: &[u8], fragment: &[u8]) -> Result, EsiAssemblyError> { + assemble_with_observer(template, fragment, || {}) +} + +/// Fastly implementation of the core cold-response assembly boundary. +pub struct FastlyTemplateAssembler; + +impl PlatformTemplateAssembler for FastlyTemplateAssembler { + fn assemble(&self, template: &[u8], fragment: &[u8]) -> Result, TemplateAssemblyError> { + assemble(template, fragment).map_err(|error| TemplateAssemblyError::Failed { + message: error.to_string(), + }) + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + + use super::*; + use trusted_server_core::publisher::AD_ASSEMBLY_SEAM; + + const FRAGMENT: &[u8] = b""; + + fn template(body: &str) -> Vec { + format!("{body}{AD_ASSEMBLY_SEAM}").into_bytes() + } + + #[test] + fn a_script_larger_than_the_parser_chunk_survives_exactly() { + let script = format!( + "", + "x".repeat(120_000) + ); + let document = template(&script); + let dispatches = Arc::new(AtomicUsize::new(0)); + let observed_dispatches = Arc::clone(&dispatches); + + let assembled = assemble_with_observer(&document, FRAGMENT, move || { + observed_dispatches.fetch_add(1, Ordering::Relaxed); + }) + .expect("should assemble a document with a large script"); + + let seam_at = document + .windows(AD_ASSEMBLY_SEAM.len()) + .position(|window| window == AD_ASSEMBLY_SEAM.as_bytes()) + .expect("should find seam"); + let mut expected = Vec::new(); + expected.extend_from_slice(&document[..seam_at]); + expected.extend_from_slice(FRAGMENT); + expected.extend_from_slice(&document[seam_at + AD_ASSEMBLY_SEAM.len()..]); + + assert_eq!( + assembled, expected, + "ESI must alter only the synthetic seam" + ); + assert_eq!(dispatches.load(Ordering::Relaxed), 1); + } + + #[test] + fn missing_and_repeated_markers_are_rejected_before_parsing() { + let missing = assemble(b"plain", FRAGMENT) + .expect_err("should reject a missing marker"); + let repeated = assemble( + format!("{AD_ASSEMBLY_SEAM}{AD_ASSEMBLY_SEAM}").as_bytes(), + FRAGMENT, + ) + .expect_err("should reject repeated markers"); + + assert!(matches!( + missing, + EsiAssemblyError::InvalidMarkerCount { count: 0 } + )); + assert!(matches!( + repeated, + EsiAssemblyError::InvalidMarkerCount { count: 2 } + )); + } + + #[test] + fn every_publisher_esi_directive_form_is_rejected_case_insensitively() { + for directive in [ + "", + "secret", + "x", + "$(HTTP_HOST)", + "text", + "", + ] { + let error = assemble(&template(directive), FRAGMENT) + .expect_err("should reject publisher-authored ESI"); + + assert!(matches!(error, EsiAssemblyError::PublisherEsiDirective)); + } + } + + #[test] + fn fragment_esi_is_emitted_verbatim_and_never_reparsed() { + let fragment = b""; + + let assembled = assemble(&template("article"), fragment).expect("should assemble"); + + assert!( + assembled + .windows(fragment.len()) + .any(|window| window == fragment), + "fragment bytes must remain data" + ); + } + + #[test] + fn dispatcher_rejects_every_url_except_the_synthetic_internal_one() { + let unexpected = fastly::Request::get("https://example.com/not-the-seam"); + let with_query = + fastly::Request::get("https://example.com/_ts/internal/reader-ad-state?publisher=1"); + + assert!(matches!( + completed_fragment_response(&unexpected, FRAGMENT), + Err(EsiAssemblyError::UnexpectedFragmentRequest { .. }) + )); + assert!(matches!( + completed_fragment_response(&with_query, FRAGMENT), + Err(EsiAssemblyError::UnexpectedFragmentRequest { .. }) + )); + } + + #[test] + fn configuration_cannot_cache_or_reparse_reader_state() { + let configuration = assembly_configuration(); + + assert!(!configuration.cache.is_includes_cacheable); + assert!(configuration.cache.includes_default_ttl.is_none()); + assert!(configuration.cache.includes_force_ttl.is_none()); + assert!(!configuration.cache.is_rendered_cacheable); + assert!(!configuration.cache.rendered_cache_control); + assert!(configuration.cache.rendered_ttl.is_none()); + assert_eq!(configuration.default_dca, DcaMode::None); + assert!(!configuration.inherit_parent_dca); + assert_eq!(configuration.max_include_depth, 1); + assert!(!configuration.enable_edge_control); + } +} diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 39d35b198..07c042ea8 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -29,11 +29,13 @@ mod app; mod backend; mod compat; mod ec_kv; +mod esi_assembly; mod logging; mod management_api; mod middleware; mod platform; mod rate_limiter; +mod template_cache; mod tinybird; use crate::app::{EcFinalizeState, TrustedServerApp, load_settings_from_config_store}; @@ -328,14 +330,7 @@ fn send_edgezero_response( mut response: HttpResponse, request_filter_effects: Option<&RequestFilterEffects>, ) { - if let Some(effects) = request_filter_effects { - effects.apply_to_response(&mut response); - } - - // Final cache guard: EC finalization and request-filter effects may have - // added a per-user Set-Cookie after `apply_finalize_headers` ran, so - // re-apply the privacy downgrade before send. - crate::middleware::enforce_set_cookie_cache_privacy(&mut response); + apply_terminal_response_effects(&mut response, request_filter_effects); let (parts, body) = response.into_parts(); @@ -364,6 +359,26 @@ fn send_edgezero_response( } } +/// Apply every late response mutation, then restore privacy invariants before headers commit. +fn apply_terminal_response_effects( + response: &mut HttpResponse, + request_filter_effects: Option<&RequestFilterEffects>, +) { + let must_remain_private = + trusted_server_core::response_privacy::is_private_or_no_store(response.headers()); + if let Some(effects) = request_filter_effects { + effects.apply_to_response(response); + } + if must_remain_private { + trusted_server_core::response_privacy::enforce_private_no_store(response); + } + + // Final cache guard: EC finalization and request-filter effects may have + // added a per-user Set-Cookie after `apply_finalize_headers` ran, so + // re-apply the privacy downgrade before send. + crate::middleware::enforce_set_cookie_cache_privacy(response); +} + const FALLBACK_UNAVAILABLE: &str = "unavailable"; const FALLBACK_NOT_SENT: &str = "not sent"; const FALLBACK_NONE: &str = "none"; @@ -485,6 +500,7 @@ mod tests { use edgezero_core::http::HeaderValue; use edgezero_core::http::response_builder; use fastly::mime; + use trusted_server_core::integrations::HeaderMutation; fn test_settings() -> Settings { Settings::from_toml( @@ -557,6 +573,36 @@ mod tests { ); } + #[test] + fn late_filter_effects_cannot_make_an_assembled_response_public() { + let mut response = response_builder() + .header("cache-control", "private, no-store") + .header("etag", "\"reader-document\"") + .body(EdgeBody::empty()) + .expect("should build response"); + let effects = RequestFilterEffects { + request_headers: Vec::new(), + response_headers: vec![ + HeaderMutation::set("cache-control", "public, s-maxage=3600"), + HeaderMutation::set("surrogate-control", "max-age=3600"), + HeaderMutation::set("cdn-cache-control", "public, max-age=3600"), + ], + }; + + apply_terminal_response_effects(&mut response, Some(&effects)); + + assert_eq!( + response + .headers() + .get("cache-control") + .and_then(|value| value.to_str().ok()), + Some("private, no-store") + ); + assert!(response.headers().get("surrogate-control").is_none()); + assert!(response.headers().get("cdn-cache-control").is_none()); + assert!(response.headers().get("etag").is_none()); + } + #[test] #[allow(clippy::panic)] fn entry_point_finalize_skips_geo_lookup_for_401() { diff --git a/crates/trusted-server-adapter-fastly/src/template_cache.rs b/crates/trusted-server-adapter-fastly/src/template_cache.rs new file mode 100644 index 000000000..3f168f574 --- /dev/null +++ b/crates/trusted-server-adapter-fastly/src/template_cache.rs @@ -0,0 +1,505 @@ +//! Fastly Core Cache backing for the shared transformed-template cache (C2). +//! +//! Only the Fastly adapter implements this; every other adapter uses +//! `UnavailableTemplateCache`, so the ESI assembly mode stays portable and only +//! the caching is Fastly-only. +//! +//! **Why Core Cache and not read-through caching.** Read-through with `after_send` + +//! `set_body_transform` looks like a better fit — it keeps HTTP semantics and derives +//! TTL and surrogate keys from origin headers for free. It is unreachable here: +//! Viceroy 0.17 stubs the entire HTTP Cache ABI and the SDK converts that into a +//! *send error*, so setting `after_send` makes every publisher origin fetch fail +//! under `fastly compute serve`, `cargo test-fastly` and the parity suite. It is also +//! silently dead whenever the origin request is in pass mode, and its closure bounds +//! (`Fn + Send + Sync`) are incompatible with a platform layer that is `!Send` by +//! construction. Recorded in the spike plan's Task 3 Step 4 so nobody re-proposes it. +//! +//! Spike-only. Remove with the spike. + +use fastly::cache::core::{CacheKey, Found, Transaction}; +use std::io::Write as _; +use std::time::Duration; +use trusted_server_core::platform::{ + PlatformTemplateCache, PlatformTemplateCacheReservation, TemplateCacheError, TemplateCacheKey, + TemplateCacheLookup, TemplateCacheMiss, TemplateCacheReservation, TemplateEntry, + TemplateMetadata, +}; + +/// Surrogate key attached to every stored template, so a single purge clears them +/// all. This is the rollback lever: without it, backing out a bad template means +/// waiting for the TTL. +const PURGE_ALL_SURROGATE_KEY: &str = "ts-template"; + +/// Fastly Core Cache implementation of the C2 template cache. +#[derive(Default)] +pub struct FastlyTemplateCache; + +impl FastlyTemplateCache { + /// Create the Fastly Core Cache implementation. + /// + /// Entry lifetime is supplied per insert after core validates origin freshness + /// and applies the operator's configured safety ceiling. + #[must_use] + pub const fn new() -> Self { + Self + } +} + +fn backend_error(message: impl Into) -> TemplateCacheError { + TemplateCacheError::Backend { + message: message.into(), + } +} + +enum ReadFoundError { + Invalid(TemplateCacheMiss), + Backend(TemplateCacheError), +} + +fn read_found(found: &Found, key: &TemplateCacheKey) -> Result { + if found.is_stale() { + return Err(ReadFoundError::Invalid(TemplateCacheMiss::NotFound)); + } + + let metadata = TemplateMetadata::decode(&found.user_metadata()).ok_or( + ReadFoundError::Invalid(TemplateCacheMiss::UnreadableMetadata), + )?; + if metadata.schema_version != key.schema_version { + return Err(ReadFoundError::Invalid(TemplateCacheMiss::SchemaMismatch)); + } + if found + .known_length() + .is_some_and(|length| length != metadata.body_len) + { + return Err(ReadFoundError::Invalid(TemplateCacheMiss::Truncated)); + } + + let body = found + .to_stream() + .map_err(|error| { + ReadFoundError::Backend(backend_error(format!( + "opening cached template body failed: {error:?}" + ))) + })? + .into_bytes(); + if body.len() as u64 != metadata.body_len { + return Err(ReadFoundError::Invalid(TemplateCacheMiss::Truncated)); + } + Ok(TemplateEntry { metadata, body }) +} + +struct FastlyTemplateReservation { + transaction: Transaction, + surrogate_keys: Vec, +} + +impl PlatformTemplateCacheReservation for FastlyTemplateReservation { + fn insert( + self: Box, + metadata: &TemplateMetadata, + body: Vec, + max_age: Duration, + ) -> Result<(), TemplateCacheError> { + if metadata.body_len != body.len() as u64 { + return Err(backend_error(format!( + "metadata body_len {} does not match the {} bytes supplied", + metadata.body_len, + body.len() + ))); + } + + let mut writer = self + .transaction + .insert(max_age) + .surrogate_keys(self.surrogate_keys.iter().map(String::as_str)) + .known_length(body.len() as u64) + .user_metadata(metadata.encode().into()) + .execute() + .map_err(|e| backend_error(format!("cache insert failed: {e:?}")))?; + writer + .write_all(&body) + .map_err(|e| backend_error(format!("writing template body failed: {e}")))?; + writer + .finish() + .map_err(|e| backend_error(format!("finishing the cached template failed: {e}")))?; + Ok(()) + } + + fn cancel(self: Box) -> Result<(), TemplateCacheError> { + self.transaction + .cancel_insert_or_update() + .map_err(|e| backend_error(format!("cancelling cache reservation failed: {e:?}"))) + } +} + +#[async_trait::async_trait(?Send)] +impl PlatformTemplateCache for FastlyTemplateCache { + async fn lookup_or_reserve( + &self, + key: &TemplateCacheKey, + ) -> Result { + let transaction = Transaction::lookup(CacheKey::from(key.to_cache_key().into_bytes())) + .execute() + .map_err(|e| backend_error(format!("transactional lookup failed: {e:?}")))?; + + if transaction.must_insert_or_update() { + return Ok(TemplateCacheLookup::Reserved( + TemplateCacheReservation::new(Box::new(FastlyTemplateReservation { + transaction, + surrogate_keys: key.surrogate_keys(), + })), + )); + } + + let found = transaction.found().ok_or_else(|| { + backend_error("transaction returned neither a hit nor an insert obligation") + })?; + Ok(match read_found(&found, key) { + Ok(entry) => TemplateCacheLookup::Hit(entry), + Err(ReadFoundError::Invalid(miss)) => TemplateCacheLookup::Invalid(miss), + Err(ReadFoundError::Backend(error)) => return Err(error), + }) + } + + async fn get(&self, key: &TemplateCacheKey) -> Result { + let cache_key = CacheKey::from(key.to_cache_key().into_bytes()); + + // A plain lookup, not a transaction: a read that does not intend to insert + // must not take an insert obligation it will never discharge, which would + // block every other client waiting on the same key until they time out. + let found = fastly::cache::core::lookup(cache_key) + .execute() + .map_err(|_| TemplateCacheMiss::NotFound)? + .ok_or(TemplateCacheMiss::NotFound)?; + + read_found(&found, key).map_err(|error| match error { + ReadFoundError::Invalid(miss) => miss, + ReadFoundError::Backend(error) => { + // This legacy method cannot expose a backend error. Production uses + // `lookup_or_reserve`, which preserves it for bounded diagnostics. + log::warn!("c2_template_cache legacy read failed: {error}"); + TemplateCacheMiss::NotFound + } + }) + } + + async fn put( + &self, + key: &TemplateCacheKey, + metadata: &TemplateMetadata, + body: Vec, + max_age: Duration, + ) -> Result<(), TemplateCacheError> { + if metadata.body_len != body.len() as u64 { + return Err(backend_error(format!( + "metadata body_len {} does not match the {} bytes supplied; storing \ + this would make every read a truncation miss", + metadata.body_len, + body.len() + ))); + } + + let cache_key = CacheKey::from(key.to_cache_key().into_bytes()); + + // Transactional insert so a cold key under load transforms once rather than + // once per concurrent request. + let tx = Transaction::lookup(cache_key) + .execute() + .map_err(|e| backend_error(format!("transactional lookup failed: {e:?}")))?; + + // Order matters. A STALE entry sets *both* `found()` and + // `must_insert_or_update()`. Testing `found()` first would return early on + // the stale bytes and never discharge the obligation, leaving every + // concurrent waiter blocked until timeout. + if !tx.must_insert_or_update() { + // Someone else already inserted a fresh entry. Nothing to do, and + // nothing to discharge. + return Ok(()); + } + + // `Transaction::insert` takes `self`, so from here there is no handle left to + // cancel the insert with. A write that fails part-way therefore cannot be + // retracted — which is why `TemplateMetadata::body_len` exists and `get` + // checks it. The metadata is written before the body, so a truncated entry + // still carries the length it was supposed to have. + let surrogate_keys = key.surrogate_keys(); + let mut writer = tx + .insert(max_age) + .surrogate_keys(surrogate_keys.iter().map(String::as_str)) + .user_metadata(metadata.encode().into()) + .execute() + .map_err(|e| backend_error(format!("cache insert failed: {e:?}")))?; + + if let Err(e) = writer.write_all(&body) { + // Deliberately not calling `finish()`. An unfinished entry has no known + // length, and even if it is observable, `get`'s length check rejects it. + return Err(backend_error(format!("writing template body failed: {e}"))); + } + + // Required. Without it the object never completes and its length stays + // unknown, so readers see a partial or absent entry. + writer + .finish() + .map_err(|e| backend_error(format!("finishing the cached template failed: {e}")))?; + + Ok(()) + } + + async fn purge_url(&self, key: &TemplateCacheKey) -> Result<(), TemplateCacheError> { + fastly::http::purge::purge_surrogate_key(&key.url_surrogate_key()) + .map_err(|e| backend_error(format!("purging invalid template failed: {e:?}"))) + } + + async fn purge_all(&self) -> Result<(), TemplateCacheError> { + fastly::http::purge::purge_surrogate_key(PURGE_ALL_SURROGATE_KEY) + .map_err(|e| backend_error(format!("purging templates failed: {e:?}"))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use trusted_server_core::creative_opportunities::AssemblyMode; + use trusted_server_core::platform::TEMPLATE_SCHEMA_VERSION; + + /// Distinct per test, so tests sharing the process cache cannot collide. + fn key(url: &str) -> TemplateCacheKey { + TemplateCacheKey { + url: url.to_string(), + request_host: "example.com".to_string(), + request_scheme: "https".to_string(), + origin_identity: "https://origin.example.com\0origin.example.com".to_string(), + assembly_mode: AssemblyMode::Esi, + vary_values: vec![trusted_server_core::platform::VaryHeaderValues { + name: "rsc".to_string(), + values: Some(vec![b"1".to_vec()]), + }], + template_fingerprint: "fp".to_string(), + schema_version: TEMPLATE_SCHEMA_VERSION, + } + } + + fn metadata_for(body: &[u8]) -> TemplateMetadata { + TemplateMetadata { + policy_headers: Vec::new(), + content_encoding: "identity".to_string(), + content_type: "text/html; charset=utf-8".to_string(), + schema_version: TEMPLATE_SCHEMA_VERSION, + body_len: body.len() as u64, + } + } + + /// The trait is `async_trait(?Send)` and this crate has no async test runtime, + /// so drive the futures directly. + fn run(fut: impl core::future::Future) -> T { + futures::executor::block_on(fut) + } + + fn cache() -> FastlyTemplateCache { + FastlyTemplateCache::new() + } + + #[test] + fn a_stored_template_reads_back_intact() { + let cache = cache(); + let key = key("https://example.com/roundtrip"); + let body = b"template".to_vec(); + let metadata = metadata_for(&body); + + run(cache.put(&key, &metadata, body.clone(), Duration::from_secs(60))) + .expect("should store"); + + let entry = run(cache.get(&key)).expect("should read back"); + assert_eq!(entry.body, body, "bytes must survive the round trip"); + assert_eq!(entry.metadata, metadata, "metadata must survive too"); + } + + #[test] + fn transactional_lookup_reserves_before_insert_then_hits() { + let cache = cache(); + let key = key("https://example.com/pre-origin-reservation"); + let body = b"collapsed".to_vec(); + let metadata = metadata_for(&body); + + let reservation = match run(cache.lookup_or_reserve(&key)).expect("lookup should work") { + TemplateCacheLookup::Reserved(reservation) => reservation, + _ => panic!("a cold transactional lookup must assign the insert obligation"), + }; + reservation + .insert(&metadata, body.clone(), Duration::from_secs(17)) + .expect("reservation should insert"); + + match run(cache.lookup_or_reserve(&key)).expect("warm lookup should work") { + TemplateCacheLookup::Hit(entry) => assert_eq!(entry.body, body), + _ => panic!("the next transactional lookup must see the inserted template"), + } + } + + #[test] + fn an_absent_key_is_a_miss_not_an_error() { + let miss = + run(cache().get(&key("https://example.com/never-stored"))).expect_err("should miss"); + assert_eq!(miss, TemplateCacheMiss::NotFound); + } + + #[test] + fn a_different_assembly_mode_does_not_read_the_same_entry() { + // The arms emit different bytes. If they shared an entry, one would serve + // the other's template. + let cache = cache(); + let esi = key("https://example.com/mode-split"); + let mut inline = esi.clone(); + inline.assembly_mode = AssemblyMode::Inline; + + let body = b"esi-template".to_vec(); + run(cache.put(&esi, &metadata_for(&body), body, Duration::from_secs(60))) + .expect("should store"); + + assert_eq!( + run(cache.get(&inline)).err(), + Some(TemplateCacheMiss::NotFound), + "inline must not read the ESI arm's template" + ); + } + + #[test] + fn a_schema_bump_reads_a_miss_rather_than_a_stale_shape() { + let cache = cache(); + let key_v1 = key("https://example.com/schema"); + let body = b"old-shape".to_vec(); + run(cache.put(&key_v1, &metadata_for(&body), body, Duration::from_secs(60))) + .expect("should store"); + + // A deploy that changes the transform bumps the constant. The old entry must + // not be assembled against. + let mut key_v2 = key_v1.clone(); + key_v2.schema_version = TEMPLATE_SCHEMA_VERSION + 1; + + assert_eq!( + run(cache.get(&key_v2)).err(), + Some(TemplateCacheMiss::NotFound), + "a bumped schema changes the key, so the old entry is simply not found" + ); + } + + #[test] + fn a_stale_but_present_entry_reads_as_a_miss_rather_than_being_served() { + // Stale-while-revalidate is a real option and deliberately not taken: it is a + // state machine `cache::core` does not implement for you, and serving stale here + // means serving a template built by an older transform or an older JS bundle. + // + // The entry has to be *present and stale*, not merely expired. A zero TTL with no + // `stale_while_revalidate` window is simply absent, so a test written that way + // passes without ever reaching `is_stale()` — verified: reverting the staleness + // check left that version green. The revalidate window is what keeps the object + // readable while stale, so this actually exercises the branch. + let key = key("https://example.com/stale"); + let body = b"stale-template".to_vec(); + let metadata = metadata_for(&body); + let cache_key = CacheKey::from(key.to_cache_key().into_bytes()); + + let mut writer = fastly::cache::core::insert(cache_key, Duration::from_secs(0)) + .stale_while_revalidate(Duration::from_secs(60)) + .user_metadata(metadata.encode().into()) + .execute() + .expect("should begin insert"); + writer.write_all(&body).expect("should write body"); + writer.finish().expect("should finish insert"); + + let miss = run(cache().get(&key)).expect_err("a stale template must not be served"); + assert_eq!(miss, TemplateCacheMiss::NotFound); + } + + #[test] + fn purge_all_clears_stored_templates() { + // The rollback lever. Without this, backing out a bad template means waiting + // for the TTL. + let cache = cache(); + let key = key("https://example.com/purge"); + let body = b"template".to_vec(); + run(cache.put(&key, &metadata_for(&body), body, Duration::from_secs(60))) + .expect("should store"); + run(cache.get(&key)).expect("should be present before purge"); + + run(cache.purge_all()).expect("should purge"); + + assert!( + run(cache.get(&key)).is_err(), + "purge must clear the template, or rollback is TTL-bound" + ); + } + + #[test] + fn a_second_put_on_a_fresh_entry_is_a_no_op() { + // Exercises the `must_insert_or_update` early return: a concurrent writer + // that finds a fresh entry must neither error nor overwrite. + let cache = cache(); + let key = key("https://example.com/second-put"); + let first = b"first".to_vec(); + run(cache.put( + &key, + &metadata_for(&first), + first.clone(), + Duration::from_secs(60), + )) + .expect("first put stores"); + + let second = b"second".to_vec(); + run(cache.put( + &key, + &metadata_for(&second), + second, + Duration::from_secs(60), + )) + .expect("second put should be a no-op, not an error"); + + assert_eq!( + run(cache.get(&key)).expect("should read").body, + first, + "a fresh entry must not be overwritten by a racing writer" + ); + } + + #[test] + fn the_cache_round_trips_through_the_platform_trait_object() { + // Every other test here calls `FastlyTemplateCache` concretely. The publisher + // never does — it reaches the cache as a `dyn PlatformTemplateCache` behind + // `RuntimeServices`. That join is what `app.rs` wires, and until this test it + // was only type-checked, never executed. + let cache: std::sync::Arc = std::sync::Arc::new(cache()); + let key = key("https://example.com/via-trait-object"); + let body = b"template".to_vec(); + + run(cache.put( + &key, + &metadata_for(&body), + body.clone(), + Duration::from_secs(60), + )) + .expect("should store"); + + assert_eq!( + run(cache.get(&key)).expect("should read back").body, + body, + "the trait object must reach the same Core Cache the concrete type does" + ); + } + + #[test] + fn a_length_mismatch_is_refused_at_write_rather_than_stored() { + // Storing metadata whose length disagrees with the body would make every + // subsequent read a truncation miss — a cache that silently never hits. + // Catch it at the write instead. + let cache = cache(); + let key = key("https://example.com/length-mismatch"); + let mut metadata = metadata_for(b"12345"); + metadata.body_len = 999; + + let err = run(cache.put(&key, &metadata, b"12345".to_vec(), Duration::from_secs(60))) + .expect_err("a length mismatch must be refused"); + assert!( + matches!(err, TemplateCacheError::Backend { .. }), + "expected a backend error, got {err:?}" + ); + } +} diff --git a/crates/trusted-server-core/Cargo.toml b/crates/trusted-server-core/Cargo.toml index c035799e9..e44d46f77 100644 --- a/crates/trusted-server-core/Cargo.toml +++ b/crates/trusted-server-core/Cargo.toml @@ -29,6 +29,7 @@ glob = { workspace = true } hex = { workspace = true } hmac = { workspace = true } http = { workspace = true } +httpdate = { workspace = true } iab_gpp = { workspace = true } jose-jwk = { workspace = true } log = { workspace = true } diff --git a/crates/trusted-server-core/benches/html_processor_bench.rs b/crates/trusted-server-core/benches/html_processor_bench.rs index 7c1303dd4..de968301c 100644 --- a/crates/trusted-server-core/benches/html_processor_bench.rs +++ b/crates/trusted-server-core/benches/html_processor_bench.rs @@ -1,5 +1,7 @@ use criterion::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main}; -use trusted_server_core::html_processor::{HtmlProcessorConfig, create_html_processor}; +use trusted_server_core::html_processor::{ + BodyCloseInjection, HtmlProcessorConfig, create_html_processor, +}; use trusted_server_core::integrations::IntegrationRegistry; use trusted_server_core::streaming_processor::StreamProcessor as _; @@ -13,6 +15,10 @@ fn make_config() -> HtmlProcessorConfig { ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, gpt_diagnostics: None, + // The benchmark measures URL rewriting, not ad injection, and + // `ad_slots_script` is `None` here — matching the previous behaviour, + // which inferred no body-close work from that. + body_close: BodyCloseInjection::None, suppress_datadome_client_side_tag: false, } } diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index cd11e1c14..fca317440 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -16,6 +16,8 @@ use crate::settings::vec_from_seq_or_map; const MAX_DYNAMIC_GAM_UNIT_PATH_BYTES: usize = 100; const MAX_SECTION_BYTES: usize = 100; +const DEFAULT_TEMPLATE_CACHE_MAX_AGE_SECONDS: u32 = 60; +const MAX_TEMPLATE_CACHE_MAX_AGE_SECONDS: u32 = 86_400; /// A single parsed segment of a [`gam_unit_path`](CreativeOpportunitySlot::gam_unit_path) template. #[derive(Debug, Clone)] @@ -191,6 +193,36 @@ const fn is_default_enabled(value: &bool) -> bool { *value == default_enabled() } +/// How per-user ad state reaches the page. +/// +/// `Inline` is the shipped behaviour: the auction result is injected before +/// `` and the root document is therefore uncacheable. `Esi` stores a +/// request-neutral shared template and fills its per-request byte seam at the edge. +/// +/// Spike-only, for the #1009 ESI validation. Remove with the spike. +/// +/// # Why the template must be request-neutral +/// +/// Under `Esi` the template is shared across visitors, so +/// nothing whose *presence* depends on the request may appear in it — not merely +/// nothing whose *value* does. `tsjs.adSlots` is the trap: its content is derived +/// from config and path, but whether it is emitted at all is gated on consent, +/// bot classification, prefetch status and the auction kill switch. A template +/// filled by the first request would freeze that request's decision for every +/// later reader. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AssemblyMode { + /// Inject bids inline before ``. Root uncacheable. Shipped behaviour. + #[default] + Inline, + /// Serve a shared template; assemble its inert marker with an exact byte split. + /// + /// The operator-facing spelling remains `esi` for continuity, but no general + /// purpose ESI parser executes on this path. + Esi, +} + /// Top-level configuration for the creative opportunities system. #[derive(Debug, Clone, Deserialize, Serialize)] #[serde(deny_unknown_fields)] @@ -261,11 +293,104 @@ pub struct CreativeOpportunitiesConfig { /// [`section_root`](Self::section_root) are omitted. #[serde(default, skip_serializing_if = "Option::is_none")] pub section_segment: Option, + /// How per-user ad state reaches the page. Absent means + /// [`AssemblyMode::Inline`], the shipped behaviour. + /// + /// `Option` rather than a bare enum, and `skip_serializing_if`, deliberately: + /// these structs use `deny_unknown_fields`, so a pushed key makes an older + /// binary fail configuration load. Keeping it absent when unset means a + /// deployment that never sets it stays rollback-compatible. + /// + /// Spike-only. See [`AssemblyMode`]. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub assembly_mode: Option, + /// Request headers the origin varies on, which the shared-template cache key must + /// cover. + /// + /// Operator-stated because a cache **lookup happens before the fetch**, so on a cold + /// key the origin's `Vary` is not yet known. See `VarySpec` for why the alternatives + /// (two-phase lookup, or storing the list and re-keying) were not taken. + /// + /// **Unset or empty means no operator-stated header is covered, so any origin + /// `Vary` other than structurally covered `Accept-Encoding` disqualifies the + /// response.** `Cookie` may never be configured: a per-cookie object violates the + /// reader-neutral template contract. This fail-closed default prevents a deployment + /// that has not stated what its origin varies on from gaining a shared cache by + /// omission. + /// + /// Spike-only. Same `Option` + `skip_serializing_if` reasoning as `assembly_mode`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub template_cache_vary: Option>, + /// Maximum time a reader-neutral transformed template may remain in C2. + /// + /// This is a safety ceiling, not freshness authorization. The origin must still + /// provide positive shared freshness, and the stored lifetime is the smaller of + /// the origin's remaining edge freshness and this value. Defaults to 60 seconds + /// and may be configured from 1 second through 1 day. + /// + /// Spike-only. Same `Option` + `skip_serializing_if` reasoning as `assembly_mode`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub template_cache_max_age_seconds: Option, + /// Operator assertion that the origin's HTML does not depend on request cookies. + /// + /// Unset or `false` disqualifies **every cookie-bearing request** from the shared + /// template cache, in both directions. That is safe and it is also very nearly a + /// disable switch: Trusted Server sets its own identity cookie, so essentially every + /// repeat visitor carries one. Left at the default, the cache can only ever serve + /// first-ever page views and cookie-less clients. + /// + /// Setting `true` asserts the origin serves the same HTML with or without cookies. + /// It is not taken on trust alone — if the origin ever declares `Vary: Cookie`, the + /// response is refused regardless of this flag or the configured key. So a wrong + /// assertion is caught whenever the origin is honest about it, and this only widens + /// the window where the origin personalizes *silently*. + /// + /// Spike-only. Same `Option` + `skip_serializing_if` reasoning as `assembly_mode`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub origin_is_cookie_independent: Option, /// Slot templates. An empty vec or `enabled = false` disables template delivery. #[serde(default, deserialize_with = "vec_from_seq_or_map")] pub slot: Vec, } +impl CreativeOpportunitiesConfig { + /// Resolved assembly mode, defaulting to [`AssemblyMode::Inline`] when unset. + #[must_use] + pub fn assembly_mode(&self) -> AssemblyMode { + self.assembly_mode.unwrap_or_default() + } + + /// Whether a cookie-bearing request may participate in the shared cache. + /// + /// Defaults to `false`, which is the conservative reading and also the one that + /// makes the cache almost inert on real traffic. See + /// [`Self::origin_is_cookie_independent`]. + #[must_use] + pub fn origin_is_cookie_independent(&self) -> bool { + self.origin_is_cookie_independent.unwrap_or(false) + } + + /// Headers the cache key covers, per operator config. + /// + /// Unset yields an empty operator spec, so any origin `Vary` other than the + /// structurally covered `Accept-Encoding` reads as a gap and the response is never + /// cached. Failing closed is deliberate: an unconfigured deployment should not + /// acquire a shared cache silently. + #[must_use] + pub fn template_cache_vary(&self) -> crate::platform::VarySpec { + crate::platform::VarySpec::new(self.template_cache_vary.clone().unwrap_or_default()) + } + + /// Safety ceiling for one shared transformed-template cache entry. + #[must_use] + pub fn template_cache_max_age(&self) -> std::time::Duration { + std::time::Duration::from_secs(u64::from( + self.template_cache_max_age_seconds + .unwrap_or(DEFAULT_TEMPLATE_CACHE_MAX_AGE_SECONDS), + )) + } +} + impl CreativeOpportunitiesConfig { /// Derives the `{section}` value for `path` under this config's section /// policy ([`section_root`](Self::section_root) and @@ -333,10 +458,32 @@ impl CreativeOpportunitiesConfig { /// Returns an error string when [`gam_network_id`](Self::gam_network_id) is /// blank but consumed by a default path or `{network_id}` template; when a /// slot has an invalid identifier, page pattern set, format list, or - /// dimensions; when a `{section}` template lacks a valid + /// dimensions; when `template_cache_max_age_seconds` falls outside 1–86,400; + /// when a `{section}` template lacks a valid /// [`section_root`](Self::section_root); or when configured values make a /// dynamic path exceed 100 UTF-8 bytes. pub fn validate_runtime(&self) -> Result<(), String> { + if self + .template_cache_max_age_seconds + .is_some_and(|seconds| !(1..=MAX_TEMPLATE_CACHE_MAX_AGE_SECONDS).contains(&seconds)) + { + return Err(format!( + "template_cache_max_age_seconds must be between 1 and {MAX_TEMPLATE_CACHE_MAX_AGE_SECONDS}" + )); + } + + if let Some(names) = &self.template_cache_vary { + crate::platform::VarySpec::try_new(names.clone()).map_err(|name| { + format!("template_cache_vary contains invalid HTTP header name `{name}`") + })?; + if names.iter().any(|name| name.eq_ignore_ascii_case("cookie")) { + return Err( + "template_cache_vary must not include Cookie; C2 templates are reader-neutral" + .to_string(), + ); + } + } + // A network ID is required only when a slot renders the default // `//` path or substitutes `{network_id}`. Static // and `{slot_id}`/`{section}`-only templates leave it inert. @@ -1197,6 +1344,10 @@ mod tests { auction_timeout_ms: None, price_granularity: PriceGranularity::default(), section_root: section_root.map(str::to_string), + assembly_mode: None, + template_cache_vary: None, + template_cache_max_age_seconds: None, + origin_is_cookie_independent: None, section_segment: None, slot: vec![slot], } @@ -1595,6 +1746,10 @@ mod tests { auction_timeout_ms: None, price_granularity: PriceGranularity::default(), section_root: None, + assembly_mode: None, + template_cache_vary: None, + template_cache_max_age_seconds: None, + origin_is_cookie_independent: None, section_segment: None, slot: Vec::new(), }; @@ -1872,6 +2027,164 @@ mod tests { ); } + #[test] + fn assembly_mode_defaults_to_inline_when_absent() { + // Arrange: the minimal config an existing deployment would have. + let toml = r#" + gam_network_id = "99999" + "#; + + // Act + let config: CreativeOpportunitiesConfig = + toml::from_str(toml).expect("should deserialize without assembly_mode"); + + // Assert + assert_eq!( + config.assembly_mode, None, + "an absent key should stay absent rather than materializing a value" + ); + assert_eq!( + config.assembly_mode(), + AssemblyMode::Inline, + "should resolve to the shipped inline behaviour" + ); + } + + #[test] + fn assembly_mode_deserializes_each_variant() { + for (raw, expected) in [("inline", AssemblyMode::Inline), ("esi", AssemblyMode::Esi)] { + let toml = format!( + r#" + gam_network_id = "99999" + assembly_mode = "{raw}" + "# + ); + let config: CreativeOpportunitiesConfig = + toml::from_str(&toml).unwrap_or_else(|e| panic!("should parse {raw}: {e}")); + assert_eq!( + config.assembly_mode(), + expected, + "should resolve `{raw}` to {expected:?}" + ); + } + + let removed_mode = r#" + gam_network_id = "99999" + assembly_mode = "client_fill" + "#; + assert!( + toml::from_str::(removed_mode).is_err(), + "client_fill is outside #1009's ESI byte-seam design and must be rejected" + ); + } + + #[test] + fn template_cache_vary_rejects_invalid_header_names() { + let config: CreativeOpportunitiesConfig = toml::from_str( + r#" + gam_network_id = "99999" + template_cache_vary = ["rsc", "not a header"] + "#, + ) + .expect("shape should deserialize before runtime validation"); + let err = config + .validate_runtime() + .expect_err("invalid field names must fail configuration validation"); + assert!(err.contains("not a header"), "unexpected error: {err}"); + + let cookie_key: CreativeOpportunitiesConfig = toml::from_str( + r#" + gam_network_id = "99999" + template_cache_vary = ["Cookie"] + "#, + ) + .expect("shape should deserialize before runtime validation"); + let err = cookie_key + .validate_runtime() + .expect_err("per-cookie templates violate the reader-neutral C2 contract"); + assert!(err.contains("Cookie"), "unexpected error: {err}"); + } + + #[test] + fn template_cache_max_age_accepts_a_positive_value_up_to_one_day() { + for seconds in [1_u32, 1_200, 86_400] { + let config: CreativeOpportunitiesConfig = toml::from_str(&format!( + r#" + gam_network_id = "99999" + template_cache_max_age_seconds = {seconds} + "# + )) + .unwrap_or_else(|error| panic!("{seconds}s should deserialize: {error}")); + + config + .validate_runtime() + .unwrap_or_else(|error| panic!("{seconds}s should validate: {error}")); + let serialized = serde_json::to_value(config).expect("should serialize config"); + assert_eq!( + serialized + .get("template_cache_max_age_seconds") + .and_then(serde_json::Value::as_u64), + Some(u64::from(seconds)), + "the configured ceiling must survive typed configuration" + ); + } + } + + #[test] + fn template_cache_max_age_rejects_zero_and_more_than_one_day() { + for seconds in [0_u32, 86_401] { + let config: CreativeOpportunitiesConfig = toml::from_str(&format!( + r#" + gam_network_id = "99999" + template_cache_max_age_seconds = {seconds} + "# + )) + .unwrap_or_else(|error| panic!("shape should deserialize before validation: {error}")); + + let error = config + .validate_runtime() + .expect_err("an unsafe template-cache ceiling must fail startup validation"); + assert!( + error.contains("template_cache_max_age_seconds"), + "unexpected validation error: {error}" + ); + } + } + + #[test] + fn unset_template_cache_max_age_is_omitted_for_rollback_compatibility() { + let config: CreativeOpportunitiesConfig = + toml::from_str("gam_network_id = \"99999\"").expect("should deserialize"); + + assert_eq!( + config.template_cache_max_age(), + std::time::Duration::from_secs(60), + "an absent ceiling must preserve the spike's existing lifetime" + ); + let serialized = toml::to_string(&config).expect("should serialize"); + + assert!( + !serialized.contains("template_cache_max_age_seconds"), + "an unset new key must not break rollback to an older binary: {serialized}" + ); + } + + #[test] + fn unset_assembly_mode_is_omitted_from_serialized_config() { + // `deny_unknown_fields` means a pushed key breaks config load on an older + // binary. A deployment that never sets this must not gain the key just by + // round-tripping through a newer one. + let config: CreativeOpportunitiesConfig = + toml::from_str("gam_network_id = \"99999\"").expect("should deserialize"); + + let serialized = toml::to_string(&config).expect("should serialize"); + + assert!( + !serialized.contains("assembly_mode"), + "unset assembly_mode must not be serialized, got:\n{serialized}" + ); + } + #[test] fn prebid_slot_params_deserializes_without_bidders_field() { let json = r#"{"bidders": {}}"#; diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index 3bff588fe..711c4e31d 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -8,7 +8,7 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use lol_html::{ - EndTagHandler, Settings as RewriterSettings, element, + EndTagHandler, Settings as RewriterSettings, doc_comments, element, end, html_content::{ContentType, EndTag}, text, }; @@ -156,6 +156,29 @@ impl StreamProcessor for HtmlWithPostProcessing { fn reset(&mut self) {} } +/// What the `` seam injects. +/// +/// This is a decision, not a side effect of whether the `` script exists. +/// An earlier shape gated body-close injection on `ad_slots_script.is_some()`, +/// which coupled two independent choices: once a shared-template mode stopped +/// emitting the head script, body-close injection silently stopped too. +/// +/// See `docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md` +/// §6.7. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub enum BodyCloseInjection { + /// Emit nothing because no slots matched under the inline path. + #[default] + None, + /// Read the auction result from `ad_bids_state` and inject it, falling back to + /// an empty payload. Today's shipped behaviour. + InlineBids, + /// Emit this markup verbatim — an inert marker the assembly step splits on. + /// Must be identical for every request that reaches the transform, or the + /// cached template is not shared-safe. + Marker(String), +} + /// Configuration for HTML processing #[derive(Clone)] pub struct HtmlProcessorConfig { @@ -176,6 +199,9 @@ pub struct HtmlProcessorConfig { pub max_buffered_body_bytes: usize, /// Request-scoped conditional diagnostics delivery decision. pub gpt_diagnostics: Option, + /// What the `` seam injects. Decided by the caller rather than inferred + /// from [`Self::ad_slots_script`]. + pub body_close: BodyCloseInjection, /// Whether to omit Trusted Server's automatic `DataDome` client-side tag. pub suppress_datadome_client_side_tag: bool, } @@ -199,6 +225,7 @@ impl HtmlProcessorConfig { ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: settings.publisher.max_buffered_body_bytes, gpt_diagnostics: None, + body_close: BodyCloseInjection::None, suppress_datadome_client_side_tag: false, } } @@ -221,6 +248,17 @@ impl HtmlProcessorConfig { self } + /// Set what the `` seam injects. + /// + /// Separate from [`with_ad_state`](Self::with_ad_state) because the two are + /// independent decisions: a shared-template mode emits no head script and + /// still needs a body-close marker. + #[must_use] + pub fn with_body_close(mut self, body_close: BodyCloseInjection) -> Self { + self.body_close = body_close; + self + } + /// Attach the request-scoped conditional diagnostics decision. #[must_use] pub fn with_gpt_diagnostics(mut self, decision: Option) -> Self { @@ -318,9 +356,44 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso let integration_registry = config.integrations.clone(); let script_rewriters = integration_registry.script_rewriters(); let ad_slots_script = config.ad_slots_script.clone(); + let body_close = config.body_close.clone(); let ad_bids_state = config.ad_bids_state.clone(); let gpt_diagnostics = config.gpt_diagnostics.clone(); + // A publisher can legitimately emit the same inert comment text as the reserved + // C2 seam, including after ``. Neutralize source comments while they are + // parsed; markup injected by the body end-tag handler is output, not reparsed, so + // the transform-owned marker remains the only exact copy. + let mut document_content_handlers = Vec::new(); + if let BodyCloseInjection::Marker(marker) = &body_close + && let Some(reserved) = marker + .strip_prefix("")) + { + let reserved = reserved.to_string(); + let escaped = format!("x{reserved}"); + document_content_handlers.push(doc_comments!(move |comment| { + if comment.text() == reserved { + comment.set_text(&escaped)?; + } + Ok(()) + })); + } + if let BodyCloseInjection::Marker(marker) = &body_close { + let marker = marker.clone(); + let injected_bids = Arc::clone(&injected_bids); + document_content_handlers.push(end!(move |document_end| { + // HTML fragments and malformed-but-renderable documents may never expose a + // body end tag. Always mint a transform-owned terminal seam in that case; + // otherwise source bytes equal to the reserved marker could be mistaken for + // ownership by the post-transform exact-count validator. + if !injected_bids.swap(true, Ordering::SeqCst) { + document_end.append(&marker, ContentType::Html); + } + Ok(()) + })); + } + let mut element_content_handlers = vec![ // Inject unified tsjs bundle once at the start of element!("head", { @@ -385,29 +458,42 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso element!("body", { let state = ad_bids_state.clone(); let injected_bids = injected_bids.clone(); - let has_slots = ad_slots_script.is_some(); + let body_close = body_close.clone(); move |el| { - if !has_slots { + if matches!(body_close, BodyCloseInjection::None) { return Ok(()); } let state = state.clone(); let injected_bids = injected_bids.clone(); + let body_close = body_close.clone(); if let Some(handlers) = el.end_tag_handlers() { let handler: EndTagHandler<'static> = Box::new(move |end_tag: &mut EndTag<'_>| { if injected_bids.swap(true, Ordering::SeqCst) { return Ok(()); } - let script_guard = state.lock().expect("should lock bid state"); - let bids_script = match &*script_guard { - Some(s) => s.clone(), - None => build_empty_bids_script(), + let markup = match &body_close { + // Verbatim, and identical on every request that + // reaches the transform — that is what makes the + // cached template shared-safe. + BodyCloseInjection::Marker(marker) => marker.clone(), + BodyCloseInjection::InlineBids => { + let script_guard = state.lock().expect("should lock bid state"); + match &*script_guard { + Some(s) => s.clone(), + None => build_empty_bids_script(), + } + } + // Unreachable: the element handler returned early + // above. Kept exhaustive rather than using `_` so a + // new variant is a compile error here. + BodyCloseInjection::None => return Ok(()), }; - end_tag.before(&bids_script, ContentType::Html); + end_tag.before(&markup, ContentType::Html); Ok(()) }); handlers.push(handler); - } else { + } else if matches!(body_close, BodyCloseInjection::InlineBids) { // No end tag (implicitly closed or EOF ``): lol_html // cannot attach an end-tag handler, so tsjs.bids/adInit() are // never injected even though adSlots was injected at ``. @@ -659,6 +745,7 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso } let rewriter_settings = RewriterSettings { + document_content_handlers, element_content_handlers, ..RewriterSettings::default() }; @@ -698,6 +785,7 @@ mod tests { fn create_test_config() -> HtmlProcessorConfig { HtmlProcessorConfig { + body_close: BodyCloseInjection::None, origin_host: "origin.example.com".to_owned(), request_host: "test.example.com".to_owned(), request_scheme: "https".to_owned(), @@ -1599,6 +1687,7 @@ mod tests { #[test] fn injects_ad_slots_at_head_open() { let config = HtmlProcessorConfig { + body_close: BodyCloseInjection::None, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), @@ -1675,6 +1764,7 @@ mod tests { let bids_script = r#""#; let state = std::sync::Arc::new(std::sync::Mutex::new(Some(bids_script.to_string()))); let config = HtmlProcessorConfig { + body_close: BodyCloseInjection::InlineBids, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), @@ -1712,6 +1802,7 @@ mod tests { let bids_script = r#""#; let state = std::sync::Arc::new(std::sync::Mutex::new(Some(bids_script.to_string()))); let config = HtmlProcessorConfig { + body_close: BodyCloseInjection::InlineBids, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), @@ -1750,6 +1841,7 @@ mod tests { let request_host = "proxy.test-publisher.example.com"; let config = HtmlProcessorConfig { + body_close: BodyCloseInjection::None, origin_host: "origin.test-publisher.example.com".to_string(), request_host: request_host.to_string(), request_scheme: "https".to_string(), @@ -1802,6 +1894,7 @@ mod tests { // (state is None) — e.g. auction timed out with zero bids. Fallback to {}. let state = std::sync::Arc::new(std::sync::Mutex::new(None)); let config = HtmlProcessorConfig { + body_close: BodyCloseInjection::InlineBids, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), @@ -1832,6 +1925,7 @@ mod tests { // unmodified (spec §8: "Existing client-side Prebid/GPT flow runs unmodified"). let state = std::sync::Arc::new(std::sync::Mutex::new(None)); let config = HtmlProcessorConfig { + body_close: BodyCloseInjection::None, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), @@ -1853,6 +1947,41 @@ mod tests { ); } + #[test] + fn bodyless_marker_mode_emits_an_owned_terminal_seam_even_after_source_bytes() { + const MARKER: &str = ""; + let config = HtmlProcessorConfig { + body_close: BodyCloseInjection::Marker(MARKER.to_string()), + origin_host: "origin.example.com".to_string(), + request_host: "example.com".to_string(), + request_scheme: "https".to_string(), + integrations: IntegrationRegistry::empty_for_tests(), + ad_slots_script: None, + ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), + max_buffered_body_bytes: 16 * 1024 * 1024, + gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, + }; + let source = + format!(r#""#); + + let mut processor = create_html_processor(config); + let output = processor + .process_chunk(source.as_bytes(), true) + .expect("should process bodyless HTML"); + let html = std::str::from_utf8(&output).expect("should be utf8"); + + assert_eq!( + html.matches(MARKER).count(), + 2, + "one source occurrence plus one transform-owned seam must reach normalization" + ); + assert!( + html.ends_with(MARKER), + "the transform-owned fallback must be unambiguously terminal" + ); + } + #[test] fn response_size_does_not_grow_disproportionately() { // Processing must not expand HTML by more than 1.1× (accounts for the diff --git a/crates/trusted-server-core/src/integrations/datadome/protection.rs b/crates/trusted-server-core/src/integrations/datadome/protection.rs index 665060014..75de88afb 100644 --- a/crates/trusted-server-core/src/integrations/datadome/protection.rs +++ b/crates/trusted-server-core/src/integrations/datadome/protection.rs @@ -58,14 +58,21 @@ impl DataDomeIntegration { return RequestFilterDecision::Continue(RequestFilterEffects::default()); } + let request_method = input.request.method().clone(); match self.filter_protection_request_inner(input).await { Ok(decision) => decision, Err(ProtectionRequestError::Setup(err)) => { - log::error!("[datadome] Protection setup failed open: {err:?}"); + log::error!( + "[datadome] protection decision=failed_open api_status=none datadome_status=unavailable method={} route=continue failure=setup error={err:?}", + request_method, + ); RequestFilterDecision::Continue(RequestFilterEffects::default()) } Err(ProtectionRequestError::Runtime(err)) => { - log::warn!("[datadome] Protection API failed open: {err:?}"); + log::warn!( + "[datadome] protection decision=failed_open api_status=none datadome_status=unavailable method={} route=continue failure=runtime error={err:?}", + request_method, + ); RequestFilterDecision::Continue(RequestFilterEffects::default()) } } @@ -183,15 +190,15 @@ impl DataDomeIntegration { if supplied_values.is_empty() { return false; } + let Some(bypass) = self.active_protection_test_bypass() else { + return false; + }; if supplied_values.len() != 1 { log::warn!( "[datadome] Multiple DataDome test bypass headers supplied; ignoring bypass" ); return false; } - let Some(bypass) = self.active_protection_test_bypass() else { - return false; - }; let store_name = StoreName::from(bypass.credential_secret_store.as_str()); let credential = match services @@ -522,31 +529,21 @@ fn log_protection_skip( reason: ProtectionSkipReason, suppress_client_tag: bool, ) { - let reason = reason.as_str(); - if suppression_skip_log_level(suppress_client_tag, is_navigation_request(input.request)) - == log::Level::Info - { - log::info!( - "[datadome] protection decision=skipped rule={} reason={} client_tag=omitted method={}", - rule_id, - reason, - input.request.method(), - ); - } else if suppress_client_tag { - log::debug!( - "[datadome] protection decision=skipped rule={} reason={} client_tag=omitted method={}", - rule_id, - reason, - input.request.method(), - ); + let level = + suppression_skip_log_level(suppress_client_tag, is_navigation_request(input.request)); + let client_tag = if suppress_client_tag { + " client_tag=omitted" } else { - log::debug!( - "[datadome] protection decision=skipped rule={} reason={} method={}", - rule_id, - reason, - input.request.method(), - ); - } + "" + }; + log::log!( + level, + "[datadome] protection decision=skipped rule={} reason={}{} method={}", + rule_id, + reason.as_str(), + client_tag, + input.request.method(), + ); } #[derive(Debug, Clone, Copy, Eq, PartialEq)] @@ -839,7 +836,7 @@ fn truncate_utf8(value: &str, limit: i32) -> String { mod tests { use std::collections::HashMap; use std::net::{IpAddr, Ipv4Addr}; - use std::sync::Arc; + use std::sync::{Arc, Mutex}; use crate::integrations::datadome::{ DataDomeConfig, ProtectionExclusionRuleConfig, ProtectionMatcherConfig, @@ -855,6 +852,8 @@ mod tests { use super::*; + static FASTLY_IS_STAGING_ENV_LOCK: Mutex<()> = Mutex::new(()); + fn protection_integration() -> Arc { let config = DataDomeConfig { enabled: true, @@ -878,6 +877,9 @@ mod tests { services: &RuntimeServices, request: &mut Request, ) -> RequestFilterDecision { + let _guard = FASTLY_IS_STAGING_ENV_LOCK + .lock() + .expect("should lock staging environment test guard"); temp_env::with_var(crate::constants::ENV_FASTLY_IS_STAGING, Some("1"), || { futures::executor::block_on(integration.filter_protection_request(RequestFilterInput { settings, @@ -1098,6 +1100,9 @@ mod tests { edgezero_core::http::HeaderValue::from_static("temporary-test-credential-32-bytes!"), ); + let _guard = FASTLY_IS_STAGING_ENV_LOCK + .lock() + .expect("should lock staging environment test guard"); let decision = temp_env::with_var( crate::constants::ENV_FASTLY_IS_STAGING, None::<&str>, diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js index 86b51ffa7..43934771d 100644 --- a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -94,8 +94,12 @@ // and deliberately identical to the bundle scheduler — the impression is // spent on a viewed tab, and the post-hydration guarantee holds whenever // the request is actually issued. - ts.scheduleInitialAdInit = function (initialBids) { + ts.scheduleInitialAdInit = function (initialBids, initialSlots) { if ((ts.navGeneration || 0) !== 0) return; + // Slots are generation-guarded for the same reason the bids are: the + // shared-template seam sends both, and an assignment made before this call + // would overwrite a committed SPA navigation's slots. + if (initialSlots) ts.adSlots = initialSlots; if (initialBids) ts.bids = initialBids; var fire = function () { if ((ts.navGeneration || 0) !== 0) return; diff --git a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs b/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs index 5bd86d19e..7a91eead4 100644 --- a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs +++ b/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs @@ -114,6 +114,83 @@ impl GptDiagnosticsRequestDecision { } } +impl GptDiagnosticsRequestDecision { + /// An active decision, for tests in other modules that need one. + /// + /// The fields are private and built by `prepare_request` from a cookie or query + /// parameter; there is no other way to obtain an active decision across a module + /// boundary. + #[cfg(test)] + #[must_use] + pub(crate) fn active_for_tests() -> Self { + Self { + active: true, + clean_browser_path_and_query: None, + cookie_action: GptDiagnosticsCookieAction::None, + } + } +} + +#[cfg(test)] +mod head_seam_invariant_tests { + use super::*; + + /// Every combination of the three fields the decision carries. + fn all_decisions() -> Vec { + let mut out = Vec::new(); + for active in [false, true] { + for clean in [None, Some("/clean".to_string())] { + for cookie_action in [ + GptDiagnosticsCookieAction::None, + GptDiagnosticsCookieAction::SetSession, + GptDiagnosticsCookieAction::ClearSession, + ] { + out.push(GptDiagnosticsRequestDecision { + active, + clean_browser_path_and_query: clean.clone(), + cookie_action, + }); + } + } + } + out + } + + #[test] + fn requires_private_no_store_is_a_superset_of_injection() { + // Load-bearing relationship, not an incidental one. Whenever this decision + // injects anything into ``, the response must also be stamped + // `private, no-store` — which is what keeps request-scoped diagnostics out + // of a shared cache if the explicit assembly-mode gate in + // `create_html_stream_processor` is ever removed or bypassed. + // + // If a future change makes a script emit without also requiring the stamp, + // this fails here rather than silently in a cached template. + for decision in all_decisions() { + let injects = + decision.bootstrap_script().is_some() || decision.module_script_tag().is_some(); + if injects { + assert!( + decision.requires_private_no_store(), + "decision injects into but does not require private/no-store: \ + {decision:?}" + ); + } + } + } + + #[test] + fn a_default_decision_injects_nothing() { + let decision = GptDiagnosticsRequestDecision::default(); + assert_eq!(decision.bootstrap_script(), None); + assert_eq!(decision.module_script_tag(), None); + assert!( + !decision.requires_private_no_store(), + "an inert decision should not force the response private" + ); + } +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum QueryDirective { Absent, diff --git a/crates/trusted-server-core/src/platform/mod.rs b/crates/trusted-server-core/src/platform/mod.rs index 287f1accf..7c80f9e12 100644 --- a/crates/trusted-server-core/src/platform/mod.rs +++ b/crates/trusted-server-core/src/platform/mod.rs @@ -12,6 +12,7 @@ //! - [`PlatformBackend`] — dynamic backend registration //! - [`PlatformHttpClient`] — outbound HTTP client //! - [`PlatformGeo`] — geographic information lookup +//! - [`PlatformTemplateAssembler`] — cold-response shared-template assembly //! //! ## Platform-Agnostic Components //! @@ -36,6 +37,8 @@ mod error; mod http; mod image_optimizer; mod kv; +mod template_assembly; +mod template_cache; #[cfg(test)] pub(crate) mod test_support; mod traits; @@ -52,6 +55,16 @@ pub use image_optimizer::{ PlatformImageOptimizerParams, PlatformImageOptimizerRegion, }; pub use kv::UnavailableKvStore; +pub use template_assembly::{ + PlatformTemplateAssembler, TemplateAssemblyError, UnavailableTemplateAssembler, +}; +pub use template_cache::REPLAYABLE_POLICY_HEADERS; +pub use template_cache::{ + PlatformTemplateCache, PlatformTemplateCacheReservation, TEMPLATE_SCHEMA_VERSION, + TemplateCacheError, TemplateCacheKey, TemplateCacheLookup, TemplateCacheMiss, + TemplateCacheReservation, TemplateEntry, TemplateMetadata, UnavailableTemplateCache, + VaryHeaderValues, VarySpec, +}; pub use traits::{PlatformBackend, PlatformConfigStore, PlatformGeo, PlatformSecretStore}; pub use types::{ ClientInfo, GeoInfo, PlatformBackendSpec, RuntimeServices, RuntimeServicesBuilder, StoreId, diff --git a/crates/trusted-server-core/src/platform/template_assembly.rs b/crates/trusted-server-core/src/platform/template_assembly.rs new file mode 100644 index 000000000..441e7ca31 --- /dev/null +++ b/crates/trusted-server-core/src/platform/template_assembly.rs @@ -0,0 +1,77 @@ +//! Platform boundary for assembling a shared template with reader-specific state. +//! +//! Core owns the cache-safety ordering and the portable byte-seam fallback. An adapter +//! may provide a richer assembler for the cold response after the reader-neutral +//! template has been stored. + +use core::fmt; + +/// Why a platform assembler could not produce a document. +#[derive(Debug, derive_more::Display)] +pub enum TemplateAssemblyError { + /// The adapter has no template assembler. + #[display("this adapter cannot assemble shared templates")] + Unsupported, + /// The platform assembler rejected or could not process the document. + #[display("template assembly failed: {message}")] + Failed { + /// Human-readable failure context. + message: String, + }, +} + +impl core::error::Error for TemplateAssemblyError {} + +/// Assembles reader-specific state into a shared HTML template. +pub trait PlatformTemplateAssembler: Send + Sync { + /// Produce the complete document served to this reader. + /// + /// # Errors + /// + /// Returns [`TemplateAssemblyError`] when the adapter cannot assemble the template. + fn assemble(&self, template: &[u8], fragment: &[u8]) -> Result, TemplateAssemblyError>; +} + +impl fmt::Debug for dyn PlatformTemplateAssembler { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("PlatformTemplateAssembler") + } +} + +/// Default assembler used by adapters that do not provide platform assembly. +#[derive(Debug, Default, Clone, Copy)] +pub struct UnavailableTemplateAssembler; + +impl PlatformTemplateAssembler for UnavailableTemplateAssembler { + fn assemble( + &self, + _template: &[u8], + _fragment: &[u8], + ) -> Result, TemplateAssemblyError> { + Err(TemplateAssemblyError::Unsupported) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unavailable_assembler_refuses_the_document() { + let error = UnavailableTemplateAssembler + .assemble(b"", b"") + .expect_err("should refuse when platform assembly is unavailable"); + + assert!(matches!(error, TemplateAssemblyError::Unsupported)); + } + + #[test] + fn assembler_contract_is_object_safe() { + let assembler: Box = Box::new(UnavailableTemplateAssembler); + + assert!(matches!( + assembler.assemble(b"template", b"fragment"), + Err(TemplateAssemblyError::Unsupported) + )); + } +} diff --git a/crates/trusted-server-core/src/platform/template_cache.rs b/crates/trusted-server-core/src/platform/template_cache.rs new file mode 100644 index 000000000..ac3761b0f --- /dev/null +++ b/crates/trusted-server-core/src/platform/template_cache.rs @@ -0,0 +1,1096 @@ +//! The shared transformed-template cache (C2) for the #1009 ESI validation spike. +//! +//! Three caches are in play and conflating them is what produced the original wrong +//! conclusion in the design doc, so this module names which one it is: +//! +//! | Cache | Contents | Owner | +//! | ----- | --------------------------------- | ------------------------------ | +//! | C1 | raw origin bytes | Fastly read-through. Not this. | +//! | C2 | post-`lol_html`, pre-assembly | **This module.** | +//! | C3 | final per-user assembled response | **Must never exist.** | +//! +//! C2 holds a *shared template*: no per-user bytes, and no decisions that depend on +//! the request. What may and may not live in it is +//! [§6.7 of the design doc](../../../../docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md), +//! and the invariant is enforced by the rendered-document byte-identity tests in +//! `publisher`. +//! +//! Spike-only. Remove with the spike. + +use core::fmt; +use std::collections::HashSet; + +use crate::creative_opportunities::AssemblyMode; + +/// Version of the transform that produced a cached template. +/// +/// Bump on **any** change to what the transform emits. Without it a deploy reads +/// yesterday's template shape and assembles against markers that moved, which fails +/// as a rendering bug far from its cause rather than as a cache miss. +/// +/// | Version | Transform | +/// | ------- | --------- | +/// | 1 | `` seam used an executable ESI include tag targeting the old fragment endpoint | +/// | 2 | Marker became the inert comment ``; the seam hands slots to `scheduleInitialAdInit` instead of assigning them | +/// | 3 | Marker became ``; canonical collision-safe key, explicit origin freshness, and complete repeated document-policy metadata | +/// | 4 | Marker is the shorter, accurate [`AD_ASSEMBLY_SEAM`](crate::publisher::AD_ASSEMBLY_SEAM) | +pub const TEMPLATE_SCHEMA_VERSION: u32 = 4; + +/// Inputs that select one cached template. +/// +/// Every field changes the emitted bytes for the same URL. A signal that changes the +/// bytes and is **not** here produces cross-served templates; a signal that is +/// per-user does not belong here at all — it belongs out of the template entirely. +/// That distinction is the whole design: the key holds per-*variant* signals, and +/// per-*user* signals are excluded from the template rather than keyed on. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TemplateCacheKey { + /// Full request URL, stated explicitly rather than inherited from an ambient + /// request, so the key cannot silently depend on what the caller happened to + /// mutate first. + pub url: String, + /// Host and scheme. The post-processed output is host-dependent by construction: + /// both reach `IntegrationHtmlContext` and drive URL rewriting. + pub request_host: String, + /// See [`Self::request_host`]. + pub request_scheme: String, + /// Publisher origin identity, including the outbound Host override. Two virtual + /// hosts can share a connection target while producing unrelated documents. + pub origin_identity: String, + /// Inline and ESI modes emit different template bytes. Without this they poison + /// each other's entries. + pub assembly_mode: AssemblyMode, + /// Values of the request headers the **origin** declares it varies on, in the + /// order the origin listed them. Not a fixed list: the origin is authoritative, + /// and hard-coding one here would silently drift when the origin's changes. + pub vary_values: Vec, + /// Digest of every setting that can shape the transformed template plus the tsjs + /// bundle. Over-invalidating is safe; omitting a shaping input cross-serves bytes. + pub template_fingerprint: String, + /// See [`TEMPLATE_SCHEMA_VERSION`]. + pub schema_version: u32, +} + +impl TemplateCacheKey { + /// Render a fixed-size opaque key for the platform cache. + /// + /// The canonical input is length-prefixed before hashing, so neither delimiters nor + /// raw request values can collide or leak into cache diagnostics. + #[must_use] + pub fn to_cache_key(&self) -> String { + use sha2::Digest as _; + + fn push(out: &mut Vec, part: &[u8]) { + out.extend_from_slice(&(part.len() as u64).to_be_bytes()); + out.extend_from_slice(part); + } + + let mut canonical = Vec::new(); + push(&mut canonical, b"ts-c2"); + push(&mut canonical, &self.schema_version.to_be_bytes()); + push( + &mut canonical, + match self.assembly_mode { + AssemblyMode::Inline => b"inline", + AssemblyMode::Esi => b"esi", + }, + ); + push(&mut canonical, self.request_scheme.as_bytes()); + push(&mut canonical, self.request_host.as_bytes()); + push(&mut canonical, self.origin_identity.as_bytes()); + push(&mut canonical, self.url.as_bytes()); + push(&mut canonical, self.template_fingerprint.as_bytes()); + push( + &mut canonical, + &(self.vary_values.len() as u64).to_be_bytes(), + ); + for varied in &self.vary_values { + push(&mut canonical, varied.name.as_bytes()); + match &varied.values { + None => push(&mut canonical, b"absent"), + Some(values) => { + push(&mut canonical, b"present"); + push(&mut canonical, &(values.len() as u64).to_be_bytes()); + for value in values { + push(&mut canonical, value); + } + } + } + } + + let digest = sha2::Sha256::digest(canonical); + format!("ts-c2-v{}-{}", self.schema_version, hex::encode(digest)) + } + + /// Surrogate keys to attach at insert, for purge-based rollback. + /// + /// `ts-template` purges every template at once, which is the rollback lever. + /// The per-URL key allows targeted invalidation. Both are needed: the broad one + /// for an incident, the narrow one for ordinary invalidation. + #[must_use] + pub fn surrogate_keys(&self) -> Vec { + vec!["ts-template".to_string(), self.url_surrogate_key()] + } + + /// Surrogate key for every variant of this publisher URL. + /// + /// Used to evict a malformed object without flushing unrelated article templates. + #[must_use] + pub fn url_surrogate_key(&self) -> String { + format!("ts-template-url-{}", digest_hex(self.url.as_bytes())) + } +} + +fn digest_hex(bytes: &[u8]) -> String { + use sha2::Digest as _; + hex::encode(sha2::Sha256::digest(bytes)) +} + +/// One configured `Vary` input exactly as it appeared on the request. +/// +/// `None` means absent. `Some(vec![vec![]])` means present with one empty field +/// value. Repeated fields stay separate and ordered; no UTF-8 conversion is involved. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VaryHeaderValues { + /// Validated, lowercase header name. + pub name: String, + /// Every raw field value in wire order, or `None` when absent. + pub values: Option>>, +} + +/// Origin response headers safe to store with a shared template and replay on a hit. +/// +/// Every one is a per-URL policy statement, identical for every reader. Nothing +/// per-reader (`Set-Cookie`) and nothing cache-controlling (`Cache-Control`, `ETag`, +/// `Surrogate-Control`) appears here, and it is an allowlist so a new origin header is +/// excluded until someone decides otherwise. +pub const REPLAYABLE_POLICY_HEADERS: &[&str] = &[ + "content-security-policy", + "content-security-policy-report-only", + "permissions-policy", + "referrer-policy", + "strict-transport-security", + "cross-origin-opener-policy", + "cross-origin-embedder-policy", + "cross-origin-resource-policy", + "origin-agent-cluster", + "reporting-endpoints", + "report-to", + "link", + "x-frame-options", + "x-content-type-options", + "content-language", + "x-robots-tag", +]; + +/// Headers the key covers by construction, whatever the operator configured. +/// +/// The shared path stores decoded identity bytes and negotiates the reader representation +/// only after assembly, so an origin declaring `Vary: Accept-Encoding` is covered without +/// reader input. This assumes those origin variants differ only by HTTP content coding; +/// operators must leave ESI disabled if an origin changes document semantics instead. +/// Without this carve-out, the ordinary declaration sent by any compressing origin reads +/// as an uncovered gap and disqualifies the response, so **C2 would never cache anything +/// against a real origin** unless the operator redundantly listed a header the transform +/// already normalizes. Found by review before it could make the spike measure a hit rate +/// of approximately zero and read that as a result. +const STRUCTURALLY_COVERED: &[&str] = &["accept-encoding"]; + +/// Request headers to include in the cache key, and where the list comes from. +/// +/// # The chicken-and-egg this resolves +/// +/// The key must cover everything the origin varies on, or two requests needing +/// different templates share one entry. But a **lookup happens before the fetch**, +/// so on a cold key the origin's `Vary` is not yet known. +/// +/// Three ways out, and the trade-off is real: +/// +/// 1. **Configure the list** — what this does. One lookup, no extra round trip, and +/// the operator states what the origin varies on. Cost: it drifts silently if the +/// origin's `Vary` changes and nobody updates config. +/// 2. **Two-phase lookup** — fetch a URL-keyed record holding the last-seen `Vary`, +/// then key properly. Correct, but doubles the lookups on every request. +/// 3. **Store the list alongside** and re-key on mismatch. Same cost as (2) plus +/// complexity. +/// +/// (1) is chosen for the spike because Step A already measured the origin's actual +/// `Vary`, the origin response is checked for drift before storage, and the configured +/// template-cache ceiling bounds how long a newly introduced mismatch can survive. +/// **This is a spike-grade choice, not a production one** — see the drift guard below. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VarySpec { + /// Header names, lowercased, in a fixed order. + names: Vec, +} + +impl VarySpec { + /// Build from configured header names. + /// + /// # Panics + /// + /// Panics when a name is not a valid HTTP field name. Runtime configuration is + /// validated with [`Self::try_new`] before this constructor is used. + #[must_use] + pub fn new(names: impl IntoIterator) -> Self { + Self::try_new(names).expect("VarySpec names should be validated at configuration load") + } + + /// Build from configured names, validating and deduplicating them. + /// + /// # Errors + /// + /// Returns the offending name when it is not a valid HTTP field name. + pub fn try_new(names: impl IntoIterator) -> Result { + let mut seen = HashSet::new(); + let mut normalized = Vec::new(); + for raw in names { + let name = http::header::HeaderName::from_bytes(raw.as_bytes()) + .map_err(|_| raw.clone())? + .as_str() + .to_string(); + if STRUCTURALLY_COVERED.contains(&name.as_str()) { + continue; + } + if seen.insert(name.clone()) { + normalized.push(name); + } + } + Ok(Self { names: normalized }) + } + + /// Configured names, lowercased. + #[must_use] + pub fn names(&self) -> &[String] { + &self.names + } + + /// Extract the key inputs from a request's headers. + /// + /// A header the origin varies on but the request omits still contributes an + /// entry, with an empty value — otherwise "absent" and "present but empty" + /// would collide, and those are different requests to the origin. + #[must_use] + pub fn values_from(&self, headers: &http::HeaderMap) -> Vec { + self.names + .iter() + .map(|name| { + let values = headers.contains_key(name.as_str()).then(|| { + headers + .get_all(name.as_str()) + .iter() + .map(|value| value.as_bytes().to_vec()) + .collect() + }); + VaryHeaderValues { + name: name.clone(), + values, + } + }) + .collect() + } + + /// Whether the origin's declared `Vary` contains anything this spec omits. + /// + /// The drift guard for choice (1) above. Called **after** the origin responds, + /// when its `Vary` is finally known: if the origin varies on something the key + /// did not cover, the template just built is unsafe to store, because a request + /// differing only in that header would read it. + /// + /// Returns the uncovered names, so the caller can log precisely which config is + /// stale rather than reporting a generic refusal. + #[must_use] + pub fn uncovered_by<'a>(&self, origin_vary: impl IntoIterator) -> Vec { + origin_vary + .into_iter() + .flat_map(|value| value.split(',')) + .map(|name| name.trim().to_ascii_lowercase()) + .filter(|name| !name.is_empty() && name != "*") + .filter(|name| !STRUCTURALLY_COVERED.contains(&name.as_str())) + .filter(|name| !self.names.contains(name)) + .collect() + } +} + +/// Metadata stored alongside the template bytes. +/// +/// `cache::core` carries **no HTTP semantics** — status, headers, encoding and +/// revalidation are all the caller's. Rather than storing origin headers and +/// replaying them, store only what is needed to rebuild a response from scratch. +/// +/// That choice is deliberate and load-bearing: the publisher path forces +/// `private, no-store` and strips validators *after* the origin send, so replaying a +/// stored origin header would fight it. Rebuilding every header on a hit means no +/// origin header is ever replayed and the `Set-Cookie` privacy net stays trivially +/// safe. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TemplateMetadata { + /// Encoding of the stored bytes. C2 writes only `identity`; retaining the field in + /// metadata makes corrupt or stale representations fail validation on read. + pub content_encoding: String, + /// Content type to rebuild the response with. + pub content_type: String, + /// Schema version the bytes were produced under. Checked on read: a mismatch is + /// a miss, not an error, so a rollback to an older binary degrades to + /// re-transforming rather than misassembling. + pub schema_version: u32, + /// Length of the template bytes as written. + /// + /// Guards against a partially written entry. `Transaction::insert` consumes the + /// transaction, so a write that fails part-way cannot cancel the insert — there + /// is no handle left to cancel it with. Recording the intended length and + /// checking it on read makes a truncated entry a miss instead of a silently + /// short template that would assemble into a broken page. + pub body_len: u64, + /// Origin response headers that are policy, not per-reader state. + /// + /// Reconstructing headers from scratch on a hit keeps origin `Set-Cookie` and caching + /// directives out of a shared cache — but it also dropped `Content-Security-Policy`, + /// framing protection and `Content-Language`, weakening the page. These are + /// per-URL and identical for every reader, so they belong with the template. + /// + /// Deliberately an allowlist: anything per-reader or cache-controlling is excluded by + /// construction rather than by remembering to strip it. + pub policy_headers: Vec<(String, String)>, +} + +impl TemplateMetadata { + /// Serialize for `user_metadata`. Deliberately a tiny hand-rolled format rather + /// than JSON — one allocation, no dependency, and a parse failure is + /// unambiguous. + #[must_use] + pub fn encode(&self) -> Vec { + let mut out = format!( + "v={}\nce={}\nct={}\nlen={}", + self.schema_version, self.content_encoding, self.content_type, self.body_len + ); + for (name, value) in &self.policy_headers { + // Header values cannot contain newlines (the HTTP parser rejects them), so a + // newline-delimited encoding cannot be broken by a header value. + out.push_str(&format!("\nh={name}:{value}")); + } + out.into_bytes() + } + + /// Parse `user_metadata`. Returns `None` on anything unexpected, which callers + /// must treat as a cache miss. + #[must_use] + pub fn decode(raw: &[u8]) -> Option { + let text = core::str::from_utf8(raw).ok()?; + let mut schema_version = None; + let mut policy_headers = Vec::new(); + let mut content_encoding = None; + let mut content_type = None; + let mut body_len = None; + for line in text.lines() { + let (key, value) = line.split_once('=')?; + match key { + "v" => { + if schema_version.replace(value.parse().ok()?).is_some() { + return None; + } + } + "ce" => { + if content_encoding.replace(value.to_string()).is_some() { + return None; + } + } + "h" => { + let (name, header_value) = value.split_once(':')?; + let name = http::header::HeaderName::from_bytes(name.as_bytes()).ok()?; + if !REPLAYABLE_POLICY_HEADERS.contains(&name.as_str()) { + return None; + } + http::HeaderValue::from_bytes(header_value.as_bytes()).ok()?; + policy_headers.push((name.as_str().to_string(), header_value.to_string())); + } + "ct" => { + if content_type.replace(value.to_string()).is_some() { + return None; + } + } + "len" => { + if body_len.replace(value.parse().ok()?).is_some() { + return None; + } + } + _ => return None, + } + } + let content_encoding = content_encoding?; + // Every template is decoded before insert. Accepting another value here would + // let corrupt metadata label plaintext bytes as gzip on a warm hit. + if content_encoding != "identity" { + return None; + } + let content_type = content_type?; + http::HeaderValue::from_bytes(content_type.as_bytes()).ok()?; + if !content_type + .split(';') + .next() + .is_some_and(|media_type| media_type.trim().eq_ignore_ascii_case("text/html")) + { + return None; + } + Some(Self { + schema_version: schema_version?, + policy_headers, + content_encoding, + content_type, + body_len: body_len?, + }) + } +} + +/// Why a template read did not produce usable bytes. +#[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)] +pub enum TemplateCacheMiss { + /// No entry for this key. + #[display("no cached template for this key")] + NotFound, + /// Found, but produced by a different transform version. + #[display("cached template has a different schema version")] + SchemaMismatch, + /// Found, but its metadata could not be parsed. + #[display("cached template metadata is unreadable")] + UnreadableMetadata, + /// Found, but shorter than the metadata says it should be — a write that failed + /// part-way. See [`TemplateMetadata::body_len`]. + #[display("cached template is truncated")] + Truncated, + /// This platform has no template cache. + #[display("no template cache on this platform")] + Unsupported, +} + +impl core::error::Error for TemplateCacheMiss {} + +/// Errors a template cache write can produce. +#[derive(Debug, derive_more::Display)] +pub enum TemplateCacheError { + /// This platform has no template cache. + #[display("no template cache on this platform")] + Unsupported, + /// The platform rejected the operation. + #[display("template cache backend error: {message}")] + Backend { + /// What the backend reported. + message: String, + }, +} + +impl core::error::Error for TemplateCacheError {} + +/// Result of the pre-origin cache transaction. +pub enum TemplateCacheLookup { + /// A fresh usable template. + Hit(TemplateEntry), + /// This request owns the obligation to provide or cancel the cold object. + Reserved(TemplateCacheReservation), + /// This adapter deliberately has no shared-template cache. + Unsupported, + /// A cache object existed but failed schema, metadata, or length validation. + Invalid(TemplateCacheMiss), +} + +/// Platform-owned insert obligation. Dropping it cancels, making every early-return +/// path safe without an async cleanup ladder in the publisher pipeline. +pub struct TemplateCacheReservation { + inner: Option>, +} + +impl core::fmt::Debug for TemplateCacheReservation { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("TemplateCacheReservation") + .finish_non_exhaustive() + } +} + +impl TemplateCacheReservation { + /// Wrap a platform reservation. + #[must_use] + pub fn new(inner: Box) -> Self { + Self { inner: Some(inner) } + } + + /// Fulfil the reservation with a validated template. + /// + /// # Errors + /// + /// Returns the platform cache error when the reservation cannot be fulfilled. + pub fn insert( + mut self, + metadata: &TemplateMetadata, + body: Vec, + max_age: std::time::Duration, + ) -> Result<(), TemplateCacheError> { + self.inner + .take() + .ok_or_else(|| TemplateCacheError::Backend { + message: "template reservation was already consumed".to_string(), + })? + .insert(metadata, body, max_age) + } + + /// Explicitly give up the reservation. Drop performs the same operation as a net. + /// + /// # Errors + /// + /// Returns the platform cache error when the reservation cannot be cancelled. + pub fn cancel(mut self) -> Result<(), TemplateCacheError> { + self.inner + .take() + .ok_or_else(|| TemplateCacheError::Backend { + message: "template reservation was already consumed".to_string(), + })? + .cancel() + } +} + +impl Drop for TemplateCacheReservation { + fn drop(&mut self) { + if let Some(inner) = self.inner.take() + && let Err(err) = inner.cancel() + { + log::warn!("c2_template_cache reservation cancellation failed: {err}"); + } + } +} + +/// Adapter-specific ownership token returned by a transactional lookup. +pub trait PlatformTemplateCacheReservation: Send { + /// Insert and discharge the obligation. + /// + /// # Errors + /// + /// Returns an adapter-specific cache error when the insert fails. + fn insert( + self: Box, + metadata: &TemplateMetadata, + body: Vec, + max_age: std::time::Duration, + ) -> Result<(), TemplateCacheError>; + + /// Cancel and allow a waiting request to take ownership. + /// + /// # Errors + /// + /// Returns an adapter-specific cache error when cancellation fails. + fn cancel(self: Box) -> Result<(), TemplateCacheError>; +} + +impl fmt::Debug for dyn PlatformTemplateCache { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("PlatformTemplateCache") + } +} + +/// A platform's shared-template cache. +/// +/// Only the Fastly adapter implements this; every other adapter uses +/// [`UnavailableTemplateCache`], which reports [`TemplateCacheMiss::Unsupported`] so +/// the caller transforms every time rather than failing. +/// +/// `Send + Sync` on the trait, `?Send` on the futures: `RuntimeServices` is held in a +/// `LazyLock` static, so the trait object must cross threads even though the futures +/// themselves never do — the platform layer is `!Send` by construction. +#[async_trait::async_trait(?Send)] +pub trait PlatformTemplateCache: Send + Sync { + /// Transactionally look up a template before origin work begins. + async fn lookup_or_reserve( + &self, + key: &TemplateCacheKey, + ) -> Result { + Ok(match self.get(key).await { + Ok(entry) => TemplateCacheLookup::Hit(entry), + Err(TemplateCacheMiss::Unsupported | TemplateCacheMiss::NotFound) => { + TemplateCacheLookup::Unsupported + } + Err(miss) => TemplateCacheLookup::Invalid(miss), + }) + } + + /// Read a template. `Err` is a miss, not a failure — every variant means + /// "transform it yourself". + async fn get(&self, key: &TemplateCacheKey) -> Result; + + /// Store a template. + /// + /// Callers must not call this without having consulted the C2 eligibility gate + /// first: this method stores what it is given and cannot tell a shared template + /// from a per-user one. + async fn put( + &self, + key: &TemplateCacheKey, + metadata: &TemplateMetadata, + body: Vec, + max_age: std::time::Duration, + ) -> Result<(), TemplateCacheError>; + + /// Purge every cached variant for one publisher URL. + async fn purge_url(&self, key: &TemplateCacheKey) -> Result<(), TemplateCacheError>; + + /// Purge every stored template. The rollback lever. + async fn purge_all(&self) -> Result<(), TemplateCacheError>; +} + +/// A template read from the cache. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TemplateEntry { + /// Metadata stored at insert. + pub metadata: TemplateMetadata, + /// The transformed template bytes. + pub body: Vec, +} + +/// The null object, used by every adapter without a template cache. +/// +/// Reporting [`TemplateCacheMiss::Unsupported`] rather than erroring means the +/// ESI assembly mode degrades to transforming per request on Cloudflare, Axum and Spin +/// instead of failing — the mode stays portable, only the caching is not. +pub struct UnavailableTemplateCache; + +#[async_trait::async_trait(?Send)] +impl PlatformTemplateCache for UnavailableTemplateCache { + async fn lookup_or_reserve( + &self, + _key: &TemplateCacheKey, + ) -> Result { + Ok(TemplateCacheLookup::Unsupported) + } + + async fn get(&self, _key: &TemplateCacheKey) -> Result { + Err(TemplateCacheMiss::Unsupported) + } + + async fn put( + &self, + _key: &TemplateCacheKey, + _metadata: &TemplateMetadata, + _body: Vec, + _max_age: std::time::Duration, + ) -> Result<(), TemplateCacheError> { + Err(TemplateCacheError::Unsupported) + } + + async fn purge_url(&self, _key: &TemplateCacheKey) -> Result<(), TemplateCacheError> { + Err(TemplateCacheError::Unsupported) + } + + async fn purge_all(&self) -> Result<(), TemplateCacheError> { + Err(TemplateCacheError::Unsupported) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + + fn key() -> TemplateCacheKey { + TemplateCacheKey { + url: "https://example.com/news/article".to_string(), + request_host: "example.com".to_string(), + request_scheme: "https".to_string(), + origin_identity: "https://origin.example.com\0origin.example.com".to_string(), + assembly_mode: AssemblyMode::Esi, + vary_values: vec![VaryHeaderValues { + name: "rsc".to_string(), + values: Some(vec![b"1".to_vec()]), + }], + template_fingerprint: "abc123".to_string(), + schema_version: TEMPLATE_SCHEMA_VERSION, + } + } + + struct CountingReservation(Arc); + + impl PlatformTemplateCacheReservation for CountingReservation { + fn insert( + self: Box, + _metadata: &TemplateMetadata, + _body: Vec, + _max_age: std::time::Duration, + ) -> Result<(), TemplateCacheError> { + Ok(()) + } + + fn cancel(self: Box) -> Result<(), TemplateCacheError> { + self.0.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + } + + #[test] + fn dropping_an_unfulfilled_reservation_cancels_exactly_once() { + let cancellations = Arc::new(AtomicUsize::new(0)); + drop(TemplateCacheReservation::new(Box::new( + CountingReservation(Arc::clone(&cancellations)), + ))); + assert_eq!(cancellations.load(Ordering::SeqCst), 1); + } + + /// Every field must change the key. A field that does not is a cross-serving + /// bug: two requests needing different templates would share one entry. + #[test] + fn every_field_changes_the_key() { + let base = key().to_cache_key(); + + let mut mode = key(); + mode.assembly_mode = AssemblyMode::Inline; + assert_ne!( + mode.to_cache_key(), + base, + "assembly mode must change the key" + ); + + let mut url = key(); + url.url = "https://example.com/other".to_string(); + assert_ne!(url.to_cache_key(), base, "url must change the key"); + + let mut host = key(); + host.request_host = "other.example.com".to_string(); + assert_ne!(host.to_cache_key(), base, "host must change the key"); + + let mut scheme = key(); + scheme.request_scheme = "http".to_string(); + assert_ne!(scheme.to_cache_key(), base, "scheme must change the key"); + + let mut origin = key(); + origin.origin_identity = "https://origin.example.com\0other.example.com".to_string(); + assert_ne!( + origin.to_cache_key(), + base, + "origin Host identity must change the key" + ); + + let mut fingerprint = key(); + fingerprint.template_fingerprint = "def456".to_string(); + assert_ne!( + fingerprint.to_cache_key(), + base, + "template fingerprint must change the key" + ); + + let mut schema = key(); + schema.schema_version = TEMPLATE_SCHEMA_VERSION + 1; + assert_ne!( + schema.to_cache_key(), + base, + "schema version must change the key" + ); + + let mut vary = key(); + vary.vary_values = vec![VaryHeaderValues { + name: "rsc".to_string(), + values: Some(vec![b"0".to_vec()]), + }]; + assert_ne!(vary.to_cache_key(), base, "vary values must change the key"); + } + + /// The reason for length prefixes rather than a delimiter. + #[test] + fn values_containing_delimiters_cannot_collide() { + let mut a = key(); + a.request_host = "a".to_string(); + a.url = "b:c".to_string(); + + let mut b = key(); + b.request_host = "a:b".to_string(); + b.url = "c".to_string(); + + assert_ne!( + a.to_cache_key(), + b.to_cache_key(), + "field values containing the delimiter must not produce the same key; a \ + collision here serves one visitor's template to another" + ); + } + + #[test] + fn rendered_key_is_fixed_size_and_contains_no_request_material() { + let rendered = key().to_cache_key(); + assert_eq!(rendered.len(), "ts-c2-v3-".len() + 64); + for sensitive in ["example.com", "/news/article", "rsc", "abc123"] { + assert!( + !rendered.contains(sensitive), + "key leaked `{sensitive}`: {rendered}" + ); + } + } + + #[test] + fn vary_header_names_are_matched_case_insensitively() { + let mut upper = key(); + upper.vary_values = vec![VaryHeaderValues { + name: "RSC".to_ascii_lowercase(), + values: Some(vec![b"1".to_vec()]), + }]; + assert_eq!( + upper.to_cache_key(), + key().to_cache_key(), + "header names are case-insensitive, so casing must not split the cache" + ); + } + + #[test] + fn vary_values_are_order_sensitive() { + // The origin lists them in a fixed order and the caller preserves it, so a + // differing order means differing inputs rather than the same request. + let mut a = key(); + a.vary_values = vec![ + VaryHeaderValues { + name: "rsc".to_string(), + values: Some(vec![b"1".to_vec()]), + }, + VaryHeaderValues { + name: "x-route".to_string(), + values: Some(vec![b"article".to_vec()]), + }, + ]; + let mut b = key(); + b.vary_values = vec![ + VaryHeaderValues { + name: "x-route".to_string(), + values: Some(vec![b"article".to_vec()]), + }, + VaryHeaderValues { + name: "rsc".to_string(), + values: Some(vec![b"1".to_vec()]), + }, + ]; + assert_ne!(a.to_cache_key(), b.to_cache_key()); + } + + #[test] + fn surrogate_keys_carry_a_global_and_a_per_url_lever() { + let keys = key().surrogate_keys(); + assert!( + keys.contains(&"ts-template".to_string()), + "a global purge lever is what makes rollback possible" + ); + assert_eq!(keys.len(), 2, "global plus per-URL"); + assert!( + !keys[1].contains(char::is_whitespace), + "surrogate keys are space-delimited; whitespace would purge more than \ + intended, got {:?}", + keys[1] + ); + assert!( + !keys[1].contains('/') && !keys[1].contains(':'), + "URL punctuation must be reduced, got {:?}", + keys[1] + ); + } + + #[test] + fn punctuation_distinct_urls_have_distinct_surrogate_keys() { + let mut slash = key(); + slash.url = "https://example.com/a/b".to_string(); + let mut colon = key(); + colon.url = "https://example.com/a:b".to_string(); + assert_ne!(slash.surrogate_keys()[1], colon.surrogate_keys()[1]); + } + + #[test] + fn an_absent_vary_header_is_distinct_from_an_empty_one() { + // "absent" and "present but empty" are different requests to the origin, so + // they must not share a template. + let spec = VarySpec::new(["RSC".to_string()]); + let absent_headers = http::HeaderMap::new(); + let absent = spec.values_from(&absent_headers); + let mut empty_headers = http::HeaderMap::new(); + empty_headers.insert("rsc", http::HeaderValue::from_static("")); + let empty = spec.values_from(&empty_headers); + assert_ne!(absent, empty); + + // The distinction that does matter: a present value differs from both. + let mut present_headers = http::HeaderMap::new(); + present_headers.insert("rsc", http::HeaderValue::from_static("1")); + let present = spec.values_from(&present_headers); + assert_ne!(present, absent); + } + + #[test] + fn repeated_and_non_utf8_vary_values_are_preserved() { + let spec = VarySpec::new(["x-route".to_string()]); + let mut headers = http::HeaderMap::new(); + headers.append("x-route", http::HeaderValue::from_static("first")); + headers.append( + "x-route", + http::HeaderValue::from_bytes(b"\xffsecond").expect("obs-text is valid field data"), + ); + assert_eq!( + spec.values_from(&headers), + vec![VaryHeaderValues { + name: "x-route".to_string(), + values: Some(vec![b"first".to_vec(), b"\xffsecond".to_vec()]), + }] + ); + } + + #[test] + fn vary_spec_lowercases_configured_names() { + assert_eq!( + VarySpec::new(["RSC".to_string(), "Accept-Encoding".to_string()]).names(), + ["rsc"] + ); + } + + #[test] + fn vary_spec_rejects_invalid_names_and_deduplicates_case_insensitively() { + assert_eq!( + VarySpec::try_new(["not a header".to_string()]), + Err("not a header".to_string()) + ); + assert_eq!( + VarySpec::try_new(["RSC".to_string(), "rsc".to_string()]) + .expect("valid names") + .names(), + ["rsc"] + ); + } + + #[test] + fn drift_is_detected_when_the_origin_varies_on_something_unconfigured() { + // The failure mode configured-Vary has: the origin adds a header to its Vary, + // nobody updates config, and requests differing only in that header start + // sharing a template. + let spec = VarySpec::new(["rsc".to_string()]); + + assert!( + spec.uncovered_by(["rsc"]).is_empty(), + "a fully covered Vary is not drift" + ); + assert_eq!( + spec.uncovered_by(["rsc, next-router-prefetch, Accept-Encoding"]), + vec!["next-router-prefetch"], + "uncovered names must be reported so the stale config is identifiable; \ + accept-encoding is excluded because the key covers it structurally" + ); + } + + #[test] + fn a_key_field_counts_as_coverage_without_being_configured() { + // The failure this prevents is silent and total: every compressing origin sends + // `Vary: Accept-Encoding`, so treating it as a gap means the cache never stores + // anything, and a spike measuring hit rate would report ~0 and look like a + // finding rather than a bug. + let spec = VarySpec::new([]); + + assert!( + spec.uncovered_by(["Accept-Encoding"]).is_empty(), + "the shared path uses one upstream encoding offer and stores identity bytes" + ); + assert_eq!( + spec.uncovered_by(["accept-encoding, rsc"]), + vec!["rsc"], + "only the genuinely uncovered name should be reported" + ); + } + + #[test] + fn a_wildcard_vary_is_not_reported_as_a_named_gap() { + // `Vary: *` means uncacheable, which the eligibility gate handles. Reporting + // it here would produce a nonsense "configure a header called *". + let spec = VarySpec::new(["rsc".to_string()]); + assert!(spec.uncovered_by(["*"]).is_empty()); + } + + #[test] + fn metadata_round_trips() { + let metadata = TemplateMetadata { + content_encoding: "identity".to_string(), + policy_headers: vec![ + ( + "content-security-policy".to_string(), + "default-src 'self'".to_string(), + ), + ( + "content-security-policy".to_string(), + "script-src 'self'".to_string(), + ), + ( + "link".to_string(), + "; rel=preload; as=script".to_string(), + ), + ], + content_type: "text/html; charset=utf-8".to_string(), + schema_version: TEMPLATE_SCHEMA_VERSION, + body_len: 42, + }; + let decoded = + TemplateMetadata::decode(&metadata.encode()).expect("should decode what it encoded"); + assert_eq!(decoded, metadata); + } + + #[test] + fn unparseable_metadata_is_a_miss_not_a_panic() { + for raw in [ + &b"not-key-value"[..], + &b"v=notanumber\nce=gzip\nct=text/html\nlen=1"[..], + &b"v=1\nce=gzip\nct=text/html"[..], + &b"v=1\nce=gzip\nct=text/html\nlen=1\nunexpected=1"[..], + &b"v=1\nv=1\nce=identity\nct=text/html\nlen=1"[..], + &b"v=1\nce=identity\nct=text/html\nlen=1\nh=cache-control:public"[..], + &b"v=1\nce=identity\nct=text/html\nlen=1\nh=not-a-policy:value"[..], + &b"v=1\nce=identity\nct=text/html\nlen=1\nh=malformed"[..], + &b"v=1\nce=identity\nct=application/json\nlen=1"[..], + &[0xff, 0xfe][..], + ] { + assert_eq!( + TemplateMetadata::decode(raw), + None, + "malformed metadata must be a miss, not a partial read: {raw:?}" + ); + } + } + + #[test] + fn the_policy_allowlist_covers_document_security_and_delivery_headers() { + for required in [ + "strict-transport-security", + "cross-origin-opener-policy", + "cross-origin-embedder-policy", + "cross-origin-resource-policy", + "origin-agent-cluster", + "reporting-endpoints", + "report-to", + "link", + ] { + assert!( + REPLAYABLE_POLICY_HEADERS.contains(&required), + "warm ESI hits must preserve {required}" + ); + } + } + + #[tokio::test] + async fn the_null_object_reports_unsupported_rather_than_failing() { + // Degrading to per-request transformation keeps the shared modes portable on + // adapters with no cache; erroring would make them Fastly-only outright. + let cache = UnavailableTemplateCache; + assert_eq!( + cache.get(&key()).await.err(), + Some(TemplateCacheMiss::Unsupported) + ); + assert!(matches!( + cache + .put( + &key(), + &TemplateMetadata { + content_encoding: "identity".to_string(), + policy_headers: Vec::new(), + content_type: "text/html".to_string(), + schema_version: TEMPLATE_SCHEMA_VERSION, + body_len: 0, + }, + Vec::new(), + std::time::Duration::from_secs(1) + ) + .await, + Err(TemplateCacheError::Unsupported) + )); + } +} diff --git a/crates/trusted-server-core/src/platform/types.rs b/crates/trusted-server-core/src/platform/types.rs index a39a26430..e9e02e524 100644 --- a/crates/trusted-server-core/src/platform/types.rs +++ b/crates/trusted-server-core/src/platform/types.rs @@ -168,6 +168,16 @@ pub struct RuntimeServices { /// per-request basis by cloning [`RuntimeServices`] with /// [`RuntimeServices::with_kv_store`]. pub(crate) kv_store: Arc, + /// Shared transformed-template cache (C2). Defaults to + /// [`UnavailableTemplateCache`], so adapters without one degrade to transforming + /// per request rather than failing. Spike-only; see + /// [`crate::platform::template_cache`]. + pub(crate) template_cache: Arc, + /// Platform-specific cold-response template assembler. + /// + /// Defaults to [`super::UnavailableTemplateAssembler`]. Core retains a portable + /// byte-seam fallback when this service is unavailable or rejects a document. + pub(crate) template_assembler: Arc, /// Dynamic backend registration and name prediction. pub(crate) backend: Arc, /// Outbound HTTP client abstraction. @@ -223,6 +233,18 @@ impl RuntimeServices { &*self.kv_store } + /// The shared transformed-template cache. Spike-only. + #[must_use] + pub fn template_cache(&self) -> &dyn super::PlatformTemplateCache { + &*self.template_cache + } + + /// Returns the platform-specific cold-response template assembler. + #[must_use] + pub fn template_assembler(&self) -> &dyn super::PlatformTemplateAssembler { + &*self.template_assembler + } + /// Returns the dynamic backend service. #[must_use] pub fn backend(&self) -> &dyn PlatformBackend { @@ -272,6 +294,29 @@ impl RuntimeServices { ..self } } + + /// Returns a clone of this instance with the template cache replaced. + /// + /// Spike-only (#1009). + #[must_use] + pub fn with_template_cache(self, cache: Arc) -> Self { + Self { + template_cache: cache, + ..self + } + } + + /// Returns a clone of this instance with the template assembler replaced. + #[must_use] + pub fn with_template_assembler( + self, + assembler: Arc, + ) -> Self { + Self { + template_assembler: assembler, + ..self + } + } } impl fmt::Debug for RuntimeServices { @@ -290,6 +335,8 @@ pub struct RuntimeServicesBuilder { config_store: Option>, secret_store: Option>, kv_store: Option>, + template_cache: Option>, + template_assembler: Option>, backend: Option>, http_client: Option>, geo: Option>, @@ -303,6 +350,8 @@ impl RuntimeServicesBuilder { config_store: None, secret_store: None, kv_store: None, + template_cache: None, + template_assembler: None, backend: None, http_client: None, geo: None, @@ -325,6 +374,23 @@ impl RuntimeServicesBuilder { self } + /// Set the shared transformed-template cache. Spike-only. + #[must_use] + pub fn template_cache(mut self, cache: Arc) -> Self { + self.template_cache = Some(cache); + self + } + + /// Set the platform-specific cold-response template assembler. + #[must_use] + pub fn template_assembler( + mut self, + assembler: Arc, + ) -> Self { + self.template_assembler = Some(assembler); + self + } + /// Set the KV store implementation. #[must_use] pub fn kv_store(mut self, kv_store: Arc) -> Self { @@ -387,6 +453,14 @@ impl RuntimeServicesBuilder { kv_store: self .kv_store .expect("should set kv_store before building RuntimeServices"), + // Defaulted rather than required: an adapter with no template cache + // should degrade to transforming per request, not fail to build. + template_cache: self + .template_cache + .unwrap_or_else(|| Arc::new(super::UnavailableTemplateCache)), + template_assembler: self + .template_assembler + .unwrap_or_else(|| Arc::new(super::UnavailableTemplateAssembler)), backend: self .backend .expect("should set backend before building RuntimeServices"), diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index e2ff3d3cf..65d5dd124 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -21,7 +21,7 @@ use std::borrow::Cow; use std::io::Write; use std::sync::{Arc, Mutex}; -use std::time::Duration; +use std::time::{Duration, Instant, SystemTime}; use brotli::Decompressor; use brotli::enc::BrotliEncoderParams; @@ -51,15 +51,19 @@ use crate::auction::types::{ use crate::consent::{consent_allows_server_side_auction, gate_eids_by_consent}; use crate::constants::{COOKIE_TS_EIDS, HEADER_X_COMPRESS_HINT}; use crate::cookies::handle_request_cookies; +use crate::creative_opportunities::{AssemblyMode, CreativeOpportunitiesConfig}; use crate::ec::EcContext; use crate::ec::kv::KvIdentityGraph; use crate::ec::registry::PartnerRegistry; use crate::error::TrustedServerError; +use crate::html_processor::BodyCloseInjection; use crate::http_util::{RequestInfo, is_navigation_request, serve_static_with_etag}; use crate::integrations::IntegrationRegistry; -use crate::platform::{GeoInfo, PlatformBackendSpec, PlatformHttpRequest, RuntimeServices}; +use crate::platform::{ + GeoInfo, PlatformBackendSpec, PlatformHttpRequest, RuntimeServices, VarySpec, +}; use crate::price_bucket::{PriceGranularity, price_bucket}; -use crate::response_privacy::enforce_synthesized_html_cache_privacy; +use crate::response_privacy::{enforce_private_no_store, enforce_synthesized_html_cache_privacy}; use crate::rsc_flight::RscFlightUrlRewriter; use crate::settings::Settings; use crate::streaming_processor::{ @@ -70,6 +74,68 @@ use crate::streaming_replacer::create_url_replacer; const SUPPORTED_ENCODING_VALUES: [&str; 3] = ["gzip", "deflate", "br"]; const DEFAULT_PUBLISHER_FIRST_BYTE_TIMEOUT: Duration = Duration::from_secs(15); +const HEADER_X_TS_C2_CACHE: &str = "x-ts-c2-cache"; +const HEADER_X_TS_ASSEMBLY: &str = "x-ts-assembly"; + +#[derive(Clone, Copy, PartialEq, Eq)] +enum C2ResponseState { + Hit, + MissReserved, + MissStored, + MissStoreError, + BypassRequest, + BypassResponse, + Unsupported, + Invalid, + BackendError, +} + +impl C2ResponseState { + const fn as_str(self) -> &'static str { + match self { + Self::Hit => "hit", + Self::MissReserved => "miss-reserved", + Self::MissStored => "miss-stored", + Self::MissStoreError => "miss-store-error", + Self::BypassRequest => "bypass-request", + Self::BypassResponse => "bypass-response", + Self::Unsupported => "unsupported", + Self::Invalid => "invalid", + Self::BackendError => "backend-error", + } + } +} + +fn set_c2_response_state(response: &mut Response, state: C2ResponseState) { + response.headers_mut().insert( + HEADER_X_TS_C2_CACHE, + HeaderValue::from_static(state.as_str()), + ); +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum AssemblyResponseState { + EsiParser, + ByteSeamFallback, + ByteSeam, +} + +impl AssemblyResponseState { + const fn as_str(self) -> &'static str { + match self { + Self::EsiParser => "esi-parser", + Self::ByteSeamFallback => "byte-seam-fallback", + Self::ByteSeam => "byte-seam", + } + } +} + +fn set_assembly_response_state(response: &mut Response, state: AssemblyResponseState) { + response.headers_mut().insert( + HEADER_X_TS_ASSEMBLY, + HeaderValue::from_static(state.as_str()), + ); +} fn body_as_reader( body: EdgeBody, @@ -201,11 +267,16 @@ fn restrict_accept_encoding(req: &mut Request) { // origin responds without compression. Adding encodings here would cause the // origin to compress its response even though the client never asked for it, // and the client would then receive content it cannot decode. + if !req.headers().contains_key(header::ACCEPT_ENCODING) { + return; + } let Some(current) = req .headers() - .get(header::ACCEPT_ENCODING) - .and_then(|value| value.to_str().ok()) - .map(str::to_owned) + .get_all(header::ACCEPT_ENCODING) + .iter() + .map(|value| value.to_str().ok()) + .collect::>>() + .map(|values| values.join(", ")) else { return; }; @@ -273,6 +344,158 @@ fn accept_encoding_qvalue(header_value: &str, target: &str) -> Option { matched_qvalue } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ReaderEncodingError { + Malformed, + NoAcceptableEncoding, +} + +fn parse_quality_value(value: &str) -> Option { + let value = value.trim(); + let (whole, fraction) = value + .split_once('.') + .map_or((value, None), |(whole, fraction)| (whole, Some(fraction))); + let fraction_is_valid = fraction.is_none_or(|fraction| { + fraction.len() <= 3 && fraction.bytes().all(|byte| byte.is_ascii_digit()) + }); + if !fraction_is_valid { + return None; + } + match whole { + "0" => value.parse().ok(), + "1" if fraction.is_none_or(|fraction| fraction.bytes().all(|byte| byte == b'0')) => { + Some(1.0) + } + _ => None, + } +} + +fn negotiate_reader_compression( + headers: &edgezero_core::http::HeaderMap, +) -> Result { + if !headers.contains_key(header::ACCEPT_ENCODING) { + return Ok(Compression::None); + } + + let mut qualities = Vec::<(String, f32)>::new(); + for field in headers.get_all(header::ACCEPT_ENCODING) { + let field = field.to_str().map_err(|_| ReaderEncodingError::Malformed)?; + for item in field + .split(',') + .map(str::trim) + .filter(|item| !item.is_empty()) + { + let mut parts = item.split(';'); + let token = parts + .next() + .map(str::trim) + .filter(|token| !token.is_empty()) + .ok_or(ReaderEncodingError::Malformed)? + .to_ascii_lowercase(); + if token != "*" && http::HeaderName::from_bytes(token.as_bytes()).is_err() { + return Err(ReaderEncodingError::Malformed); + } + let mut quality = 1.0; + let mut saw_quality = false; + for parameter in parts { + let (name, value) = parameter + .trim() + .split_once('=') + .ok_or(ReaderEncodingError::Malformed)?; + if !name.trim().eq_ignore_ascii_case("q") || saw_quality { + return Err(ReaderEncodingError::Malformed); + } + quality = parse_quality_value(value).ok_or(ReaderEncodingError::Malformed)?; + saw_quality = true; + } + if qualities.iter().any(|(seen, _)| seen == &token) { + return Err(ReaderEncodingError::Malformed); + } + qualities.push((token, quality)); + } + } + + let explicit = |name: &str| { + qualities + .iter() + .find_map(|(candidate, quality)| (candidate == name).then_some(*quality)) + }; + let wildcard = explicit("*"); + let quality_for = |name: &str| explicit(name).or(wildcard).unwrap_or(0.0); + // Identity is implicitly acceptable at q=1 unless explicitly excluded, or a + // wildcard q=0 excludes every unlisted coding. + let identity_quality = + explicit("identity").unwrap_or_else(|| if wildcard == Some(0.0) { 0.0 } else { 1.0 }); + + let candidates = [ + (Compression::Brotli, quality_for("br")), + (Compression::Gzip, quality_for("gzip")), + (Compression::Deflate, quality_for("deflate")), + (Compression::None, identity_quality), + ]; + let mut selected = None; + for (compression, quality) in candidates { + if quality > 0.0 && selected.is_none_or(|(_, best)| quality > best) { + selected = Some((compression, quality)); + } + } + selected + .map(|(compression, _)| compression) + .ok_or(ReaderEncodingError::NoAcceptableEncoding) +} + +fn set_response_compression(response: &mut Response, compression: Compression) { + let encoding = match compression { + Compression::None => None, + Compression::Gzip => Some("gzip"), + Compression::Deflate => Some("deflate"), + Compression::Brotli => Some("br"), + }; + if let Some(encoding) = encoding { + response + .headers_mut() + .insert(header::CONTENT_ENCODING, HeaderValue::from_static(encoding)); + } else { + response.headers_mut().remove(header::CONTENT_ENCODING); + } + let varies_on_encoding = response + .headers() + .get_all(header::VARY) + .iter() + .any(|value| { + value.to_str().is_ok_and(|value| { + value + .split(',') + .any(|name| name.trim().eq_ignore_ascii_case("accept-encoding")) + }) + }); + if !varies_on_encoding { + response + .headers_mut() + .append(header::VARY, HeaderValue::from_static("Accept-Encoding")); + } + response.headers_mut().remove(header::CONTENT_LENGTH); +} + +fn response_compression(response: &Response) -> Compression { + response + .headers() + .get(header::CONTENT_ENCODING) + .and_then(|value| value.to_str().ok()) + .map(Compression::from_content_encoding) + .unwrap_or(Compression::None) +} + +fn encode_complete_body( + body: Vec, + compression: Compression, +) -> Result, Report> { + let mut encoder = BodyStreamEncoder::new(compression); + let mut encoded = encoder.encode_chunk(body)?; + encoded.extend_from_slice(&encoder.finish()?); + Ok(encoded) +} + /// Unified tsjs static serving: `/static/tsjs=` /// /// Serves two types of bundles: @@ -361,6 +584,8 @@ struct ProcessResponseParams<'a> { suppress_datadome_client_side_tag: bool, gpt_diagnostics: Option<&'a crate::integrations::gpt_diagnostics::GptDiagnosticsRequestDecision>, + /// See [`HtmlStreamProcessorParams::shared_template_authorized`]. + shared_template_authorized: bool, } struct PublisherBodyProcessor { @@ -384,9 +609,10 @@ impl PublisherBodyProcessor { settings, integration_registry, ad_slots_script: params.ad_slots_script.as_deref().map(str::to_string), - ad_bids_state: Arc::clone(¶ms.ad_bids_state), + ad_bids_state: Arc::clone(params.ad_bids_state.script_cell()), suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, gpt_diagnostics: params.gpt_diagnostics.clone(), + shared_template_authorized: params.template_cache_key.is_some(), })?) } else if is_rsc_flight { Box::new(RscFlightUrlRewriter::new( @@ -428,6 +654,7 @@ fn process_response_streaming( body: EdgeBody, output: &mut W, params: &ProcessResponseParams, + output_compression: Compression, ) -> Result<(), Report> { let is_html = is_html_content_type(params.content_type); let is_rsc_flight = @@ -443,7 +670,7 @@ fn process_response_streaming( let compression = Compression::from_content_encoding(params.content_encoding); let config = PipelineConfig { input_compression: compression, - output_compression: compression, + output_compression, chunk_size: 8192, }; // Bound how much decoded gzip output may sit in the heap at once, using the @@ -465,6 +692,7 @@ fn process_response_streaming( ad_bids_state: params.ad_bids_state.clone(), suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, gpt_diagnostics: params.gpt_diagnostics.cloned(), + shared_template_authorized: params.shared_template_authorized, })?; StreamingPipeline::new(config, processor) .with_max_pending_decoded_bytes(max_pending_decoded_bytes) @@ -509,13 +737,21 @@ async fn process_response_streaming_async( params.content_encoding ); - let compression = Compression::from_content_encoding(¶ms.content_encoding); + let input_compression = Compression::from_content_encoding(¶ms.content_encoding); + // A C2 template is always identity bytes. Decode during the transform instead of + // recompressing and immediately decoding the entire buffered result afterwards. + let output_compression = if params.template_cache_key.is_some() { + Compression::None + } else { + input_compression + }; let mut processor = PublisherBodyProcessor::new(params, settings, integration_registry)?; process_body_chunks_async( body, output, &mut processor, - compression, + input_compression, + output_compression, settings.publisher.max_buffered_body_bytes, ) .await @@ -557,11 +793,12 @@ async fn process_body_chunks_async( body: EdgeBody, writer: &mut W, processor: &mut P, - compression: Compression, + input_compression: Compression, + output_compression: Compression, max_body_bytes: usize, ) -> Result<(), Report> { - let mut decoder = BodyStreamDecoder::new(compression, max_body_bytes); - let mut encoder = BodyStreamEncoder::new(compression); + let mut decoder = BodyStreamDecoder::new(input_compression, max_body_bytes); + let mut encoder = BodyStreamEncoder::new(output_compression); let mut source = BodyChunkSource::new(body, STREAM_CHUNK_SIZE).with_max_bytes(max_body_bytes); while let Some(segments) = @@ -950,6 +1187,137 @@ struct HtmlStreamProcessorParams<'a> { ad_bids_state: Arc>>, suppress_datadome_client_side_tag: bool, gpt_diagnostics: Option, + /// Whether a shared template was authorized for this response. + /// + /// Carried rather than re-derived so both seams see the same answer. See + /// [`effective_assembly_mode`]. + shared_template_authorized: bool, +} + +/// The diagnostics decision the template may carry. +/// +/// Diagnostics is request-scoped — activated by a cookie or query parameter, and +/// documented as an immutable per-request decision — so it must not reach a shared +/// template. +/// +/// It does not leak today even without this gate, but only by coincidence: +/// `requires_private_no_store()` is a strict superset of the conditions under which +/// a script is emitted, and that stamp lands before the C2 gate reads response +/// headers, so the gate refuses. Two independent conditions that happen to align, +/// with nothing enforcing the relationship. This makes the guarantee explicit; +/// `requires_private_no_store_is_a_superset_of_injection` keeps the coincidence as a +/// backstop if this gate is ever removed. +pub(crate) fn template_gpt_diagnostics( + mode: AssemblyMode, + decision: Option, +) -> Option { + match mode { + AssemblyMode::Inline => decision, + AssemblyMode::Esi => None, + } +} + +/// The marker emitted at the `` seam under [`AssemblyMode::Esi`], reserving the +/// place this reader's slots and bids are spliced into. +/// +/// An inert HTML comment, deliberately. Template schema v1 used an executable ESI include +/// tag here, when the `esi` crate resolved it at the edge. That crate was removed from the +/// render path because it truncates any element larger than its 16 KB chunk size, and +/// nothing has parsed ESI since. What remained was a tag that *looked* executable, would +/// have been executed by any ESI-enabled layer in front of us, and renders as text in a +/// browser if assembly is ever skipped. A comment cannot do any of those things: an +/// unassembled template degrades to a page with no ads rather than a page with a visible +/// tag. +/// +/// Carries no URL. Every byte here is a byte every reader of the shared template +/// receives, so nothing request-scoped may appear, and keeping a URL out also removes +/// any escaping question at the seam. +pub const AD_ASSEMBLY_SEAM: &str = ""; + +/// The mode the operator asked for, before availability is taken into account. +/// +/// Spelled once, because the mode has to mean the same thing at the cache key, at the +/// seam, and at both hit finalizers. Every one of those re-derived it from the same +/// `Option` chain, and the finalizers had no way to ask at all — which is why they +/// demanded a seam marker of a mode that emits none. +fn configured_assembly_mode(settings: &Settings) -> AssemblyMode { + settings + .creative_opportunities + .as_ref() + .map(CreativeOpportunitiesConfig::assembly_mode) + .unwrap_or_default() +} + +/// Whether this mode's `` seam emits [`AD_ASSEMBLY_SEAM`]. +/// +/// The property that decides whether a template is *expected* to have a hole in it, and +/// therefore whether the absence of one is a defect or the design. Only `Esi` splices per +/// reader. +/// +/// Matched exhaustively rather than compared against `Esi`, so a new mode has to state +/// its answer here instead of silently inheriting one. +fn mode_emits_seam_marker(mode: AssemblyMode) -> bool { + match mode { + AssemblyMode::Inline => false, + AssemblyMode::Esi => true, + } +} + +/// The assembly mode this response will actually be delivered under. +/// +/// The configured mode says what the operator wants; the cache key says whether it is +/// available. A shared mode with no key means the gate refused this response — the +/// origin set a cookie, declared a `Vary` the key does not cover, returned a non-200, +/// and so on — so there is no shared template to build and nothing downstream will +/// assemble one. +/// +/// When that happens the request falls back to [`AssemblyMode::Inline`] **entirely**, +/// at every seam. Falling back at one seam and not another is what produced the failure +/// this function exists to prevent: the `` seam emitted a legacy ESI tag because +/// the mode was `Esi`, while assembly was skipped because there was no key, so the reader +/// received a document with unresolved executable ESI markup in it and no bids at all. +/// +/// Bypassing is the *normal* case against a real origin, not an edge case, so this path +/// runs far more often than the shared one. +fn effective_assembly_mode(settings: &Settings, shared_template_authorized: bool) -> AssemblyMode { + let configured = configured_assembly_mode(settings); + if matches!(configured, AssemblyMode::Inline) || shared_template_authorized { + return configured; + } + log::debug!( + "assembly mode {configured:?} is unavailable for this response (no shared template \ + was authorized); falling back to inline" + ); + AssemblyMode::Inline +} + +/// What the `` seam should inject, given the assembly mode. +/// +/// Explicit rather than inferred. The previous shape read +/// `ad_slots_script.is_some()` inside the element handler, which silently coupled +/// two independent decisions: once [`template_ad_slots_script`] stopped emitting a +/// head script under a shared mode, body-close injection stopped with it. +/// +/// `Esi` emits [`AD_ASSEMBLY_SEAM`], an inert HTML comment marking where this reader's +/// slots and bids are spliced in. Assembly is a byte split on that comment, performed by +/// this crate on both the miss and the hit path; no ESI layer is involved. +pub(crate) fn body_close_injection( + mode: AssemblyMode, + head_script_present: bool, +) -> BodyCloseInjection { + match mode { + // Per-navigation and never shared, so gating on slot presence is correct. + AssemblyMode::Inline => { + if head_script_present { + BodyCloseInjection::InlineBids + } else { + BodyCloseInjection::None + } + } + // Constant across every request that reaches the transform — which is what + // makes it safe in a shared template. + AssemblyMode::Esi => BodyCloseInjection::Marker(AD_ASSEMBLY_SEAM.to_string()), + } } fn create_html_stream_processor( @@ -963,10 +1331,18 @@ fn create_html_stream_processor( params.origin_host, params.request_host, params.request_scheme, - ) - .with_ad_state(params.ad_slots_script, params.ad_bids_state) - .with_gpt_diagnostics(params.gpt_diagnostics) - .with_datadome_client_tag_suppression(params.suppress_datadome_client_side_tag); + ); + + let assembly_mode = effective_assembly_mode(params.settings, params.shared_template_authorized); + let body_close = body_close_injection(assembly_mode, params.ad_slots_script.is_some()); + + let gpt_diagnostics = template_gpt_diagnostics(assembly_mode, params.gpt_diagnostics); + + let config = config + .with_ad_state(params.ad_slots_script, params.ad_bids_state) + .with_gpt_diagnostics(gpt_diagnostics) + .with_body_close(body_close) + .with_datadome_client_tag_suppression(params.suppress_datadome_client_side_tag); Ok(create_html_processor(config)) } @@ -1000,6 +1376,27 @@ pub enum PublisherResponse { /// Parameters for [`process_response_streaming`]. params: Box, }, + /// A shared template read from C2, to be assembled on the way out. + /// + /// Distinct from [`Self::Stream`] because the bytes are **already transformed** — + /// running them through `lol_html` again would inject a second tsjs `", + html_escape_for_script(slots_json), + html_escape_for_script(&bids) + ) +} + +/// The slot definitions a shared-mode seam must carry, as JSON. +/// +/// Mirrors [`template_ad_slots_script`]'s gating: same `should_run_ad_stack` condition, +/// same slot set. The difference is only *where* it is delivered — the seam, per +/// request, rather than the head, into a shared template. +pub(crate) fn seam_ad_slots_json( + mode: AssemblyMode, + should_run_ad_stack: bool, + settings: &Settings, + matched_slots: &[crate::creative_opportunities::CreativeOpportunitySlot], + request_path: &str, +) -> Option { + if matches!(mode, AssemblyMode::Inline) || !should_run_ad_stack { + return None; + } + let co_config = settings.creative_opportunities.as_ref()?; + let section = co_config.section_for_path(request_path); + let slots: Vec = matched_slots + .iter() + .filter_map(|slot| build_slot_json(slot, co_config, §ion)) + .collect(); + Some( + serde_json::to_string(&slots) + .expect("serde_json::to_string of Vec should be infallible"), + ) +} + /// Build the empty-bids `origin" + .to_vec(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("cache-control", "public, max-age=300"), + ( + "content-security-policy", + "default-src 'self'; script-src 'nonce-reader-nonce'", + ), + ], + ); + } + + let _ = body_of(run(&settings, &services, navigation_request()).await).await; + let _ = body_of(run(&settings, &services, navigation_request()).await).await; + + assert_eq!( + stub.recorded_request_uris().len(), + 2, + "a response-bound CSP nonce and its HTML must never be reused from C2" + ); + assert!( + cache + .entries + .lock() + .expect("should lock entries") + .is_empty() + ); + } + + #[tokio::test] + async fn a_post_is_never_answered_from_a_cached_get() { + // `handle_publisher_request` is the `*`-method fallback route, so a publisher + // path that renders a page on GET and accepts a form or webhook on POST reaches + // here for both. Serving the cached GET to the POST swallows the mutating + // request entirely: the origin never sees it, the caller gets 200 and a page, + // and nothing anywhere reports a problem. + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + queue_shareable_html(&stub); + queue_shareable_html(&stub); + + // Warm the cache with a GET. + let _ = run(&settings, &services, navigation_request()).await; + assert_eq!(stub.recorded_request_uris().len(), 1); + + let post = HttpRequest::builder() + .method(Method::POST) + .uri("https://ts.example.com/article") + .header(header::HOST, "ts.example.com") + .body(EdgeBody::from("field=value")) + .expect("should build post request"); + let _ = run(&settings, &services, post).await; + + assert_eq!( + stub.recorded_request_uris().len(), + 2, + "the POST must reach the origin rather than being answered from the \ + cached GET" + ); + assert_eq!( + cache.entries.lock().expect("should lock entries").len(), + 1, + "and it must not store a template of its own" + ); + } + + #[tokio::test] + async fn an_authenticated_request_is_not_served_a_shared_template() { + // The stored template is perfectly cacheable; this request is not entitled + // to it. The store gate cannot express that, because it is a property of + // the reader rather than of the bytes — which is why the lookup re-checks. + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + queue_shareable_html(&stub); + queue_shareable_html(&stub); + + let _ = run(&settings, &services, navigation_request()).await; + assert_eq!( + cache.entries.lock().expect("should lock entries").len(), + 1, + "the cold request should have populated the cache" + ); + + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let consent = crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + }; + let mut ec_context = EcContext::new_for_test(None, consent); + let authenticated = HttpRequest::builder() + .method(Method::GET) + .uri("https://ts.example.com/article") + .header(header::HOST, "ts.example.com") + .header("sec-fetch-dest", "document") + .header(header::AUTHORIZATION, "Basic dXNlcjpwYXNz") + .body(EdgeBody::empty()) + .expect("should build authenticated request"); + let _ = handle_publisher_request( + &settings, + &services, + None, + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &[article_slot()], + registry: None, + }, + authenticated, + ) + .await + .expect("should proxy publisher request"); + + assert_eq!( + stub.recorded_request_uris().len(), + 2, + "an authenticated request must reach the origin rather than read a \ + shared template" + ); + } + } + + mod c2_gate_tests { + //! `cache::core` stores whatever it is handed and rejects nothing, so every + //! one of these conditions is the caller's to enforce. Each is a leak vector + //! or an eligibility rule, not a preference. + + use super::*; + use crate::creative_opportunities::AssemblyMode; + use edgezero_core::http::HeaderName; + + fn headers(pairs: &[(HeaderName, &str)]) -> edgezero_core::http::HeaderMap { + let mut map = edgezero_core::http::HeaderMap::new(); + for (name, value) in pairs { + map.insert( + name.clone(), + HeaderValue::from_str(value).expect("should build header value"), + ); + } + map + } + + fn shareable() -> edgezero_core::http::HeaderMap { + headers(&[(header::CACHE_CONTROL, "max-age=60")]) + } + + /// The shipped default: no operator has stated what the origin varies on, so the + /// key covers nothing. Responses without a `Vary` are unaffected; any `Vary` at + /// all disqualifies. + fn nothing_covered() -> VarySpec { + VarySpec::new([]) + } + + #[test] + fn an_unconfigured_deployment_never_caches_a_varying_response() { + // The fail-closed default. An operator who has not stated the origin's Vary + // must not acquire a shared cache by omission — and a real origin varies on + // something, so this is the common path, not an edge case. + // Deliberately not `Accept-Encoding`: the shared path normalizes supported + // content codings to one identity template, so that header is covered + // whatever the operator configured. Using it here would test the + // structural-coverage carve-out rather than the drift guard. + let mut varying = shareable(); + varying.insert(header::VARY, HeaderValue::from_static("rsc")); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &varying, + ¬hing_covered(), + ), + Some(C2BypassReason::VaryNotCovered(VaryGap(vec![ + "rsc".to_string() + ]))), + "an unstated Vary must disqualify rather than silently under-key" + ); + } + + #[test] + fn an_origin_that_varies_on_cookie_is_refused_even_when_declared_independent() { + // The backstop that makes `origin_is_cookie_independent` safe to offer. The + // operator asserts their origin ignores cookies; if the origin then says + // otherwise, the assertion loses. Without this, a wrong assertion would + // silently cross-serve personalized HTML. + let mut varying = shareable(); + varying.insert(header::VARY, HeaderValue::from_static("Cookie")); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + // The operator's assertion has already been applied here: this is + // `false` precisely because they declared independence. + false, + StatusCode::OK, + "text/html", + &varying, + &VarySpec::new(["cookie".to_string()]), + ), + Some(C2BypassReason::VaryCookie), + "the origin's declaration must override both cookie independence and an \ + accidentally configured per-cookie key" + ); + } + + #[test] + fn a_private_directive_on_a_second_cache_control_line_is_refused() { + // `HeaderMap::get` returns the first value only. An origin that sends + // `Cache-Control: public, max-age=300` and then `Cache-Control: private` on a + // separate line means exactly what one comma-joined line would mean, but the + // second line was invisible — so a response the origin marked private was + // written to a cache shared between readers. The `Vary` reads a few lines up + // already use `get_all` for the same reason. + let mut split = edgezero_core::http::HeaderMap::new(); + split.append( + header::CACHE_CONTROL, + HeaderValue::from_static("public, max-age=300"), + ); + split.append(header::CACHE_CONTROL, HeaderValue::from_static("private")); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &split, + ¬hing_covered(), + ), + Some(C2BypassReason::OriginNotShareable), + "a directive on any Cache-Control line must disqualify the response" + ); + } + + #[test] + fn a_no_store_directive_on_a_second_cache_control_line_is_refused() { + // Same defect, the other directive that matters — `no-store` is the one an + // origin uses for a response that must not be written down anywhere. + let mut split = edgezero_core::http::HeaderMap::new(); + split.append( + header::CACHE_CONTROL, + HeaderValue::from_static("max-age=60"), + ); + split.append(header::CACHE_CONTROL, HeaderValue::from_static("no-store")); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &split, + ¬hing_covered(), + ), + Some(C2BypassReason::OriginNotShareable) + ); + } + + #[test] + fn cdn_specific_cache_policy_cannot_be_overridden_by_public_cache_control() { + for name in crate::response_privacy::CDN_CACHE_HEADERS { + let mut split = shareable(); + split.insert( + header::HeaderName::from_static(name), + HeaderValue::from_static("no-store"), + ); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &split, + ¬hing_covered(), + ), + Some(C2BypassReason::OriginNotShareable), + "C2 must fail closed on the CDN-specific policy header {name}" + ); + } + } + + #[test] + fn unsupported_vendor_freshness_does_not_authorize_c2() { + for name in crate::response_privacy::CDN_CACHE_HEADERS + .iter() + .filter(|name| **name != "surrogate-control") + { + let mut split = shareable(); + split.insert( + header::HeaderName::from_static(name), + HeaderValue::from_static("max-age=60"), + ); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &split, + ¬hing_covered(), + ), + Some(C2BypassReason::OriginNotShareable), + "the Fastly exception must not authorize the vendor policy {name}" + ); + } + } + + #[test] + fn observed_fastly_surrogate_policy_uses_edge_freshness_capped_by_configuration() { + let publisher_headers = headers(&[ + (header::CACHE_CONTROL, "public, max-age=60"), + ( + header::HeaderName::from_static("surrogate-control"), + "max-age=1200, stale-while-revalidate=21600, stale-if-error=604800", + ), + ]); + + assert_eq!( + c2_cache_ttl( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &publisher_headers, + &C2CachePolicy::for_test(¬hing_covered(), Duration::from_secs(300),), + ), + Ok(Duration::from_secs(300)), + "Fastly's edge freshness should take precedence over the shorter browser \ + lifetime, while the configured safety ceiling remains authoritative" + ); + } + + #[test] + fn fastly_surrogate_freshness_takes_precedence_over_standard_freshness() { + for (cache_control, surrogate_control, expected) in [ + ("public, max-age=300", "max-age=30", 30), + ("public, max-age=30", "max-age=300", 300), + ] { + let publisher_headers = headers(&[ + (header::CACHE_CONTROL, cache_control), + ( + header::HeaderName::from_static("surrogate-control"), + surrogate_control, + ), + ]); + + assert_eq!( + c2_cache_ttl( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &publisher_headers, + &C2CachePolicy::for_test(¬hing_covered(), Duration::from_secs(600),), + ), + Ok(Duration::from_secs(expected)) + ); + } + } + + #[test] + fn surrogate_stale_windows_do_not_extend_fresh_reuse() { + let publisher_headers = headers(&[ + (header::CACHE_CONTROL, "public, max-age=300"), + ( + header::HeaderName::from_static("surrogate-control"), + "max-age=20, stale-while-revalidate=600, stale-if-error=1200", + ), + (header::AGE, "10"), + ]); + + assert_eq!( + c2_cache_ttl( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &publisher_headers, + &C2CachePolicy::for_test(¬hing_covered(), Duration::from_secs(600),), + ), + Ok(Duration::from_secs(10)), + "stale windows are validated metadata, not fresh C2 lifetime" + ); + } + + #[test] + fn ambiguous_or_unsupported_surrogate_policy_fails_closed() { + for (policy, expected) in [ + ("max-age", C2BypassReason::MalformedCachePolicy), + ( + "max-age=30, max-age=60", + C2BypassReason::MalformedCachePolicy, + ), + ("max-age=tomorrow", C2BypassReason::MalformedCachePolicy), + ("max-age=30, public", C2BypassReason::MalformedCachePolicy), + ("stale-if-error=60", C2BypassReason::NoPositiveFreshness), + ("max-age=0", C2BypassReason::NoPositiveFreshness), + ("max-age=30,", C2BypassReason::MalformedCachePolicy), + ] { + let mut publisher_headers = shareable(); + publisher_headers.insert( + header::HeaderName::from_static("surrogate-control"), + HeaderValue::from_str(policy).expect("should build Surrogate-Control"), + ); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &publisher_headers, + ¬hing_covered(), + ), + Some(expected), + "`{policy}` must fail closed" + ); + } + } + + #[test] + fn restrictive_surrogate_policy_is_never_overridden_by_standard_freshness() { + for directive in ["private", "no-store", "no-cache"] { + let mut publisher_headers = shareable(); + publisher_headers.insert( + header::HeaderName::from_static("surrogate-control"), + HeaderValue::from_str(directive).expect("should build Surrogate-Control"), + ); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &publisher_headers, + ¬hing_covered(), + ), + Some(C2BypassReason::OriginNotShareable), + "`{directive}` must remain authoritative" + ); + } + } + + #[test] + fn restrictive_standard_policy_is_never_overridden_by_surrogate_freshness() { + for directive in ["private", "no-store", "no-cache"] { + let publisher_headers = headers(&[ + ( + header::CACHE_CONTROL, + &format!("public, max-age=60, {directive}"), + ), + ( + header::HeaderName::from_static("surrogate-control"), + "max-age=1200", + ), + ]); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &publisher_headers, + ¬hing_covered(), + ), + Some(C2BypassReason::OriginNotShareable), + "standard `{directive}` must refuse C2 even with positive edge freshness" + ); + } + } + + #[test] + fn surrogate_control_can_authorize_fastly_edge_freshness_without_browser_freshness() { + let publisher_headers = headers(&[( + header::HeaderName::from_static("surrogate-control"), + "max-age=1200", + )]); + + assert_eq!( + c2_cache_ttl( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &publisher_headers, + &C2CachePolicy::for_test(¬hing_covered(), Duration::from_secs(300),), + ), + Ok(Duration::from_secs(300)), + "Fastly edge freshness should not require browser freshness" + ); + } + + #[test] + fn repeated_cache_control_lines_without_a_disqualifier_still_cache() { + // The other direction: reading every value must not turn an ordinary + // multi-line `Cache-Control` into a bypass, or the fix would disable the + // cache instead of tightening it. + let mut split = edgezero_core::http::HeaderMap::new(); + split.append(header::CACHE_CONTROL, HeaderValue::from_static("public")); + split.append( + header::CACHE_CONTROL, + HeaderValue::from_static("max-age=60"), + ); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &split, + ¬hing_covered(), + ), + None + ); + } + + #[test] + fn origin_freshness_is_positive_age_adjusted_and_capped() { + let fresh_headers = headers(&[(header::CACHE_CONTROL, "public, max-age=300")]); + assert_eq!( + c2_cache_ttl( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &fresh_headers, + &C2CachePolicy::for_test(¬hing_covered(), Duration::from_secs(60),), + ), + Ok(Duration::from_secs(60)) + ); + + let aged = headers(&[ + (header::CACHE_CONTROL, "s-maxage=50, max-age=300"), + (header::AGE, "35"), + ]); + assert_eq!( + c2_cache_ttl( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &aged, + &C2CachePolicy::for_test(¬hing_covered(), Duration::from_secs(60),), + ), + Ok(Duration::from_secs(15)) + ); + + let old_date_without_age = headers(&[ + (header::CACHE_CONTROL, "public, max-age=60"), + (header::DATE, "Wed, 12 Aug 2026 08:00:00 GMT"), + ]); + let one_minute_later = httpdate::parse_http_date("Wed, 12 Aug 2026 08:01:00 GMT") + .expect("should parse fixture time"); + assert_eq!( + origin_shared_ttl_at( + &old_date_without_age, + one_minute_later, + Duration::from_secs(60), + ), + Err(C2BypassReason::NoPositiveFreshness), + "an old Date is apparent age even when an upstream omitted Age" + ); + } + + #[test] + fn zero_exhausted_missing_and_malformed_freshness_are_refused() { + for (map, expected) in [ + ( + headers(&[(header::CACHE_CONTROL, "max-age=0")]), + C2BypassReason::NoPositiveFreshness, + ), + ( + headers(&[(header::CACHE_CONTROL, "max-age=60"), (header::AGE, "60")]), + C2BypassReason::NoPositiveFreshness, + ), + ( + headers(&[(header::CACHE_CONTROL, "public")]), + C2BypassReason::NoPositiveFreshness, + ), + ( + headers(&[(header::CACHE_CONTROL, "max-age=tomorrow")]), + C2BypassReason::MalformedCachePolicy, + ), + ( + headers(&[(header::CACHE_CONTROL, "max-age=\"60")]), + C2BypassReason::MalformedCachePolicy, + ), + ( + headers(&[(header::CACHE_CONTROL, "max-age=+60")]), + C2BypassReason::MalformedCachePolicy, + ), + ] { + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &map, + ¬hing_covered(), + ), + Some(expected) + ); + } + } + + #[test] + fn expires_can_authorize_but_never_extend_an_expired_response() { + let now = httpdate::parse_http_date("Wed, 12 Aug 2026 08:00:00 GMT") + .expect("should parse fixture time"); + let fresh = headers(&[ + (header::DATE, "Wed, 12 Aug 2026 08:00:00 GMT"), + (header::EXPIRES, "Wed, 12 Aug 2026 08:00:30 GMT"), + ]); + assert_eq!( + origin_shared_ttl_at(&fresh, now, Duration::from_secs(60)), + Ok(Duration::from_secs(30)) + ); + + let expired = headers(&[ + (header::DATE, "Wed, 12 Aug 2026 08:01:00 GMT"), + (header::EXPIRES, "Wed, 12 Aug 2026 08:00:30 GMT"), + ]); + assert_eq!( + origin_shared_ttl_at(&expired, now, Duration::from_secs(60)), + Err(C2BypassReason::NoPositiveFreshness) + ); + } + + #[test] + fn request_semantics_bypass_c2_except_for_a_max_age_zero_reload() { + for (name, value) in [ + (header::CACHE_CONTROL, "no-cache"), + (header::CACHE_CONTROL, "max-age=30"), + (header::CACHE_CONTROL, "max-age=\"0"), + (header::CACHE_CONTROL, "min-fresh=10"), + (header::CACHE_CONTROL, "no-store"), + (header::PRAGMA, "no-cache"), + (header::PRAGMA, "legacy-extension, no-cache"), + (header::RANGE, "bytes=0-99"), + (header::IF_NONE_MATCH, "\"etag\""), + (header::IF_MODIFIED_SINCE, "Wed, 12 Aug 2026 08:00:00 GMT"), + ] { + let map = headers(&[(name.clone(), value)]); + assert!(request_bypasses_c2(&map), "{name}: {value} must bypass"); + } + assert!( + !request_bypasses_c2(&headers(&[(header::CACHE_CONTROL, "max-age=0")])), + "a browser reload may reuse C2 because the assembled response and auction \ + are still rebuilt for this reader" + ); + assert!(!request_bypasses_c2(&headers(&[( + header::CACHE_CONTROL, + "public" + )]))); + } + + #[test] + fn a_wildcard_vary_is_refused() { + // `VarySpec::uncovered_by` filters `*` out, with a comment saying the + // eligibility gate handles it. It did not — nothing rejected the wildcard, so + // a response the origin said no key can select was shareable. + let mut varying = shareable(); + varying.insert(header::VARY, HeaderValue::from_static("*")); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &varying, + ¬hing_covered(), + ), + Some(C2BypassReason::VaryWildcard) + ); + } + + #[test] + fn a_fully_covered_vary_is_cacheable() { + let mut varying = shareable(); + varying.insert( + header::VARY, + HeaderValue::from_static("rsc, Accept-Encoding"), + ); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &varying, + &VarySpec::new(["rsc".to_string(), "accept-encoding".to_string()]), + ), + None, + "a key covering everything the origin varies on is safe to store" + ); + } + + #[test] + fn config_drift_names_the_missing_header() { + // The failure this guards: the origin adds a header to its Vary, nobody + // updates config, and requests differing only in that header start sharing a + // template. The reason must name it, or diagnosing means a bisect. + let mut varying = shareable(); + varying.insert( + header::VARY, + HeaderValue::from_static("rsc, next-router-prefetch"), + ); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &varying, + &VarySpec::new(["rsc".to_string()]), + ), + Some(C2BypassReason::VaryNotCovered(VaryGap(vec![ + "next-router-prefetch".to_string() + ]))), + "the uncovered header must be named" + ); + } + + #[test] + fn a_vary_split_across_repeated_headers_is_still_checked() { + // Vary is a list header, so an origin may send it once or many times. Reading + // only the first would let the rest through unkeyed. + let mut varying = shareable(); + varying.append(header::VARY, HeaderValue::from_static("rsc")); + varying.append(header::VARY, HeaderValue::from_static("cookie")); + + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &varying, + &VarySpec::new(["rsc".to_string()]), + ), + Some(C2BypassReason::VaryCookie), + "a repeated Vary header must not hide names behind the first value" + ); + } + + #[test] + fn a_plain_shareable_html_200_is_cacheable() { + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &shareable(), + ¬hing_covered(), + ), + None, + "ESI shareable HTML 200 should be eligible" + ); + } + + #[test] + fn inline_mode_never_writes_a_template() { + assert_eq!( + c2_bypass_reason( + AssemblyMode::Inline, + false, + false, + StatusCode::OK, + "text/html", + &shareable(), + ¬hing_covered(), + ), + Some(C2BypassReason::InlineMode), + "inline has no shared template to write" + ); + } + + #[test] + fn an_authorized_request_is_never_cached() { + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + true, + false, + StatusCode::OK, + "text/html", + &shareable(), + ¬hing_covered(), + ), + Some(C2BypassReason::AuthorizedRequest), + "an authenticated response must not enter a shared cache" + ); + } + + #[test] + fn a_forwarded_request_cookie_disqualifies_even_without_set_cookie() { + // The dangerous case: session established on an earlier request, so this + // response carries no Set-Cookie, has no Cache-Control at all, is a 200, + // and is HTML — yet is personalized because TS forwarded the Cookie to + // origin unchanged. Every other condition reports it cacheable. + let no_cache_control = edgezero_core::http::HeaderMap::new(); + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + true, + StatusCode::OK, + "text/html", + &no_cache_control, + ¬hing_covered(), + ), + Some(C2BypassReason::CookieForwarded), + "cookie-personalized HTML must not become a shared template" + ); + } + + #[test] + fn an_origin_set_cookie_is_never_cached() { + let with_cookie = headers(&[ + (header::CACHE_CONTROL, "max-age=60"), + (header::SET_COOKIE, "sid=abc; Path=/"), + ]); + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &with_cookie, + ¬hing_covered(), + ), + Some(C2BypassReason::OriginSetCookie), + "caching this would replay one visitor's cookie to the next" + ); + } + + #[test] + fn non_shareable_cache_control_is_refused_case_insensitively() { + for directive in [ + "private", + "no-store", + "no-cache", + "Private, max-age=60", + "NO-STORE", + "public, No-Cache", + ] { + let map = headers(&[(header::CACHE_CONTROL, directive)]); + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &map, + ¬hing_covered(), + ), + Some(C2BypassReason::OriginNotShareable), + "`{directive}` should disqualify the response" + ); + } + } + + #[test] + fn a_datadome_block_is_refused_by_the_status_check() { + // DataDome replaces the document with a 403 + // (`integrations/datadome/protection.rs:778`). There is no separate + // marker to detect, and none is needed. + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::FORBIDDEN, + "text/html", + &shareable(), + ¬hing_covered(), + ), + Some(C2BypassReason::NonOkStatus), + "a blocked document must not become the shared template" + ); + } + + #[test] + fn non_html_is_refused() { + for content_type in ["text/x-component", "application/json", ""] { + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + content_type, + &shareable(), + ¬hing_covered(), + ), + Some(C2BypassReason::NotHtml), + "`{content_type}` has no HTML template to transform" + ); + } + } + + #[test] + fn unsupported_content_encoding_is_refused_before_representation_headers_change() { + let map = headers(&[ + (header::CACHE_CONTROL, "public, max-age=60"), + (header::CONTENT_ENCODING, "zstd"), + ]); + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &map, + ¬hing_covered(), + ), + Some(C2BypassReason::UnsupportedContentEncoding) + ); + + let mut repeated = headers(&[(header::CACHE_CONTROL, "public, max-age=60")]); + repeated.append(header::CONTENT_TYPE, HeaderValue::from_static("text/html")); + repeated.append( + header::CONTENT_TYPE, + HeaderValue::from_static("application/json"), + ); + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &repeated, + ¬hing_covered(), + ), + Some(C2BypassReason::MalformedRepresentationHeaders) + ); + } + + #[test] + fn leak_vectors_are_reported_before_mere_ineligibility() { + // A response that fails several conditions should name the most serious + // one, so an operator reading the log sees the security reason rather + // than a content-type quibble. + let map = headers(&[ + (header::CACHE_CONTROL, "private"), + (header::SET_COOKIE, "sid=abc"), + ]); + assert_eq!( + c2_bypass_reason( + AssemblyMode::Esi, + true, + false, + StatusCode::FORBIDDEN, + "application/json", + &map, + ¬hing_covered(), + ), + Some(C2BypassReason::AuthorizedRequest), + "authorization is the most serious disqualifier and should win" + ); + } + } + + mod template_neutrality_tests { + //! The gate for #1009's shared-template design. + //! + //! An "absence of per-user values" scan is not sufficient here: the bug + //! that nearly shipped was a *conditionally present* element whose own + //! content was per-URL. These tests assert byte-identity across requests + //! that differ only in the gating decision. + + use super::*; + use crate::creative_opportunities::{ + AssemblyMode, CreativeOpportunityFormat, CreativeOpportunitySlot, + }; + + pub(super) fn slot() -> CreativeOpportunitySlot { + CreativeOpportunitySlot { id: "atf".to_string(), - formats: vec![AdFormat { - media_type: MediaType::Banner, + gam_unit_path: Some("/99999/example/home".to_string()), + div_id: Some("ad-atf".to_string()), + page_patterns: vec!["/**".to_string()], + formats: vec![CreativeOpportunityFormat { width: 300, height: 250, + media_type: MediaType::Banner, }], floor_price: None, targeting: Default::default(), - bidders: Default::default(), - }], - publisher: PublisherInfo { - domain: "test-publisher.com".to_string(), - page_url: Some("https://test-publisher.com/article".to_string()), - }, - user: UserInfo { - id: None, - consent: None, - eids: None, - }, - device: None, - site: None, - context: Default::default(), + providers: Default::default(), + compiled_patterns: Vec::new(), + compiled_unit: None, + } } - } - - fn build_request(method: Method, uri: &str) -> HttpRequest { - HttpRequest::builder() - .method(method) - .uri(uri) - .body(EdgeBody::empty()) - .expect("should build test request") - } - - #[test] - fn stream_publisher_body_injects_active_diagnostics_for_materialized_html() { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config("gpt_diagnostics", &serde_json::json!({ "enabled": true })) - .expect("should enable diagnostics"); - let integration_registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); - let mut request = HttpRequest::builder() - .method(Method::GET) - .uri("https://publisher.example/article?ts_console=1") - .header("sec-fetch-dest", "document") - .body(EdgeBody::empty()) - .expect("should build activation request"); - let decision = - crate::integrations::gpt_diagnostics::prepare_request(&settings, &mut request) - .expect("should prepare diagnostics request"); - let mut params = make_stream_params(&settings, ""); - params.content_type = "text/html".to_owned(); - params.gpt_diagnostics = Some(decision); - let mut output = Vec::new(); - - stream_publisher_body( - EdgeBody::from("Example"), - &mut output, - ¶ms, - &settings, - &integration_registry, - ) - .expect("should process materialized HTML"); - - let html = String::from_utf8(output).expect("should produce UTF-8 HTML"); - assert!( - html.contains("__tsjs_gpt_diagnostics_active"), - "should inject the activation flag" - ); - assert!( - html.contains("tsjs-gpt_diagnostics.min.js"), - "should inject the standalone diagnostics module" - ); - } - - #[test] - fn stream_publisher_body_round_trips_gzip() { - let settings = create_test_settings(); - let integration_registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); - let input = b"{\"asset\":\"https://origin.test-publisher.com/path/file.js\"}"; - let compressed = gzip_encode(input); - let params = make_stream_params(&settings, "gzip"); - let mut output = Vec::new(); - - stream_publisher_body( - EdgeBody::from(compressed), - &mut output, - ¶ms, - &settings, - &integration_registry, - ) - .expect("should stream gzip response through rewrite pipeline"); - let decoded = gzip_decode(&output); - let decoded = String::from_utf8(decoded).expect("should decode rewritten gzip payload"); - assert!( - decoded.contains("https://test-publisher.com/path/file.js"), - "should rewrite origin URLs to the request host" - ); - assert!( - !decoded.contains("origin.test-publisher.com"), - "should remove the origin hostname from the rewritten payload" - ); - } + pub(super) fn settings_with_slots() -> Settings { + let mut settings = crate::test_support::tests::create_test_settings(); + // Construct the section rather than mutating it if present: the shared + // fixture does not carry `[creative_opportunities]`, and an `if let + // Some(..)` here would silently no-op and make the inline assertion + // below vacuous. + settings.creative_opportunities = Some(CreativeOpportunitiesConfig { + enabled: true, + gam_network_id: "99999".to_string(), + auction_timeout_ms: Some(500), + price_granularity: Default::default(), + section_root: None, + assembly_mode: None, + template_cache_vary: None, + template_cache_max_age_seconds: None, + origin_is_cookie_independent: None, + section_segment: None, + slot: vec![slot()], + }); + settings + } - #[test] - fn stream_publisher_body_round_trips_brotli() { - let settings = create_test_settings(); - let integration_registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); - let input = b"{\"asset\":\"https://origin.test-publisher.com/path/file.css\"}"; - let compressed = brotli_encode(input); - let params = make_stream_params(&settings, "br"); - let mut output = Vec::new(); + #[test] + fn shared_modes_emit_no_head_script_regardless_of_the_gating_decision() { + let settings = settings_with_slots(); + let slots = [slot()]; + let mode = AssemblyMode::Esi; + let ran = template_ad_slots_script(mode, true, &settings, &slots, "/"); + let did_not_run = template_ad_slots_script(mode, false, &settings, &slots, "/"); - stream_publisher_body( - EdgeBody::from(compressed), - &mut output, - ¶ms, - &settings, - &integration_registry, - ) - .expect("should stream brotli response through rewrite pipeline"); + assert_eq!( + ran, did_not_run, + "{mode:?}: the template must be byte-identical whether or not the ad \ + stack ran; a cached object cannot carry one request's consent, bot, \ + prefetch or kill-switch decision" + ); + assert_eq!( + ran, None, + "{mode:?}: adSlots belongs in the per-request seam, not the template" + ); + } - let decoded = brotli_decode(&output); - let decoded = String::from_utf8(decoded).expect("should decode rewritten brotli payload"); - assert!( - decoded.contains("https://test-publisher.com/path/file.css"), - "should rewrite origin URLs to the request host" - ); - assert!( - !decoded.contains("origin.test-publisher.com"), - "should remove the origin hostname from the rewritten payload" - ); - } + #[test] + fn inline_mode_keeps_its_request_dependent_behaviour() { + // Inline responses are per-navigation and never shared, so gating is + // correct there. This guards against "fixing" the shared-mode bug by + // breaking the shipped path. + let settings = settings_with_slots(); + let slots = [slot()]; - #[test] - fn request_ec_uses_cookie_not_header() { - let settings = create_test_settings(); - let header_ec = format!("{}.HdrId1", "a".repeat(64)); - let cookie_ec = format!("{}.CkId01", "b".repeat(64)); - let req = Request::builder() - .method(Method::GET) - .uri("https://test.example.com/page") - .header("x-ts-ec", &header_ec) - .header("cookie", format!("ts-ec={cookie_ec}; other=value")) - .body(EdgeBody::empty()) - .expect("should build test request"); + assert!( + template_ad_slots_script(AssemblyMode::Inline, true, &settings, &slots, "/") + .is_some(), + "inline should emit adSlots when the ad stack runs" + ); + assert_eq!( + template_ad_slots_script(AssemblyMode::Inline, false, &settings, &slots, "/"), + None, + "inline should emit nothing when the ad stack does not run" + ); + } - let ec_context = EcContext::read_from_request(&settings, &req, &noop_services()) - .expect("should read EC context"); + #[test] + fn shared_modes_are_neutral_across_differing_slot_matches() { + // Slot matching folds in the request path. Under a shared mode even + // that must not reach the template. + let settings = settings_with_slots(); - assert_eq!( - ec_context.ec_value(), - Some(cookie_ec.as_str()), - "should resolve request EC ID from cookie" - ); - assert!( - ec_context.cookie_was_present(), - "should detect cookie was present" - ); - assert_eq!( - ec_context.existing_cookie_ec_id(), - Some(cookie_ec.as_str()), - "should return cookie EC value for revocation" - ); - } + let matched = template_ad_slots_script( + AssemblyMode::Esi, + true, + &settings, + &[slot()], + "/news/article", + ); + let unmatched = + template_ad_slots_script(AssemblyMode::Esi, true, &settings, &[], "/other"); - /// Drive `handle_publisher_request` with no creative opportunities — a plain - /// proxy with no server-side auction. Hides the auction/EC wiring so callers - /// read like a simple `(settings, services, req)` proxy. - async fn run_publisher_proxy( - settings: &Settings, - services: &RuntimeServices, - req: Request, - ) -> PublisherResponse { - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - let mut ec_context = - EcContext::read_from_request(settings, &req, services).expect("should read EC context"); - handle_publisher_request( - settings, - services, - None, - &mut ec_context, - AuctionDispatch { - orchestrator: &orchestrator, - slots: &[], - registry: None, - }, - req, - ) - .await - .expect("should proxy publisher request") + assert_eq!( + matched, unmatched, + "the template must not vary with slot matching under a shared mode" + ); + } } mod ssat_cache_policy_tests { @@ -5024,6 +11633,7 @@ mod tests { match response { PublisherResponse::Buffered(response) | PublisherResponse::Stream { response, .. } + | PublisherResponse::AssembleTemplate { response, .. } | PublisherResponse::PassThrough { response, .. } => response.into_parts().0, } } @@ -5353,7 +11963,9 @@ mod tests { // Assert let response = match response { PublisherResponse::Buffered(response) => response, - PublisherResponse::PassThrough { .. } | PublisherResponse::Stream { .. } => { + PublisherResponse::PassThrough { .. } + | PublisherResponse::Stream { .. } + | PublisherResponse::AssembleTemplate { .. } => { panic!("unexpected origin 304 should return a buffered response") } }; @@ -5436,7 +12048,9 @@ mod tests { // Assert let response = match response { PublisherResponse::Buffered(response) => response, - PublisherResponse::PassThrough { .. } | PublisherResponse::Stream { .. } => { + PublisherResponse::PassThrough { .. } + | PublisherResponse::Stream { .. } + | PublisherResponse::AssembleTemplate { .. } => { panic!("noneligible origin 304 should remain buffered") } }; @@ -5510,7 +12124,8 @@ mod tests { *response.body_mut() = body; response } - PublisherResponse::Stream { response, .. } => response, + PublisherResponse::Stream { response, .. } + | PublisherResponse::AssembleTemplate { response, .. } => response, }; assert_eq!(response.status(), StatusCode::OK); @@ -6222,7 +12837,7 @@ mod tests { read_count: Arc::clone(&read_count), body_close_processed_at: Arc::clone(&body_close_processed_at), }; - let ad_bids_state = Arc::new(Mutex::new(None)); + let ad_bids_state = AdBidsState::default(); let ctx = AuctionCollectCtx { dispatched, telemetry: AuctionTelemetryCarry { @@ -6267,7 +12882,7 @@ mod tests { let settings = create_test_settings(); let services = noop_services(); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - let ad_bids_state = Arc::new(Mutex::new(None)); + let ad_bids_state = AdBidsState::default(); let mut state = AuctionHoldState::new( DispatchedAuctionGuard::new(DispatchedAuction::empty_for_test( test_auction_request(), @@ -6316,6 +12931,7 @@ mod tests { ); assert!( ad_bids_state + .script_cell() .lock() .expect("should lock bid state") .is_none(), @@ -6333,6 +12949,7 @@ mod tests { ); assert!( ad_bids_state + .script_cell() .lock() .expect("should lock bid state") .is_some(), @@ -6764,6 +13381,59 @@ mod tests { ); } + #[test] + fn esi_reader_encoding_negotiation_honours_quality_identity_and_repeated_fields() { + let headers = |values: &[&str]| { + let mut headers = edgezero_core::http::HeaderMap::new(); + for value in values { + headers.append( + header::ACCEPT_ENCODING, + HeaderValue::from_str(value).expect("should build accept-encoding"), + ); + } + headers + }; + + assert_eq!( + negotiate_reader_compression(&headers(&[])), + Ok(Compression::None) + ); + assert_eq!( + negotiate_reader_compression(&headers(&["gzip;q=0.8", "br;q=0.4, identity;q=0.1"])), + Ok(Compression::Gzip) + ); + assert_eq!( + negotiate_reader_compression(&headers(&["gzip, br"])), + Ok(Compression::Brotli), + "server preference breaks an equal-quality tie" + ); + assert_eq!( + negotiate_reader_compression(&headers(&["gzip;q=0.5"])), + Ok(Compression::None), + "implicit identity has q=1" + ); + assert_eq!( + negotiate_reader_compression(&headers(&["zstd, identity;q=0"])), + Err(ReaderEncodingError::NoAcceptableEncoding) + ); + assert_eq!( + negotiate_reader_compression(&headers(&["gzip;q=invalid"])), + Err(ReaderEncodingError::Malformed) + ); + for malformed in [ + "gzip;q=1e-1", + "gzip;q=0.1234", + "gzip;q=1.001", + "not a coding;q=1", + ] { + assert_eq!( + negotiate_reader_compression(&headers(&[malformed])), + Err(ReaderEncodingError::Malformed), + "{malformed} is not valid Accept-Encoding syntax" + ); + } + } + #[test] fn tsjs_dynamic_returns_not_found_for_unknown_filename() { let settings = create_test_settings(); @@ -6982,6 +13652,9 @@ mod tests { let body = EdgeBody::from(compressed); let params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: "gzip".to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -6989,7 +13662,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/css".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -7031,6 +13704,9 @@ mod tests { IntegrationRegistry::new(&settings).expect("should create integration registry"); let params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7038,7 +13714,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/html; charset=utf-8".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -7069,6 +13745,9 @@ mod tests { let registry = IntegrationRegistry::new(&settings).expect("should create integration registry"); let params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7076,7 +13755,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/html; charset=utf-8".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -7185,6 +13864,9 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7192,7 +13874,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/css".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -7239,6 +13921,9 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: "gzip".to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7246,7 +13931,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/css".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -7296,6 +13981,9 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: "deflate".to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7303,7 +13991,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/css".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -7353,6 +14041,9 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: "br".to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7360,7 +14051,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/css".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -7410,6 +14101,9 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: "br".to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7417,7 +14111,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/css".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -7455,6 +14149,9 @@ mod tests { fn non_html_stream_params(content_encoding: &str) -> OwnedProcessResponseParams { OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: content_encoding.to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7462,7 +14159,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/css".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -7642,8 +14339,11 @@ mod tests { IntegrationRegistry::new(&settings).expect("should create integration registry"); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); - let state = Arc::new(Mutex::new(None)); + let state = AdBidsState::default(); let mut params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7707,8 +14407,11 @@ mod tests { IntegrationRegistry::new(&settings).expect("should create integration registry"); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); - let state = Arc::new(Mutex::new(None)); + let state = AdBidsState::default(); let mut params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: "gzip".to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7776,6 +14479,9 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7783,7 +14489,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/css".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: Some(test_auction_request()), dispatched_auction: Some(DispatchedAuction::empty_for_test( @@ -7836,6 +14542,9 @@ mod tests { .body(EdgeBody::empty()) .expect("should build response"); let params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: content_encoding.to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7843,7 +14552,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/css".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -7970,6 +14679,9 @@ mod tests { dispatched_auction: Option, ) -> OwnedProcessResponseParams { OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: content_encoding.to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -7980,7 +14692,7 @@ mod tests { r#""# .to_string(), ), - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: dispatched_auction.as_ref().map(|_| test_auction_request()), dispatched_auction, @@ -8316,6 +15028,9 @@ mod tests { let ec_context = EcContext::new_for_test(None, crate::consent::types::ConsentContext::default()); OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -8323,7 +15038,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/html; charset=utf-8".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: Some(AuctionObservationContext::from_parts( AuctionSource::SpaNavigation, "proxy.example.com", @@ -8499,6 +15214,9 @@ mod tests { .map(bytes::Bytes::copy_from_slice) .collect(); let params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: "gzip".to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -8509,7 +15227,7 @@ mod tests { r#""# .to_string(), ), - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: Some(test_auction_request()), dispatched_auction: Some(DispatchedAuction::empty_for_test( @@ -8568,8 +15286,11 @@ mod tests { IntegrationRegistry::new(&settings).expect("should create integration registry"); let bids_script = r#""#; - let state = Arc::new(Mutex::new(Some(bids_script.to_string()))); + let state = AdBidsState::with_script(bids_script); let params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -8624,6 +15345,9 @@ mod tests { // Claim gzip encoding but feed non-gzip bytes. The GzDecoder will // error as soon as it tries to read the gzip header. let params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: "gzip".to_string(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -8631,7 +15355,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/html".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -8733,6 +15457,9 @@ mod tests { let body = EdgeBody::from(html.to_vec()); let params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -8740,7 +15467,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/html; charset=utf-8".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -8791,6 +15518,9 @@ mod tests { // Small, single-fragment RSC script — placeholder path (not fallback). let html = br#""#; let params = OwnedProcessResponseParams { + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), @@ -8798,7 +15528,7 @@ mod tests { request_scheme: "https".to_string(), content_type: "text/html".to_string(), ad_slots_script: None, - ad_bids_state: Arc::new(Mutex::new(None)), + ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, @@ -8835,8 +15565,9 @@ mod tests { #[cfg(test)] mod creative_opportunities_tests { use super::super::{ - MatchedSlotsContext, build_ad_slots_script, build_auction_request, build_bid_map, - build_bids_script, diagnostics_auction_id, html_escape_for_script, write_bids_to_state, + AdBidsState, MatchedSlotsContext, build_ad_slots_script, build_auction_request, + build_bid_map, build_bids_script, diagnostics_auction_id, html_escape_for_script, + write_bids_to_state, }; use crate::auction::types::{ApsRendererV1, ApsTagType, Bid, BidRenderer, MediaType}; use crate::consent::ConsentContext; @@ -8862,6 +15593,10 @@ mod tests { auction_timeout_ms: Some(500), price_granularity: PriceGranularity::Dense, section_root: None, + assembly_mode: None, + template_cache_vary: None, + template_cache_max_age_seconds: None, + origin_is_cookie_independent: None, section_segment: None, slot: Vec::new(), } @@ -9147,7 +15882,7 @@ mod tests { ), ); - let state = std::sync::Arc::new(std::sync::Mutex::new(None)); + let state = AdBidsState::default(); write_bids_to_state( &winning_bids, PriceGranularity::Dense, @@ -9158,6 +15893,7 @@ mod tests { Some(&auction_request.id), ); let script = state + .script_cell() .lock() .expect("should lock initial bid state") .clone() @@ -9192,6 +15928,7 @@ mod tests { Some(&auction_request.id), ); let empty_script = state + .script_cell() .lock() .expect("should lock empty initial bid state") .clone() diff --git a/crates/trusted-server-core/src/response_privacy.rs b/crates/trusted-server-core/src/response_privacy.rs index 21ba9f20b..0fe7650dd 100644 --- a/crates/trusted-server-core/src/response_privacy.rs +++ b/crates/trusted-server-core/src/response_privacy.rs @@ -9,7 +9,7 @@ //! cache such as Cloudflare would otherwise serve an operator/origin //! `Cache-Control: public` on a cookie-bearing response as-is. -use edgezero_core::http::{HeaderName, HeaderValue, Response, header}; +use edgezero_core::http::{HeaderMap, HeaderName, HeaderValue, Response, header}; use crate::settings::Settings; @@ -30,21 +30,67 @@ fn strip_cdn_cache_headers(response: &mut Response) { } } -/// Forces synthesized HTML to be private and non-storable. +/// Whether `Cache-Control` already forbids shared caching. /// -/// Use this exact policy whenever Trusted Server changes an origin HTML -/// representation with request-specific content: force `private, no-store`, -/// remove origin validators, and remove all CDN-targeted cache directives. -pub(crate) fn enforce_synthesized_html_cache_privacy(response: &mut Response) { +/// Extracted because both arms of the cookie-privacy net below need it. +/// +/// `publisher::c2_bypass_reason` deliberately does **not** call this and keeps its own +/// copy: it additionally treats `no-cache` as non-shareable, because "revalidate before +/// reuse" is correct for an HTTP cache and too permissive for a spike-owned one. The +/// duplicate is the stricter of the two, so consolidating them would loosen the shared- +/// template gate rather than tidy it. +/// +/// Directives are case-insensitive (RFC 9111 §5.2), so `No-Store` and `Private` +/// count. `no-cache` deliberately does **not**: it requires revalidation before +/// reuse, not a refusal to store, so a `no-cache` response is still shareable. +/// Callers needing the stricter reading must check it themselves. +#[must_use] +pub fn is_private_or_no_store(headers: &HeaderMap) -> bool { + headers.get_all(header::CACHE_CONTROL).iter().any(|value| { + value.to_str().is_ok_and(|value| { + value.split(',').any(|directive| { + let name = directive + .split_once('=') + .map_or(directive, |(name, _)| name); + matches!( + name.trim().to_ascii_lowercase().as_str(), + "private" | "no-store" + ) + }) + }) + }) +} + +/// Reassert the terminal privacy invariant for a synthesized per-reader response. +/// +/// Call this after every configurable response mutation. It deliberately overwrites +/// `Cache-Control` and strips validators, expiry metadata, and CDN-specific cache +/// directives so a later integration cannot turn an assembled document into C3. +pub fn enforce_private_no_store(response: &mut Response) { response.headers_mut().insert( header::CACHE_CONTROL, HeaderValue::from_static("private, no-store"), ); - response.headers_mut().remove(header::ETAG); - response.headers_mut().remove(header::LAST_MODIFIED); + for name in [ + header::ETAG.as_str(), + header::LAST_MODIFIED.as_str(), + header::EXPIRES.as_str(), + header::AGE.as_str(), + ] { + response.headers_mut().remove(name); + } strip_cdn_cache_headers(response); } +/// Forces synthesized HTML to be private and non-storable. +/// +/// Use this exact policy whenever Trusted Server changes an origin HTML +/// representation with request-specific content: force `private, no-store`, +/// remove origin validators, and remove all CDN-targeted cache directives. +pub(crate) fn enforce_synthesized_html_cache_privacy(response: &mut Response) { + enforce_private_no_store(response); +} + /// Forces cookie-bearing responses to stay private to shared caches. /// /// Any response that sets a per-user cookie (notably the EC identity cookie) @@ -63,14 +109,7 @@ pub fn enforce_set_cookie_cache_privacy(response: &mut Response) { // independent of Cache-Control and would otherwise let a shared cache store // and replay one visitor's Set-Cookie. strip_cdn_cache_headers(response); - // Cache-Control directives are case-insensitive (RFC 9111 §5.2), so match - // against a lowercased copy — `No-Store` / `Private` must count. - let already_uncacheable = response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|v| v.to_str().ok()) - .map(str::to_ascii_lowercase) - .is_some_and(|v| v.contains("private") || v.contains("no-store")); + let already_uncacheable = is_private_or_no_store(response.headers()); if !already_uncacheable { response.headers_mut().insert( header::CACHE_CONTROL, @@ -96,12 +135,7 @@ pub fn enforce_set_cookie_cache_privacy(response: &mut Response) { pub fn apply_response_headers_with_cache_privacy(settings: &Settings, response: &mut Response) { enforce_set_cookie_cache_privacy(response); - let response_is_uncacheable = response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|v| v.to_str().ok()) - .map(str::to_ascii_lowercase) - .is_some_and(|v| v.contains("private") || v.contains("no-store")); + let response_is_uncacheable = is_private_or_no_store(response.headers()); for (key, value) in &settings.response_headers { if response_is_uncacheable @@ -296,6 +330,40 @@ mod tests { } } + #[test] + fn terminal_private_stamp_removes_every_cache_and_validator_header() { + let mut response = response_builder() + .header(header::CACHE_CONTROL, "public, s-maxage=600") + .header(header::ETAG, "\"origin\"") + .header(header::LAST_MODIFIED, "Wed, 12 Aug 2026 00:00:00 GMT") + .header(header::EXPIRES, "Wed, 12 Aug 2026 01:00:00 GMT") + .header(header::AGE, "30") + .header("surrogate-control", "max-age=600") + .header("cdn-cache-control", "public, max-age=600") + .body(edgezero_core::body::Body::empty()) + .expect("should build response"); + + enforce_private_no_store(&mut response); + + assert_eq!( + response.headers()[header::CACHE_CONTROL], + "private, no-store" + ); + for name in [ + header::ETAG.as_str(), + header::LAST_MODIFIED.as_str(), + header::EXPIRES.as_str(), + header::AGE.as_str(), + "surrogate-control", + "cdn-cache-control", + ] { + assert!( + !response.headers().contains_key(name), + "terminal private stamp must strip {name}" + ); + } + } + #[test] fn applies_operator_headers_on_cookieless_response() { let settings = settings_with_response_headers(&[("x-operator", "value")]); diff --git a/crates/trusted-server-js/lib/src/core/types.ts b/crates/trusted-server-js/lib/src/core/types.ts index 0c68d43fe..624b4ef92 100644 --- a/crates/trusted-server-js/lib/src/core/types.ts +++ b/crates/trusted-server-js/lib/src/core/types.ts @@ -452,8 +452,18 @@ export interface TsjsApi { * Lives in the bundle so the lifecycle is executable under test and shares * [`navGeneration`] with the SPA auction hook; `gpt_bootstrap.js` installs * a minimal fallback for pages where the bundle fails to load. + * + * `initialSlots` exists for the shared-template `` seam, which is the + * only place slot definitions arrive with the bids rather than from the head + * script. Passing them here rather than assigning `tsjs.adSlots` before the + * call puts them behind the same generation guard: an assignment made ahead + * of the guard would clobber a committed SPA navigation's slots with the SSR + * document's, and then be read by that route's `adInit()`. */ - scheduleInitialAdInit?: (initialBids?: Record) => void; + scheduleInitialAdInit?: ( + initialBids?: Record, + initialSlots?: AuctionSlot[] + ) => void; /** Read-only GPT lifecycle diagnostics API, present only in an activated tab. */ gptDiagnostics?: GptDiagnosticsApi; /** diff --git a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts index ebf7e225e..bfafd6e6a 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts @@ -665,7 +665,15 @@ function installInitialLoadDetector(ts: TsjsApi): void { * SSR bootstrap as current. For the same reason the initial bids payload is * passed in and applied here, generation-guarded — assigning it * unconditionally at body end would clobber the live bids a faster SPA - * navigation already applied. When a navigation has committed since — or + * navigation already applied. + * + * `initialSlots` is passed in for exactly the same reason and was missing it. + * Only the shared-template `` seam sends slots — under `inline` they + * come from the head script, which runs before any navigation can commit — and + * that seam assigned `tsjs.adSlots` on the line *before* calling this. The + * guard protected the bids and `adInit()` while the assignment it was meant to + * protect had already happened, so a committed SPA navigation kept its bids and + * lost its slots. When a navigation has committed since — or * commits while the deferred callback is pending — the SSR payload is * dropped and `adInit()` is not run: running anyway would re-run the newer * route's live slots/bids, destroying and redefining that route's TS slots @@ -685,8 +693,12 @@ function installInitialLoadDetector(ts: TsjsApi): void { * holds whenever the request is actually issued. */ function installScheduleInitialAdInit(ts: TsjsApi): void { - ts.scheduleInitialAdInit = function (initialBids?: Record) { + ts.scheduleInitialAdInit = function ( + initialBids?: Record, + initialSlots?: AuctionSlot[] + ) { if ((ts.navGeneration ?? 0) !== 0) return; + if (initialSlots) ts.adSlots = initialSlots; if (initialBids) ts.bids = initialBids; const runUnlessNavigated = (): void => { if ((ts.navGeneration ?? 0) !== 0) return; diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts index d3e1d7099..69cb65bfc 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts @@ -159,6 +159,36 @@ describe('gpt_bootstrap.js fallback', () => { expect(adInit).not.toHaveBeenCalled(); }); + it('fallback scheduler guards the SSR slot definitions with the same generation check', () => { + // The shared-template seam hands slots to the scheduler rather than assigning + // them itself, so the fallback has to honour the same guard as the bundle. If it + // applied them unconditionally, a page whose bundle failed to load would take the + // stale SSR slots over a committed navigation's. + runBootstrap(); + const ts = (window as TestWindow).tsjs!; + ts.adInit = vi.fn(); + const liveSlot = { + id: 'live_slot', + gam_unit_path: '/123/live', + div_id: 'div-live', + formats: [[300, 250]] as Array<[number, number]>, + }; + const ssrSlot = { + id: 'ssr_slot', + gam_unit_path: '/123/ssr', + div_id: 'div-ssr', + formats: [[728, 90]] as Array<[number, number]>, + }; + + ts.scheduleInitialAdInit!({ ssr_slot: { hb_pb: '1.00' } }, [ssrSlot]); + expect(ts.adSlots).toEqual([ssrSlot]); + + ts.adSlots = [liveSlot]; + ts.navGeneration = 1; + ts.scheduleInitialAdInit!({ ssr_slot: { hb_pb: '1.00' } }, [ssrSlot]); + expect(ts.adSlots).toEqual([liveSlot]); + }); + it('fallback adInit defines, targets, and displays a TS slot through the command queue', () => { const mockSlot = { addService: vi.fn().mockReturnThis(), diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/schedule_initial_ad_init.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/schedule_initial_ad_init.test.ts index 999c60c55..830bac217 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/schedule_initial_ad_init.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/schedule_initial_ad_init.test.ts @@ -310,6 +310,69 @@ describe('scheduleInitialAdInit', () => { expect(adInit).toHaveBeenCalledTimes(1); }); + it('applies the SSR slot definitions on the initial document', async () => { + // Under a shared-template mode the head script emits no `tsjs.adSlots`, so the + // `` seam is the only source of slot definitions. They must arrive, or + // `adInit()` iterates an empty list and the page defines no TS slots at all. + await importGptModule(); + const ts = (window as TestWindow).tsjs!; + ts.adInit = vi.fn(); + const ssrSlot = { + id: 'ssr_slot', + gam_unit_path: '/123/ssr', + div_id: 'div-ssr', + formats: [[728, 90]] as Array<[number, number]>, + }; + + ts.scheduleInitialAdInit!({ ssr_slot: { hb_pb: '1.00' } }, [ssrSlot]); + + expect(ts.adSlots).toEqual([ssrSlot]); + expect(ts.bids).toEqual({ ssr_slot: { hb_pb: '1.00' } }); + }); + + it('drops the SSR slot definitions when a navigation has already committed', async () => { + // The guard covered the bids and the adInit call, but the shared-template seam + // assigned `tsjs.adSlots` on the line *before* calling the scheduler — outside the + // guard entirely. A navigation that committed while the SSR document was still + // streaming therefore kept its own bids and silently lost its slots to the stale + // SSR payload, and the next `adInit()` for that route defined the wrong slots. + fetchStub.mockResolvedValue({ + ok: true, + json: async () => ({ slots: [], bids: {} }), + }); + await importGptModule(); + const ts = (window as TestWindow).tsjs!; + const adInit = vi.fn(); + ts.adInit = adInit; + + history.pushState({}, '', '/b'); + await flushAsync(); + expect(ts.navGeneration).toBe(1); + const liveSlot = { + id: 'live_slot', + gam_unit_path: '/123/live', + div_id: 'div-live', + formats: [[300, 250]] as Array<[number, number]>, + }; + ts.adSlots = [liveSlot]; + + ts.scheduleInitialAdInit!({ ssr_slot: { hb_pb: '1.00' } }, [ + { + id: 'ssr_slot', + gam_unit_path: '/123/ssr', + div_id: 'div-ssr', + formats: [[728, 90]], + }, + ]); + + expect(ts.adSlots).toEqual([liveSlot]); + + window.dispatchEvent(new Event('load')); + flushFrame(); + flushFrame(); + expect(adInit).not.toHaveBeenCalled(); + }); + it('cancels queued GPT work when a navigation commits before the command queue drains', async () => { // adInit() only queues its slot work on googletag.cmd, which drains when // GPT itself loads — possibly long after the generation check that diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index b763cac24..0b94587c4 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -1382,6 +1382,122 @@ loader: TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__ENABLED=false ``` +### Shared template assembly (`assembly_mode = "esi"`) + +`assembly_mode` controls how initial-page slot and bid state is delivered: + +- `inline` (default) transforms every origin response and injects the current + reader's slots and bids directly. +- `esi` opts into a reader-neutral transformed-template cache on Fastly. The + cache stores identity bytes containing one inert, versioned comment. On an + authorized cold miss, Fastly replaces that comment in a private working copy + with one synthetic ESI include and resolves it from the already-built reader + state using the pinned `stackpop/esi` parser. No HTTP fragment request occurs. + Warm hits use an exact byte split instead, preserving the fast article-prefix + stream while the auction finishes. + +This is deliberately not general publisher-controlled ESI. A transformed origin +document containing any ` **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Turn off the redundant origin cache bypass that the spec identifies as the +actual TTFB cost, behind an operator flag, and establish the measurement baseline that +later work is compared against. + +> **This plan does not close #1009.** It contains no ESI arm and no client-fill arm, so +> completing it cannot answer whether ESI separates cacheable content from per-user +> state. It is a **supporting optimisation and the experimental control** for +> [the ESI validation spike](./2026-08-10-1009-esi-validation-spike.md), which is where +> #1009 is actually decided. Scoped and framed this way after external review on +> 2026-08-10. + +**Architecture:** Two investigation tasks that produce recorded findings and no code; one +code task that adds a config-gated timing log and makes the cache bypass operator- +controlled; and one config change that flips it, gated on the first investigation. +Nothing here touches the auction, the `` hold, or bid delivery — those are +Stages 1–2 in the spec and are explicitly out of scope. + +**Tech Stack:** Rust 2024 edition, `wasm32-wasip1`, Fastly Compute, `web_time::Instant` +for wasm-safe timing, `log` for instrumentation, Viceroy for adapter tests. + +**Spec:** `docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md` +(§3 Steps A/B/C and §4 Stage 0). Read §4 and §5 before starting Task 5. + +**Before pushing, run both documentation gates:** + +```bash +cd docs && npm run format && npm run build && cd .. +``` + +`npm run build` is not optional — `format` passes on documents with dead links, and that +shipped a broken docs build on this branch once already. + +**Two prettier gotchas, both hit while writing this plan.** CI gate 7 +(`cd docs && npm run format`) runs `prettier --check` over all of `docs/`, so both bite. + +1. **Not idempotent on embedded markdown fences.** The first `--write` reformats the + outer document and the embedded ` ```markdown ` block only settles on a second pass. + If `--check` still warns immediately after a `--write`, run `--write` again before + concluding anything is wrong. +2. **It mangles bare `snake_case` identifiers inside fences**, reading the underscores as + emphasis and rewriting `origin_fetch_ms` to `origin*fetch_ms`. **Always wrap + identifiers in backticks**, including inside fenced blocks and table cells. + +--- + +## Background an implementer needs + +Trusted Server proxies a publisher's origin, rewrites the HTML at the edge to inject ad +slot definitions and a JS bundle, and runs a server-side ad auction. For requests that +are eligible for that ad stack, `publisher.rs` currently does three things to the origin +request and response that together make the page uncacheable: + +1. strips conditional and range headers so the origin must return a full body, +2. sets a **cache bypass** so the Fastly read-through cache is skipped entirely, and +3. strips every cacheability header from the response. + +The spec establishes that (2) is redundant given (1) — by the time the request reaches +the cache it is already unconditional, so a cache HIT returns a full body anyway — and +that (2) is the dominant cost. This plan makes (2) operator-controlled and then turns it +off, after first confirming that is safe. + +**Why it might not be safe:** RSC (React Server Component) requests and ordinary HTML +navigations share the same URL and are distinguished only by request headers. RSC +requests are not classified as navigations, so they already flow through the cache while +HTML navigations bypass it. Removing the bypass puts both under one cache key. If the +origin does not declare `Vary` for those headers, the cache could serve one +representation in response to a request for the other. Task 1 checks this. + +**Terms:** _POP_ = Fastly edge point of presence. _shield_ = a designated POP that +backs other POPs. _read-through cache_ = Fastly's cache on the backend request path. +_bypass / `Pass`_ = skip that cache. + +--- + +## File structure + +| File | Responsibility in this plan | +| ---------------------------------------------------------------- | ---------------------------------------------------------------------------------- | +| `docs/superpowers/plans/2026-08-08-1009-measurement-findings.md` | **Create.** Recorded output of Tasks 1–2. Gates Task 5. | +| `crates/trusted-server-core/src/publisher.rs` | **Modify.** Timing log and the bypass flag (Task 3); tests (Task 5). | +| `crates/trusted-server-core/src/settings.rs` | **Modify.** `publisher.bypass_origin_cache` and `debug.publisher_timing` (Task 3). | +| `trusted-server.example.toml` | **Modify.** Document the new key (Task 5). | + +No new modules. No adapter changes: the `bypass_cache` platform capability and its +per-adapter mappings stay in place and keep their tests — the publisher-path call site +becomes operator-controlled rather than unconditional. + +## Task order and dependencies + +Only one edge is real. Do not serialize the rest. + +``` +Task 1 (origin Vary check) ──────┬──> Task 2 (appends to the findings file Task 1 creates) + │ + ├──> Task 5 (flip the flag) +Task 3 (instrumentation + flag) ─┘ +``` + +**Task 1 is externally blocked.** It needs the publisher origin hostname, which lives in +the operator's gitignored `trusted-server.toml`. Arrange access before starting, or the +plan stalls on its first step. + +Task 3 is independent and can start immediately. Task 2 only needs Task 1 far enough to +have created the findings document. Task 5 needs Task 1's verdict **and** Task 3's config +flag to exist. + +--- + +## Task 1: Step A — origin `Vary` check + +**Files:** + +- Create: `docs/superpowers/plans/2026-08-08-1009-measurement-findings.md` + +This task is an investigation. It writes no code and gates Task 5. + +- [ ] **Step 1: Get the origin URL** + +The publisher origin is operator config, not in the repo. Read it from the deployed +service config or ask the operator. Do **not** hardcode it into any committed file — the +findings document records the _result_, not the hostname. + +```bash +# The key is `publisher.origin_url` in the operator's trusted-server.toml +# (gitignored). Confirm the value before proceeding. +``` + +- [ ] **Step 2: Request the HTML representation and capture `Vary`** + +```bash +curl -sS -D - -o /dev/null "https:///" \ + -H 'Sec-Fetch-Dest: document' \ + -H 'Accept: text/html' +``` + +Expected: response headers. Record whether a `Vary` header is present and its value. + +- [ ] **Step 3: Request the RSC representation at the same URL** + +```bash +curl -sS -D - -o /dev/null "https:///" \ + -H 'RSC: 1' \ + -H 'Accept: text/x-component' +``` + +Expected: a different `Content-Type` (`text/x-component`) than Step 2, proving the two +representations share a URL. Record `Vary` again. + +- [ ] **Step 4: Probe the `Next-Router-*` headers** + +Do not skip this. The PASS criterion below names these headers, and an implementer who +tests only HTML and `RSC` can record a PASS that is wrong — which routes to Task 5a, the +one outcome this plan calls dangerous. + +```bash +for H in 'Next-Router-Prefetch: 1' 'Next-Router-State-Tree: %5B%22%22%5D'; do + echo "--- $H" + curl -sS -D - -o /dev/null "https:///" \ + -H 'RSC: 1' -H "$H" \ + | grep -iE '^(vary|content-type|content-length|cache-control|set-cookie):' +done +``` + +Compare `Content-Type` and `Content-Length` against the plain `RSC: 1` request from +Step 3. If either differs, the origin varies on that header and `Vary` must name it. + +Capture `Cache-Control` and `Set-Cookie` on every request in this task, not just this +one — see Step 5. + +- [ ] **Step 5: Probe cookie personalization — the bigger hole** + +The representation check above covers RSC-vs-HTML. It does **not** cover the larger +class: TS forwards client cookies to origin unchanged, so any cookie-personalized HTML +(logged-in state, paywall meter, publisher-side A/B assignment) becomes cross-servable +once the cache is on. + +**Do not compare body hashes.** Verified on the live origin: this page regenerates +~170 ad-slot container IDs as fresh 32-hex UUIDs on every request, so three requests give +three different hashes with byte-identical lengths, cookie or not. A hash comparison +reports a false FAIL every time. + +Normalize per-request identifiers, establish the no-cookie baseline drift first, then ask +whether the cookie arm differs by _more_ than that baseline: + +```bash +ORIGIN="https://"; HOSTH="Host: " +norm() { sed -E 's/[0-9a-f]{32}/UUID/g' "$1"; } + +for n in a b; do + curl -sS "$ORIGIN/" -H "$HOSTH" \ + -H 'Sec-Fetch-Dest: document' -H 'Accept: text/html' > "nc_$n.html" +done +curl -sS "$ORIGIN/" -H "$HOSTH" \ + -H 'Sec-Fetch-Dest: document' -H 'Accept: text/html' \ + -H 'Cookie: ' > ck.html + +echo "baseline drift: $(diff <(norm nc_a.html) <(norm nc_b.html) | grep -c '^[<>]')" +echo "with cookie: $(diff <(norm nc_a.html) <(norm ck.html) | grep -c '^[<>]')" +diff <(norm nc_a.html) <(norm ck.html) | head -20 +``` + +Send the `Host` override — the origin is a shared vhost and will not return the right +document without it. Read it from `publisher.origin_host_header_override`. + +**Step A has been run once and returned a PROVISIONAL PASS**, which is **not** sufficient +to flip the flag. See [the findings](./2026-08-08-1009-measurement-findings.md) for the +five untested conditions. Complete them and record a `FINAL PASS` before Task 5. + +Three failure shapes, any of which blocks Stage 0 independently of the `Vary` verdict: + +- Bodies differ by cookie **and** `Vary` does not name `Cookie` → cross-serving of + personalized HTML. +- Origin emits `Set-Cookie` alongside a shared-cacheable `Cache-Control` → the cache can + replay one visitor's cookie to the next. TS's privacy net does not help; it downgrades + **TS's** response, after the cache has already stored the origin's. +- The deployment is `Authorization`-gated (as #1009 describes) and authorized responses + are cacheable → same problem, different header. + +- [ ] **Step 6: Request with the experiment header, if the operator uses one** + +Repeat Step 2 with the publisher's experiment header set to two different values. +Record whether the bodies differ and whether `Vary` names that header. + +- [ ] **Step 7: Record the finding** + +Create `docs/superpowers/plans/2026-08-08-1009-measurement-findings.md`: + +```markdown +# #1009 measurement findings + +## Step A — origin `Vary` declaration + +**Date:** · **Checked by:** + +| Representation | `Content-Type` returned | `Content-Length` | `Vary` present? | `Vary` value | +| --------------------- | ----------------------- | ---------------- | --------------- | ------------ | +| HTML navigation | | | | | +| RSC | | | | | +| RSC + `Next-Router-*` | | | | | +| Experiment variant | | | | | + +**Cookie / auth exposure:** bodies differ by cookie? `Vary: Cookie` present? origin +`Set-Cookie` on a shared-cacheable response? `Authorization`-gated responses cacheable? + +**Verdict:** FINAL PASS / PROVISIONAL PASS / FAIL + +`FINAL PASS` = `Vary` names every request header the origin varies on (`RSC`, any +`Next-Router-*` or experiment header whose value changed the body, **and `Cookie` if +bodies differ by cookie**), no `Set-Cookie` rides a shared-cacheable response, **and** all +five conditions in Task 5's gate are recorded — a real authenticated session cookie, Basic +Auth through TS, the experiment variant, representative routes, and cached-hit +slot/render attribution. + +`PROVISIONAL PASS` = the `Vary` and cookie checks hold, but one or more of those five is +untested. **Not a release gate.** A first pass lands here. + +`FAIL` = any `Vary` or `Set-Cookie` criterion is unmet. + +**Consequence:** `FINAL PASS` → Task 5a (flip the flag). `PROVISIONAL PASS` → close the +gaps before Task 5 starts. `FAIL` → Task 5b (cache-key discriminator). See spec §4. + +**A FAIL is also a live production defect, not only a Stage 0 blocker.** RSC fetches are +not navigations, so they never set the bypass and **already transit the read-through +cache today**. If the origin varies undeclared on `Next-Router-*`, TS is cross-serving RSC +variants in production right now. File it immediately rather than deferring with Task 5b. +``` + +- [ ] **Step 8: Commit** + +CI gate 7 runs `prettier --check` across all of `docs/`, so format the findings file +before staging it — a filled-in markdown table will not be prettier-clean by hand. + +```bash +cd docs && npx prettier --write superpowers/plans/2026-08-08-1009-measurement-findings.md && cd .. +git add docs/superpowers/plans/2026-08-08-1009-measurement-findings.md +git commit -m "Record origin Vary findings for #1009 Stage 0 gate" +``` + +--- + +## Task 2: Step B — what consumes TS's own response headers + +**Files:** + +- Modify: `docs/superpowers/plans/2026-08-08-1009-measurement-findings.md` — **created by + Task 1 Step 6.** If Task 1 has not reached that step, create the file with just its + `# #1009 measurement findings` heading rather than blocking. + +Investigation. Determines whether the spec's Stage 3b has a consumer. Does not gate +Task 5, but it appends to Task 1's findings document — do not run the two concurrently +against that file. + +- [ ] **Step 1: Pick a path that already emits shared-cache headers** + +`serve_static_with_etag` emits `public, max-age=300, s-maxage=300` plus +`Surrogate-Control` — see `crates/trusted-server-core/src/http_util.rs:294-311`. It backs +the `/static/tsjs=` bundle route (`publisher.rs:303`, `:322`). Use that URL against +the deployed service. + +- [ ] **Step 2: Request it twice and inspect for cache markers** + +```bash +URL="https:///static/tsjs=" +curl -sS -D - -o /dev/null "$URL" | grep -iE 'x-cache|age:|x-served-by|hit-state' +sleep 2 +curl -sS -D - -o /dev/null "$URL" | grep -iE 'x-cache|age:|x-served-by|hit-state' +``` + +Expected on the second request: if a cache sits in front of the Compute service, an `age` +greater than zero or an `x-cache` containing `HIT`. + +**The probe above is weak evidence** — absence of `age` is equally consistent with "no +cache" and "cold cache". **The topology check below is the actual answer; run it first and +skip the probe if it is conclusive.** + +```bash +fastly service list +fastly service-version list --service-id +# Look for a Delivery service fronting the Compute service, and for shielding +# configured on the service rather than only on the origin backend. +``` + +A Compute service with no Delivery service in front and no fronting shield does not have +its own output cached — that is the configuration the spec assumes, and this step exists +to confirm or refute it rather than to leave it assumed. + +**While you have the service open, answer a second question that matters more than this +task does:** is the _publisher backend_ shielded on the TS service? + +```bash +fastly backend list --service-id --version active +# Look for a shield on the publisher origin backend. +``` + +#1009's entire off-TS advantage came from a **shield** HIT, not a POP HIT. Whether +Stage 0 recovers a shield HIT or only a single-POP HIT changes the size of the win +materially, and nothing else in this plan establishes it. + +- [ ] **Step 3: Record the finding** + +Append to the findings document: + +```markdown +## Step B — consumers of TS's own response headers + +**Verdict:** SHARED CACHE PRESENT / NO SHARED CACHE + +**Evidence:** + +**Consequence:** NO SHARED CACHE → spec Stage 3b is inert until a topology change; +deprioritize it and ship only Stage 3a (browser caching). SHARED CACHE PRESENT → +Stage 3b gains a consumer AND the per-user `x-geo-*` header leak in spec §7 becomes an +active privacy exposure rather than a theoretical one. Escalate immediately in that case. +``` + +- [ ] **Step 4: Commit** + +```bash +cd docs && npx prettier --write superpowers/plans/2026-08-08-1009-measurement-findings.md && cd .. +git add docs/superpowers/plans/2026-08-08-1009-measurement-findings.md +git commit -m "Record response-header cache consumer findings for #1009" +``` + +--- + +## Task 3: Step C — origin fetch timing, and the bypass flag + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-core/src/settings.rs` — `publisher.bypass_origin_cache`, + `default_bypass_origin_cache`, `debug.publisher_timing`, the `Publisher` `Default` impl, + eight test literals, and the `origin_host` doctest +- Modify: `crates/trusted-server-core/src/test_support.rs` (log-capture helper) +- Test: inside `mod ssat_cache_policy_tests` at `crates/trusted-server-core/src/publisher.rs:4541` + +**Use `web_time::Instant`, not `std::time::Instant`** — the workspace targets +`wasm32-wasip1` and `web_time` is the wasm-safe clock already used at +`crates/trusted-server-core/src/auction/orchestrator.rs:7`. + +### Two timings, and why these two + +Measure **`hold_wait_ms`** and **`origin_fetch_ms`**. Not the rewrite. + +`hold_wait_ms` is the decision. The hold's cost is literally the duration of one +`.await` — `collect_stream_auction` at `publisher.rs:793`, plus the two EOF variants in +`hold_finish_ready_segments` (`:869`) and `hold_finish_tail_segments` (`:896`). Two +`Instant`s around those calls answer "does the hold block?" directly, instead of +inferring it by comparing origin fetch against auction duration. + +`origin_fetch_ms` is attribution — how much of any win Stage 0 can claim. + +`rewrite_ms` decides nothing. Step C's verdict compares origin fetch against auction +collect, and the ceiling argument in spec §6.4 is structural — it needs no number. +Measuring the rewrite would mean instrumenting two finalizers +(`buffer_publisher_response_async` at `publisher.rs:1114`, and the +`async_stream::try_stream!` block at `publisher.rs:1286`), working around moves out of +`params` inside that block, and finding a correlation key that does not exist — +`OwnedProcessResponseParams` (`publisher.rs:1065-1087`) has no `request_path`, and adding +one means touching all 26 construction sites. + +None of that buys a decision. Skip it. If a rewrite figure is later wanted to set a +target, add it as a separate follow-on once the verdict is known. + +**Why a log line and not `Server-Timing`:** for `origin_fetch_ms` alone a response header +would in fact work — the value is known before headers commit. A log line is still +preferred because it is server-side (no dependence on a browser harness to collect it), +`log` is this project's instrumentation crate per `CLAUDE.md`, and the auction path +already measures itself the same way. The spec previously claimed `Server-Timing` cannot +work at all; that overbroad claim has already been corrected there. + +### Log volume — gate it + +The line sits after the origin send, so it fires for every publisher request that reaches +origin — tagged `ad_stack=false` for ineligible ones, not only for eligible navigations. +That is more useful for comparison and more log spend, and the instrumentation is +temporary either way. Gate it behind the existing debug surface rather than +emitting unconditionally: add a `#[serde(default)] pub publisher_timing: bool` to +`DebugConfig` (`crates/trusted-server-core/src/settings.rs:1872`), following +`ja4_endpoint_enabled` and `auction_html_comment` alongside it. Default `false`; enable +via `ts config push` for the measurement window, then disable. + +This also means the Step 1 test must set that flag in its settings fixture. + +The split is also what makes the Step 1 test achievable — `run_with_slots` +(`publisher.rs:4769`) invokes only `handle_publisher_request` and never drives either +finalizer, so a test asserting on a combined line could never pass. + +**What `origin_fetch_ms` actually measures.** `publisher.rs:2863-2865` sets +`.with_stream_response()` when the adapter supports it, so on Fastly `send()` returns at +response _headers_, not after the body downloads. `origin_fetch_ms` is therefore **origin +TTFB**, not full download time. Name it that way in the findings document. It is still +the correct before/after signal for Stage 0 — the bypass affects whether the request hits +a cache at all — but when comparing against auction `total_time_ms` in Step 9, compare +like with like and say which quantity each column holds. + +- [ ] **Step 1: Write the failing test** + +**Placement matters.** Add the test **inside `mod ssat_cache_policy_tests`** +(`publisher.rs:4541`), not the outer `mod tests` (`:4035`). Every helper it uses is +private to that nested module: `settings_with_enabled_auction_and_creative_opportunities` +(`:4684`), `article_slot` (`:4721`), `conditional_navigation_request` (`:4740`), +`queue_cacheable_html_response` (`:4752`), `run_with_slots` (`:4769`). Placed in the outer +module it will not resolve — and because two _other_ `article_slot` functions exist +(`:9593`, `:10276`) returning a different type, the failure surfaces as a confusing type +error rather than a missing-name error. + +**First, add the log-capture helper.** `crates/trusted-server-core/src/test_support.rs` +has none. Note its shape: the whole file is `#[cfg(test)] pub mod tests { … }`, so the +path is `crate::test_support::tests::capture_logs`, not `crate::test_support::capture_logs` +— see existing consumers at `auth.rs:103` and `config_payload.rs:48`. + +Two constraints the helper must respect or the test fails for unrelated reasons: + +- `log::set_boxed_logger` succeeds **once per process**. Install via a `OnceLock`/`Once` + and have `capture_logs()` return a guard that clears and then reads a shared buffer. +- Call `log::set_max_level(log::LevelFilter::Info)` or higher, or `log::info!` is filtered + out before it reaches the logger. +- **Do not have the guard hold the buffer's own `Mutex`.** The test body runs code that + calls `log::info!` on the same thread, and the logger must lock that same mutex to + append — `std::sync::Mutex` is not reentrant, so this **hangs** rather than failing. + Use two locks: a separate process-wide serialization mutex held by the guard, and the + buffer's own mutex taken and released per line by the logger. +- The buffer is process-global and every other concurrently-running `trusted-server-core` + test logs into it, so a `got: {captured}` diagnostic will be large. Assert with + `contains`, not equality. +- `log::set_max_level` is global for the test binary. Setting it to `Info` is fine, but it + affects every test in the process. + +```rust +#[tokio::test] +async fn eligible_navigation_logs_origin_fetch_duration() { + // Arrange + let logs = crate::test_support::tests::capture_logs(); + let mut settings = settings_with_enabled_auction_and_creative_opportunities(); + // The log line is gated; without this the assertions below can never pass. + settings.debug.publisher_timing = true; + let stub = Arc::new(StubHttpClient::new()); + queue_cacheable_html_response(&stub); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let slots = [article_slot()]; + + // Act + let _ = run_with_slots(&settings, &services, &slots, conditional_navigation_request()).await; + + // Assert + let captured = logs.contents(); + assert!( + captured.contains("publisher_timing"), + "eligible navigation should emit a publisher_timing log line, got: {captured}" + ); + assert!( + captured.contains("origin_fetch_ms="), + "publisher_timing should record origin_fetch_ms, got: {captured}" + ); +} +``` + +This test deliberately asserts only on the `publisher_timing` line. `run_with_slots` never +drives a finalizer, so `publisher_rewrite` is out of its reach — cover that separately if +at all, rather than contorting this test. + +- [ ] **Step 2: Run the test to verify it fails** + +```bash +cargo test -p trusted-server-core --target aarch64-apple-darwin \ + eligible_navigation_logs_origin_fetch_duration -- --nocapture +``` + +Expected: FAIL — no `publisher_timing` in the captured logs. (Substitute your host +triple; core tests run natively for fast iteration. The Viceroy run comes in Step 6.) + +- [ ] **Step 3: Time the origin fetch** + +In `publisher.rs`, at the top with the other imports, add: + +```rust +use web_time::Instant; +``` + +Then wrap the origin send. The current code is at `publisher.rs:2870`: + +```rust +let mut response = match services.http_client().send(platform_request).await { +``` + +Change it to: + +```rust +let origin_fetch_start = Instant::now(); +let mut response = match services.http_client().send(platform_request).await { +``` + +and immediately after the `match` completes (after the existing `};` that closes it, +before the existing `log::debug!("Publisher origin response received: ...")` at `:2888`): + +```rust +let origin_fetch_ms = u64::try_from(origin_fetch_start.elapsed().as_millis()).unwrap_or(u64::MAX); +``` + +**Make the bypass config-driven in the same change.** This is what lets Stage 0 ship as a +config flip rather than a second deploy — see Task 5. Replace the block at +`publisher.rs:2866-2868`: + +```rust +// Single source of truth for the request and the log line below. Operator- +// controlled so the read-through cache can be re-enabled without a release; +// see docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md §4. +let cache_bypass = should_run_ad_stack && settings.publisher.bypass_origin_cache; +if cache_bypass { + platform_request = platform_request.with_cache_bypass(); +} +``` + +Add the setting to `Publisher` in `crates/trusted-server-core/src/settings.rs:29`, +**defaulting to today's behaviour** so this change is a no-op until deliberately flipped: + +```rust +/// Bypass the platform read-through cache on ad-eligible publisher navigations. +/// +/// `true` preserves the historical behaviour introduced by the SSAT 304-prevention +/// design. `false` lets those navigations use the read-through cache; the +/// conditional-header strip already guarantees a complete body on a cache HIT. +/// Temporary operator control for the Stage 0 rollout — remove once settled. +#[serde(default = "default_bypass_origin_cache")] +pub bypass_origin_cache: bool, +``` + +```rust +fn default_bypass_origin_cache() -> bool { + true +} +``` + +**Adding this field breaks nine sites. Update them in the same commit or Step 2 fails to +compile before it can produce the intended RED failure:** + +- The hand-written `Default` impl at `settings.rs:81-97`. +- Eight exhaustive test literals. The line numbers below anchor each + `let publisher = Publisher {` **opening**, not a field — add the new field inside each + brace: `settings.rs:3553`, `:3564`, `:3575`, `:3586`, `:3597`, `:3608`, `:3621`, + `:3635`. `clippy-fastly` runs `--all-targets`, so these gate lint too. +- The rustdoc example for `origin_host`, whose literal opens at `settings.rs:130`. + **This is a live doctest** and the host-triple test command below does not skip + doctests. + +While there, mirror the existing default-agreement test +`publisher_default_max_buffered_body_bytes_matches_config_default` (`settings.rs:3648`) — +it exists to catch a hand-written `Default` diverging from a serde default, which is +exactly the shape this field re-introduces. One assertion. + +Then emit the line, immediately after computing `origin_fetch_ms`, gated on the debug +flag from the section above: + +```rust +if settings.debug.publisher_timing { + log::info!( + "publisher_timing origin_fetch_ms={origin_fetch_ms} \ + cache_bypass={cache_bypass} ad_stack={should_run_ad_stack}" + ); +} +``` + +- [ ] **Step 4: Instrument `hold_wait_ms` — the decision metric** + +This is the number the whole effort turns on, and it needs **one edit in one function**. + +`collect_stream_auction` (`publisher.rs:2431`) is the only function that awaits the +auction collect, and all three call sites reach it: + +| Call site | Path | +| ------------------- | ------------------------------------------------------------ | +| `publisher.rs:793` | `hold_collect_close_tail` — Fastly lazy stream | +| `publisher.rs:2257` | `body_close_hold_loop`, EOF arm — Axum, Cloudflare, Spin | +| `publisher.rs:2311` | `body_close_hold_loop`, mid-stream arm — same three adapters | + +Instrument the callee, not the callers. It already destructures `settings` out of +`AuctionCollectDeps` (`:2436`), so the debug flag is in scope with no new plumbing, and +one edit covers every adapter. + +Wrap the `collect_dispatched_auction` await at `:2447-2449`: + +```rust + let hold_wait_start = Instant::now(); + let result = orchestrator + .collect_dispatched_auction(dispatched, services, &collect_ctx) + .await; + if settings.debug.publisher_timing { + let hold_wait_ms = + u64::try_from(hold_wait_start.elapsed().as_millis()).unwrap_or(u64::MAX); + log::info!("publisher_hold hold_wait_ms={hold_wait_ms}"); + } +``` + +`settings` here is `&&Settings` from the destructure — deref as needed; the compiler will +say so. + +**Do not instrument `hold_finish_ready_segments` (`:869`) or `hold_finish_tail_segments` +(`:896`).** Neither awaits the collect. The first returns `close_found` for its caller to +act on; the second delegates to `hold_collect_close_tail` at `:909`. Instrumenting them +would double-count. + +**Do not instrument the auction itself.** `OrchestrationResult::total_time_ms` +(`orchestrator.rs:285`, struct at `:1449`, per-provider at `:365`) already flows to +`auction_events_raw`. `hold_wait_ms` measures something different and more useful: how +long the _response_ waited, which is near zero when the auction finished during transfer +even though `total_time_ms` is large. + +- [ ] **Step 5: Run the test to verify it passes** + +```bash +cargo test -p trusted-server-core --target aarch64-apple-darwin \ + eligible_navigation_logs_origin_fetch_duration -- --nocapture +``` + +Expected: PASS. + +- [ ] **Step 6: Run the full publisher test module under the real target** + +A format-changing edit to this file can break tests far from the one you added, and the +Viceroy runner aborts on the first panic — so run the whole suite, not a filtered subset. + +```bash +cargo test-fastly +``` + +Expected: PASS. `app::tests` DNS `Error` lines in the output are pre-existing noise. + +- [ ] **Step 7: Verify format and lint** + +```bash +cargo fmt --all -- --check +cargo clippy-fastly +``` + +Expected: both clean. + +- [ ] **Step 8: Commit** + +```bash +git add crates/trusted-server-core/src/publisher.rs \ + crates/trusted-server-core/src/settings.rs \ + crates/trusted-server-core/src/test_support.rs +git commit -m "Add an operator switch for the origin cache bypass and log origin fetch time" +``` + +Staging without `settings.rs` leaves a tree that does not compile. + +- [ ] **Step 9: Deploy and collect** + +Deploy first. Then enable the log — it is gated and off by default: + +```bash +# In the operator's trusted-server.toml, under [debug]: +# publisher_timing = true +ts config push +``` + +**Deploy before pushing, not after.** `Settings`, `Publisher`, and `DebugConfig` all carry +`#[serde(deny_unknown_fields)]`, and `ts config push` validates against the typed schema +(`crates/trusted-server-cli` → `run_config_push_typed::`). So the +`ts` binary must be rebuilt from this commit (`cargo install-cli`), and pushing the new +keys before the new WASM is live would break config load on the deployed build. +`trusted-server.example.toml:121-125` records this same hazard for +`auction.rewrite_creatives`. + +Then capture the **bypass-on baseline only**. Do not try to collect an off arm here — +turning the bypass off _is_ Task 5, which is gated on Task 1's verdict and forbidden on a +FAIL. The off arm is collected in Task 5 Step 8. + +Capture enough navigations to separate the medians with confidence, across both a homepage and an article path, with the bypass +both on and off. Record the N alongside the result. + +Append to the findings document: + +```markdown +## Step C — server-side latency breakdown + +**N per arm:** · **Paths:** · **Date:** + +| Arm | `origin_fetch_ms` = origin TTFB (median) | auction `total_time_ms` (median) | `rewrite_ms` (median) | +| ---------- | ---------------------------------------- | -------------------------------- | --------------------- | +| bypass on | | | | +| bypass off | | | | + +Read the asymmetry carefully. `origin_fetch_ms` is origin **TTFB** — the send returns at +response headers because `.with_stream_response()` is set — whereas `total_time_ms` is +the auction's full duration. The comparison below is still the right one, but it is not +comparing two like quantities. + +**Verdict:** HOLD IS FREE / HOLD IS COSTING + +Read it off `hold_wait_ms` directly — no model, no comparison against auction duration. + +HOLD IS FREE = `hold_wait_ms` median near zero. The auction finishes during body +transfer. Proceed as staged in spec §7: Stage 0 primary, Stage 2 protects its win. + +HOLD IS COSTING = `hold_wait_ms` median materially non-zero. **Staging inverts** — +Stage 2 becomes primary and Stage 0 secondary. The work does not change, only its order. +Spec §6.2 argues for the first outcome but explicitly does not prove it, so treat the +second as a live possibility. +``` + +- [ ] **Step 10: Commit the findings** + +```bash +cd docs && npx prettier --write superpowers/plans/2026-08-08-1009-measurement-findings.md && cd .. +git add docs/superpowers/plans/2026-08-08-1009-measurement-findings.md +git commit -m "Record server-side latency breakdown for #1009" +``` + +--- + +> **Task 4 (spec correction) was completed while this plan was being written.** §3's +> mechanism bullet, §4's operator-flag framing, and the `unexpected_origin_304` watch are +> all already in the spec. Nothing to do; the task is removed rather than left as a +> no-op an implementer would stall on. + +--- + +## Task 5: Stage 0 — turn the origin cache bypass off + +**Gate:** do not flip the flag until Task 1 has recorded a **`FINAL PASS`**. There are +three verdicts, not two. + +- **`FINAL PASS`** → Task 5a (config flip). +- **`PROVISIONAL PASS`** → **stop.** Not a release gate. This is the current state. It + means the representation split is declared correctly under the conditions tested, and + that those conditions were too narrow to flip production on. +- **`FAIL`** → Task 5b. Do **not** flip; it can serve an RSC payload to an HTML + navigation. + +**`FINAL PASS` requires all five, each recorded in the findings document:** + +| Condition | Why the provisional run is insufficient | +| -------------------------------------------------------- | ---------------------------------------------------- | +| A real authenticated or state-bearing session cookie | `sessionid=abc123` is synthetic and proves nothing | +| Basic Auth exercised **through TS**, not just the origin | #1009 describes a gated deployment | +| The experiment variant named in #1009 | Absent from the origin's `Vary`; unexplained | +| Representative routes — article, section, search | Only the homepage was probed | +| Cached-hit slot and render attribution | The randomized div IDs are an unverified interaction | + +Any one of these unrecorded means the verdict stays `PROVISIONAL PASS` and Task 5 does +not start. + +Because Task 3 made the bypass config-driven, Stage 0 ships as a **config change on an +already-deployed build** — no second release, and the read path reverts with another +config push rather than a revert. That matters here: the failure mode this gates on is +cache poisoning, where minutes of exposure are worse than a slow rollout. + +**But a config push is not a full rollback.** It stops HTML navigations reading from +cache; it evicts nothing already stored. See Step 4's rollback sequence — flip, then purge +or roll a versioned namespace, then observe past the origin TTL. Until a C1 purge path +exists, the tail is "wait out the origin TTL," and that must be an accepted, recorded +risk before the flip. + +### Task 5a: flip the flag (Task 1 verdict = `FINAL PASS`) + +**Files:** + +- Modify: the operator's `trusted-server.toml` (gitignored) +- Modify: `crates/trusted-server-core/src/publisher.rs` — the test, and later the default +- Modify: `trusted-server.example.toml` — document the key + +- [ ] **Step 1: Add a test covering the flag in both positions** + +The existing test at `publisher.rs:4824` +(`eligible_navigation_bypasses_cache_and_returns_non_storable_html`) asserts `vec![true]` +and must **keep passing** while the default is `true` — it now documents the default +rather than the only behaviour. Leave it, and add a sibling next to it: + +```rust +#[tokio::test] +async fn eligible_navigation_uses_read_through_cache_when_bypass_disabled() { + // Arrange + let mut settings = settings_with_enabled_auction_and_creative_opportunities(); + settings.publisher.bypass_origin_cache = false; + let stub = Arc::new(StubHttpClient::new()); + queue_cacheable_html_response(&stub); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let slots = [article_slot()]; + + // Act + let response = + run_with_slots(&settings, &services, &slots, conditional_navigation_request()).await; + let response_head = response_head(response); + + // Assert + assert_eq!( + stub.recorded_cache_bypass_flags(), + vec![false], + "disabling bypass_origin_cache should let the navigation use the read-through \ + cache; the conditional-header strip already guarantees a full body on a HIT" + ); + assert_eq!( + recorded_header( + stub.recorded_request_headers().first().expect("should record request"), + header::IF_NONE_MATCH.as_str() + ), + None, + "conditional headers must still be stripped with the bypass disabled" + ); + assert!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|v| v.to_str().ok()) + .is_some_and(|v| v.contains("no-store")), + "the synthesized document must stay non-storable regardless of the bypass flag" + ); +} +``` + +Those last two assertions are the point of the test: the flag must change **only** the +cache mode, leaving the conditional-header strip and the response non-storability intact. + +**Leave `publisher.rs:4941` and `:5160` unchanged** — they already assert `vec![false]` +for non-eligible requests and must keep doing so. `Range`/`If-Range` stripping is covered +by `eligible_range_navigation_fetches_complete_html` (`publisher.rs:4883`), unaffected. + +- [ ] **Step 2: Run both tests** + +```bash +cargo test -p trusted-server-core --target aarch64-apple-darwin \ + eligible_navigation -- --nocapture +``` + +Expected: both the existing default-behaviour test and the new flag-disabled test PASS. +If Task 3's `bypass_origin_cache` field is not yet in place, the new test will not +compile — land Task 3 first. + +- [ ] **Step 3: Document both keys in the example config** + +Add to `trusted-server.example.toml` under `[debug]` (line 149, alongside +`ja4_endpoint_enabled` and `auction_html_comment`): + +```toml +# Emit a `publisher_timing` log line per publisher origin fetch. Temporary +# instrumentation for the #1009 latency measurement; leave false in production. +publisher_timing = false +``` + +And under `[publisher]`: + +```toml +# Bypass the platform read-through cache on ad-eligible navigations. +# `true` is the historical default. Set `false` to let those navigations use the +# read-through cache — only after confirming the origin declares `Vary` for every +# header it varies on (see the Stage 0 precondition). +bypass_origin_cache = true +``` + +- [ ] **Step 4: Flip it in the operator config and push** + +```bash +# In the operator's trusted-server.toml, under [publisher]: +# bypass_origin_cache = false +ts config push +``` + +Note from prior operational experience in this repo: the environment-variable overlay is +scalar-only **and** only overrides keys that already exist in the TOML. Adding the key to +the operator's file is required; setting only an env var will be silently dropped. + +**Rollback is a config push plus an eviction — not a config push alone.** Pushing `true` +again stops HTML navigations reading from cache, but evicts nothing: objects already +cached, including those RSC and other request classes keep reading, persist until they +expire. The origin's `max-age=60` bounds that, but does not remove it. + +Full rollback: + +1. Push `bypass_origin_cache = true`. +2. Purge — **and note this is C1, not C2.** `InsertBuilder::surrogate_keys` belongs to + the Core Cache API and applies to the transformed-template cache the ESI spike builds. + It has no effect on the HTTP read-through cache that Stage 0 turns on. Purging C1 + requires either surrogate keys the **origin** supplies on its responses, or the HTTP + cache's own request/candidate surrogate-key surface. Confirm which is available before + relying on it. + + **Neither is wired today.** If the flip ships before one exists, the rollback story is + "wait out the origin TTL" — roughly a minute, per the Step A findings. That is + survivable, but it must be an accepted risk recorded before the flip rather than a + discovery during an incident. + +3. Observe past the origin TTL before declaring the incident closed. + +- [ ] **Step 5: Run the full suite across every adapter** + +```bash +cargo test-fastly && cargo test-axum && cargo test-cloudflare && cargo test-spin +``` + +Expected: all PASS. If `platform/test_support.rs:797` or `:888` fail, they are testing +the stub's own recording behaviour rather than publisher behaviour — read them before +changing anything. + +- [ ] **Step 6: Format and lint every target** + +```bash +cargo fmt --all -- --check +cargo clippy-fastly && cargo clippy-axum && cargo clippy-cloudflare \ + && cargo clippy-cloudflare-wasm && cargo clippy-spin-native && cargo clippy-spin-wasm +``` + +Expected: all clean. + +- [ ] **Step 7: Commit the code and config-template changes** + +```bash +git add crates/trusted-server-core/src/publisher.rs trusted-server.example.toml +git commit -m "Add an operator switch for the publisher origin cache bypass" +``` + +- [ ] **Step 8: Watch for the failure modes, not just the win** + +After the flip, check three things before declaring success. The first two are regression +signals, not confirmations. + +1. **`unexpected_origin_304` abandonment telemetry.** This reason + (`publisher.rs:2896`, emitted via `emit_abandoned_auction` at `:2360`) exists because + the ad-stack path refuses cached and conditional origin responses. Re-enabling the + cache is precisely what could revive it. **Any non-zero rate is a rollback signal** — + it means a 304 is reaching TS, which the conditional-header strip was supposed to make + impossible. Push `true` and investigate before continuing. +2. **Representation mixing.** Spot-check that HTML navigations still return HTML and RSC + fetches still return `text/x-component`. A mismatch is the Task 1 risk having + materialized despite a PASS verdict — roll back immediately, this is cache poisoning. +3. **`origin_fetch_ms` and `cache_bypass=false`** in the `publisher_timing` logs. This is + the win, and it is the _last_ thing to check, not the first. + +- [ ] **Step 9: Record and commit** + +Append the before/after medians and the three checks above to the findings document, +format it, and commit. + +- [ ] **Step 10: Retire the flag (follow-up, not now)** + +Once the flip has held for a sustained period, flip the default to `false` in +`default_bypass_origin_cache`, then remove the setting and the branch entirely. Track it; +a temporary flag left in place becomes permanent configuration surface. + +### Task 5b: cache-key discriminator (Task 1 verdict = FAIL) + +**Do not implement from this plan.** A FAIL means the origin serves multiple +representations at one URL without declaring `Vary`, so removing the bypass requires TS +to add its own cache-key discriminator — a feature, not a deletion, and materially larger +than Stage 0 as scoped here. + +Escalate with the Task 1 findings and write a separate plan. Two things that plan must +address, both from spec §4: + +1. The discriminator must key on the request headers that actually distinguish the + representations (`RSC`, `Next-Router-*`, the experiment header), **not** on the + navigation classification. `is_navigation_request` + (`crates/trusted-server-core/src/http_util.rs:73-98`) falls back to the `Accept` + header when Fetch Metadata is absent, and its own comment warns that `fetch()` can set + `Accept: text/html` — so a fetch-based request can be misclassified as a navigation. +2. Whether the origin should simply be asked to declare `Vary`, which is cheaper than + building the discriminator and fixes the problem for every consumer rather than only + for TS. + +--- + +## Out of scope + +Named so nobody widens this plan mid-flight. All are specified in the spec. + +- **Stages 1–2** — moving bid delivery off the response body and deleting the `` + hold. Spec §7 and §8 put these behind the correctness defects. Spec §5 explains why + starting them casually produces a silent revenue loss. +- **Stages 3a/3b** — response cacheability. 3b is additionally gated on Task 2. +- **Stages 4–5** — purge capability, TS-owned template cache, ESI. +- **Removing the `bypass_cache` platform capability.** Task 5a removes one call site only. + +--- + +## Definition of done + +- [ ] Findings document records verdicts for Steps A, B, and C, each with its date, its + N where applicable, and the consequence spelled out. +- [ ] `publisher_timing` and `publisher_hold` lines are emitted in production and + readable, and `hold_wait_ms` has a recorded median. +- [ ] Task 1 recorded a **`FINAL PASS`** — all five conditions in Task 5's gate closed, + not merely the provisional run. +- [ ] Either Task 5a is shipped, or Task 1 returned FAIL and both a production defect and + a follow-up plan for 5b exist. +- [ ] A purge path or versioned cache-key namespace exists **before** the flip, or the + "wait out the TTL" rollback is explicitly accepted and recorded as a risk. +- [ ] **The win is measured client-side, not from `origin_fetch_ms`.** That figure is + origin TTFB and excludes body download, rewrite, and post-processing — it is + attribution, not the outcome. #1009 already has a working tester-cookie browser A/B + measuring the TTFB the publisher actually complained about; use it for before/after. +- [ ] `unexpected_origin_304` rate is zero and representations are not mixed (Task 5a + Step 8) — both checked **before** the win is claimed. +- [ ] All CI gates pass: `cargo fmt --all -- --check`; the six clippy targets; the four + adapter test suites; the parity suite; JS build, test, and format; docs format. diff --git a/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md b/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md new file mode 100644 index 000000000..fa3e90544 --- /dev/null +++ b/docs/superpowers/plans/2026-08-08-1009-measurement-findings.md @@ -0,0 +1,503 @@ +# #1009 measurement findings + +Recorded output of the checks in +[the plan](./2026-08-08-1009-measurement-and-stage-0.md). Results only — the origin +hostname is operator config and is deliberately not reproduced here. + +> **Final implementation note, 2026-08-12.** The opt-in `esi` mode now uses Fastly +> Core Cache plus an exact inert byte seam. The parser and client-fill experiments were +> removed. Real-origin numbers in this file remain evidence about the observed path, not +> a clean before/after benchmark. Current operational semantics are documented in +> [the configuration guide](../../guide/configuration.md). + +## Step A — origin `Vary` declaration and cookie exposure + +**Date:** 2026-08-08 · **Method:** direct `curl` against the publisher origin with the +configured `origin_host_header_override`, homepage path. + +### Representation split + +| Representation | `Content-Type` | `Cache-Control` | `Set-Cookie` | +| ------------------------------------ | ------------------ | --------------- | ------------ | +| HTML navigation | `text/html` | `max-age=60` | none | +| `RSC: 1` | `text/x-component` | `max-age=60` | none | +| `RSC: 1` + `Next-Router-Prefetch: 1` | `text/x-component` | `max-age=60` | none | + +`Vary`, identical on every response: + +``` +vary: rsc, next-router-state-tree, next-router-prefetch, next-router-segment-prefetch, Accept-Encoding +``` + +The origin declares **every** header that distinguishes the representations, including +`next-router-segment-prefetch`, which the plan's probe list did not think to check. The +HTML/RSC split at one URL is real and correctly declared. + +### Cookie personalization + +Hash comparison was useless here and the plan's probe as written would have produced a +false FAIL — see the method note below. After normalizing per-request identifiers: + +| Comparison | Differing lines | +| ------------------------------ | --------------- | +| no-cookie A vs no-cookie B | 2 | +| no-cookie A vs **with cookie** | 2 | + +Both diffs are the same single `generationTimestamp` field in the RSC payload. **The +cookie changes nothing.** Byte lengths were identical across all three responses +(1,432,944). + +Cookie sent: `ts-tester=true; sessionid=abc123; ts-ec=probe`. + +### Verdict: **PROVISIONAL PASS** — not sufficient to gate a production flip + +Downgraded 2026-08-10 after external review. Everything below held under the conditions +tested; the conditions tested are narrower than the gate requires. + +What passed: + +- `Vary` names every request header the origin varies on. ✅ +- Bodies did not differ by the cookie sent, so `Vary: Cookie` was not required **for + that cookie**. ✅ +- No `Set-Cookie` on a shared-cacheable response. ✅ +- Origin returns 200 without credentials at this layer. ✅ + +**What was not tested, and each of these can flip the verdict:** + +| Gap | Why it matters | +| ---------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `sessionid=abc123` is not a real session | A synthetic value proves nothing about a state-bearing publisher session. An authenticated or paywall-metered session is exactly the case that would personalize. | +| One route (homepage) only | Article, section, and search routes may personalize differently. | +| Experiment variant never exercised | #1009 says the origin varies on one. It is absent from `Vary` — see Residual uncertainty below. | +| Basic Auth through TS untested | #1009 describes a gated deployment. Only the origin was probed directly. | +| Cached-hit slot resolution untested | The randomized div IDs below are an unverified interaction, not a cleared one. | + +**Consequence:** Stage 0 still takes the operator-flag path rather than the cache-key +discriminator, and no live cross-serving defect is indicated. But this is **not** a +release gate. Close the table above before flipping the flag in production. + +## Two findings the checks were not looking for + +### 1. The origin already intends this page to be shared-cached + +`cache-control: max-age=60` with a correct `Vary` and no `Set-Cookie`. The origin has +been cacheable all along; Trusted Server opted out of it. That is the spec's §4 framing +confirmed from the other side, and it strengthens the case that the bypass was +belt-and-braces rather than load-bearing. + +It also bounds the win: a 60-second TTL means Stage 0 buys a cache hit only within that +window. Whether that translates into a meaningful hit rate depends on request volume per +URL, which is not measured here. + +### 2. Ad-slot div IDs are randomized per request — and this interacts with Stage 0 + +The only per-request variance in the document is ~170 lines of ad-slot container IDs, +each a fresh 32-hex UUID: + +``` +ad-in_content-f75fa7fba54a4fc2a2d787f51c1837dd-in_content-0 +ad-in_content-a968b27e3ee2424f8bb1c19560abf2b1-in_content-0 ← same slot, next request +``` + +Under the bypass, Trusted Server sees fresh IDs on every request. **Once the cache is on, +every visitor within a 60-second window receives the same IDs.** + +This is very likely fine — `tsjs.adSlots` is built from configured slot definitions, not +scraped from origin markup, and injection is a prefix match on the configured `div_id`. +But it is an untested interaction between Stage 0 and the slot-matching path, and it was +not in anyone's risk list. **This is a release gate, not a note.** Verify slot matching resolves against a cached +document before flipping the flag, and watch TS-attributed renders across the flip +rather than only `origin_fetch_ms`. + +## Method note — a defect in the plan's Step A probe + +The plan's cookie check compares `shasum` of the response bodies. On this origin that +test always fails, cookie or not, because of the randomized div IDs above. Three requests +produced three different hashes with byte-identical lengths. + +**Correct method:** normalize per-request identifiers before comparing, e.g. +`sed -E 's/[0-9a-f]{32}/UUID/g'`, and diff the normalized bodies rather than hashing +them. Establish the no-cookie baseline drift first, then compare the cookie arm against +that baseline — a cookie arm is only interesting if it differs by _more_ than the +baseline does. Fix the plan before anyone re-runs this. + +## Residual uncertainty + +#1009 states the origin varies on an experiment header as well as `rsc` and +`next-router-*`. **No experiment header appears in the origin's `Vary` list**, and the +RSC payload's `experiments` key did not differ across any of the requests made here. + +Three readings, unresolved: the issue was imprecise; experiments are assigned +client-side; or they key on a cookie value this probe did not supply. The `Vary` +declaration is authoritative for cache correctness and it is thorough enough to name four +Next-specific headers, so this is unlikely to be a cache-safety gap. Worth one question +to whoever wrote that line in #1009 rather than further probing. + +## Rollback caveat, added 2026-08-10 + +The plan described flipping the flag back as a seconds-long rollback. That is +incomplete. Re-enabling the bypass stops **HTML navigations** reading from cache; it +evicts nothing. Objects already cached — including those RSC and other request classes +continue to read — persist until they expire. + +Two mitigations, both real: + +- The origin's `max-age=60` bounds read-through exposure to roughly a minute. +- Purge exists in-process — `fastly::http::purge::purge_surrogate_key`. An earlier claim + that TS had no purge capability was wrong; it has no _wiring_, which is buildable. + +**But note which cache.** `InsertBuilder::surrogate_keys` belongs to the **Core Cache** +API and applies to the transformed-template cache the ESI spike would build (C2). It has +**no effect on the HTTP read-through cache** that Stage 0 turns on (C1). Purging C1 needs +surrogate keys the _origin_ supplies on its responses, or the HTTP cache's own +request/candidate surrogate-key surface. Confirm which is available before relying on it — +an earlier revision of this document conflated the two. + +**C2's purge is locally testable; C1's is not.** Verified 2026-08-10: Viceroy 0.17 +implements `purge_surrogate_key` against the same in-process cache it serves reads from +(`viceroy-lib-0.17.0/src/wiggle_abi/fastly_purge_impl.rs:10-32`), soft purge included. So +the purge-based rollback for the C2 template cache the spike builds can be exercised end +to end without a Fastly service. That does nothing for Stage 0, whose exposure is C1. + +Rollback is therefore: flip the flag, **then** purge C1 by whichever mechanism is actually +available (or roll a versioned key namespace), **then** observe past the origin TTL before +declaring the incident closed. With no C1 purge path wired, the tail is the TTL itself — +roughly a minute here, and a recorded risk rather than a surprise. + +## ESI spike Task 1 — does `esi` 0.7 build on this toolchain? + +**Date:** 2026-08-10 · **Verdict: PASS.** The cheapest falsifier for the ESI question +clears. #1009 is not closed by a toolchain limit. + +| Check | Result | +| ------------------------------------------------------------------------------------------ | -------------------- | +| `cargo add esi@0.7 --package trusted-server-adapter-fastly` | resolved `esi 0.7.1` | +| `cargo check-fastly` (Rust 1.95.0 / `wasm32-wasip1`) | clean | +| `cargo fmt --all -- --check` | clean | +| All six clippy targets (fastly, axum, cloudflare, cloudflare-wasm, spin-native, spin-wasm) | clean | +| `cargo check --manifest-path crates/trusted-server-integration-tests/Cargo.toml --tests` | clean | + +**Nine new transitive dependencies:** `esi 0.7.1`, `nom 8.0.0`, `rand 0.10.2`, +`rand_core 0.10.1`, `chacha20 0.10.1`, `cpufeatures 0.3.0`, `atoi 2.0.0`, +`html-escape 0.2.15`, `md5 0.8.1`. + +**No existing shared dependency moved.** `regex` stays 1.12.4, `bytes` 1.12.0, `log` +0.4.33. `nom` and `rand` gain new majors that coexist with the existing 7.1.3 / 0.8.6 / +0.9.4 rather than replacing them — the best available outcome, since a forced bump on a +shared dep is what would have made this expensive. + +### A claim in the spike plan was wrong + +Task 1 Step 3 told the implementer to check for a desync between the root `Cargo.lock` and +`crates/trusted-server-integration-tests/Cargo.lock`. **That second lockfile does not +exist.** The integration-tests crate is a workspace member (root `Cargo.toml:10`) and +shares the root lockfile, so the desync hazard cannot arise in that form. The plan has +been corrected. The dual-lockfile constraint was real at some earlier point; it is not the +current layout. + +### Viceroy 0.17 supports the whole Core Cache surface this spike needs + +**Date:** 2026-08-10 · **Verdict: PASS.** Probed directly under +`cargo test -p trusted-server-adapter-fastly --target wasm32-wasip1`, then removed: + +| API | Result | +| ------------------------------------------------------------------------ | ------ | +| `cache::core::insert(key, ttl).execute()` → write → `finish()` | works | +| `cache::core::lookup(key).execute()` → `Found::to_stream()` | works | +| `Transaction::lookup(key).execute()` → `must_insert_or_update()` | works | +| `Transaction::insert(ttl).surrogate_keys([…]).execute_and_stream_back()` | works | +| Second transactional lookup reports a hit, no obligation | works | + +That is the entire API surface the spike's Task 3 Step 4 specifies, including the +transaction and stream-back shapes. + +**Consequence: provisioning a Fastly service is not a prerequisite.** An earlier revision +of the spike plan made it Task 2 and a blocker on everything downstream. Almost all of the +correctness and safety work — the C2 cache logic, the transform, template byte-identity, +ESI assembly, DCA and dispatcher refusal, fragment-failure degradation, header ordering, +and the leakage gates — runs locally. The plan is re-sequenced accordingly. + +**What still needs a real service:** shielding behaviour, POP-level cache tiering, +request collapsing under genuine concurrency (Viceroy is a single instance, so a passing +`Transaction` test proves the API works and not that collapsing is correct under load), +stale revalidation timing, and **every performance number in the decision rule**. Local +timings are meaningless for the decision. + +### Not yet verified + +Compiling and a cache round-trip are not an implementation. Nothing yet exercises the +`lol_html` transform into C2, ESI assembly, or any runtime behaviour on the publisher path, +and the `esi` dependency is added but unused. + +## ESI spike Task 3 — implementation progress + +**Date:** 2026-08-10. All of it behaviour-neutral under the default +`AssemblyMode::Inline`; nothing here changes a shipped code path. + +| Step | State | +| -------------------------------- | ------------------------------------------------------------------- | +| 1 — `AssemblyMode` setting | **Done.** `Option` on `CreativeOpportunitiesConfig`. | +| 2 — head-seam neutrality gate | **Done.** `template_ad_slots_script`, three byte-identity tests. | +| 2b — body-close decoupling | **Done.** `BodyCloseInjection`, `body_close_injection`. | +| 2c — emit the marker under `Esi` | **Not done.** Blocked on the fragment endpoint; see below. | +| 3 — C2 eligibility gate | **Done.** `c2_bypass_reason`, eight tests. Logs only, no cache I/O. | +| 4 — C2 cache read/write | **Not started.** Design choice open; see below. | + +### What is deliberately absent + +**No marker is emitted under `Esi`.** The marker must point at a fragment endpoint +returning an **executable script**. `/_ts/page-bids` returns JSON +(`publisher.rs`, `handle_page_bids`) and ESI splices fragment bytes verbatim, so aiming +at it would put raw JSON where a script belongs. That endpoint does not exist, and a +marker with nothing behind it is worse than no marker. A test pins the current answer so +it changes deliberately. + +**No cache read or write.** `c2_bypass_reason` has a real call site that logs its verdict, +which makes the decision observable during the spike without mutating anything. Task 3 +Step 4 is blocked on choosing between read-through-with-body-transform and explicit +`cache::core` — the plan names that as a decision to make before writing code, and it is +under investigation rather than assumed. + +### A defect this work introduced and then caught + +Gating the head seam on neutrality made `ad_slots_script` `None` under the shared modes. +The body-close element handler read exactly that value to decide whether to inject at all, +so shared modes silently stopped injecting anything at `` — a side effect of a +`` change. Safe, since emitting nothing cannot leak, but wrong in the way the spec +warns about: the gate has to be "did this response carry bids", not "does this page have +slots". + +Found by reading the handler while starting the next step, not by a failing test. Fixed by +replacing the inference with a named decision. The test that now guards it asserts +body-close is identical whether or not the head script is present — a decision that read +the head script would be _accidentally_ correct today, because that script is always +absent under shared modes, and wrong the moment that changes. + +Worth recording because it is the same shape as the bug the whole task exists to prevent: +something that looks correct and quietly does nothing. + +### Coverage and its limits + +Fourteen new tests. `fmt`, all six clippy targets, and all four adapter suites pass, with +1850 core tests under Viceroy. `clippy --all-targets` caught a benchmark construction site +that all four test suites missed — the suites are not the whole gate. + +**The neutrality guarantee is narrower than it looks.** The tests prove `tsjs.adSlots` is +neutral. They say nothing about the other things injected at the same seam — integration +`head_inserts`, the gpt-diagnostics bootstrap, the RSC placeholder rewriter — which the +spec flags as needing an audit and which that audit has not yet covered. Until it does, +treat request-neutrality as asserted for one element rather than established for the +template. + +## Code review of the Task 3 commits — three HIGH findings, all closed + +**Date:** 2026-08-11. An independent review of the four implementation commits found +three HIGH issues. The default `Inline` path was verified unchanged byte-for-byte, so +none was a live regression — but all three were invariants this branch exists to +establish and none was enforced or tested. + +### 1. The auction dispatched under shared modes with nothing to consume it + +`assembly_mode` was computed _after_ the dispatch decision, so flipping to `client_fill` +or `esi` would still have sent real SSP bid requests, held the response for the full +auction budget, and discarded the result — because both injection seams now return +nothing — with no error, no warning and no log. + +Exactly the silent-waste signature §5 of the design doc is about, reached by an +incomplete feature flag rather than by removing the hold. Fixed by hoisting +`assembly_mode` above the dispatch and gating on `root_auction_is_useful`. + +The test derives the invariant rather than asserting per-variant: a root auction is +useful exactly when a seam will consume its result. A new mode cannot make the dispatch +gate and the injection decisions disagree without failing it. + +### 2. The C2 gate ignored the forwarded client `Cookie` + +TS forwards client cookies to origin unchanged — there is no `Cookie` strip on the +publisher path. So a response can be cookie-personalized while carrying no `Set-Cookie` +itself (session established earlier), no `Cache-Control` at all, status 200, HTML — and +every condition in the gate reported it cacheable. + +§4 of the design doc names this. The plan's own Task 3 Step 3 checklist missed it, so +the implementation matching the checklist exactly still had the hole. Now disqualifying +until an origin `Vary` covering `Cookie` is verified. + +### 3. Request-neutrality was asserted for one element, not the seam + +The head seam still injected integration `head_inserts` and the GPT-diagnostics +bootstrap unconditionally. + +Audited both. **`head_inserts` is clean** — all three implementations (datadome, didomi, +gpt) take the context parameter unused, so output depends on configuration, not the +request. **GPT diagnostics is not** — cookie- or query-activated, and documented as an +immutable request-scoped decision. + +It did not leak, but only by coincidence: `requires_private_no_store()` is a strict +superset of the conditions under which either script is emitted, and that stamp lands +before the C2 gate reads response headers, so the gate refused. Two independent +conditions that happened to align, with nothing enforcing the relationship. + +Fixed on both sides — the processor receives no diagnostics decision under shared modes, +**and** a test enumerates every combination of the decision's three fields asserting that +anything which injects also requires the stamp. The gate is the guarantee; the invariant +test is the backstop if the gate is ever removed. + +### What this says about the tests that existed + +All three findings were in code the existing tests covered — and passed. The tests +exercised the pure decision functions with hand-built inputs and never the rendered +``/`` bytes. That is still true: **no test renders a full document through +`create_html_processor` and compares two requests byte-for-byte.** The plan's Task 3 +Step 2 requires exactly that, and it remains the most valuable missing test. + +### Reviewer's gate, adopted + +Do not proceed to Task 3 Step 4 (actual C2 read/write) or expose `AssemblyMode` to any +test or staging traffic until the full-document byte-identity test exists. The three +fixes above close the known holes; that test is what would catch the next one. + +## Task 3 complete — the C2 cache engages end to end + +`2db10639` (store), `2a2e6c6a` (lookup), plus `b688d667`/`577eb85a` for the `Vary` +handling. A second request for the same URL is now served without touching the origin, +byte-identical to what was stored. + +**Three problems only appeared once the code had to run**, none of them visible in the +plan or in review: + +1. **The key needed the origin's `Vary`, but a lookup precedes the fetch.** Resolved with + an operator-stated list plus a post-response drift guard that refuses to store under a + key that missed something. Spike-grade: a two-phase lookup is the correct answer and + doubles the lookups. +2. **The key carried the encoding the _origin_ chose**, which also does not exist at + lookup time — storing under `br`, looking up under `gzip, br`, a cache that never hits. + Now keyed on what was sent to the origin. +3. **Storing needs every transformed byte; streaming does not collect them.** Shared modes + take the buffered finalizer, branching on the store authorization rather than the + assembly mode, so `Inline` cannot reach it. + +Each was a case where the design read as complete and the implementation had a hole in +it. That is the same pattern as the three review findings above, arriving one layer down. + +**Verified by mutation, not just by green tests.** Disabling the lookup fails the hit +test, so the hit is the cache answering rather than the fixture answering twice; dropping +the `Authorization` re-check fails the authenticated test; reading only the first `Vary` +header value, and disabling the drift guard, each fail their own tests. The reviewer's +gate above was satisfied first: the byte-identity tests it demanded exist and were +themselves mutation-checked. + +**Still not deployable.** `ClientFill` and `Esi` render a template with a hole and +nothing filling it — Task 4 and Task 5. A cache that works is necessary, not sufficient. + +## Local end-to-end run — the Esi arm renders + +`viceroy serve` against a stub origin, config pushed into a scratchpad `fastly.toml` so +nothing tracked was modified. Served document: + +```html +

Stub article

+
+

Body copy.

+ +``` + +No executable ESI tag. One origin fetch for two requests. `private, no-store` on the hit. +Cached template 353 bytes against 467 served, so the cache holds the pre-assembly +template. All three fragment formats behave: script, JSON, and `400` on a typo. `Inline` +unaffected — two fetches for two requests, no C2 activity, no markers. + +### The bug only a running server could find + +With the auction **enabled**, C2 never engaged: two origin fetches, marker unresolved. + +TS stamps its own `private, no-store` when `should_run_ad_stack` is true. The C2 gate ran +after that stamp, read it as the origin's declaration, concluded `OriginNotShareable`, and +refused — **on every page that serves ads**, which is every page that matters. + +The more important half is why no test caught it. The fixture left the auction disabled +and passed `slots: &[]`, so `should_run_ad_stack` was false in every test, the stamp never +fired, and the ordering was unobservable. Every C2 assertion had been made against the one +configuration where C2's hardest condition does not apply. + +Demonstrated both ways: with the old fixture, reintroducing the bug passes all seven +tests; with the corrected fixture it fails six. + +### Pattern across this branch + +Five bugs now share one shape — compiled, passed every existing test, and were wrong: + +1. The head-seam gate silently disabled body-close injection (`d9e05973`). +2. The key held the encoding the origin _chose_, so the cache could never hit (`2a2e6c6a`). +3. A C2 hit served with no `Cache-Control` at all (`0adb578e`). +4. A C2 hit dropped its in-flight auction, billing SSPs for nothing (`b3ac59a6`). +5. The gate read TS's own header as the origin's (`4c557347`). + +Three were found by writing the test the plan asked for. One needed a running server. None +were found by review — including my own, twice over on the same gate. + +The stale-cache test is the same failure in miniature: it passed while never reaching +`is_stale()`, and only mutation testing exposed that. A test that passes for the wrong +reason is worse than no test, because it is counted as coverage. + +## Independent review — two blockers, and two reasons the cache would have measured nothing + +An independent reviewer read `main...HEAD` and, importantly, **demonstrated** findings by +running code rather than inferring them. Four things it found that review-by-reading had +not. + +**A POST was answered from a cached GET.** `handle_publisher_request` is the `*`-method +fallback route, so a publisher path that renders on GET and accepts a form or webhook on +POST reaches it for both. The origin never saw the mutating request; the caller got `200` +and a page. Fixed at key construction, since the key governs lookup and store alike. + +**`Vary: Accept-Encoding` disqualified everything.** The key has a dedicated +`accept_encoding` field, so such an origin is already keyed correctly — but the coverage +check consulted only the operator-configured list and reported a gap. Every compressing +origin sends that header, so **C2 would have stored nothing against any real origin**. +This is worse than a plain bug: the spike would have measured a hit rate near zero and +reported it as a result. + +**Cookies excluded essentially every repeat visitor.** Any cookie disqualified in both +directions, and TS sets its own identity cookie. The population that could ever see a warm +hit was roughly first-ever page views and cookie-less clients. The design notes called +this the "first-nav exception"; it is the common case, not the exception. Now opt-in via +`origin_is_cookie_independent`, with the `Vary: Cookie` drift guard overriding a wrong +assertion. + +**`ClientFill` had no end-to-end coverage.** The reviewer reintroduced a diagnostics leak +scoped to that mode and all 1889 tests passed. Investigation showed that specific mutation +is unreachable — `requires_private_no_store()` is a strict superset of the injection +condition and stamps before the gate reads headers — but only by a coincidence between two +independent conditions. Both the coverage gap and the coincidence are now pinned by tests. + +### What the review says about the review process + +The reviewer's confirmed findings all came from **running** something. Its clean bills — +no leak in the template itself, no `Inline` regression — came with positive evidence: +tracing that integration context types carry no per-reader field at all, and separately +proving the leakage test has teeth by breaking the store/assemble order and watching it +fail. + +Two of my own comments were wrong, and it caught both by checking rather than reading: +one claimed three call sites where there are two, the other described a dispatcher +mechanism that stopped existing when assembly moved to `CompletedRequest`. + +Verified afterwards against a running server with the origin advertising +`Vary: Accept-Encoding`: a cookie-bearing repeat visitor now costs one origin fetch across +two requests, and a POST still reaches the origin. + +## Step B — consumers of TS's own response headers + +Not yet run. + +## Step C — hold and origin fetch timings + +Not yet run. diff --git a/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md b/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md new file mode 100644 index 000000000..5b9a455e7 --- /dev/null +++ b/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md @@ -0,0 +1,967 @@ +# #1009 ESI Validation Spike + +> **HISTORICAL SPIKE — DO NOT IMPLEMENT.** This document records the investigation, +> including executable ESI tags, parser/subrequests, and a client-fill arm that were all +> removed. Every unchecked item below is historical, not remaining work. The accepted +> implementation keeps the public `esi` spelling but uses Fastly C2 plus exact byte-seam +> assembly. See +> [the merge-hardening design](../specs/2026-08-12-1009-esi-merge-hardening-design.md) and +> [implementation plan](./2026-08-12-1009-esi-merge-hardening.md). + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development +> (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps +> use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Decide #1009 on evidence. Build a shared-template pipeline behind a flag, run +ESI and client-fill against it, and produce a decision record that either adopts ESI, +adopts client-fill, or rejects both — with the Fastly-only maintenance cost priced in. + +**Architecture:** +`origin → lol_html transform → fastly::cache::core → finalize headers → stream assembly`. + +Headers finalize **before** assembly, not after — streaming responses on this adapter +commit headers first and then pipe chunks, so nothing can be set once assembly starts. + +The transform emits **one unconditional marker at the body-close seam**. Not two: the +head seam is not a template hole, because `tsjs.adSlots` presence is request-gated +(Task 3 Step 2). The cached object is a shared template with no per-user bytes and no +request-dependent decisions. Assembly is either the `esi` crate (edge) or a client fetch +(browser), selected per request by the arm allocator so both are measured on one build. + +**Tech Stack:** Rust 2024, `wasm32-wasip1`, `fastly` 0.12.1 (`cache::core`, `http::purge`), +`esi` 0.7, `lol_html`, a real Fastly test service for cache behaviour. + +**Spec:** `docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md` — +read the 2026-08-10 correction at the top and +[§6.6](../specs/2026-08-08-esi-cacheable-root-validation-design.md#66-the-esi-pipeline-corrected) +before writing any code. + +**Control:** [the Stage 0 plan](./2026-08-08-1009-measurement-and-stage-0.md). Its +instrumentation and its bypass flag are prerequisites — this plan compares against them +and does not duplicate them. + +--- + +## Why this plan exists + +An earlier revision of the spec concluded ESI was structurally impossible. It was wrong: +`fastly::cache::core` provides the cache boundary natively, and purge runs inside Compute. +That correction reopens #1009 as an empirical question, and this plan is how it gets +answered. + +**What is genuinely uncertain**, and what each arm is for: + +1. Does a shared template plus per-request assembly beat today's inline path enough to + matter? +2. Does **edge** assembly (ESI) beat **client** assembly (a fetch) by enough to justify a + Fastly-only rendering path that must be maintained alongside the portable one? +3. Can per-user leakage be excluded across cold MISS, warm HIT, stale revalidation, + transform failure, and fragment failure? + +Question 3 is a gate, not a metric. A win on 1 and 2 with a failure on 3 is a rejection. + +## Three caches, never conflated + +The original error came from treating these as one thing. Every task below names which it +means. + +| # | Cache | Contents | Status | +| --- | --------------------------------- | ----------------------------- | ----------------------------------- | +| C1 | Origin read-through | raw origin bytes | Exists. Stage 0 turns it back on. | +| C2 | Shared transformed template | post-`lol_html`, pre-assembly | **New.** What this plan builds. | +| C3 | Assembled-response delivery cache | final per-user output | **Must never exist.** Not proposed. | + +If a task appears to require C3, stop — that is the leakage failure mode, not a design +option. + +## Arms + +Five, but only four are treatable as equivalent. + +| Arm | Root | Bids | Notes | +| ------- | ----------------------- | ---------------- | ---------------------------------------------------------- | +| **A0** | inline, C1 bypassed | inline `` | Today. The baseline. | +| **A1** | inline, C1 on | inline `` | Stage 0. Isolates the bypass from the template change. | +| **A2** | shared template from C2 | client fetch | Portable. Works on all four adapters. | +| **A3** | shared template from C2 | ESI at the edge | Fastly-only. The thing #1009 proposed. | +| **REF** | origin direct, TS off | publisher's own | **Reference, not an arm.** Different work, not comparable. | + +A0→A1 measures the bypass. A1→A2 measures the template split. A2→A3 measures edge versus +client assembly — **that difference is the entire case for ESI**, and it is the number +this plan exists to produce. + +**Do not compare A2 and A3 on root TTFB.** They serve the same C2 template, so their root +timings should be near-identical by construction; a null result there proves nothing. +ESI's claimed advantage is that bids arrive without a client round-trip, so measure: +**bids-ready time**, **`adInit` fire time**, and **first TS-attributed creative paint**. +Root TTFB stays as a guard that the template path did not regress, not as the comparison. + +REF is included because #1009 anchors on it, and excluded from pass/fail because TS-off +does no auction and no injection. Comparing against it measures the feature's existence, +not its implementation. + +--- + +## Task order and dependencies + +``` +Task 1 (esi compiles) ── DONE, PASS ──┐ + ├──> Task 3 (C2 cache) ─┬──> Task 4 (A2 client-fill) +Stage 0 plan (flag + instrumentation) ┘ ├──> Task 5 (A3 ESI) + └──> Task 6 (safety gates) + │ + Task 2 (real service) ─────────────────────────────┴──> Task 7 (decision) +``` + +**Task 2 is not a blocker on Tasks 3–6.** Everything those tasks need is exercisable under +Viceroy 0.17 — verified, see Task 2. The real service is required only for the +measurements Task 7 decides on, so provision it once there is something worth measuring. + +Task 6 runs against every arm, not once at the end. + +--- + +## Task 1: Confirm `esi` 0.7 builds on this toolchain + +Cheapest possible falsification. Do this before anything else. + +**Files:** `crates/trusted-server-adapter-fastly/Cargo.toml` + +- [ ] **Step 1: Add the dependency** + +```bash +cargo add esi@0.7 --package trusted-server-adapter-fastly +``` + +It belongs in the **Fastly adapter**, never in `trusted-server-core` — the crate is +hard-bound to `fastly::{Request, Response, Backend}` and core must stay portable. + +- [ ] **Step 2: Check it compiles for the real target** + +```bash +cargo check-fastly +``` + +Expected: clean. The crate declares edition 2021 with no `rust-version`, and pulls recent +`rand` and `nom`, so this is a genuine question on Rust 1.95.0 / `wasm32-wasip1`. + +- [ ] **Step 3: Check no shared dependency was forced to move** + +```bash +git diff --stat Cargo.lock +cargo check --manifest-path crates/trusted-server-integration-tests/Cargo.toml --tests \ + --target "$(rustc -vV | sed -n 's/^host: //p')" +``` + +**Correction, verified 2026-08-10:** an earlier revision of this step warned about a +desync between the root `Cargo.lock` and `crates/trusted-server-integration-tests/Cargo.lock`. +**That second lockfile does not exist** — the crate is a workspace member (root +`Cargo.toml:10`) and shares the root lockfile. The hazard cannot arise in that form. + +What does matter is whether adding `esi` forces an **existing** shared dependency to a new +version, since `regex`, `bytes`, and `log` are used across the workspace. Adding a new +major that coexists is harmless; moving an existing one is not. If one moves, fix with a +targeted `cargo update -p --precise ` — **never a full update**. + +**Already run and recorded** in [the findings](./2026-08-08-1009-measurement-findings.md): +no existing shared dependency moved. + +- [ ] **Step 4: Record and commit, or stop** + +**Task 1 is complete — verdict PASS, recorded 2026-08-10.** `esi` 0.7.1 compiles clean on +Rust 1.95.0 / `wasm32-wasip1`, all six clippy targets pass, and no existing shared +dependency moved. See [the findings](./2026-08-08-1009-measurement-findings.md). + +Had Step 2 failed, this plan would have stopped here with #1009 answered "not on this +toolchain." It did not. + +```bash +git add crates/trusted-server-adapter-fastly/Cargo.toml Cargo.lock +git commit -m "Add the esi crate to the Fastly adapter for the #1009 validation spike" +``` + +--- + +## Task 2: Local validation first, real service only for what needs it + +**Verified 2026-08-10 under Viceroy 0.17: the entire Core Cache surface this spike uses +works locally.** A probe exercised `cache::core::insert`, `lookup`, `finish`, `to_stream`, +and — the shape Task 3 Step 4 actually specifies — `Transaction::lookup`, +`must_insert_or_update`, `insert(...).surrogate_keys(...).execute_and_stream_back()`, and +hit-after-insert semantics. All passed. Recorded in +[the findings](./2026-08-08-1009-measurement-findings.md). + +That reorders this plan. An earlier revision made provisioning a Fastly service Task 2 and +a blocker on everything after it. It is not a blocker: **almost all of the correctness and +safety work is local**, and only the numbers and the cache topology need real +infrastructure. + +| Work | Where | +| ------------------------------------------------------------ | ------------ | +| C2 insert / lookup / transaction logic (Task 3) | **Local** | +| The `lol_html` transform and template byte-identity (Task 3) | **Local** | +| ESI assembly — the crate is pure Rust over `BufRead`/`Write` | **Local** | +| DCA off, dispatcher allowlist, injection refusal (Task 5) | **Local** | +| Fragment-failure degradation (Task 5) | **Local** | +| Header-finalization ordering, no-C3 assertions (Task 6) | **Local** | +| Cross-user leakage / request-neutrality gates (Task 6) | **Local** | +| Shielding behaviour | Real service | +| POP-level cache tiering (`x-cache`, `hit-state`, `age`) | Real service | +| Request collapsing under genuine concurrency | Real service | +| Stale revalidation timing at the edge | Real service | +| **Every performance number in Task 7's decision rule** | Real service | + +**So: build and prove correctness locally through Tasks 3, 5, and 6 before provisioning +anything.** If the design is wrong or leaks, that surfaces locally for free, and the +service is only needed once there is something worth measuring. + +Two caveats on the local scope. Viceroy is a single instance, so a passing `Transaction` +test proves the API works, **not** that collapsing behaves correctly under load. And local +timings are meaningless for the decision — do not let a fast local run substitute for +Task 7 evidence. + +### When the real service is needed + +- [ ] **Step 1: Provision it — after local correctness passes, not before** + +Separate from production. Confirm and record: whether the publisher backend is +**shielded**, and whether any Delivery service fronts the Compute service. Both change +what the numbers mean. + +```bash +fastly service list +fastly backend list --service-id --version latest +``` + +The shielding answer also settles an open question from the Stage 0 findings: #1009's +off-TS win came from a shield HIT, so whether the test service has one determines whether +its numbers transfer to production at all. + +- [ ] **Step 2: Extend the harness for lineage, not just correlation** + +The existing tester-cookie A/B has no way to join server timings to browser timings. A +root-only request ID is not enough either: under A3 the auction happens in a **fragment +subrequest**, so a root ID never reaches the auction telemetry. + +Propagate a **lineage ID plus the experiment arm** through the whole chain: + +``` +root request → C2 lookup → fragment subrequest → auction telemetry → browser render event +``` + +Generated at TS entry, forwarded into the fragment request, attached to the +`auction_events_raw` row, echoed as `x-ts-request-id`, and exposed to the browser harness +so render events carry it. Every timing log line includes both fields. + +Without this the experiment cannot join hold time, origin time, auction telemetry, browser +TTFB, and render outcome for the same pageview. **That is the difference between an +experiment and a pile of numbers.** + +- [ ] **Step 3: Capture C1 and C2 status separately** + +`x-cache`, `hit-state`, and `age` describe the **HTTP read-through cache (C1)**. They say +nothing about the **transformed-template cache (C2)**, which is a `cache::core` object +with no HTTP semantics. Recording only the former and calling it "cache status" would +attribute C2 hits and misses to the wrong tier. + +Emit both: the C1 headers as-is, plus an explicit `x-ts-c2` field carrying HIT / MISS / +STALE / BYPASS from the transaction outcome. Record the serving POP alongside. A median +that mixes cold-MISS and warm-HIT requests is meaningless, and arms cannot be compared +unless the mix is known — per tier. + +- [ ] **Step 4: Build a request-scoped arm allocator** + +`AssemblyMode` as specified in Task 3 is a **global** setting, but the sample plan below +requires randomized, non-sequential allocation. A global flip gives sequential blocks +instead, which confounds arm with time of day, cache warmth, and traffic mix. + +Allocate per request: hash the lineage ID into buckets, or key off the tester cookie. +The global setting stays as the kill switch and as the way to force a single arm; the +allocator is what the experiment actually uses. Record the assigned arm on every log line +and every telemetry row. + +- [ ] **Step 5: Define the sample plan before collecting anything** + +Write all of this into the findings document **before** the first measurement, and treat +it as fixed: + +| Element | What to state | +| -------------------- | -------------------------------------------------------------------------------------------------------------------------- | +| Allocation | Requests per arm per route, and how arms are assigned | +| Randomization | Randomized or blocked by route and cache state — not sequential runs | +| Pilot variance | A small pilot to estimate variance, before sizing the real run | +| MDE and power | The smallest difference worth detecting, and the N that detects it | +| CI method | Which interval, computed how | +| Warmup and carryover | How cold MISS is forced, how warm HIT is confirmed, and how one arm's cache state is prevented from contaminating the next | + +Rationale: this whole effort exists because #1009 drew a causal conclusion from N=4 that +did not survive contact with the code. Repeating that with more arms and no power +calculation would be worse, not better — it would look rigorous while being equally +unfalsifiable. + +--- + +## Task 3: Build C2 — the shared transformed-template cache + +The core of the spike. Behind a flag, default off. + +**Files:** + +- `crates/trusted-server-core/src/publisher.rs` — emit **one** unconditional marker at the body-close seam (see Step 2; the head seam is not a template hole) +- `crates/trusted-server-core/src/settings.rs` — the mode flag +- `crates/trusted-server-adapter-fastly/src/` — the `cache::core` read/write + +- [ ] **Step 1: Add the assembly-mode setting** + +```rust +/// How per-user ad state reaches the page. +/// +/// `Inline` is today's behaviour: bids injected before ``, root uncacheable. +/// `ClientFill` and `Esi` both serve a shared template from the transformed-template +/// cache and fill the holes afterwards. Spike-only — remove with the spike. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AssemblyMode { + #[default] + Inline, + ClientFill, + Esi, +} +``` + +Default `Inline` so the flag is a no-op until set. Note the hazards the Stage 0 plan +already documents: `Settings` carries `#[serde(deny_unknown_fields)]`, `ts config push` is +typed, and `Publisher` has a hand-written `Default` plus eight exhaustive test literals +and a live doctest. + +- [ ] **Step 2: Make the template strictly request-neutral** + +**The obvious design is wrong and would leak.** An earlier draft kept `tsjs.adSlots` in +the shared template on the grounds that it is per-URL. Its _content_ is per-URL; its +_presence_ is not. It is gated on `should_run_ad_stack` (`publisher.rs:2920-2927`), which +is `is_get && is_navigation && !is_prefetch && !is_bot && has_matched_slots && +consent_allows_auction && auction_enabled`. + +So the first request to fill C2 would freeze **its own** consent decision, bot +classification, prefetch status, and kill-switch state into an object every later visitor +reads. A consent-denied first fill serves a no-ads template to consenting users; a +consenting first fill serves ad markup to a user who refused. + +**Rule: the template contains an unconditional inert placeholder and nothing else.** + +| Element | Where it lives | +| ------------------------- | -------------------------------------------------- | +| tsjs bundle script tag | Template — content-hashed, genuinely per-URL | +| URL rewrites | Template — per-host, in the cache key | +| `tsjs.adSlots` | **Fragment** — its presence is request-dependent | +| `tsjs.bids` | **Fragment** | +| GPT diagnostics bootstrap | **Fragment** — gated on a per-request cookie/query | + +Emit **one** unconditional marker at the body-close seam, identical on every request that +reaches the transform. Under `Esi` it is an executable ESI include tag; under +`ClientFill` it is nothing at all, with the client fetching unprompted. + +- [ ] **Step 3: Bypass C2 for anything that must not be shared** + +`cache::core` is not an HTTP cache — it will happily store whatever you hand it. Nothing +rejects private or authenticated responses for you. Refuse to insert when **any** holds: + +- The origin response carries `Set-Cookie`. +- The origin response is `private`, `no-store`, or `no-cache`. +- The request carried `Authorization`. +- The response is not 200 with an HTML content type. +- DataDome's request filter replaced the document. + +Audit every request-dependent rewrite before declaring the template neutral — the +integration head-inserts and the GPT-diagnostics bootstrap are both request-scoped and +must not reach C2. + +**Assert it, do not assume it.** A unit test over the transform output must fail on any +of: a bid value, an EC ID, a consent string, a geo value, a diagnostics bootstrap, or a +`Set-Cookie`. Then a second test must assert the template is **byte-identical** for two +requests differing in consent, bot classification, and prefetch status. That second test +is the one that catches this class of bug; the first would have passed on the broken +design. + +- [ ] **Step 4: Write and read C2 — with the real API** + +The builder is move-based and the insert and read handles are different objects. Naïve +code does not compile: + +```rust +// WRONG — surrogate_keys consumes the builder and returns it; this discards the +// return value and then uses a moved binding. And execute() gives a WRITE stream, +// so there is nothing to read back from it. +let mut insert = cache::core::insert(key, ttl); +insert.surrogate_keys(["ts-template"]); +let body = insert.execute()?; +``` + +Correct shape, using a transaction so a cold cache under load transforms once: + +```rust +use fastly::cache::core::{Transaction, CacheKey}; + +let tx = Transaction::lookup(CacheKey::from(key_bytes)).execute()?; + +// Order matters: a STALE entry sets BOTH found() and must_insert_or_update(). +// Testing found() first would serve the stale bytes and silently never fulfil the +// update obligation, leaving every concurrent waiter blocked until timeout. +let template: Body = if tx.must_insert_or_update() { + // Fetch and prepare BEFORE consuming `tx`. After `insert()` the transaction is + // gone and `cancel_insert_or_update()` is unreachable, so anything that can fail + // and does not need the writer belongs here. + let origin = match fetch_and_prepare_origin() { + Ok(origin) => origin, + Err(e) => { + tx.cancel_insert_or_update()?; // releases the obligation to a waiter + return fallback_uncached(e); + } + }; + + // `Transaction::insert(self)` consumes `tx` from this line on. + let (mut writer, found) = tx + .insert(template_ttl) + .surrogate_keys(["ts-template", &url_surrogate_key]) // chained, not discarded + .user_metadata(metadata_envelope) + .execute_and_stream_back()?; + + match stream_lol_html_output(origin, &mut writer) { + Ok(()) => { + writer.finish()?; // REQUIRED, and consumes `writer` + found.to_stream()? // fallible; there is no `to_body()` + } + Err(e) => { + // Also consumes `writer`, marking an unsuccessful end so no partial + // template is served. (A `StreamingBody` dropped without `finish()` is + // aborted anyway, but say it explicitly.) + writer.abandon()?; + return fallback_uncached(e); + } + } +} else if let Some(found) = tx.found() { + found.to_stream()? // C2 HIT — skip origin fetch and transform +} else { + unreachable!("a transaction is either obliged to insert or has found an item") +}; +``` + +Two ownership rules this shape exists to respect, both of which an earlier draft broke: +`Transaction::insert(self)` **consumes** the transaction, so a helper taking `&tx` cannot +call it and `cancel_insert_or_update` is unreachable afterwards; and `finish`/`abandon` +each consume the writer, so neither can be referenced from an arm that did not bind it. + +**Decide the stale policy explicitly.** `Found::is_stale()` and `is_usable()` exist, and +`stale_while_revalidate` can be set at insert. Serving stale while revalidating is a real +option — but it is a state machine, and `cache::core` implements none of it for you. The +spike should start by treating stale as a miss and only add stale-serve if the numbers +justify it. + +**`cache::core` carries no HTTP semantics.** Status, headers, content encoding, and +revalidation are all yours. Serialize what you need into `user_metadata` — at minimum the +content encoding, the transform schema version, and the origin `Vary` values the key was +built from — and decide explicitly whether the stored template is compressed. + +**Cache key must include**, beyond the origin's declared `Vary` (`rsc`, +`next-router-state-tree`, `next-router-prefetch`, `next-router-segment-prefetch`, +`Accept-Encoding` — measured, see the Stage 0 findings): + +- The full URL, explicitly. Do not rely on an ambient request key. +- **The assembly mode.** A2 and A3 emit different template bytes and would otherwise + poison each other's entries. +- **A template schema version**, bumped whenever the transform changes, so a deploy does + not read yesterday's shape. +- Request host and scheme, the enabled-integration set, and the tsjs content hash. + +Per-user signals must never appear in the key. If a signal cannot be excluded from the +template, it does not belong in C2 at all. + +### Design decided 2026-08-10: `cache::core`. Do not revisit read-through. + +An earlier revision left this open between `cache::core` and read-through caching with +`after_send` + `set_body_transform`. Investigated and verified against the pinned SDK and +Viceroy 0.17 source. **Read-through is not viable here** — not on preference, on three +hard blockers: + +1. **Viceroy stubs the entire HTTP Cache ABI**, and the SDK converts that into a _send + error_ rather than a fallback. `is_request_cacheable` returns + `Err(NotAvailable("HTTP Cache API primitives"))` + (`viceroy-lib-0.17.0/src/wiggle_abi/http_cache.rs:108-114`; 26 such stubs in that + file), which makes `must_use_host_caching()` true, which with a send hook set returns + `Err(SendErrorCause::HttpCacheApiUnsupported)` + (`fastly-0.12.1/src/http/request.rs:626-632`). **Setting `after_send` makes every + publisher origin fetch fail** under `fastly compute serve`, `cargo test-fastly`, and + the parity suite. The whole local loop dies. +2. **`with_cache_bypass` makes the hook silently dead.** `get_caching_mode` checks + `cache_override.is_pass()` **first** (`request.rs:612-615`) and returns host caching, so + `after_send` is never invoked and no error is raised. On exactly the requests in scope, + today, the hook would do nothing quietly. +3. **The closure bounds are incompatible with this codebase.** `with_after_send` requires + `Fn + Send + Sync + 'static` (`request.rs:545-550`). Everything the rewriter needs is + `!Send` by construction — `edgezero_core::body::Body` wraps a `LocalBoxStream` + deliberately, which is why the platform layer is `#[async_trait(?Send)]` throughout. + And `set_body_transform` is synchronous, so it could never await the auction collect. + +Read-through's appeal was real — `CandidateResponse::apply_and_stream_back` is +`execute_and_stream_back` with HTTP semantics attached, and TTL/SWR/vary/surrogate keys +derived from origin headers for free. It is simply unreachable from here. + +**Also settled: core cannot reach it at all.** `PlatformHttpRequest` +(`platform/http.rs:16-37`) is a plain data struct with no callback slot, and carrying one +would name `fastly::http::CandidateResponse` in portable core, breaking the other three +adapters. + +### Follow the existing null-object pattern + +`cache::core` fits the shape the repo already uses four times for a Fastly-only capability +behind a portable trait: `UnavailableHttpClient` (`platform/http.rs:216-243`), +`UnavailableKvStore` (`platform/kv.rs:14-17`), and the `RuntimeServices.kv_store` +field/accessor/builder (`platform/types.rs:170,222,269,330`). Add +`PlatformTemplateCache` the same way, and follow +`crates/trusted-server-adapter-fastly/src/ec_kv.rs` — 140 lines, the repo's only real +edge-storage read/write — rather than inventing a shape. + +**Return `EdgeBody`, not `Vec`.** `EdgeBody::Stream` exists, +`fastly_body_to_edge_stream` (`adapter-fastly/src/platform.rs:503`) already converts, and +`PublisherResponse::Buffered` tolerates a live stream (`publisher.rs:1019-1022`). + +### Exact insertion point + +**Immediately before `let mut platform_request = PlatformHttpRequest::new(...)`** — the +last line before `req` is consumed, and a few lines before the origin send. Everything +needed is in scope there: `settings`, `services`, the final URI and Host, `backend_name`, +`request_path`, `matched_slots`, `should_run_ad_stack`, `request_had_authorization`, +`request_host`, `request_scheme`. + +**One required move:** `assembly_mode` is currently computed _after_ the send, for the +logging call site. It depends only on `settings`, so hoist it above the insertion point. + +**Tee-ing is not needed.** With any post-processor registered — and the Next.js +integration always registers one — `HtmlWithPostProcessing` emits nothing until the final +chunk and then returns the whole transformed document as one contiguous buffer +(`html_processor.rs:92-97,148`). Two `write_all` calls on the same slice; no tee +abstraction, no extra copy. Still use `execute_and_stream_back`, but for transaction +correctness and request collapsing rather than for memory. On a hit the processor is never +built at all. + +- [ ] **Step 4b: close the risks the design investigation surfaced** + +Four, all specific to this codebase rather than to `cache::core` in general. + +**`Vary` is in the key list but nothing consumes it.** `c2_bypass_reason` checks +`Set-Cookie`, `Cache-Control`, `Authorization`, status and content type — **not `Vary`**. +Viceroy supports `WriteOptions.vary_rule`, so the mechanism exists; the gate has to use +it. Until then the key is missing a signal the origin explicitly declares, and Step A's +verdict is a `PROVISIONAL PASS`, not a release gate. + +**Resolved — `VarySpec`, commit `b688d667`.** Building the key exposed a problem this +plan states but does not solve: the key must cover everything the origin varies on, but +**a lookup happens before the fetch**, so on a cold key the origin's `Vary` is not yet +known. Three ways out — configure the list; two-phase lookup against a URL-keyed record +holding the last-seen `Vary`; or store the list alongside and re-key on mismatch. The +latter two are correct and double the lookups on every request. + +Configured is taken, **as a spike-grade choice rather than a production one**: Step A +already measured the origin's actual `Vary`, and a 60s TTL bounds drift to a minute +rather than indefinitely. + +The drift is guarded rather than merely accepted. `VarySpec::uncovered_by` runs _after_ +the origin responds, when its `Vary` is finally known, and names which headers the +configured spec missed. A template built under a key that did not cover something the +origin varies on **must not be stored** — a request differing only in that header would +read it. Naming the specific headers makes a stale config identifiable instead of +producing a generic refusal. + +Two decisions worth their tests. An absent header and a present-but-empty one key the +same, because the origin sees no difference between them. And `Vary: *` is not reported +as a named gap — it means uncacheable, which the eligibility gate handles, and reporting +it would produce a nonsense instruction to configure a header called `*`. + +Still open: wiring `uncovered_by` into `c2_bypass_reason` as a bypass reason, which +happens with the store call site. + +**Store bytes plus a metadata envelope; rebuild every header on a hit.** The publisher +path forces `private, no-store` and strips `ETag`/`Last-Modified`/CDN headers _after_ the +send. Replaying stored origin headers would fight that. Store only the transformed body +and a small `user_metadata` envelope — content encoding, content type, schema version, +tsjs hash — and construct every response header from scratch on a hit. Then no origin +header is ever replayed and the `Set-Cookie` privacy net is trivially safe. +`get_user_metadata` is implemented in Viceroy. + +**Content-Encoding belongs in the key.** The streaming pipeline pairs input encoding to +the same output encoding, so the transformed bytes inherit whatever the origin negotiated +from the client's `Accept-Encoding` — still gzip, deflate, br or identity after +`restrict_accept_encoding` narrows it. Either key on the negotiated encoding or normalize +to identity in the cache and re-encode on read. Getting this wrong serves brotli bytes to +a client that asked for gzip. + +**Host and scheme belong in the key.** The post-processed output is host-dependent by +construction: `request_host` and `request_scheme` reach `IntegrationHtmlContext`. + +- [ ] **Step 4c: file the wasted-dispatch follow-up** + +The auction is dispatched _before_ the insertion point. Under `Esi` and `ClientFill` the +root injects nothing, so that dispatch is already pure waste on this branch — and on a C2 +hit it is waste that must be cleaned up via `emit_abandoned_auction` or it leaks +telemetry. + +Keeping the lookup at the insertion point above is right for the spike: minimal diff, and +lookup latency overlaps the in-flight auction. Moving it earlier would eliminate the +wasted dispatch but serialize the lookup ahead of dispatch. **File it; do not fix it +here.** Suppressing root-level dispatch under the shared modes is Task 4's job, where it +also has to be reconciled with the exactly-one-auction gate. + +- [ ] **Step 5: Unit tests, then the target suite** + +```bash +cargo test -p trusted-server-core --target aarch64-apple-darwin assembly_mode +cargo test-fastly && cargo test-axum && cargo test-cloudflare && cargo test-spin +cargo fmt --all -- --check && cargo clippy-fastly +``` + +`ClientFill` must work on all four adapters. `Esi` is Fastly-only and must not break the +others' compilation. + +- [x] **Step 6: the call site — DONE.** `2db10639` (store), `2a2e6c6a` (lookup). + +The cache now engages end to end: a second request for the same URL is served without +touching the origin, and is byte-identical to what was stored. Verified by mutation — +disabling the lookup fails the hit test, so the hit is the cache answering rather than +the fixture answering twice. + +**Wiring the lookup corrected the key.** It carried the content encoding the _origin_ +chose, which does not exist at lookup time. That meant storing under `br` and looking up +under `gzip, br` — a cache that never hits. The field is now the `Accept-Encoding` sent +to the origin. Sound because negotiation is a function of what the origin was offered, +so identical offers yield identical choices; the chosen encoding stays in the metadata +and is what the served response declares. + +That made every key field request-derived, so **the key is built before the fetch** and +the response gate only authorizes storing it. A key that needed the response could only +ever authorize a store, never satisfy a read. + +**The lookup re-checks the request-derived disqualifications, and only those.** The +store gate is response-derived and cannot re-run, but need not: anything in the cache +passed it on the way in. What must re-run are properties of the _reader_ rather than of +the bytes — an authenticated request must not be served a shared template even when that +template is perfectly cacheable. + +**Shared modes take the buffered finalizer.** Storing needs every transformed byte and +streaming does not collect them. The branch keys on the store authorization rather than +on the assembly mode, so `Inline` never reaches it and the spike cannot regress the +shipped path by construction. A C2 _miss_ therefore buffers — the right trade, since a +miss is already paying an origin fetch and a full transform, and what the spike measures +is the hit, where there is no origin fetch to stream from at all. + +Every response header on a hit is constructed, never replayed, so no origin header can +reach a second visitor through the cache. + +The publisher tests use an in-memory cache double, so they prove the wiring rather than +the backing. The join they leave untested is the one `app.rs` makes: the publisher +reaches the cache as a `dyn PlatformTemplateCache` behind `RuntimeServices`, never as +the concrete type the Fastly tests exercise. That join is now executed under Viceroy +against the real Core Cache rather than only type-checked. + +**What this does not establish.** `ClientFill` and `Esi` still render a template with a +hole and nothing filling it. Task 4 and Task 5 remain the blockers on anything +deployable — a cache that works is necessary, not sufficient. + +--- + +## Task 4: Arm A2 — client-fill + +Mostly already specified. See +[the spec's Appendix B](../specs/2026-08-08-esi-cacheable-root-validation-design.md#appendix-b--stage-1-plumbing-condensed) +for the client plumbing, the two-condition join gate, and the server contract; and +[§5](../specs/2026-08-08-esi-cacheable-root-validation-design.md#5-the-trap-in-the-deferred-work--read-this-before-scheduling-stages-12) +for the silent-empty-bids trap, which applies in full. + +- [ ] **Step 1: Hoist the closure-trapped client state** — `pageBidsEndpoint`, + `requestPageBids`, and the `inflight`/`currentPath`/`lastAppliedPath` state, per + Appendix B. Do **not** route the initial load through `onNavigate`. +- [ ] **Step 2: Make `installScheduleInitialAdInit` a hydration-ready AND bids-settled + join**, with a bounded timeout that fires `adInit` untargeted rather than stranding + the slot. Derive the timeout from measured fetch latency, not a constant. +- [ ] **Step 3: Suppress the navigation-path dispatch** so exactly one auction runs per + pageview. Add a new `AuctionSource` for initial loads **plus the mechanism that + delivers it** — a header behind the same-origin gate, not a query parameter. +- [ ] **Step 4: Relocate terminal telemetry.** Navigation `Completed` is emitted only from + the collect functions; the `ts-debug` dump rides the same string. Both move. +- [ ] **Step 5: Verify exactly one auction per pageview** in `auction_events_raw`. Two is + a doubling of SSP spend and an immediate fail. + +--- + +## Task 5: Arm A3 — ESI at the edge + +- [x] **Step 0: the mechanism works — DONE.** `9539061e`, hardened in `0597f54e`. + +Verified under Viceroy with the real `esi` 0.7 crate rather than argued from docs: a +template carrying the `` seam's own ESI include tag comes back with the fragment +spliced in its place and no unresolved tag left. + +**The async/sync obstacle is dissolved, not worked around.** `esi`'s fragment dispatcher +is synchronous and this codebase's fragment producer is `async`; calling one from the +other means a nested executor, which panics. +`PendingFragmentContent::CompletedRequest` lets the dispatcher hand back an +already-built response, so the caller resolves the fragment in the normal async flow and +the dispatcher performs **no I/O at all** — no subrequest, no backend, no self-call, +nothing for Viceroy to stub. That also removes the need for a self-referencing backend +this plan would otherwise have required. + +**Step 2's instruction was right, and reading the crate showed why.** +`CacheConfig::is_includes_cacheable` defaults to **`true`**. A fragment carries one +visitor's bids, so the default caches per-user data and serves it to the next visitor — +silently, on a hit. `includes_force_ttl` is worse where set: it caches everything, +ignoring `private`, `no-store` and `Set-Cookie` alike. Both now stated explicitly, along +with `default_dca`/`inherit_parent_dca` (fragment bytes are data, never re-parsed as +ESI), `max_include_depth = 1`, and rendered caching / `edge_control` off because the +publisher path owns those headers. + +Nine tests. Four assert the configuration; the rest assert behaviour, including that a +fragment containing its own nested ESI include is spliced as text rather than dispatched, so +auction data cannot drive fragment requests. + +**What remains is the call site**, below. Emitting the include and resolving it are both +proven; connecting them is not done. + +- [ ] **Step 1: Wire `process_stream`, not the wrappers** + +`process_response` and `process_response_streaming` consume `self` _and_ send the response +themselves, which takes ownership away from the finalize / `ec_finalize` / apply-effects +ordering. `process_stream(&mut self, src: impl BufRead, out: &mut impl Write, …)` keeps it. + +Source is the C2 body. Sink is the client response body. + +**The ordering an earlier draft described is impossible.** It said EC cookie, geo, and the +privacy net run _after_ assembly. They cannot: streaming responses on this adapter +**commit headers first and then pipe chunks** +(`adapter-fastly/src/main.rs`, `send_edgezero_response`). Once ESI starts writing, no +header can change. + +The correct invariant: + +> **Finalize every header before a single body byte is written** — EC `Set-Cookie`, geo +> suppression, and an unconditional `Cache-Control: private, no-store` — **then** stream +> the assembly with no further header mutation. + +That means `private, no-store` is set unconditionally up front rather than derived from +what the assembly turns out to contain. Deriving it after the fact is not available, and +assuming it was is how a per-user response ends up shared-cacheable. + +- [ ] **Step 2: Disable DCA explicitly and allowlist the dispatcher** + +```rust +let config = esi::Configuration::default() + .with_escaped(false) + .with_default_dca(esi::DcaMode::None) // call the setter; do not rely on the default + .with_inherit_parent_dca(false); +``` + +Comments are not configuration. An earlier draft said DCA "stays at its default" — on a +pre-1.0 crate whose default could move in a patch release, and where this setting fails +**open**, that is not good enough. Call the setters. + +Also disable **fragment caching** explicitly, or mark the include `no-store="on"`. A +cached auction fragment is a per-user object in a shared cache — the C3 failure mode by +another route. + +The dispatcher must be **exact-path allowlisted**: a fragment URL that is not the bids +endpoint is refused, not fetched. The built-in dispatcher builds a dynamic backend per URL +host and panics on a hostless URL — never use it. + +Rationale in the spec's §2: bid payloads carry partner-controlled creative markup, so a +recursive parse would let an SSP make the edge fetch an arbitrary URL. **Add a unit test +that feeds a partner-controlled ESI include targeting `http://attacker.example/` through +a creative payload and asserts no fetch is attempted.** + +- [ ] **Step 3: The fragment must be a script, not the JSON endpoint** + +**`/_ts/page-bids` cannot be the ESI target.** It returns +`serde_json::json!({"slots":…, "bids":…})` (`publisher.rs:3987`), and ESI splices fragment +bytes in literally — the page would contain raw JSON where an executable script belongs. +Nothing would call `scheduleInitialAdInit`. + +Add a **dedicated fragment endpoint** returning the executable script — the same shape +`build_bids_script` produces today, plus the `adSlots` assignment that moved out of the +template in Task 3 Step 2. Either that, or use the `esi` crate's fragment-response +processor to wrap the JSON; the dedicated endpoint is simpler and easier to assert on. + +Three more things the naïve marker gets wrong: + +- **The same-origin gate will reject it.** `page_bids_request_allowed` + (`publisher.rs:3644`) requires `Sec-Fetch-Site: same-origin` or the `X-TSJS-Page-Bids` + header. An internal ESI subrequest carries neither. Give the fragment endpoint an + internal contract and a fixed backend rather than weakening that gate — it exists to + stop third parties burning SSP quota. +- **Parent context does not propagate.** EC identity, consent state, client IP, geo, User + Agent, and the correlation ID all live on the parent request. Forward an **explicitly + approved allowlist** of them into the fragment request. Forwarding everything is how a + fragment ends up more privileged than the parent. +- **Root dispatch must be suppressed.** The navigation path already dispatches an + auction. If A3 does not suppress it, every pageview runs two — doubling SSP and APS + spend. This applies to **A2 and A3 alike**. + +- [ ] **Step 4: Validate the whole URL, not the path** + +An exact-path allowlist alone permits `https://attacker.example/_ts/page-bids`. Validate +**scheme, authority, method, path, and query** — or better, ignore the marker's URL +entirely and dispatch to a fixed internal backend, treating the ESI include as a signal +rather than an address. + +Add a test that feeds an ESI include targeting +`https://attacker.example/_ts/page-bids` through a creative payload and asserts no +outbound fetch is attempted. + +- [ ] **Step 5: Deterministic synthetic fragment first** + +Before wiring the real auction, point the include at a fixed-content endpoint. This +separates "does the pipeline assemble correctly" from "does the auction behave," and the +two fail very differently. Only once assembly is proven does the fragment become the real +one. + +- [ ] **Step 6: Handle the flush hazard** + +`esi` flushes its output writer after each parse batch. Fastly's `StreamingBody` is a +`BufWriter`, so anything between esi and it must propagate `flush()` or nothing leaves the +Wasm heap. + +- [ ] **Step 7: Fragment failure must degrade, not break** + +Assert that a fragment timeout or non-2xx yields a page with empty bids rather than a 5xx +or a truncated document. Note the crate's non-obvious semantics: `alt` is attempted before +`onerror="continue"`, and `` runs **all** attempts and concatenates every +non-failed output — it is not first-success-wins. + +--- + +## Task 6: Safety gates — run against every arm + +Not a phase. Every one of these is a hard fail, independent of any performance result. + +- [x] **Zero cross-user leakage.** DONE — `76df2469`. Two synthetic users differing in EC + identity, consent jurisdiction and geo store a byte-identical template, each against + a fresh cache so the first cannot answer for the second. Forbidden-substring checks + are the second layer, since byte-identity also holds if both leak the same thing. + Mutation-verified: leaking `adSlots` through the head seam fails it. +- [x] **Cold MISS, warm HIT, stale revalidation** DONE — `76df2469`, and end to end under + `viceroy serve` (below). Stale reads as a miss; serving stale would mean serving a + template built by an older transform or bundle. + + The first stale test passed for the wrong reason and had to be rewritten: a zero TTL + produces an *absent* entry, not a stale one, so `is_stale()` was never reached — + confirmed by reverting the check and watching it stay green. Only a + `stale_while_revalidate` window makes an entry present-and-stale. + +- [x] **Transform failure** DONE — `76df2469`. A partial template in C2 is the worst + outcome available: a truncated document served to every later visitor, indefinitely, + with no error after the first request. Mutation-verified by storing before the cap + check. +- [ ] **Request collapsing** works: concurrent cold requests transform once. +- [x] **DCA disabled** DONE — `0597f54e`. Config asserted _and_ behaviour: a fragment + carrying its own nested ESI include is spliced as text rather than dispatched. + +- [ ] **Request collapsing** — not tested, and not testable here. Viceroy is + single-threaded, so the concurrent cold-request case cannot be produced. The racing + _writer_ path is covered (`a_second_put_on_a_fresh_entry_is_a_no_op`), which is the + correctness half; the collapsing half needs real concurrency. +- [ ] **Exactly one auction per pageview**, from `auction_events_raw`. +- [ ] **Cookie and privacy finalization ran BEFORE assembly**, not after — EC + `Set-Cookie` on first visit, geo suppression, and an unconditional + `Cache-Control: private, no-store`. Headers commit before the body streams on this + adapter, so "finalize after assembly" is not available; asserting it that way is how + a per-user response ends up shared-cacheable. ESI's streaming mode dropping + `$add_header` is a consequence of the same constraint, not a separate hazard. +- [ ] **Slot and bid attribution unchanged.** Same slots matched, same bids applied, same + renders attributed. Use TS-attributed renders — the SSAT line item, non-empty + `ts.bids`, `hb_adid` presence — **never slot fill**, which is blind to empty bids + because `adInit` defines slots regardless. +- [x] **No C3 — assert positively, not by absence.** DONE — `0adb578e`, and this gate's + wording caught a live bug. A C2 hit returns before the point where the publisher path + stamps `private, no-store`, so it served HTML with **no `Cache-Control` at all** — + heuristically cacheable, and therefore a shared cache of an assembled per-user + response. Checking for the _absence_ of `public`/`s-maxage`/`Surrogate-Control` would + have reported it as safe, because there was nothing present to forbid. Covered for + returning visitors specifically, where the cookie-privacy net never fires. + + Original wording, retained because it is what made the difference: Forbidding `public`, `s-maxage`, and + `Surrogate-Control` is **not sufficient**: a bare `Cache-Control: max-age=60` passes + that check and is still shared-cacheable, and that is exactly what the measured + origin sends. Require instead that every assembled response carries + `Cache-Control: private, no-store` and that `Expires`, `ETag`, `Last-Modified`, and + all four CDN cache directives are stripped. Test it for **returning** users + specifically — they set no EC cookie, so the cookie privacy net never fires and is + not a backstop here. + +--- + +## Task 7: The decision record + +**Files:** `docs/superpowers/plans/2026-08-10-1009-esi-decision-record.md` + +- [ ] **Step 1: Record every arm** with N, confidence interval, cache-tier mix, route mix, + and POP. Any arm missing those is not reportable. + +- [ ] **Step 2: Apply the decision rule, stated here before the data exists** + +**Adopt ESI only if all three hold:** + +1. Every Task 6 gate passes on A3. +2. A3 beats A2 on **bids-ready time, `adInit` fire time, and first TS-attributed creative + paint** — by a margin the reviewers ratify **before** collection, not chosen after + seeing the numbers. **Not root TTFB:** A2 and A3 serve the same C2 template, so their + root timings are near-identical by construction and a difference there would be noise. + Root TTFB is a non-regression guard only. +3. Render outcomes on A3 are non-inferior to A0. + +**Otherwise adopt A2 (client-fill)** if its gates pass and it beats A1. It is portable +across all four adapters and carries no Fastly-only maintenance burden. + +**Otherwise keep A1** — Stage 0 alone — and record #1009 as answered in the negative with +evidence. + +The margin in (2) exists because A3's cost is not its diff. It is a second rendering +architecture, Fastly-only, on a pre-1.0 crate, in the critical render path. A small win +does not pay for that. + +- [ ] **Step 3: Record what would change the answer**, so this does not get re-litigated + from scratch. At minimum: React #418 / [#938](https://github.com/IABTechLab/trusted-server/issues/938) + being fixed such that `adInit` can run synchronously, which is what would make edge + assembly's round-trip saving actually worth something. + +- [ ] **Step 4: Clean up.** Remove the spike flag or promote it to a real setting; purge + C2 (`purge_surrogate_key` on `ts-template`); remove the synthetic fragment endpoint; + and either land or delete the `esi` dependency. **A spike flag left in place becomes + permanent configuration surface.** + +--- + +## Reproducibility metadata + +Record with every result, or it cannot be re-run or trusted: commit SHA; `esi` and +`fastly` crate versions; Fastly service and version IDs; whether the backend is shielded; +`template_ttl`; the origin's `Cache-Control` and `Vary` at collection time; assembly mode; +routes; N per arm; and the cache-tier mix. + +## Out of scope + +- **Stages 1–2 of the spec** as production work. This spike may build parts of the + client-fill path to measure it; shipping it is a separate decision behind the + correctness defects. +- **Full RSC/flight partitioning.** `rsc_flight.rs` has no static/dynamic split. +- **Publisher-authored ESI.** Breaks the no-origin-changes promise. +- **A C3 delivery cache.** Not a deferred item — a thing that must not exist. + +## Definition of done + +- [ ] Task 1 verdict recorded: `esi` 0.7 builds on Rust 1.95.0 / `wasm32-wasip1`, or it + does not and the spike stopped. +- [ ] All four arms measured on one build, with correlation IDs joining server and browser + timings, and cache tier recorded per request. +- [ ] Every Task 6 gate has an explicit pass/fail per arm. +- [ ] Decision record exists, applies the pre-ratified rule, and names what would change + the answer. +- [ ] Cleanup complete: flag resolved, C2 purged, synthetic endpoint removed, dependency + landed or dropped. +- [ ] All CI gates pass: `cargo fmt --all -- --check`; the six clippy targets; the four + adapter test suites; the parity suite; JS build, test, and format; docs format. diff --git a/docs/superpowers/plans/2026-08-12-1009-esi-merge-hardening.md b/docs/superpowers/plans/2026-08-12-1009-esi-merge-hardening.md new file mode 100644 index 000000000..a2b388594 --- /dev/null +++ b/docs/superpowers/plans/2026-08-12-1009-esi-merge-hardening.md @@ -0,0 +1,294 @@ +# #1009 ESI Merge and Hardening Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use `superpowers:executing-plans` to implement +> this plan task-by-task. This plan is intentionally executed inline because the operator +> explicitly prohibited subagents. + +**Goal:** Merge current `main` and make the opt-in ESI byte-seam/shared-template path correct, +private, cache-semantic, compressed, observable, and operationally reversible. + +**Architecture:** Fastly Core Cache holds identity-encoded reader-neutral templates behind a +transaction acquired before origin work. Every request assembles its own slots and structured bid +map at an exact inert seam, encodes the result for that client, and receives a final immutable +private/no-store policy. + +**Tech Stack:** Rust 1.95, Fastly Compute/Core Cache, `edgezero_core` HTTP types, `lol_html`, +TypeScript/Vitest, Viceroy, shell harness. + +> **Implementation status, 2026-08-12:** Tasks 1–12 are complete on the branch. The Viceroy +> harness passed in both modes after running outside the filesystem sandbox so it could read the +> macOS native-certificate keychain. + +--- + +### Task 1: Merge live main and preserve auction contracts + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-core/src/integrations/gpt_diagnostics.rs` +- Modify: `crates/trusted-server-js/lib/src/core/types.ts` +- Modify: `crates/trusted-server-js/lib/src/integrations/gpt/index.ts` +- Test: adjacent Rust and Vitest modules + +- [x] Merge `origin/main` with `git merge --no-ff origin/main`. +- [x] Resolve the `AdBidsState`/`write_bids_to_state` conflict by building one structured map with + `auction_id`, storing both map and script, and returning its delivered slot IDs. +- [x] Add/adjust tests proving ESI and inline retain `hb_auction_id`, APS renderer metadata, and + delivered-winner attribution. +- [x] Run the focused Rust and GPT tests. +- [x] Complete the merge commit. + +### Task 2: Remove mechanisms outside the approved ESI byte-seam design + +**Files:** + +- Modify: `crates/trusted-server-core/src/creative_opportunities.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Delete: `crates/trusted-server-core/src/platform/template_assembly.rs` +- Modify: `crates/trusted-server-core/src/platform/mod.rs` +- Modify: `crates/trusted-server-core/src/platform/types.rs` +- Delete: `crates/trusted-server-adapter-fastly/src/esi_assembly.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/app.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` +- Modify: `crates/trusted-server-adapter-fastly/Cargo.toml` +- Modify: `Cargo.lock` + +- [x] Update mode tests to specify only `inline` and `esi`; watch the old client-fill expectations + fail or stop compiling. +- [x] Remove `ClientFill`, executable fragment serialization, assembler traits/registration, and + the `esi` crate. +- [x] Update comments to call the production path byte-seam assembly. +- [x] Run focused configuration, publisher, and Fastly adapter tests. +- [x] Commit the scope cleanup. + +### Task 3: Canonicalize and bound the template key + +**Files:** + +- Modify: `crates/trusted-server-core/src/platform/template_cache.rs` +- Modify: `crates/trusted-server-core/src/creative_opportunities.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` + +- [x] Write failing tests for absent versus empty `Vary`, repeated raw values, invalid configured + names, punctuation-colliding purge URLs, changed origin host override, and changed creative + configuration. +- [x] Replace string pairs with a typed canonical `Vary` value preserving presence and all bytes. +- [x] Hash a length-prefixed canonical key and hash the URL-specific surrogate key. +- [x] Include publisher origin identity and the complete template-shaping fingerprint. +- [x] Run focused key/configuration tests and commit. + +### Task 4: Enforce request and origin cache semantics + +**Files:** + +- Modify: `crates/trusted-server-core/src/platform/template_cache.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-core/Cargo.toml` if HTTP-date parsing needs a direct dependency + +- [x] Write failing tests for response `max-age=0`, positive max age, repeated/malformed cache + directives, `Age` exhaustion, expired/malformed `Expires`, missing freshness, invalid `Vary`, + and request no-cache/no-store/range/conditional bypasses. +- [x] Add a typed cache eligibility result carrying the positive remaining TTL. +- [x] Parse relevant response directives fail-closed and cap, never extend, origin freshness. +- [x] Add request-side bypass classification before lookup. +- [x] Make unsupported/backend-failed cache lookups fall back to inline processing on non-Fastly + adapters rather than buffering a cacheless ESI path. +- [x] Run focused eligibility tests and commit. + +### Task 5: Move request collapse before the origin fetch + +**Files:** + +- Modify: `crates/trusted-server-core/src/platform/template_cache.rs` +- Modify: `crates/trusted-server-core/src/platform/types.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/template_cache.rs` + +- [x] Verify the reservation is acquired before origin work and the Fastly transaction contract + blocks same-key waiters. Viceroy is single-threaded, so it cannot directly reproduce two + truly concurrent cold requests. +- [x] Introduce a lookup outcome with an opaque insert reservation and explicit cancellation. +- [x] Implement Fastly `Transaction::lookup` before origin work and consume/cancel its obligation + on every exit path. +- [x] Ensure invalid fresh entries become replaceable rather than causing repeated refetches. +- [x] Run focused Core Cache/Viceroy tests and commit. + +### Task 6: Make privacy and policy-header parity final + +**Files:** + +- Modify: `crates/trusted-server-core/src/platform/template_cache.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-core/src/response_privacy.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` +- Modify: `crates/trusted-server-core/src/integrations/registry.rs` tests + +- [x] Write failing tests for repeated CSP/CSP-Report-Only, omitted COOP/COEP/CORP/HSTS/Link, + unknown cached header metadata, duplicate required metadata fields, and a late integration + changing `Cache-Control` to public. +- [x] Capture all ordered values, expand the safe allowlist, and decode metadata strictly. +- [x] Replay with `append`, then apply the assembled-response privacy policy last. +- [x] Preserve and reassert private/no-store after request-filter effects in Fastly's final send. +- [x] Run focused header/privacy tests and commit. + +### Task 7: Bypass shared templates for request-private diagnostics + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` + +- [x] Write a failing warm-cache test activated by diagnostics query and another by diagnostics + cookie. +- [x] Make `requires_private_no_store()` a lookup/store disqualifier. +- [x] Verify ordinary diagnostics-disabled requests still hit C2. +- [x] Run focused diagnostics/C2 tests and commit. + +### Task 8: Re-encode assembled responses for the reader + +**Files:** + +- Modify: `crates/trusted-server-core/src/platform/template_cache.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` + +- [x] Write failing cold/warm tests requiring gzip/br clients to receive a matching encoded body + and proving reader encoding no longer partitions the stored template. +- [x] Keep the origin offer within the reader's supported codings so a response-gate bypass remains + lossless, while decoding every stored template to identity. +- [x] Carry the selected response encoding separately from identity template metadata. +- [x] Encode buffered assembly after splicing and stream hit prefix/seam/suffix through one encoder. +- [x] Handle `identity;q=0` without serving an unacceptable representation. +- [x] Emit the correct `Vary: Accept-Encoding` response semantics after final encoding. +- [x] Run focused compression tests and commit. + +### Task 9: Make marker failures safe + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-core/src/platform/template_cache.rs` + +- [x] Write failing tests for HTML with no explicit ``, a publisher-authored marker + collision, and a corrupt cached marker. +- [x] Record/validate a schema-bound seam location or use a collision-resistant marker contract. +- [x] Cancel storage and fall back safely when the optimization cannot produce one seam. +- [x] Run focused miss/hit assembly tests and commit. + +### Task 10: Add operational observability and harden the harness + +**Files:** + +- Modify: `crates/trusted-server-core/src/platform/template_cache.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/template_cache.rs` +- Modify: `scripts/c2-local-test.sh` +- Modify: `.github/workflows/test.yml` + +- [x] Write failing tests for distinct backend-error versus not-found status and C2 response-state + reporting. +- [x] Preserve backend errors and emit bounded C2 status without exposing key material. +- [x] Change the harness to operate on a temporary manifest and fail on missing/non-numeric probe + output or empty response bodies. +- [x] Test both cold and warm integrity and execute the generated scheduler payload contract. +- [x] Add the ESI harness to CI where Viceroy prerequisites are available. +- [x] Run shell syntax/static checks and commit. + +### Task 11: Document configuration, semantics, and rollback + +**Files:** + +- Modify: `trusted-server.example.toml` +- Modify: `docs/guide/configuration.md` +- Modify: `docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md` +- Modify: `docs/superpowers/specs/2026-08-11-1009-streaming-assembly-architecture.md` +- Modify: relevant #1009 findings documents + +- [x] Document that `esi` means Fastly C2 plus byte-seam assembly, not parser execution or final + HTTP shared caching. +- [x] Document `template_cache_vary`, cookie independence, freshness, metrics, purge, rollback + ordering, and limitations on non-Fastly adapters. +- [x] Close or supersede stale spike checkboxes and remove claims contradicted by the final code. +- [x] Run docs format/build and commit. + +### Task 12: Full verification + +**Files:** none expected beyond fixes discovered by verification + +- [x] Run `cargo fmt --all -- --check`. +- [x] Run all four adapter test aliases and the parity suite. +- [x] Run all six clippy aliases. +- [x] Build the Fastly release WASM. +- [x] Run JS tests, build, and format under pinned Node 24.12.0. +- [x] Run docs format/build. +- [x] Run `scripts/c2-local-test.sh esi` and `inline` if the environment exposes the required + local certificate store; otherwise report the exact environment blocker. +- [x] Run `git diff --check`, inspect the merge graph, and confirm the worktree contains only + intended changes. + +### Task 13: Interpret Fastly Surrogate-Control conservatively + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `docs/guide/configuration.md` +- Modify: `docs/superpowers/specs/2026-08-12-1009-esi-merge-hardening-design.md` + +- [x] Write a failing gate test using `Cache-Control: max-age=60` plus the observed publisher + `Surrogate-Control` policy (`max-age=1200`, `stale-while-revalidate=21600`, and + `stale-if-error=604800`). +- [x] Write failing tests proving the shorter standard/surrogate freshness wins, stale windows do + not extend fresh reuse, restrictive directives are refused, and unknown, duplicate, or + malformed directives fail closed. +- [x] Parse only Fastly's supported `max-age`, `stale-while-revalidate`, and `stale-if-error` + directives; continue refusing every other vendor CDN policy field. +- [x] Keep request `Cache-Control: max-age=0` as an intentional C2 bypass so reload preserves its + revalidation semantics. +- [x] Run focused tests, `cargo test-fastly`, target-matched formatting/clippy, both local harness + modes, and verify the observed publisher policy progresses from `miss-stored` to `hit` in + the local Fastly runtime on an ordinary navigation. + +### Task 14: Allow browser reloads to reuse a fresh ESI template + +Task 14 supersedes Task 13's conservative request `max-age=0` bypass after end-to-end testing +proved that C2 reuses only the neutral template and still creates a new private response and +auction. + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `docs/superpowers/specs/2026-08-12-1009-esi-merge-hardening-design.md` + +- [x] Write a failing end-to-end test proving `Cache-Control: max-age=0` reruns the auction but + does not refetch the reader-neutral publisher template. +- [x] Treat only a valid zero request max age as compatible with C2; continue bypassing positive + or malformed constraints and every explicit revalidation directive. +- [x] Verify the focused tests, formatting, and Fastly clippy, then commit independently. + +### Task 15: Make the ESI template-cache ceiling configurable + +Task 15 supersedes Task 13's shorter-of-standard-and-surrogate rule. The final behavior follows +Fastly edge precedence while retaining restrictive directives as hard refusals. + +**Files:** + +- Modify: `crates/trusted-server-core/src/creative_opportunities.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/template_cache.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/app.rs` +- Modify: `trusted-server.example.toml` +- Modify: `docs/guide/configuration.md` +- Modify: `docs/superpowers/specs/2026-08-12-1009-esi-merge-hardening-design.md` + +- [x] Add failing configuration tests for the 60-second default, an explicit 1,200-second ceiling, + zero, values above one day, and omission from serialized rollback-compatible config. +- [x] Add failing freshness tests proving Fastly precedence, age deduction, and the configured + ceiling for the observed `Cache-Control: max-age=60` plus + `Surrogate-Control: max-age=1200` response. +- [x] Implement `template_cache_max_age_seconds` under `[creative_opportunities]` and thread its + resolved duration into C2 eligibility. +- [x] Remove the Fastly adapter's second hard-coded 60-second cap; the already-authorized + per-entry max age becomes the sole insertion lifetime. +- [x] Update the example and operator guide, without editing the tracked deployment + `fastly.toml`. +- [x] Run focused red/green tests, full adapter tests and clippy gates, documentation checks, and + inspect the final diff with `fastly.toml` excluded. diff --git a/docs/superpowers/plans/2026-08-14-1009-esi-parser-assembly.md b/docs/superpowers/plans/2026-08-14-1009-esi-parser-assembly.md new file mode 100644 index 000000000..a4c04b9f8 --- /dev/null +++ b/docs/superpowers/plans/2026-08-14-1009-esi-parser-assembly.md @@ -0,0 +1,104 @@ +# #1009 ESI Parser Assembly Implementation Plan + +> **Execution note:** Implemented inline in the current checkout, without a worktree or +> subagents, as requested. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Use the repaired ESI parser on authorized cold C2 misses without changing the existing warm-hit streaming behavior. + +**Architecture:** C2 retains the inert schema-v4 seam. Core delegates cold assembly through a platform trait; Fastly converts the seam to one synthetic ESI include and resolves it from the already-collected per-reader script. Parser failure falls back to core's validated byte split, while warm hits continue to stream by byte seam. + +**Tech Stack:** Rust 1.95, `wasm32-wasip1`, Fastly Compute/Viceroy, `stackpop/esi` pinned by Git revision, `error-stack`. + +--- + +### Task 1: Restore a platform assembly boundary + +**Files:** + +- Create: `crates/trusted-server-core/src/platform/template_assembly.rs` +- Modify: `crates/trusted-server-core/src/platform/mod.rs` +- Modify: `crates/trusted-server-core/src/platform/types.rs` +- Test: `crates/trusted-server-core/src/platform/template_assembly.rs` + +- [x] Add a failing object-safety/default-behavior test for `PlatformTemplateAssembler`. +- [x] Run the focused core test and confirm it fails because the boundary is absent. +- [x] Add the trait, error type, unavailable default, runtime service field, builder method, + accessor, and test support. +- [x] Run the focused tests and confirm they pass. + +### Task 2: Delegate only cold-miss assembly + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Test: `crates/trusted-server-core/src/publisher.rs` + +- [x] Add a recording assembler to the C2 end-to-end tests. +- [x] Add a test asserting one platform call on a cold miss and no additional call on the + subsequent warm hit. +- [x] Add a test asserting platform failure returns a complete byte-seam response. +- [x] Add tests for `x-ts-assembly` values on parser, fallback, and warm paths. +- [x] Run each test first and confirm the expected failure. +- [x] Change `assemble_if_shared` to call the platform assembler after storage, fall back + to the validated byte split on error, and return the assembly method. +- [x] Set `x-ts-assembly` without changing `x-ts-c2-cache` or privacy headers. +- [x] Re-run the C2 end-to-end test module. + +### Task 3: Add the repaired Fastly ESI adapter + +**Files:** + +- Modify: `crates/trusted-server-adapter-fastly/Cargo.toml` +- Modify: `Cargo.lock` +- Create: `crates/trusted-server-adapter-fastly/src/esi_assembly.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/app.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` +- Test: `crates/trusted-server-adapter-fastly/src/esi_assembly.rs` + +- [x] Add failing adapter tests for a large Next.js script followed by the seam, an + unexpected publisher ESI directive, an unexpected dispatcher URL, and verbatim + fragment content. +- [x] Run the focused Fastly test filter and confirm the missing module/implementation + fails. +- [x] Pin `https://github.com/stackpop/esi.git` at + `4c53feab4d22ad9a84641b4c46f3f63bc6d197e2`. +- [x] Implement the explicit no-cache/no-DCA ESI configuration and synthetic completed + fragment dispatcher. +- [x] Register `FastlyTemplateAssembler` in per-request runtime services. +- [x] Run the focused Fastly tests and confirm they pass. + +### Task 4: Preserve cache schema and documentation truth + +**Files:** + +- Modify: `docs/superpowers/specs/2026-08-11-1009-streaming-assembly-architecture.md` +- Modify: `docs/guide/configuration.md` +- Modify: `scripts/c2-local-test.sh` +- Test: `crates/trusted-server-core/src/platform/template_cache.rs` + +- [x] Add/adjust tests proving schema version 4 and the inert stored marker remain + unchanged. +- [x] Extend the local harness to require `esi-parser` on the miss and `byte-seam` on the + hit. +- [x] Update architecture and operator documentation to describe the hybrid path and + pinned fork accurately. +- [x] Run formatting and the harness's static checks. + +### Task 5: Full verification and signed commit + +**Files:** + +- Review every modified file. + +- [x] Run `cargo fmt --all -- --check`. +- [x] Run every target-matched Clippy alias from `CLAUDE.md`. +- [x] Run `cargo test-fastly`, `cargo test-axum`, `cargo test-cloudflare`, and + `cargo test-spin`. +- [x] Run the integration parity test. +- [x] Run JS tests/build/format and docs format. +- [x] Run the C2 local harness when Viceroy and its certificate environment are + available; otherwise report that environmental gap explicitly. +- [x] Run `git diff --check`, inspect staged scope, and confirm no operator configuration + or secrets are staged. +- [x] Create one SSH-signed commit only after every required gate is green. +- [x] Verify the commit signature locally and report the exact commit ID and test counts. diff --git a/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md b/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md new file mode 100644 index 000000000..031ecc50c --- /dev/null +++ b/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md @@ -0,0 +1,864 @@ +# The Cacheable Root: Latency Diagnosis and Stage 0 Design + +_Filename retains its original `esi-` prefix; the commit history and every +cross-reference point at it. The subject moved, the path did not._ + +**Issue:** IABTechLab/trusted-server#1009 · **Date:** 2026-08-08 · + +> **HISTORICAL RECORD — NOT THE CURRENT IMPLEMENTATION.** This document preserves the +> measurement and feasibility investigation. Every executable ESI tag, parser, and +> subrequest described below belongs to a rejected spike; do not use those sections to +> infer current runtime behavior. The final branch retains `assembly_mode = "esi"` only as +> the operator spelling for Fastly C2 plus exact byte-seam assembly. See +> [the merge-hardening design](./2026-08-12-1009-esi-merge-hardening-design.md). + +**Revised:** 2026-08-10 +**Baseline:** citations verified at `cfb98f4`; unchanged as of `b0ce56c3`. + +> ## ⚠️ Correction, 2026-08-10 — this document's original ESI verdict was wrong +> +> The first revision concluded that ESI was **structurally blocked**: that it +> presupposed a TS-owned template cache which did not exist, and that such a cache was +> in turn blocked on purge capability the platform did not offer. **Both claims are +> false**, and an external review was right to reject them. +> +> Verified against the pinned `fastly` 0.12.1: +> +> - **The cache boundary is native.** `fastly::cache::core` provides +> `insert(key, max_age).execute() -> StreamingBody` for arbitrary bytes, `lookup()` / +> `found()` to read them back, and `Transaction` with `must_insert()` for request +> collapsing. The two-stage design needs no separate KV or template service. +> - **Purge exists in-process.** `fastly::http::purge::purge_surrogate_key` purges from +> inside Compute; the management-API token scope cited in the original is irrelevant to +> it. Note which cache, though: `InsertBuilder::surrogate_keys([...])` is the **Core +> Cache** API and keys the transformed-template cache (C2). It does **not** key the HTTP +> read-through cache (C1) that Stage 0 turns on — purging that needs origin-supplied +> keys or the HTTP cache's own surrogate-key surface. +> - **The original pipeline ordering was backwards.** It said "order esi → lol*html, +> never the reverse." `lol_html` \_emits* the ESI include tags, so ESI must run after it. +> Correct order is in [§6.6](#66-the-esi-pipeline-corrected). +> +> The error was inspecting what this repository does and reporting it as what the +> platform permits — the same mistake this document criticises #1009 for making in the +> other direction. +> +> **ESI is therefore feasible and unvalidated, not rejected.** Validating it is +> [a separate plan](../plans/2026-08-10-1009-esi-validation-spike.md). What survives +> here is the latency re-diagnosis and the Stage 0 optimisation, which are worth doing +> and are **not** an answer to #1009. + +## Document map — read this first + +#1009 is answered across three documents, not one. This is the only place that says +which owns what. + +| Document | Owns | +| ------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------- | +| **This spec** | Why the TTFB regression happens, what Stage 0 is and why, and the corrected ESI feasibility verdict | +| [Stage 0 plan](../plans/2026-08-08-1009-measurement-and-stage-0.md) | Implementing the measurement and the cache-bypass flag. **Does not close #1009.** | +| [ESI validation spike](../plans/2026-08-10-1009-esi-validation-spike.md) | **Where #1009 is actually decided.** Four arms, safety gates, decision rule. | +| [Findings](../plans/2026-08-08-1009-measurement-findings.md) | Recorded results. Currently: Step A only, at `PROVISIONAL PASS`. | + +**If you want the ESI answer**, it is [§2](#2-why--the-three-findings) for the verdict, +[§6.6](#66-the-esi-pipeline-corrected) for the pipeline, and the spike plan for how it +gets validated. Everything else here is Stage 0 and the latency analysis behind it. + +**Decision requested:** approve the four items in §1. + +> **Orientation.** #1009 asks whether Edge Side Includes (ESI) can cache page fragments +> so that cacheable publisher HTML is separated from per-user ad state, recovering a +> TTFB regression that Trusted Server (TS) adds to navigations on a Next.js App Router +> publisher running on Fastly Compute. ESI can do this; whether it should is not settled +> here. Separately, the regression has a cheaper cause than the issue assumes. +> +> **This document deliberately carries no performance measurements.** Every conclusion +> below is derived from code at the pinned baseline, so it can be checked by reading the +> repository rather than by trusting a benchmark. Where a quantity is needed and unknown, +> it is named as unknown and [§3](#3-monday-morning) says how to obtain it. +> +> Terms used throughout: **the hold** = TS holding the HTTP response open at `` +> until the server-side auction (SSAT) resolves. **React #418** = the React +> hydration-mismatch error raised when `adInit()` mutates ad-slot subtrees during +> hydration; it is why bid application is deferred to `window.load`. It is a React error +> number, **not** a repository issue — the tracker is +> [#938](https://github.com/IABTechLab/trusted-server/issues/938). **The SSAT price +> defect** = a live mispricing bug named in #1009 (prices reading 100× high) — cited +> from #1009 and prior investigation, not re-verified here. + +--- + +## 1. Decision requested + +| # | Decision | Owner needed | +| --- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------- | +| D1 | **ESI is feasible and unvalidated.** Validate it via [the spike plan](../plans/2026-08-10-1009-esi-validation-spike.md), not by deferring it. | Eng + product | +| D2 | **Fund ~3 days of measurement** (§3). No dependencies. Can start immediately. | Eng | +| D3 | **Approve Stage 0** — an operator flag disabling the origin cache bypass, subject to a **`FINAL PASS`** in §3. A `Vary` check alone is a `PROVISIONAL PASS` and is not a release gate. Rollback needs a purge path, not only a config push. | Eng | +| D4 | **Stages 1–2 queue behind the SSAT price defect and #938.** Stages 3b–4 unscheduled. ESI is not in this queue — see §7. | Product | + +Rationale for D4 in [§8](#8-priority). Everything this document recommends _against_ +doing is in [§7](#7-deferred-work-specified-not-scheduled), at deliberately lower +detail than the work it recommends. + +--- + +## 2. Why — the three findings + +**ESI is buildable on the pinned SDK, and unvalidated.** `lol_html` emits executable ESI +include tags into a shared template; `fastly::cache::core` stores that template; the +`esi` crate assembles per request on the way out. Everything that requires is +already a dependency. The real open questions are empirical, not architectural: does it +beat a plain client fetch by enough to justify a Fastly-only rendering path, and can +per-user leakage be excluded under cold MISS, warm HIT, stale revalidation, and fragment +failure. [The spike plan](../plans/2026-08-10-1009-esi-validation-spike.md) answers +those; [§6.6](#66-the-esi-pipeline-corrected) gives the pipeline. + +Two constraints stay true regardless. ESI is **Fastly-only at every API level**, so it +is a per-platform accelerator rather than the architecture, and its maintenance cost +belongs in the decision. And its Dynamic Content Assembly must be **explicitly disabled** +— bid payloads carry partner-controlled creative markup, so under `DcaMode::Esi` an SSP +could embed an ESI include targeting an arbitrary URL and make the edge fetch it. Details in +[Appendix E](#appendix-e--esi-notes-condensed). + +**The auction is already out of band; the hold is ~free.** It is dispatched _before_ +the origin fetch and does not block — dispatched at `publisher.rs:2751-2755`, sent at `:2870` — +with a 500 ms budget. The actual cost is `with_cache_bypass` +(`publisher.rs:2867`), +which forces every ad-eligible navigation to miss the Fastly readthrough cache. + +**The two fixes are multiplicative.** Removing the bypass alone lets the previously +hidden auction surface as the new bottleneck. Removing the hold alone changes nothing, +because the auction was never the bottleneck. **Shipping the hold removal without the +bypass removal will measure no improvement and will read as the effort having failed** — +the most likely way this work gets judged unfairly. + +**Ordering is established; magnitude is not.** The ordering above follows from code and +needs no measurement. The _size_ of the win does — and the one quantity it depends on, +the origin build time under `Pass`, has never been measured. #1009's timings do not +supply it: they compare cached fetches against each other, not against an origin build. +**Quote no figure to a publisher until §3 Step C runs.** Full reasoning in +[§6](#6-the-analysis). + +--- + +## 3. Monday morning + +Three checks, ordered cheapest-first. Each needs a named owner before starting. + +**Step A — origin `Vary` and cookie check (minutes for the first pass).** `curl` the +origin with and without `RSC`, `Next-Router-*`, and the experiment header; inspect `Vary`, +`Cache-Control`, and `Set-Cookie`. **This first pass yields a `PROVISIONAL PASS` only** — +it is not what gates the flip. A `FINAL PASS` additionally requires a real authenticated +session, Basic Auth through TS, the experiment variant, representative routes, and +cached-hit render attribution. Do the cheap pass first because it is the +cheapest thing that unblocks anything. + +**Step B — what consumes TS's own response headers (under a day).** Request a TS-served +path that already emits `public, s-maxage` +(`http_util.rs:294-311`) +twice and look for `x-cache`/`age` on TS's _own_ response. **Gates the Stage 3a/3b +split** — see [§7](#7-deferred-work-specified-not-scheduled). + +**Step C — measure the hold directly (1 day + a measurement window).** + +The hold's cost is literally the duration of one `.await`: `collect_stream_auction` at +`publisher.rs:793`, plus the +two EOF variants in `hold_finish_ready_segments` and `hold_finish_tail_segments`. Two +`Instant`s around it yield **`hold_wait_ms`** — the number this entire document is +arguing about, measured rather than modelled. + +Emit two timings per ad-eligible navigation: + +| Metric | Why | +| ----------------- | ---------------------------------------------------------------------- | +| `hold_wait_ms` | **The decision.** How long the response was actually held for bids. | +| `origin_fetch_ms` | Attribution — how much of the win Stage 0 can claim. Origin TTFB only. | + +`hold_wait_ms` replaces the proxy comparison an earlier draft proposed. Comparing `O` +against `A` was an indirect way of asking "does the hold block?"; this asks it directly, +costs less to build, and removes the modelling error corrected in +[§6.2](#62-what-the-hold-actually-costs). + +Deliberately not measured: auction collect duration is already instrumented +(`OrchestrationResult::total_time_ms`, `auction/orchestrator.rs:285`, flowing to +`auction_events_raw`) — read it, don't rebuild it. Rewrite duration decides nothing and +would mean touching two finalizers. + +- **Mechanism: a `log::info!` line behind a debug flag, not `Server-Timing`.** A response + header would in fact work for the origin-fetch figure — that value is known before + headers commit — but a server-side log needs no browser harness to collect it, `log` is + this project's instrumentation crate, and the auction path already measures itself with + `web_time::Instant`. Gate it behind config: one line per eligible navigation is real log + spend and the instrumentation is temporary. +- **Sample: enough navigations per arm to separate the medians with confidence**, across + both page types, and state the N alongside any result. #1009's sample was small enough + that its conclusion did not survive contact with the code; replacing it with another + underpowered sample would repeat the error. + +**Step C has two outcomes, both actionable:** + +| `hold_wait_ms` median | Meaning | Effect on staging | +| --------------------- | ----------------------- | ------------------------------------------------------------ | +| Near zero | The hold is free | Proceed as staged: Stage 0 primary, Stage 2 protects its win | +| Materially non-zero | The hold **is** costing | **Staging inverts** — Stage 2 primary, Stage 0 secondary | + +The work does not change; its order and justification do. **The staging in §7 is +conditional on this measurement**, and the second outcome is a live possibility rather +than a formality — §6.2's argument for the first is weaker than an earlier draft claimed. + +Stage 1's bids-fetch timeout still needs a measured client-side figure rather than an +invented constant, but Step C is server-side and does not supply it. Capture it from the +browser harness when Stage 1 is actually scheduled. + +--- + +## 4. Stage 0 — the only build item recommended now + +Stop bypassing the read-through cache on ad-eligible navigations +(`publisher.rs:2867`). + +**Ship it as an operator flag, not a deletion.** Add +`publisher.bypass_origin_cache`, defaulting to today's behaviour, in the same release as +the Step C instrumentation. Then turn it off with `ts config push`. + +The diff is slightly larger than deleting a line, and that is the point. The risk being +gated here is **cache poisoning** — serving one representation in response to a request +for another. For that class of failure, rollback speed dominates diff size: a config push +reverts the read path in seconds where a release does not — but a config push **evicts +nothing**, so full rollback is flip, then purge or roll a versioned key namespace, then +observe past the origin TTL. The flag also buys an A/B on a byte-identical +build, removing build difference as a confound in the very measurement this depends on, +and allows flipping for a tester-cookie population before all traffic. + +Retire the flag once the change has held: flip the default, then delete the setting and +its branch. A temporary flag left in place becomes permanent configuration surface. + +### What to watch after the flip + +Two regression signals, both checked before the win is: + +- **`unexpected_origin_304` abandonment rate.** That reason + (`publisher.rs:2894-2916`, + emitted via `emit_abandoned_auction` at `:2360`) exists precisely because the ad-stack + path refuses cached and conditional origin responses. Re-enabling the cache is what + could revive it. **Any non-zero rate is a rollback signal** — it means a 304 is reaching + TS that the conditional-header strip was supposed to make impossible. +- **Representation mixing.** Spot-check that HTML navigations still return HTML and RSC + fetches still return `text/x-component`. A mismatch means the `Vary` risk materialized + despite a PASS verdict. Roll back immediately; this is cache poisoning, not a + performance regression. + +**Why it is safe in principle.** The conditional-header strip runs 34 lines earlier +under the same gate (`publisher.rs:2832-2836`, +which also strips `Range`/`If-Range`), so the request already reaches the cache +unconditional and a HIT returns a full body. [The 304-prevention design](./2026-07-22-ssat-root-document-304-prevention-design.md) +added the bypass as belt-and-braces and listed the TTFB cost under its own Risks. The +strip alone satisfies its invariant. + +**But it carries a risk that design never considered — and this is the blocking +precondition.** RSC fetches are not navigations +(`is_navigation_request` +requires `Sec-Fetch-Dest: document`), so they never set the bypass and **already flow +through the readthrough cache**, while HTML navigations are `PASS`. Removing the bypass +puts both representations under one cache key. #1009 states the origin varies on +`rsc`, `next-router-*`, and a publisher-specific experiment header — if that variance is +not declared via `Vary`, the +cache can serve a flight payload to an HTML navigation. + +The classification is also not airtight: `is_navigation_request` falls back to the +`Accept` header when Fetch Metadata is absent, and its own comment warns _"this path is +weaker — `fetch()` can set Accept: text/html"_ +(`http_util.rs:84-88`). + +**A FAIL is not merely a Stage 0 blocker — it is a live production defect.** RSC fetches +already transit the read-through cache today, because they never set the bypass. If the +origin varies on `Next-Router-*` without declaring it, TS is cross-serving RSC variants +right now. On a FAIL, file that immediately and treat "ask the origin to declare `Vary`" +as urgent rather than as the cheaper of two options. + +**The `Vary` check is necessary but not sufficient.** Turning the read-through cache on +for HTML navigations exposes three things a representation check does not cover, and all +three are a larger class than the RSC split: + +- **Client `Cookie`.** TS forwards client cookies to origin unchanged — there is no + `COOKIE` strip on the publisher path. Any cookie-personalized HTML (logged-in state, + paywall meter, publisher-side A/B assignment) becomes cross-servable unless the origin + declares `Vary: Cookie` or marks those responses private. +- **Origin `Set-Cookie`.** If the origin emits `Set-Cookie` alongside a shared-cacheable + `Cache-Control`, the read-through cache can replay one visitor's cookie to the next. + TS's own privacy net downgrades **TS's** response — it runs after the cache has already + stored the origin's. +- **`Authorization`.** #1009 describes a basic-auth-gated deployment. Responses to + authorized requests entering a shared cache needs its own check. + +So Step A must capture `Cache-Control` and `Set-Cookie` too, and repeat each request with +and without a session cookie. Same minutes of work; closes the bigger hole. + +**Two effort branches, and Step A's `Vary` result decides which** — note this selects the +_shape_ of Stage 0, while the `FINAL PASS` conditions decide _whether it ships at all_: + +| Step A result | Stage 0 is… | Effort | +| ---------------------- | --------------------------------------------- | ------ | +| Origin declares `Vary` | the flag, its tests, then a config push | 1–2 d | +| Origin does **not** | a TS-side cache-key discriminator — a feature | 4–8 d | + +The discriminator is the safer design either way, because it keys on the headers that +actually distinguish the representations rather than on the navigation classification. + +**Two benefits beyond TTFB, worth stating to a publisher:** + +- **Origin load drops.** The 304-prevention design explicitly accepted _"increasing + origin load"_ as a cost. This reverses it. +- **`stale-if-error` becomes reachable.** Under `Pass` an origin outage is a hard + failure. This needs a decision rather than a default: stale HTML carries stale slot + markup, and whether that beats an error is a product call. + +--- + +## 5. The trap in the deferred work — read this before scheduling Stages 1–2 + +The hold is load-bearing for something other than latency. The invariant is: + +> `ad_bids_state` must be `Some(..)` when `lol_html` processes the `` end tag. + +The end-tag handler (`html_processor.rs:381-395`) +locks that mutex once and falls back to `build_empty_bids_script()` on `None`. + +**Removing the hold without relocating collection renders a normal page with +`tsjs.bids = {}` and no server-side ads** — no error, no non-2xx, no ERROR log. On +Axum, Cloudflare, and Spin the loss is fully silent: +`publisher.rs:2248` holds a +bare `Option` with no guard, so not even a drop warning fires. **The +SSPs are billed regardless.** + +This is why Stage 2 is gated on three companions and a production soak, and why slot +fill cannot be the canary — see [§7](#7-deferred-work-specified-not-scheduled). + +--- + +## 6. The analysis + +### 6.1 Corrections to #1009's premises + +| # | #1009 states | Verified against `cfb98f4` | +| --- | -------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | Two per-user injection seams | Partly. `tsjs.adSlots` **content** is per-URL — `build_slot_json` emits config- and path-derived fields only. But its **presence** is gated on `should_run_ad_stack` (consent, bot, prefetch, kill switch), so it is request-dependent and **must not live in a shared template**. See §6.7. | +| 2 | Identity off-inline is a prerequisite | Privacy net is cookie-gated and returning navs set no cookie (`ec/finalize.rs:86-94`). **First-visit only** — but note the corollary: because returning navigations set no cookie, that net never fires for them and is **not** a backstop against shared-caching a per-user response. | +| 3 | Stamp at `:2882-2888` | `:2945-2963`, `private, no-store`, also removing `ETag`/`Last-Modified`/four CDN headers. **Seven headers.** | +| 4 | Three cacheability killers | Two more: `bypass_cache` and the `304→502 guard`. **The bypass is the cost.** | +| 5 | Two `!Send` pipelines is the hard part | `?Send` already pervasive. **Not the obstacle.** | +| 6 | Goal: root as a shared Fastly HIT | Nothing caches TS's own response on Compute; the A/B's `x-cache` is the **backend readthrough** cache. **Reframes the goal.** | + +Rows on #1009's `esi` compatibility check (holds), its drifted line numbers, and its +two broken `#1`/`#3` cross-references are in [Appendix A](#appendix-a--full-1009-correction-table). + +**Credit where due.** #1009 names the hold as blocker 1 and states it correctly. What +changes here is its _causal weight_. Likewise, #1009's own observation that TS _"shifts +the auction cost from client-side to server-side rather than adding new work"_ is the +argument for client-fill, which the issue then declines in favour of ESI. + +### 6.2 What the hold actually costs + +**An earlier draft of this section claimed a stronger argument than the code supports. +It was wrong, and the correction matters.** + +The hold does not key off `lol_html` at all. `BodyCloseHoldBuffer::push` +(`publisher.rs:2190-2202`) +scans the **decoded origin input** for ` Dispatch precedes the origin fetch, so the hold costs `max(0, A − T)`, where `A` is the +> auction collect duration and `T` is origin TTFB plus body transfer up to the `` +> byte. Since `` sits at the end of a document, `T` is close to the full download. + +`A` is bounded by `auction_timeout_ms`, resolved as +`creative_opportunities.auction_timeout_ms` falling back to `auction.timeout_ms` +(`publisher.rs:2680-2684`) +— check the resolution order against your own config rather than trusting a number; the +shipped example sets different values at each level. + +**This is a claim requiring measurement, not a proof.** §3 Step C measures the hold's +cost directly rather than inferring it. + +A finding that does survive, and belongs with [the ceiling](#64-the-ceiling): because +`HtmlWithPostProcessing` withholds all output until the final chunk, the streaming-prefix +design at `publisher.rs:1343-1348` +— whose comment promises "the client receives the document up to `` while the +auction rides alongside transfer" — is **inert on a Next.js publisher**. Every +`step.ready` yields empty bytes. That comment is misleading on exactly the publisher +under discussion. + +### 6.3 The quantity nobody has measured + +Write the fetch time under `Pass` as `O`. Recovery depends on it, and it has never been +captured. #1009's timings cannot supply it: they compare a POP hit against a +shield-served fetch, both of which are _cached_ paths, whereas `CacheOverride::Pass` +bypasses TS's read-through cache and its shield. + +Note `Pass` bypasses **TS's** caches only. It has no authority over any CDN the publisher +runs in front of their own origin — and #1009's `x-cache: MISS, MISS` on the TS-on arm +hints one may exist. So `O` may not be origin build time at all. Since `O` is the single +quantity this model depends on, that ambiguity is worth resolving in Step C rather than +assuming. + +What follows from code alone, without any number: + +| Configuration | Long pole after the change | Recovery | +| ------------------- | -------------------------- | ------------------------------ | +| Hold removal only | origin (still `PASS`) | **none** | +| Bypass removal only | the auction budget | partial — the auction surfaces | +| **Both** | the rewrite | **the full available win** | + +That ordering is what the staging rests on, and it is measurement-independent. The +magnitude of each row is not, and §3 Step C supplies it. + +### 6.4 The ceiling + +#1009 targets "approach the TS-off warm numbers." **Unreachable, structurally.** Those +numbers are TS-off _streaming_ a POP HIT. TS buffers the whole document before emitting +a byte (16 MB cap), so its floor is `full origin body download + full rewrite` — above a +streamed hit by construction, whatever the timings turn out to be. Set the target from +Step C's measured rewrite cost rather than from the TS-off baseline. Going below the +floor requires true origin streaming (#849), out of scope. A non-Next.js publisher with +no post-processor takes the streaming path and would see a lower floor. + +### 6.5 Confidence + +**High on the structural claims.** §6.2's argument, the bypass forcing a cache miss, the +the silent-empty-bids failure mode, the geo and `Vary` blockers, +and the fill-canary blindness are all read directly out of the code at `cfb98f4`. Anyone +can check them without running anything. + +**None on magnitude.** `O` is unmeasured and the rewrite cost is unmeasured. This +document does not estimate them, and no figure in it should be quoted as one. + +Worth stating plainly: #1009 reached the opposite causal conclusion from a small sample. +That is a caution about small samples generally, not only about that one — which is why +§3 Step C specifies the measurement rather than this document supplying a substitute +for it. + +### 6.6 The ESI pipeline, corrected + +An earlier revision of this document said "order esi → lol*html, never the reverse." +That is backwards. `lol_html` is what \_emits* the ESI include tags; ESI cannot process +tags that do not exist yet. The correct order: + +``` +origin → lol_html transform → fastly::cache::core → finalize headers → stream esi assembly → client + (one unconditional marker (shared template, (EC cookie, geo, (per request, + at the body-close seam; surrogate-keyed, unconditional fetch the + the head seam is NOT a TS-chosen TTL) private/no-store) fragment) + hole — adSlots presence + is request-gated, §6.7) nothing may change + after this point +``` + +The push/pull mismatch that the earlier revision treated as a blocker is real but +irrelevant: `lol_html` pushes, `esi` pulls, and **the cache is the buffer between them**. +That is not an obstacle to the two-stage design — it _is_ the two-stage design, which is +what #1009 proposed in the first place. + +Mechanism, all present in the pinned `fastly` 0.12.1: + +| Need | API | +| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| Store the template | `cache::core::insert(key, max_age).execute() -> StreamingBody` | +| Read it back | `cache::core::lookup(key)` → `found()` | +| Avoid a thundering herd | `cache::core::Transaction` — `must_insert()` / `must_insert_or_update()` | +| Invalidate **C2 only** | `InsertBuilder::surrogate_keys([...])` (Core Cache) + `fastly::http::purge::purge_surrogate_key`. Does **not** key C1 — see the row below. | +| Invalidate C1 | Origin-supplied surrogate keys, or the HTTP cache's own surrogate-key surface. Not the Core Cache API. | + +Purge runs **inside Compute**. The management-API token scope cited under +[Stage 4](#7-deferred-work-specified-not-scheduled) governs a different surface and does +not gate this. + +**Three caches, kept distinct.** Conflating them is what produced the original error: + +1. **Origin read-through** — raw origin bytes. What Stage 0 turns back on. +2. **Shared transformed template** — post-`lol_html`, pre-ESI, no per-user data. The ESI + target, and new. +3. **Assembled-response delivery cache** — the final per-user output. **Must never + exist.** Nothing in this document or the spike proposes one. + +**Validation constraint.** Viceroy 0.17 cannot exercise the customized read-through hooks +end to end. Unit tests can cover the transform and the security properties; MISS / HIT / +stale / shielding behaviour must run against a real Fastly test service. + +--- + +### 6.7 What may and may not live in a shared template + +A correction to §6.1 row 1, and the constraint that governs any shared-template design. + +The original framing — "`adSlots` is per-URL, so there is one per-user hole, not two" — +is half right and dangerously so. `build_slot_json` really does emit only config- and +path-derived fields. But whether the script is emitted **at all** is gated on +`should_run_ad_stack` (`publisher.rs:2920-2927`), which is +`is_get && is_navigation && !is_prefetch && !is_bot && has_matched_slots && +consent_allows_auction && auction_enabled`. + +So the _content_ is per-URL and the _presence_ is per-request. A shared object filled by +the first request would freeze that request's consent decision, bot classification, +prefetch status, and kill-switch state for every later reader. A consent-denied fill +serves a no-ads template to consenting users; a consenting fill serves ad markup to +someone who refused. + +**The rule for anything cached and shared:** + +| May live in the template | Must live in the per-request fragment | +| --------------------------------------- | ---------------------------------------------- | +| tsjs bundle script tag (content-hashed) | `tsjs.adSlots` — presence is request-gated | +| URL rewrites (per-host, in the key) | `tsjs.bids` | +| | GPT diagnostics bootstrap (cookie/query-gated) | +| | Integration head-inserts (request-scoped) | + +The test that catches this class is **byte-identity of the template across requests +differing in consent, bot classification, and prefetch status** — not an absence-of- +per-user-values scan, which the broken design would have passed. + +This applies to any shared-template work, ESI or client-fill alike. The +[spike plan](../plans/2026-08-10-1009-esi-validation-spike.md) implements it. + +--- + +## 7. Deferred work, specified not scheduled + +**The full sequence, in one place.** Stage 0 is specified in [§4](#4-stage-0--the-only-build-item-recommended-now) +rather than repeated here; everything below it is deferred. + +| Stage | What | Status | +| ----- | ----------------------------------------------- | ---------------------------------------------- | +| **0** | Operator flag disabling the origin cache bypass | Recommended now. Gated on a `FINAL PASS`. §4. | +| 1 | Bid delivery off the response body | Deferred behind the correctness defects | +| 2 | Delete the `` hold | Deferred; one-way, needs a Stage 1 soak | +| 3a | Browser caching (`private, max-age` + `ETag`) | Specified, low risk, unscheduled | +| 3b | Shared cacheability | Blocked on geo suppression, `Vary`, and Step B | +| 4 | Purge wiring | Prerequisite for any TS-owned cache | + +**ESI is not a stage here.** It was Stage 5 in an earlier revision, queued behind the +rest. It no longer queues: it is feasible on the pinned SDK and is decided by +[its own spike plan](../plans/2026-08-10-1009-esi-validation-spike.md), which runs +independently of Stages 1–4. The shared template cache it needs is `fastly::cache::core` +([§6.6](#66-the-esi-pipeline-corrected)), not a new service. + +Lower detail below is deliberate. Full specifications are in the appendices. + +**Stage 1 — bid delivery off the response body.** The client fetches `/_ts/page-bids` at +navigation generation 0. Endpoint, same-origin gate, wire shape, and client consumer +already exist. Three decisions must be made before planning: the `slots: []` precedence +rule when head-open already injected a non-empty `ts.adSlots`; the new terminal-event +emission point; and whether the dispatch/collect split survives at all. Plumbing detail +in [Appendix B](#appendix-b--stage-1-plumbing-condensed). Estimated 8–13 d, low-to-medium +confidence, uncertainty concentrated client-side. + +Three companions are mandatory, not optional: **suppress the server bids script +entirely** (not an empty one), **fail loud** (the end-tag handler takes bids by value so +a missing auction is a compile error), and **relocate telemetry** (navigation +`Completed` rows are emitted only from the collect functions, and the `ts-debug` dump +rides the same string). Behaviour change to accept: under client-fill the auction runs +only if the browser executes the fetch, so bots and JS-disabled clients stop triggering +server-side auctions — revenue-relevant, sign unknown. + +**Stage 2 — delete the hold.** 5–8 d. **Rollback is one-way**: it deletes the hold, the +dispatch/collect split, and twelve tests, so the only revert is a release. Ships only +after Stage 1 has run flag-on in production for a window defined _before_ Stage 1 +starts, with TS-attributed renders flat and `auction_events_raw` navigation rows intact. +Secondary wins: removes the duplicated per-codec decoder/encoder wiring, six compression +imports, and the non-parser-context `` runs _all_ attempts and concatenates every non-failed output** — not + first-success-wins, so primary/fallback pairs render both. Least obvious behaviour in + the crate. +- Single include, not per-slot: the auction is one operation producing all slots' bids. + +--- + +## Appendix F — deferred open items (condensed) + +Implementation-level, for unscheduled work only. Decisions needing a human are in +[§9](#9-decisions-needed-from-this-review). + +Should `collect_non_html_auction` (`publisher.rs:2388`) go with the hold or stay? Is +`body_close_hold_loop_stream` (`:2109`, no production caller) safe to delete, or is the +buffered-adapter streaming cutover (#495) still live? Does hidden-tab rAF behaviour +interact badly with a bids timeout? What are Fastly's pending-request semantics when a +`DispatchedAuction` drops mid-flight? Does `stale-if-error` on a cached root serve +acceptable content given stale slot markup? And the googletag shim discards listeners +queued before it loads (#1009 Part 1) — not filed, should be. + +--- + +## Appendix G — code-grounded seams + +All pinned to `cfb98f4`. + +| Concern | Location | +| --------------------------------------------------------- | ------------------------------------------------------------------------------- | +| Eligibility decision | `publisher.rs:2651`, `:2660` | +| `is_navigation_request` | `http_util.rs:73-98` | +| Auction dispatch (pre-origin, non-blocking) | `publisher.rs:2698-2760` | +| Auction overlap intent | `auction/orchestrator.rs:950-952` | +| Auction timeout resolution | `publisher.rs:2680-2684` (creative_opportunities, else auction.timeout_ms) | +| Conditional/range header strip | `publisher.rs:2832-2836` | +| Origin cache bypass | `publisher.rs:2866-2868` | +| Origin 304 → 502 guard | `publisher.rs:2894-2916` | +| `adSlots` build (content per-URL, presence request-gated) | `publisher.rs:2920`, `:3558-3577`, `:3501-3525` | +| Uncacheable stamp | `publisher.rs:2945-2963` | +| `` hold — sync / async / Fastly lazy | `publisher.rs:2235` / `:2109` / `:1318-1390` | +| Hold buffer | `publisher.rs:2177-2218` | +| Auction collect (HTML / non-HTML) | `publisher.rs:2431` (emits `:2456`) / `:2388` (emits `:2410`) | +| Abandonment emitter | `publisher.rs:2360` | +| Bids script build | `publisher.rs:3438-3491` | +| `/_ts/page-bids` | `publisher.rs:3611`, handler `:3723`, auction `:3903` | +| Injection seams (head / body-close) | `html_processor.rs:310-363` / `:381-395` | +| Post-processor buffering | `html_processor.rs:62-94` | +| Next.js post-processor registration | `integrations/nextjs/mod.rs:107` | +| Max buffered body (16 MB) | `settings.rs:77-79` | +| EC cookie issuance policy | `ec/finalize.rs:86-107` | +| Cookie-privacy net | `response_privacy.rs:20-61` | +| Geo response headers | `adapter-fastly/src/middleware.rs:194-200` | +| Cacheable-header precedent | `http_util.rs:294-311` | +| Management token lacks purge (wrong surface) | `adapter-fastly/src/management_api.rs:12` | +| In-process purge / surrogate keys | `fastly` 0.12.1 `cache::core`, `http::purge::purge_surrogate_key` | +| Client initial-ad gate | `js/lib/src/integrations/gpt/index.ts:536-555` | +| `adInit` bid application | `js/lib/src/integrations/gpt/index.ts:566`, `:652`, `:657-661` | +| Client SPA auction hook | `js/lib/src/integrations/gpt/index.ts:806`, `:859`, `:892-949` | +| Prior design that introduced the killers | `docs/superpowers/specs/2026-07-22-ssat-root-document-304-prevention-design.md` | diff --git a/docs/superpowers/specs/2026-08-11-1009-streaming-assembly-architecture.md b/docs/superpowers/specs/2026-08-11-1009-streaming-assembly-architecture.md new file mode 100644 index 000000000..5b593a15d --- /dev/null +++ b/docs/superpowers/specs/2026-08-11-1009-streaming-assembly-architecture.md @@ -0,0 +1,260 @@ +# Streaming assembly: the architecture #1009 actually needs + +**Date:** 2026-08-11 +**Status:** Decision record. Supersedes the delivery half of the +[ESI validation spike](../plans/2026-08-10-1009-esi-validation-spike.md); the cache half +stands. +**Issue:** IABTechLab/trusted-server#1009 + +> **Implementation update, 2026-08-14.** Warm C2 hits still use Design C exactly as +> specified here. Authorized cold misses now also validate the repaired +> `stackpop/esi` parser, pinned by commit: the inert C2 marker becomes one synthetic +> include only in a private working copy, resolved from the already-built reader +> fragment without an HTTP request. Parser failure falls back to the byte seam. See +> [the hybrid implementation design](./2026-08-14-1009-esi-parser-assembly-design.md). +> Sections 2–2c and Design B remain investigation history; the native self-subrequest +> design is still not implemented. + +--- + +## 1. The correction this document exists for + +An earlier reading of the latency, recorded in +[the measurement findings](../plans/2026-08-08-1009-measurement-findings.md), said the +`` hold costs approximately nothing and the whole cost is the origin-cache bypass. + +That is true **today, and only today.** It is true for a reason that stops holding the +moment the rest of this work lands: + +| | Origin fetch | Auction | Reader waits | +| ----------------------------------- | ------------ | -------------------- | ------------ | +| Today | ~650 ms | hidden inside it | ~650–800 ms | +| Cached root, **buffered** assembly | 0 | fully exposed | ~auction cap | +| Cached root, **streaming** assembly | 0 | overlapped with send | ~ms | + +The auction is dispatched before the origin fetch and both run concurrently, so the hold +costs `max(0, auction − origin)` — zero while the origin is slow. Make the root cacheable +and the origin fetch disappears; the auction then has nothing left to hide behind and +becomes the _entire_ remaining cost. + +**So the two problems are coupled, and neither fix shows a win alone.** That is why the +issue is right to treat both as prerequisites, and why measuring one at a time misleads. + +Two distinct problems get bundled in the issue as one blocker. They need different fixes: + +1. **Bids live in the response body** → the page is _uncacheable_. Fixed by templatizing. + **Done.** +2. **The response is held for the auction** → the page is _slow_. Fixed by streaming the + shell and filling the seam late. **Not done** — this document. + +## 2. What the current implementation gets wrong + +On a C2 hit, `collect_and_assemble_cached_template` awaits the auction, then assembles, +then returns a fully buffered `PublisherResponse::Buffered`. The reader receives nothing +until bids resolve. + +That relocates the hold rather than removing it, and on a hit it is _worse than today_ in +one respect: there is no origin fetch left to hide it behind, so the full auction latency +lands on first byte. + +The routing decision that caused it — shared modes take the buffered finalizer — was made +because **a store needs complete transformed bytes.** True on a miss. Irrelevant on a hit, +where the template is already materialized. + +## 2b. Demonstrated, not argued + +Run locally under `viceroy serve` against a stub origin with a **self-imposed 1.5 s bid +endpoint**. These are synthetic numbers from a delay chosen to be observable — not a +measurement of any real deployment, and not comparable to publisher data. + +| Request | Cache | TTFB | Total | Origin fetched | +| ------- | ----- | --------- | --------- | -------------- | +| 1 | miss | ~injected | ~injected | yes | +| 2 | hit | ~injected | ~injected | **no** | +| 3 | hit | ~injected | ~injected | **no** | + +Two things are visible, and both matter more than the absolute values: + +1. **The cache works.** One origin fetch across three requests; the C2 log shows one + miss, one store, two hits. +2. **The reader waits exactly as long anyway.** Time-to-first-byte equals total on every + request, so nothing streams — the entire response lands at once, after the auction. + On the hits the origin fetch is gone and first byte still tracks the injected bid + delay. + +That is the claim in §1 and §2 reproduced on demand: a cached root delivers **no latency +benefit to the reader** while the response is held for the auction. It also gives the +harness a pass/fail shape for the change this document proposes — under streaming +assembly, TTFB must fall away from total by approximately the injected delay. + +## 2c. Measured against the shipped path — a ~100x TTFB regression + +`scripts/c2-local-test.sh` runs both modes against the same stub, with a self-imposed +1.5 s bid endpoint. Synthetic numbers, not a measurement of any deployment. + +| Mode | TTFB | Total | +| ------------------------- | ----------------- | ------- | +| `inline` (shipped) | **0.010–0.019 s** | ~1.51 s | +| `esi` (buffered assembly) | **1.524–1.532 s** | ~1.53 s | + +**The shipped path already streams correctly.** First byte in ~10 ms; the article paints +while the auction runs; only `` waits. Buffered assembly turns that into a wait +for the whole auction before the first byte — roughly **100x worse TTFB than doing +nothing**. + +This corrects §1 and §2, which framed buffered assembly as capturing the origin-fetch +saving and merely failing to add the streaming benefit. It is worse than that: it +**removes** a benefit today's code already delivers. The origin-fetch saving is +irrelevant beside losing the stream. + +It also sharpens where production's latency actually goes. Locally the stub origin +answers in ~2 ms, so `inline` TTFB is ~10 ms. In production the origin fetch is slow and +uncached, and TS cannot send a first byte until the origin sends one — so production TTFB +is the **origin fetch**, with the auction hidden behind the remainder of the body plus the +`` hold. The fix is therefore a fast origin _while keeping the stream_: exactly +Design C, and exactly what buffered assembly gives up. + +**Consequence for the plan:** `esi` mode must not be exposed to any traffic in its current +form. It is not a smaller win than hoped, it is a regression. + +### A harness bug worth recording + +The first version of this comparison reported `inline` fetching the origin zero times — +nonsense that still printed four passes. Viceroy was launched inside a subshell, so `$!` +was the subshell rather than the server; cleanup killed the wrapper and orphaned viceroy. +The next run then failed to bind and **silently answered from the previous run's process**, +carrying that run's config and warm cache. + +A harness that answers from the wrong server is worse than one that crashes, because its +output looks like data. Fixed with no subshell, a pre-flight port check, and a startup +wait that fails loudly. The regression above was invisible until the control worked. + +## 3. The decisive facts + +Three, all verified in the codebase rather than assumed: + +1. **The existing streaming path already implements stream-then-stall-at-the-seam.** + `publisher.rs` builds an `async_stream::try_stream!` that streams body chunks and holds + **only** at `` for the auction (`hold_auction`, `AuctionHoldState`). This is + shipping behaviour, not new work. +2. **`EdgeBody::Stream` is an async stream** — consumers call `stream.next().await` — so an + `await` may sit between chunks. Nothing needs a nested executor. +3. **`BodyCloseInjection::Marker(String)` already exists**, and the streaming finalizers + already strip `Content-Length`. + +## 4. Three designs + +| | Streams | Auctions | Requires | Adapters | +| ----------------------------------- | ------- | -------------- | ------------------------ | --------- | +| **A** — buffered assembly (current) | No | 1 | nothing | Fastly | +| **B** — native ESI subrequest | Yes | 1, in fragment | self-referencing backend | Fastly | +| **C** — cached shell + seam split | Yes | 1 | nothing | **All 4** | + +### Design B, for the record + +`PendingFragmentContent::PendingRequest` is what the `esi` crate is built for: the +dispatcher fires a real subrequest and the processor blocks on the handle. Fastly's +`send_async`/`wait` is **synchronous**, so this sidesteps the sync-dispatcher problem +without any executor. + +It also vindicates the _original_ dispatch gate. Under B the root must **not** dispatch, +because the fragment request runs the auction. The later reversal to +`root_auction_is_useful(Esi) = true` is correct for buffered assembly and wrong for +streaming. **Dispatch-usefulness is a function of the delivery mechanism**, which is the +non-obvious coupling in this design space. + +### Design C — the recommendation + +The template carries an **inert HTML comment sentinel** where the reader's ad slots and +bids go, emitted by the existing `Marker` variant: + +``` + +``` + +On a C2 hit: + +``` +commit headers (private, no-store; no Content-Length) ← must precede any byte on Fastly +stream template[..sentinel] ← the article paints here +await the auction ← the only stall, at the very end +write the bids script +stream template[sentinel+len..] +``` + +Since a hit has the whole template in hand, this is a `split_once`, not a streaming +search. Three yields from a `try_stream!`. + +**Why a comment sentinel rather than a byte offset in metadata.** An offset is O(1), but +capturing it means plumbing the writer position into a `lol_html` end-tag handler, and it +does not survive re-encoding. A `find` over a ~100 KB buffered template is free by +comparison. + +**Why a comment rather than executable ESI markup.** An HTML comment is inert. If +assembly ever fails to substitute, the reader sees nothing; an unresolved ESI include +tag renders as visible text. Failure degrades to "no ads" instead of "broken page". + +**Why not re-run `lol_html` over the cached template.** It would inject a second tsjs +`