From 6bfb8c16d3d767caa37e23666982cdf3fe808e6f Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 17 Oct 2026 11:53:12 +0100 Subject: [PATCH 01/18] The Axum, Cloudflare and Spin adapters run the request filters and the device module Each of the three adapters now runs the request cycle the Fastly adapter runs. After the modules' preparers, a named route other than the inspection endpoints, and the publisher fallback, classify the device with the module [device] selects (derive_device_signals), read the Edge Cookie context with that classification (build_ec_context), and run the integration request filters (run_pre_route_filters) before the route. A filter that answers stands in for the route, its request headers reach the route, and its response headers are left on the response for FinalizeResponseMiddleware, which writes them after the finalize headers and then restores the privacy guards (apply_terminal_response_effects). robots.txt is answered once the preparers have run and before the Edge Cookie steps and the filters, with an empty Edge Cookie context. Axum's named routes and fallback go through execute_named and execute_fallback where they went through execute_handler. Cloudflare and Spin register the same paths and methods as before, each handled by named_route_handler, and the fallback by execute_fallback. On Spin the discovery, signature verification and first-party routes and the page-bids preflight now run the preparers, which they skipped. These adapters supply no TLS or HTTP/2 signals, so [device] module = "platform" builds the built-in module there. src/app/request_cycle_tests.rs in each adapter checks, through the router its composition root builds and a stub origin, that a filter's answer stands in for the page and the origin is not asked, that a filter's request header reaches a module route and its response header is written after the operator's, that the selected device module's classification reaches a module route's call, and that robots.txt is answered with a filter that answers everything registered. cargo test-axum (1,357 passed), cargo test-cloudflare (61) and cargo test-spin (95) pass, and so do the Cloudflare wasm32 check, the Spin native and wasm32-wasip1 checks and its release build, clippy with warnings denied on the three adapters and cargo fmt. --- crates/trusted-server-adapter-axum/src/app.rs | 517 ++++++++---- .../src/app/request_cycle_tests.rs | 402 ++++++++++ .../src/middleware.rs | 32 +- .../src/app.rs | 590 ++++++++++---- .../src/app/request_cycle_tests.rs | 405 ++++++++++ .../src/middleware.rs | 32 +- crates/trusted-server-adapter-spin/src/app.rs | 744 +++++++++++------- .../src/app/request_cycle_tests.rs | 405 ++++++++++ .../src/middleware.rs | 32 +- 9 files changed, 2575 insertions(+), 584 deletions(-) create mode 100644 crates/trusted-server-adapter-axum/src/app/request_cycle_tests.rs create mode 100644 crates/trusted-server-adapter-cloudflare/src/app/request_cycle_tests.rs create mode 100644 crates/trusted-server-adapter-spin/src/app/request_cycle_tests.rs diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 72331ae00f..a56b229d73 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -1,4 +1,3 @@ -use core::future::Future; use std::sync::Arc; use std::sync::atomic::AtomicBool; @@ -6,7 +5,7 @@ use edgezero_core::app::Hooks; use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; use edgezero_core::http::{ - HandlerFuture, HeaderValue, Method, Request, Response, StatusCode, header, + HandlerFuture, HeaderMap, HeaderValue, Method, Request, Response, StatusCode, header, }; use edgezero_core::router::RouterService; use error_stack::Report; @@ -20,16 +19,22 @@ use trusted_server_core::auction::{ use trusted_server_core::cache_policy::EdgeCacheHeader; use trusted_server_core::closed_paths::closed_path_response; use trusted_server_core::ec::EcContext; +use trusted_server_core::ec::device::{ + BuiltinDeviceModule, DeviceModule, DeviceSignals, build_device_module, +}; use trusted_server_core::ec::module::ensure_module_available; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; +use trusted_server_core::evidence::BorrowedRequestInfo; use trusted_server_core::inspect::config::{CONFIG_JSON_PATH, CONFIG_PAGE_PATH, handle_config}; use trusted_server_core::inspect::data::{DATA_PAGE_PATH, handle_data}; use trusted_server_core::inspect::permissions::{ PERMISSIONS_JSON_PATH, PERMISSIONS_PAGE_PATH, handle_permissions, }; use trusted_server_core::integrations::{ - IntegrationBuilder, IntegrationRegistry, ProxyDispatchInput, + IntegrationBuilder, IntegrationRegistry, ProxyDispatchInput, RequestFilterEffects, + RequestFilterRegistryInput, RequestFilterRegistryOutcome, }; +use trusted_server_core::module_context::{ModuleContext, ModuleRequest}; use trusted_server_core::proxy::{ handle_first_party_click, handle_first_party_proxy, handle_first_party_proxy_rebuild, handle_first_party_proxy_sign, @@ -359,38 +364,60 @@ pub(crate) fn http_error(report: &Report) -> Response { } // --------------------------------------------------------------------------- -// Shared handler executor +// Request cycle // --------------------------------------------------------------------------- -async fn execute_handler( - state: Arc, - ctx: RequestContext, - handler: F, -) -> Result -where - F: FnOnce(Arc, RuntimeServices, Request) -> Fut, - Fut: Future>>, -{ - let services = state.services_for_request(&ctx); - let mut req = ctx.into_request(); - // The single preparation point for the adapter. Every route in the table - // except `/health` is registered to `named_route_handler` or - // `fallback_handler`, and both wrap this function, so each request has its - // modules' preparers run exactly once and always before routing. - if let Err(error) = state.registry.prepare_request(&state.settings, &mut req) { - return Ok(http_error(&error)); +/// Classifies the request with the device module `[device]` selects. +/// +/// A module a registration supplies, which the request's services carry, is +/// shown the `User-Agent` and the cookies, which carry what a page gathers in +/// the browser. Otherwise [`build_device_module`] gives the built-in module, +/// which reads only the `User-Agent`. This adapter supplies no TLS or HTTP/2 +/// signals, so `platform` classifies as the built-in module does. The module +/// is handed a module context carrying the request, that evidence, the +/// settings and the services. +async fn derive_device_signals( + settings: &Settings, + services: &RuntimeServices, + req: &Request, +) -> DeviceSignals { + let registered = services.device_module(); + let mut headers = HeaderMap::new(); + for (name, shown) in [ + (header::USER_AGENT, true), + (header::COOKIE, registered.is_some()), + ] { + if shown && let Some(value) = req.headers().get(&name) { + headers.insert(name, value.clone()); + } } - Ok(handler(state, services, req) + let client_ip = services + .client_info() + .client_ip + .map(|ip| ip.to_string()) + .unwrap_or_default(); + let request_info = BorrowedRequestInfo::new(&client_ip, None).with_headers(&headers); + let built; + let module: &dyn DeviceModule = match registered.as_deref() { + Some(module) => module, + None => { + built = build_device_module(settings, || { + Box::new(BuiltinDeviceModule::new()) as Box + }); + built.as_ref() + } + }; + let context = ModuleContext::new(ModuleRequest::of(req, services.client_info())) + .with_evidence(&request_info) + .with_settings(settings) + .with_services(services); + module + .detect(context.call(module.id(), module.required_permissions())) .await - .unwrap_or_else(|e| http_error(&e))) } -// --------------------------------------------------------------------------- -// EC context -// --------------------------------------------------------------------------- - -/// Builds the geo-aware [`EcContext`] for consent-gated endpoints (`/auction`, -/// `/_ts/page-bids`, and the publisher fallback). +/// Builds the geo-aware [`EcContext`] every route in the request cycle is +/// handed, carrying the request's device classification. /// /// The geo lookup runs inside /// [`EcContext::read_from_request_resolving_geo`], so every adapter reports the @@ -416,39 +443,240 @@ async fn build_ec_context( services: &RuntimeServices, req: &Request, ) -> Result> { - EcContext::read_from_request_resolving_geo(&state.settings, req, services).await + let device_signals = derive_device_signals(&state.settings, services, req).await; + let mut context = + EcContext::read_from_request_resolving_geo(&state.settings, req, services).await?; + context.set_device_signals(device_signals); + Ok(context) +} + +/// Result of the request-filter step that runs before a route. +enum PreRoute { + /// A filter answered, or failed, and that answer stands in for the + /// route's. Carries the response effects the filters left. + ShortCircuit { + response: Response, + effects: RequestFilterEffects, + }, + /// The route runs, and these response effects are applied afterwards. + Continue { effects: RequestFilterEffects }, +} + +/// Runs the integration request filters before the route. +/// +/// Request header changes are applied to `req`, so the route sees them, and +/// the response effects are returned for [`FinalizeResponseMiddleware`] to +/// apply once the response is finalized. A filter that answers, such as a +/// bot challenge, stands in for the route. Every filter reads the location +/// and the permissions resolved for the request's Edge Cookie context. +async fn run_pre_route_filters( + state: &AppState, + services: &RuntimeServices, + req: &mut Request, + ec_context: &EcContext, +) -> PreRoute { + match state + .registry + .filter_request(RequestFilterRegistryInput { + settings: &state.settings, + services, + req, + geo_info: ec_context.geo_info(), + permissions: Some(ec_context.permissions()), + }) + .await + { + Ok(RequestFilterRegistryOutcome::Continue(effects)) => PreRoute::Continue { effects }, + Ok(RequestFilterRegistryOutcome::Respond { response, effects }) => PreRoute::ShortCircuit { + response: *response, + effects, + }, + Err(report) => { + log::error!("Failed to run integration request filters: {report:?}"); + PreRoute::ShortCircuit { + response: http_error(&report), + effects: RequestFilterEffects::default(), + } + } + } +} + +/// Leaves non-empty request-filter response effects on `response`, for +/// [`FinalizeResponseMiddleware`] to apply after it finalizes the response. +fn attach_request_filter_effects( + mut response: Response, + effects: RequestFilterEffects, +) -> Response { + if !effects.response_headers.is_empty() { + response.extensions_mut().insert(effects); + } + response +} + +/// Runs a named route's request through the request cycle. +/// +/// The modules' preparers run first. The inspection endpoints are read only, +/// so they are answered next, before the Edge Cookie context is read and the +/// request filters run. Every other route is handed the Edge Cookie context +/// and runs once the request filters let the request continue. +async fn execute_named( + state: Arc, + ctx: RequestContext, + handler: NamedRouteHandler, +) -> Result { + let services = state.services_for_request(&ctx); + let mut req = ctx.into_request(); + if let Err(error) = state.registry.prepare_request(&state.settings, &mut req) { + return Ok(http_error(&error)); + } + + match handler { + NamedRouteHandler::Permissions => { + return Ok(handle_permissions(&state.settings, &services, &req) + .await + .unwrap_or_else(|error| http_error(&error))); + } + NamedRouteHandler::Config => return Ok(handle_config(&state.settings, &req)), + NamedRouteHandler::Data => { + return Ok(handle_data(None, None, services.public_prefix(), &req) + .unwrap_or_else(|error| http_error(&error))); + } + _ => {} + } + + let mut ec_context = match build_ec_context(&state, &services, &req).await { + Ok(context) => context, + Err(report) => return Ok(http_error(&report)), + }; + let effects = match run_pre_route_filters(&state, &services, &mut req, &ec_context).await { + PreRoute::ShortCircuit { response, effects } => { + return Ok(attach_request_filter_effects(response, effects)); + } + PreRoute::Continue { effects } => effects, + }; + + let response = run_named_route(&state, &services, req, handler, &mut ec_context) + .await + .unwrap_or_else(|error| http_error(&error)); + Ok(attach_request_filter_effects(response, effects)) +} + +async fn run_named_route( + state: &AppState, + services: &RuntimeServices, + req: Request, + handler: NamedRouteHandler, + ec_context: &mut EcContext, +) -> Result> { + match handler { + NamedRouteHandler::TrustedServerDiscovery => { + handle_trusted_server_discovery(&state.settings, services, req) + } + NamedRouteHandler::VerifySignature => { + handle_verify_signature(&state.settings, services, req) + } + NamedRouteHandler::Permissions | NamedRouteHandler::Config | NamedRouteHandler::Data => { + unreachable!("the inspection endpoints are answered before the request cycle") + } + NamedRouteHandler::Auction => { + // The auction consent gate reads the caller's jurisdiction from + // the geo-aware context, where `EcContext::default()` would fail it + // closed for consented users. + handle_auction( + &state.settings, + &state.orchestrator, + None, + None, + ec_context, + services, + req, + ) + .await + } + NamedRouteHandler::PageBids => { + // SPA re-auction endpoint. `OPTIONS` is a CORS preflight for this + // side-effecting GET and is always denied so the GET handler's + // `X-TSJS-Page-Bids` gate stays trustworthy. + if req.method() == Method::OPTIONS { + return Ok(page_bids_preflight_denied()); + } + let auction = AuctionDispatch { + orchestrator: &state.orchestrator, + slots: state.settings.creative_opportunity_slots(), + registry: None, + }; + handle_page_bids(&state.settings, services, None, auction, ec_context, req).await + } + NamedRouteHandler::FirstPartyProxy => { + handle_first_party_proxy(&state.settings, services, req).await + } + NamedRouteHandler::FirstPartyClick => { + handle_first_party_click(&state.settings, services, req).await + } + NamedRouteHandler::FirstPartySign => { + handle_first_party_proxy_sign(&state.settings, services, req).await + } + NamedRouteHandler::FirstPartyProxyRebuild => { + handle_first_party_proxy_rebuild(&state.settings, services, req).await + } + } } // --------------------------------------------------------------------------- -// Fallback dispatcher (tsjs / integration proxy / publisher) +// Fallback dispatcher (robots.txt / tsjs / integration proxy / publisher) // --------------------------------------------------------------------------- -/// Dispatches the fallback routes: tsjs assets, integration proxy routes, and -/// the publisher origin. +async fn execute_fallback( + state: Arc, + ctx: RequestContext, +) -> Result { + let services = state.services_for_request(&ctx); + let mut req = ctx.into_request(); + if let Err(error) = state.registry.prepare_request(&state.settings, &mut req) { + return Ok(http_error(&error)); + } + Ok(dispatch_fallback(&state, &services, req).await) +} + +/// Dispatches the fallback routes: robots.txt, tsjs assets, integration proxy +/// routes, and the publisher origin. /// -/// The request arrives already prepared, and is not to a closed path. Every -/// route that reaches here is registered to [`fallback_handler`], which -/// answers a closed path itself and runs any other request through -/// [`execute_handler`], and [`execute_handler`] calls -/// `registry.prepare_request` before it calls this function, so there is -/// nothing left to prepare here. +/// The request arrives prepared, and is not to a closed path, because +/// [`fallback_handler`] answers a closed path itself and runs any other +/// request through [`execute_fallback`], which runs the preparers. async fn dispatch_fallback( state: &AppState, services: &RuntimeServices, - req: Request, -) -> Result> { + mut req: Request, +) -> Response { let path = req.uri().path().to_string(); let method = req.method().clone(); - if method == Method::GET && path.starts_with("/static/tsjs=") { - return handle_tsjs_dynamic(&req, &state.registry, EDGE_CACHE_HEADER); + // A crawler asking for robots.txt is answered once the modules have + // prepared the request, before the Edge Cookie steps and the request + // filters, so nothing is read from its cookies and no filter's answer + // stands in for the file. + if asks_for_robots_txt(&state.settings, &method, &path) { + return answer_robots_txt(state, services, req, &method).await; } - if state.registry.has_route(&method, &path) { + let mut ec_context = match build_ec_context(state, services, &req).await { + Ok(context) => context, + Err(report) => return http_error(&report), + }; + let effects = match run_pre_route_filters(state, services, &mut req, &ec_context).await { + PreRoute::ShortCircuit { response, effects } => { + return attach_request_filter_effects(response, effects); + } + PreRoute::Continue { effects } => effects, + }; + + let result = if method == Method::GET && path.starts_with("/static/tsjs=") { + handle_tsjs_dynamic(&req, &state.registry, EDGE_CACHE_HEADER) + } else if state.registry.has_route(&method, &path) { // A module route is handed what was resolved for its request, the // permissions among them, as the publisher path is. - let mut ec_context = build_ec_context(state, services, &req).await?; - return state + state .registry .handle_proxy(ProxyDispatchInput { method: &method, @@ -464,41 +692,100 @@ async fn dispatch_fallback( Err(Report::new(TrustedServerError::BadRequest { message: format!("Unknown integration route: {path}"), })) - }); - } - - // Run the server-side auction with the configured creative-opportunity - // slots; `handle_publisher_request` matches them against the request path. - let mut ec_context = build_ec_context(state, services, &req).await?; - let auction = AuctionDispatch { - orchestrator: &state.orchestrator, - slots: state.settings.creative_opportunity_slots(), - registry: None, + }) + } else { + // Run the server-side auction with the configured creative-opportunity + // slots; `handle_publisher_request` matches them against the request + // path. + let auction = AuctionDispatch { + orchestrator: &state.orchestrator, + slots: state.settings.creative_opportunity_slots(), + registry: None, + }; + match handle_publisher_request( + AppContext { + settings: &state.settings, + integration_registry: &state.registry, + }, + services, + None, + &mut ec_context, + auction, + req, + EDGE_CACHE_HEADER, + ) + .await + { + // Async finalize so the dispatched auction is collected and its + // bids are injected before `` (the sync buffer path would + // drop them). + Ok(publisher_response) => { + buffer_publisher_response_async( + publisher_response, + &method, + &state.settings, + &state.registry, + &state.orchestrator, + services, + ) + .await + } + Err(report) => Err(report), + } }; - let publisher_response = handle_publisher_request( + + attach_request_filter_effects(result.unwrap_or_else(|error| http_error(&error)), effects) +} + +/// Whether the request is a crawler's `GET` or `HEAD` of the robots.txt that +/// `[robots-txt]` makes. +fn asks_for_robots_txt(settings: &Settings, method: &Method, path: &str) -> bool { + settings.robots_txt.is_some() + && path == trusted_server_core::robots_txt::ROBOTS_TXT_PATH + && matches!(*method, Method::GET | Method::HEAD) +} + +/// Answers robots.txt through the publisher path with an empty Edge Cookie +/// context and no auction, so none of the crawler's cookies or consent +/// signals are read. +async fn answer_robots_txt( + state: &AppState, + services: &RuntimeServices, + req: Request, + method: &Method, +) -> Response { + let answer = handle_publisher_request( AppContext { settings: &state.settings, integration_registry: &state.registry, }, services, None, - &mut ec_context, - auction, + &mut EcContext::default(), + AuctionDispatch { + orchestrator: &state.orchestrator, + slots: &[], + registry: None, + }, req, EDGE_CACHE_HEADER, ) - .await?; - // Async finalize so the dispatched auction is collected and its bids are - // injected before `` (the sync buffer path would drop them). - buffer_publisher_response_async( - publisher_response, - &method, - &state.settings, - &state.registry, - &state.orchestrator, - services, - ) - .await + .await; + let response = match answer { + Ok(answer) => { + buffer_publisher_response_async( + answer, + method, + &state.settings, + &state.registry, + &state.orchestrator, + services, + ) + .await + } + Err(report) => Err(report), + }; + response.unwrap_or_else(|report| http_error(&report)) } fn fallback_handler( @@ -511,11 +798,7 @@ fn fallback_handler( return Box::pin(async move { Ok::<_, EdgeError>(response) }); } let state = Arc::clone(&state); - Box::pin(execute_handler( - state, - ctx, - |state, services, req| async move { dispatch_fallback(&state, &services, req).await }, - )) + Box::pin(execute_fallback(state, ctx)) } } @@ -638,79 +921,7 @@ fn named_route_handler( ) -> impl Fn(RequestContext) -> HandlerFuture + Clone + Send + Sync + 'static { move |ctx: RequestContext| { let state = Arc::clone(&state); - Box::pin(execute_handler( - state, - ctx, - move |state, services, req| async move { - match handler { - NamedRouteHandler::TrustedServerDiscovery => { - handle_trusted_server_discovery(&state.settings, &services, req) - } - NamedRouteHandler::VerifySignature => { - handle_verify_signature(&state.settings, &services, req) - } - NamedRouteHandler::Permissions => { - handle_permissions(&state.settings, &services, &req).await - } - NamedRouteHandler::Config => Ok(handle_config(&state.settings, &req)), - NamedRouteHandler::Data => { - handle_data(None, None, services.public_prefix(), &req) - } - NamedRouteHandler::Auction => { - // Build the geo-aware EC context so the auction consent - // gate sees the caller's jurisdiction — `EcContext::default()` - // fails it closed for consented users. - let mut ec_context = build_ec_context(&state, &services, &req).await?; - handle_auction( - &state.settings, - &state.orchestrator, - None, - None, - &mut ec_context, - &services, - req, - ) - .await - } - NamedRouteHandler::PageBids => { - // SPA re-auction endpoint. `OPTIONS` is a CORS preflight - // for this side-effecting GET and is always denied so the - // GET handler's `X-TSJS-Page-Bids` gate stays trustworthy. - if req.method() == Method::OPTIONS { - Ok(page_bids_preflight_denied()) - } else { - let mut ec_context = build_ec_context(&state, &services, &req).await?; - let auction = AuctionDispatch { - orchestrator: &state.orchestrator, - slots: state.settings.creative_opportunity_slots(), - registry: None, - }; - handle_page_bids( - &state.settings, - &services, - None, - auction, - &mut ec_context, - req, - ) - .await - } - } - NamedRouteHandler::FirstPartyProxy => { - handle_first_party_proxy(&state.settings, &services, req).await - } - NamedRouteHandler::FirstPartyClick => { - handle_first_party_click(&state.settings, &services, req).await - } - NamedRouteHandler::FirstPartySign => { - handle_first_party_proxy_sign(&state.settings, &services, req).await - } - NamedRouteHandler::FirstPartyProxyRebuild => { - handle_first_party_proxy_rebuild(&state.settings, &services, req).await - } - } - }, - )) + Box::pin(execute_named(state, ctx, handler)) } } @@ -890,8 +1101,9 @@ impl TrustedServerApp { } } -/// Answers the attestation pages without the request preparers, because -/// the pages are read only. +/// Answers the attestation pages without the request preparers, the Edge +/// Cookie steps or the request filters, because the pages are read only and +/// a verifier fetching one from a server must not be turned away as a bot. fn attestation_route_handler( state: Arc, ) -> impl Fn(RequestContext) -> HandlerFuture + Clone + Send + Sync + 'static { @@ -966,6 +1178,9 @@ fn build_router(state: &Arc) -> RouterService { router.build() } +#[cfg(test)] +mod request_cycle_tests; + #[cfg(test)] mod tests { use edgezero_core::http::request_builder; diff --git a/crates/trusted-server-adapter-axum/src/app/request_cycle_tests.rs b/crates/trusted-server-adapter-axum/src/app/request_cycle_tests.rs new file mode 100644 index 0000000000..dd576ee018 --- /dev/null +++ b/crates/trusted-server-adapter-axum/src/app/request_cycle_tests.rs @@ -0,0 +1,402 @@ +//! The request cycle this adapter runs for a routed request, checked through +//! its router: the request filters run before the route and their response +//! effects after the finalize headers, the route's module call carries the +//! device classification, and robots.txt is answered ahead of both. + +use std::sync::Arc; + +use edgezero_core::body::Body; +use edgezero_core::http::{Request, Response, StatusCode, request_builder}; +use edgezero_core::router::RouterService; +use error_stack::Report; +use trusted_server_core::ec::device::{DeviceModule, DeviceSignals}; +use trusted_server_core::error::TrustedServerError; +use trusted_server_core::evidence::RequestInfo; +use trusted_server_core::integrations::{ + HeaderMutation, IntegrationBuilder, IntegrationEndpoint, IntegrationProxy, + IntegrationRegistration, IntegrationRequestFilter, RequestFilterDecision, RequestFilterEffects, + RequestFilterInput, +}; +use trusted_server_core::module_context::ModuleCall; +use trusted_server_core::platform::PlatformHttpClient; +use trusted_server_core::platform::test_support::{ + StubHttpClient, build_services_with_http_client, +}; +use trusted_server_core::settings::Settings; + +use super::TrustedServerApp; + +/// The id and module name of the probe, which registers a filter that lets +/// the request continue, a device module and a route reporting what its +/// module call was handed. +const PROBE_ID: &str = "cycle_probe"; +const PROBE_MODULE: &str = "testing.cycle-probe"; + +/// The id and module name of a filter that answers every request. +const ANSWERING_ID: &str = "answering_filter"; +const ANSWERING_MODULE: &str = "testing.answering-filter"; + +/// The response header both filters write as an effect. +const EFFECT_HEADER: &str = "x-cycle-probe"; +/// The request header the continuing filter adds. +const FILTERED_HEADER: &str = "x-cycle-probe-filtered"; +/// What the answering filter answers. +const ANSWER: &str = "answered by the filter"; + +struct AnsweringFilter; + +#[async_trait::async_trait(?Send)] +impl IntegrationRequestFilter for AnsweringFilter { + fn integration_id(&self) -> &'static str { + ANSWERING_ID + } + + async fn filter_request( + &self, + _input: RequestFilterInput<'_>, + ) -> Result> { + let mut response = Response::new(Body::from(ANSWER)); + *response.status_mut() = StatusCode::FORBIDDEN; + Ok(RequestFilterDecision::Respond { + response: Box::new(response), + effects: RequestFilterEffects { + request_headers: Vec::new(), + response_headers: vec![HeaderMutation::set(EFFECT_HEADER, "answered")], + }, + }) + } +} + +struct ContinuingFilter; + +#[async_trait::async_trait(?Send)] +impl IntegrationRequestFilter for ContinuingFilter { + fn integration_id(&self) -> &'static str { + PROBE_ID + } + + async fn filter_request( + &self, + _input: RequestFilterInput<'_>, + ) -> Result> { + Ok(RequestFilterDecision::Continue(RequestFilterEffects { + request_headers: vec![HeaderMutation::set(FILTERED_HEADER, "1")], + response_headers: vec![HeaderMutation::set(EFFECT_HEADER, "continued")], + })) + } +} + +/// A device module that marks its classification with the cookies it was +/// shown, so a test can tell that it ran and what it saw. +struct MarkingDevice; + +impl MarkingDevice { + fn mark(&self, request_info: &dyn RequestInfo) -> DeviceSignals { + let mut signals = DeviceSignals::derive_ua_only(request_info.user_agent()); + signals.platform_class = Some(format!( + "marked:{}", + request_info.header("cookie").unwrap_or_default() + )); + signals + } +} + +#[async_trait::async_trait(?Send)] +impl DeviceModule for MarkingDevice { + fn id(&self) -> &'static str { + PROBE_MODULE + } + + async fn detect(&self, call: ModuleCall<'_>) -> DeviceSignals { + call.inject(self, Self::mark) + .unwrap_or_else(|_| DeviceSignals::unknown()) + } +} + +/// A route answering with the device classification its module call was +/// handed and whether the continuing filter's request header reached it. +struct ReportRoute; + +impl ReportRoute { + fn report( + &self, + req: Request, + device: Option<&DeviceSignals>, + ) -> Result> { + let (parts, _body) = req.into_parts(); + let report = serde_json::json!({ + "device": device.and_then(|signals| signals.platform_class.clone()), + "filtered": parts + .headers + .get(FILTERED_HEADER) + .and_then(|value| value.to_str().ok()), + }); + Ok(Response::new(Body::from(report.to_string()))) + } +} + +#[async_trait::async_trait(?Send)] +impl IntegrationProxy for ReportRoute { + fn integration_name(&self) -> &'static str { + PROBE_ID + } + + fn routes(&self) -> Vec { + vec![self.get("/report")] + } + + async fn handle( + &self, + call: ModuleCall<'_>, + req: Request, + ) -> Result> { + call.inject_with(self, req, Self::report)? + } +} + +fn register_probe( + _settings: &Settings, +) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder(PROBE_ID) + .without_js() + .with_request_filter(Arc::new(ContinuingFilter)) + .with_device_module(PROBE_MODULE, Arc::new(MarkingDevice)) + .with_proxy(Arc::new(ReportRoute)) + .build(), + )) +} + +fn register_answering( + _settings: &Settings, +) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder(ANSWERING_ID) + .without_js() + .with_request_filter(Arc::new(AnsweringFilter)) + .build(), + )) +} + +fn always(_settings: &Settings) -> Result> { + Ok(true) +} + +fn probe_builder() -> IntegrationBuilder { + IntegrationBuilder::new(PROBE_ID, "adapter-tests", register_probe, always) + .with_module_name(PROBE_MODULE) +} + +fn answering_builder() -> IntegrationBuilder { + IntegrationBuilder::new(ANSWERING_ID, "adapter-tests", register_answering, always) + .with_module_name(ANSWERING_MODULE) +} + +/// Settings for the publisher the requests are for, with `extra` appended. +fn settings_with(extra: &str) -> Settings { + Settings::from_toml(&format!( + r#" + [source] + module = "origin" + + [source.origin] + url = "https://origin.test-publisher.example.com" + + [publisher] + domain = "test-publisher.example.com" + cookie_domain = ".test-publisher.example.com" + proxy_secret = "request-cycle-test-proxy-secret" + + [ec] + module = "hmac" + + [ec.hmac] + passphrase = "test-secret-key-32-bytes-minimum" + + [geo] + assume_single_jurisdiction = true + + {extra} + "# + )) + .expect("should parse the request cycle test settings") +} + +/// The router built through the composition root with `builders`, every +/// request sent to `origin`. +fn router_with( + settings: Settings, + builders: &[IntegrationBuilder], + origin: &Arc, +) -> RouterService { + TrustedServerApp::routes_with_registrations_and_services( + settings, + builders, + build_services_with_http_client(Arc::clone(origin) as Arc), + ) + .expect("should build the router") +} + +fn get(path: &str, cookie: Option<&str>) -> Request { + let mut builder = request_builder() + .method("GET") + .uri(format!("https://test-publisher.example.com{path}")) + .header("host", "test-publisher.example.com"); + if let Some(cookie) = cookie { + builder = builder.header("cookie", cookie); + } + builder + .body(Body::empty()) + .expect("should build the request") +} + +async fn route(router: &RouterService, request: Request) -> Response { + router + .oneshot(request) + .await + .expect("should route the request") +} + +fn header<'a>(response: &'a Response, name: &str) -> Option<&'a str> { + response + .headers() + .get(name) + .and_then(|value| value.to_str().ok()) +} + +fn text(response: Response) -> String { + let bytes = response + .into_body() + .into_bytes() + .expect("should hold the body in memory"); + String::from_utf8(bytes.to_vec()).expect("the body should be text") +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_filter_that_answers_stands_in_for_the_page() { + let origin = Arc::new(StubHttpClient::new()); + let router = router_with( + settings_with("[testing]\nmodules = [\"answering-filter\"]"), + &[answering_builder()], + &origin, + ); + + let response = route(&router, get("/article", None)).await; + + assert_eq!( + response.status(), + StatusCode::FORBIDDEN, + "the filter's answer should stand in for the page" + ); + assert_eq!( + header(&response, EFFECT_HEADER), + Some("answered"), + "the effects the answering filter left should be applied" + ); + assert_eq!(text(response), ANSWER); + assert!( + origin.recorded_request_uris().is_empty(), + "the publisher's origin should not be asked" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_filter_s_request_header_reaches_the_route_and_its_effect_is_written_last() { + let origin = Arc::new(StubHttpClient::new()); + let mut settings = settings_with("[testing]\nmodules = [\"cycle-probe\"]"); + settings + .response_headers + .insert(EFFECT_HEADER.to_owned(), "operator".to_owned()); + let router = router_with(settings, &[probe_builder()], &origin); + + let response = route(&router, get("/integrations/cycle_probe/report", None)).await; + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + header(&response, EFFECT_HEADER), + Some("continued"), + "the filter's response effect should be written after the operator's headers" + ); + let report: serde_json::Value = + serde_json::from_str(&text(response)).expect("the report should be JSON"); + assert_eq!( + report["filtered"], "1", + "the filter's request header should reach the route" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn the_device_module_s_classification_reaches_the_module_call() { + let origin = Arc::new(StubHttpClient::new()); + let router = router_with( + settings_with(&format!( + "[device]\nmodule = \"{PROBE_MODULE}\"\n\n[testing]\nmodules = [\"cycle-probe\"]" + )), + &[probe_builder()], + &origin, + ); + + let response = route( + &router, + get( + "/integrations/cycle_probe/report", + Some("example_width=1280"), + ), + ) + .await; + + assert_eq!(response.status(), StatusCode::OK); + let report: serde_json::Value = + serde_json::from_str(&text(response)).expect("the report should be JSON"); + assert_eq!( + report["device"], "marked:example_width=1280", + "the selected device module should classify the request, shown its cookies" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn robots_txt_is_answered_before_the_request_filters() { + let origin = Arc::new(StubHttpClient::new()); + origin.push_response_with_headers( + 200, + b"User-agent: *\nAllow: /\n".to_vec(), + vec![("content-type", "text/plain")], + ); + let router = router_with( + settings_with( + r#" + [testing] + modules = ["answering-filter"] + + [robots-txt] + modules = "refuse_all" + always_allow = ["/ads.txt"] + + [[fetch]] + media_type = "text/plain" + path = "/robots.txt" + middleware = ["robots_txt"] + "#, + ), + &[answering_builder()], + &origin, + ); + + let response = route(&router, get("/robots.txt", None)).await; + + assert_eq!( + response.status(), + StatusCode::OK, + "no filter's answer should stand in for robots.txt" + ); + assert_eq!( + header(&response, EFFECT_HEADER), + None, + "the filters should not run for robots.txt" + ); + assert_eq!( + text(response), + "User-agent: *\nAllow: /ads.txt\nDisallow: /\n" + ); + assert_eq!(origin.recorded_request_methods(), ["GET"]); +} diff --git a/crates/trusted-server-adapter-axum/src/middleware.rs b/crates/trusted-server-adapter-axum/src/middleware.rs index 70781b9f4b..3eb75b8dbe 100644 --- a/crates/trusted-server-adapter-axum/src/middleware.rs +++ b/crates/trusted-server-adapter-axum/src/middleware.rs @@ -8,6 +8,11 @@ use edgezero_core::middleware::{Middleware, Next}; use trusted_server_core::http_util::{ ROUTE_NAMESPACE, sanitize_trusted_client_ip_headers, strip_deployment_headers, }; +use trusted_server_core::integrations::RequestFilterEffects; +use trusted_server_core::response_privacy::{ + TerminalPrivateResponse, enforce_private_no_store, enforce_set_cookie_cache_privacy, + enforce_uncacheable_cache_privacy, +}; use trusted_server_core::settings::Settings; // --------------------------------------------------------------------------- @@ -56,7 +61,8 @@ impl Middleware for SanitizeRequestMiddleware { /// in a local dev context. /// /// Registered directly inside [`SanitizeRequestMiddleware`], so that every -/// outgoing response carries a consistent set of headers. +/// outgoing response carries a consistent set of headers. Once the response +/// is finalized it applies the response effects the request filters left. pub struct FinalizeResponseMiddleware { settings: Arc, } @@ -74,13 +80,37 @@ impl Middleware for FinalizeResponseMiddleware { async fn handle(&self, ctx: RequestContext, next: Next<'_>) -> Result { let path = ctx.request().uri().path().to_owned(); let mut response = next.run(ctx).await?; + let request_filter_effects = response.extensions_mut().remove::(); // Before finalizing, so a header finalizing writes is kept. strip_deployment_headers(&path, ROUTE_NAMESPACE, &mut response); apply_finalize_headers(&self.settings, &mut response); + apply_terminal_response_effects(&mut response, request_filter_effects.as_ref()); Ok(response) } } +/// Applies the request filters' response effects to a finalized response, +/// then restores the privacy guards a late header could loosen. +fn apply_terminal_response_effects( + response: &mut Response, + request_filter_effects: Option<&RequestFilterEffects>, +) { + let must_remain_private = response + .extensions() + .get::() + .is_some(); + if let Some(effects) = request_filter_effects { + effects.apply_to_response(response); + } + if must_remain_private { + enforce_private_no_store(response); + } + // A filter's header can add a per-user `Set-Cookie`, or a `private` or + // `no-store` directive, after the finalize headers ran. + enforce_set_cookie_cache_privacy(response); + enforce_uncacheable_cache_privacy(response); +} + // --------------------------------------------------------------------------- // apply_finalize_headers — extracted for unit testing // --------------------------------------------------------------------------- diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index c230dfb7da..24950d2334 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -6,7 +6,7 @@ use std::sync::atomic::AtomicBool; use edgezero_core::app::Hooks; use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; -use edgezero_core::http::{HeaderValue, Method, Request, Response, header}; +use edgezero_core::http::{HeaderMap, HeaderValue, Method, Request, Response, header}; use edgezero_core::router::RouterService; use error_stack::Report; use trusted_server_core::addressing::{RequestAddressing, addressed_router}; @@ -25,14 +25,20 @@ use trusted_server_core::config_payload::{ DEFAULT_SECRET_STORE_ID, settings_from_config_blob_with, }; use trusted_server_core::ec::EcContext; +use trusted_server_core::ec::device::{ + BuiltinDeviceModule, DeviceModule, DeviceSignals, build_device_module, +}; use trusted_server_core::ec::module::{EdgeCookieModule, build_reusable_module}; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; +use trusted_server_core::evidence::BorrowedRequestInfo; use trusted_server_core::inspect::config::{CONFIG_PATHS, handle_config}; use trusted_server_core::inspect::data::{DATA_PAGE_PATH, handle_data}; use trusted_server_core::inspect::permissions::{PERMISSIONS_PATHS, handle_permissions}; use trusted_server_core::integrations::{ - IntegrationBuilder, IntegrationRegistry, ProxyDispatchInput, + IntegrationBuilder, IntegrationRegistry, ProxyDispatchInput, RequestFilterEffects, + RequestFilterRegistryInput, RequestFilterRegistryOutcome, }; +use trusted_server_core::module_context::{ModuleContext, ModuleRequest}; use trusted_server_core::platform::RuntimeServices; #[cfg(target_arch = "wasm32")] use trusted_server_core::platform::StoreName; @@ -475,11 +481,60 @@ impl AppState { } // --------------------------------------------------------------------------- -// Per-request RuntimeServices +// Request cycle // --------------------------------------------------------------------------- -/// Builds the geo-aware [`EcContext`] for consent-gated endpoints (`/auction`, -/// `/_ts/page-bids`, and the publisher fallback). +/// Classifies the request with the device module `[device]` selects. +/// +/// A module a registration supplies, which the request's services carry, is +/// shown the `User-Agent` and the cookies, which carry what a page gathers in +/// the browser. Otherwise [`build_device_module`] gives the built-in module, +/// which reads only the `User-Agent`. This adapter supplies no TLS or HTTP/2 +/// signals, so `platform` classifies as the built-in module does. The module +/// is handed a module context carrying the request, that evidence, the +/// settings and the services. +async fn derive_device_signals( + settings: &Settings, + services: &RuntimeServices, + req: &Request, +) -> DeviceSignals { + let registered = services.device_module(); + let mut headers = HeaderMap::new(); + for (name, shown) in [ + (header::USER_AGENT, true), + (header::COOKIE, registered.is_some()), + ] { + if shown && let Some(value) = req.headers().get(&name) { + headers.insert(name, value.clone()); + } + } + let client_ip = services + .client_info() + .client_ip + .map(|ip| ip.to_string()) + .unwrap_or_default(); + let request_info = BorrowedRequestInfo::new(&client_ip, None).with_headers(&headers); + let built; + let module: &dyn DeviceModule = match registered.as_deref() { + Some(module) => module, + None => { + built = build_device_module(settings, || { + Box::new(BuiltinDeviceModule::new()) as Box + }); + built.as_ref() + } + }; + let context = ModuleContext::new(ModuleRequest::of(req, services.client_info())) + .with_evidence(&request_info) + .with_settings(settings) + .with_services(services); + module + .detect(context.call(module.id(), module.required_permissions())) + .await +} + +/// Builds the geo-aware [`EcContext`] every route in the request cycle is +/// handed, carrying the request's device classification. /// /// The geo lookup runs inside /// [`EcContext::read_from_request_resolving_geo`], so every adapter reports the @@ -504,7 +559,73 @@ async fn build_ec_context( services: &RuntimeServices, req: &Request, ) -> Result> { - EcContext::read_from_request_resolving_geo(settings, req, services).await + let device_signals = derive_device_signals(settings, services, req).await; + let mut context = EcContext::read_from_request_resolving_geo(settings, req, services).await?; + context.set_device_signals(device_signals); + Ok(context) +} + +/// Result of the request-filter step that runs before a route. +enum PreRoute { + /// A filter answered, or failed, and that answer stands in for the + /// route's. Carries the response effects the filters left. + ShortCircuit { + response: Response, + effects: RequestFilterEffects, + }, + /// The route runs, and these response effects are applied afterwards. + Continue { effects: RequestFilterEffects }, +} + +/// Runs the integration request filters before the route. +/// +/// Request header changes are applied to `req`, so the route sees them, and +/// the response effects are returned for [`FinalizeResponseMiddleware`] to +/// apply once the response is finalized. A filter that answers, such as a +/// bot challenge, stands in for the route. Every filter reads the location +/// and the permissions resolved for the request's Edge Cookie context. +async fn run_pre_route_filters( + state: &AppState, + services: &RuntimeServices, + req: &mut Request, + ec_context: &EcContext, +) -> PreRoute { + match state + .registry + .filter_request(RequestFilterRegistryInput { + settings: &state.settings, + services, + req, + geo_info: ec_context.geo_info(), + permissions: Some(ec_context.permissions()), + }) + .await + { + Ok(RequestFilterRegistryOutcome::Continue(effects)) => PreRoute::Continue { effects }, + Ok(RequestFilterRegistryOutcome::Respond { response, effects }) => PreRoute::ShortCircuit { + response: *response, + effects, + }, + Err(report) => { + log::error!("Failed to run integration request filters: {report:?}"); + PreRoute::ShortCircuit { + response: http_error(&report), + effects: RequestFilterEffects::default(), + } + } + } +} + +/// Leaves non-empty request-filter response effects on `response`, for +/// [`FinalizeResponseMiddleware`] to apply after it finalizes the response. +fn attach_request_filter_effects( + mut response: Response, + effects: RequestFilterEffects, +) -> Response { + if !effects.response_headers.is_empty() { + response.extensions_mut().insert(effects); + } + response } // --------------------------------------------------------------------------- @@ -541,8 +662,10 @@ where } } -/// Answers the attestation pages without the request preparers, so no -/// preparer can change or refuse the request whose host and nonce are signed. +/// Answers the attestation pages without the request preparers, the Edge +/// Cookie steps or the request filters, so no preparer can change or refuse +/// the request whose host and nonce are signed, and a verifier fetching one +/// from a server is not turned away as a bot. fn attestation_route_handler( state: Arc, ) -> impl Fn(RequestContext) -> BoxedHandlerFuture + Clone + 'static { @@ -558,6 +681,290 @@ fn attestation_route_handler( } } +// --------------------------------------------------------------------------- +// Named routes +// --------------------------------------------------------------------------- + +/// The routes that run in the request cycle, each registered by +/// [`build_router`] on the paths and methods it answers. +#[derive(Clone, Copy)] +enum NamedRouteHandler { + TrustedServerDiscovery, + VerifySignature, + Auction, + PageBids, + FirstPartyProxy, + FirstPartyClick, + FirstPartySign, + FirstPartyProxyRebuild, +} + +fn named_route_handler( + state: Arc, + handler: NamedRouteHandler, +) -> impl Fn(RequestContext) -> BoxedHandlerFuture + Clone + 'static { + move |ctx: RequestContext| { + let state = Arc::clone(&state); + Box::pin(execute_named(state, ctx, handler)) as BoxedHandlerFuture + } +} + +/// Runs a named route's request through the request cycle. +/// +/// The modules' preparers run first. The route is then handed the Edge +/// Cookie context and runs once the request filters let the request +/// continue. +async fn execute_named( + state: Arc, + ctx: RequestContext, + handler: NamedRouteHandler, +) -> Result { + let services = state.services_for_request(&ctx); + let mut req = ctx.into_request(); + if let Err(error) = state.registry.prepare_request(&state.settings, &mut req) { + return Ok(http_error(&error)); + } + + let mut ec_context = match build_ec_context(&state.settings, &services, &req).await { + Ok(context) => context, + Err(report) => return Ok(http_error(&report)), + }; + let effects = match run_pre_route_filters(&state, &services, &mut req, &ec_context).await { + PreRoute::ShortCircuit { response, effects } => { + return Ok(attach_request_filter_effects(response, effects)); + } + PreRoute::Continue { effects } => effects, + }; + + let response = run_named_route(&state, &services, req, handler, &mut ec_context) + .await + .unwrap_or_else(|error| http_error(&error)); + Ok(attach_request_filter_effects(response, effects)) +} + +async fn run_named_route( + state: &AppState, + services: &RuntimeServices, + req: Request, + handler: NamedRouteHandler, + ec_context: &mut EcContext, +) -> Result> { + match handler { + NamedRouteHandler::TrustedServerDiscovery => { + handle_trusted_server_discovery(&state.settings, services, req) + } + NamedRouteHandler::VerifySignature => { + handle_verify_signature(&state.settings, services, req) + } + NamedRouteHandler::Auction => { + // The auction consent gate reads the caller's jurisdiction from + // the geo-aware context, where `EcContext::default()` would fail it + // closed for consented users. + handle_auction( + &state.settings, + &state.orchestrator, + None, + None, + ec_context, + services, + req, + ) + .await + } + NamedRouteHandler::PageBids => { + // The OPTIONS preflight is denied so the GET handler's + // `X-TSJS-Page-Bids` gate stays trustworthy. + if req.method() == Method::OPTIONS { + return Ok(page_bids_preflight_denied()); + } + let auction = AuctionDispatch { + orchestrator: &state.orchestrator, + slots: state.settings.creative_opportunity_slots(), + registry: None, + }; + handle_page_bids(&state.settings, services, None, auction, ec_context, req).await + } + NamedRouteHandler::FirstPartyProxy => { + handle_first_party_proxy(&state.settings, services, req).await + } + NamedRouteHandler::FirstPartyClick => { + handle_first_party_click(&state.settings, services, req).await + } + NamedRouteHandler::FirstPartySign => { + handle_first_party_proxy_sign(&state.settings, services, req).await + } + NamedRouteHandler::FirstPartyProxyRebuild => { + handle_first_party_proxy_rebuild(&state.settings, services, req).await + } + } +} + +// --------------------------------------------------------------------------- +// Fallback dispatcher (robots.txt / tsjs / integration proxy / publisher) +// --------------------------------------------------------------------------- + +async fn execute_fallback( + state: Arc, + ctx: RequestContext, +) -> Result { + let services = state.services_for_request(&ctx); + let mut req = ctx.into_request(); + if let Some(response) = closed_path_response(&req) { + return Ok(response); + } + if let Err(error) = state.registry.prepare_request(&state.settings, &mut req) { + return Ok(http_error(&error)); + } + Ok(dispatch_fallback(&state, &services, req).await) +} + +/// Dispatches the fallback routes: robots.txt, tsjs assets (`GET` only), +/// integration proxy routes, and the publisher origin. +/// +/// The request arrives prepared, and is not to a closed path, because +/// [`execute_fallback`] answers a closed path and runs the preparers first. +async fn dispatch_fallback( + state: &AppState, + services: &RuntimeServices, + mut req: Request, +) -> Response { + let path = req.uri().path().to_owned(); + let method = req.method().clone(); + + // A crawler asking for robots.txt is answered once the modules have + // prepared the request, before the Edge Cookie steps and the request + // filters, so nothing is read from its cookies and no filter's answer + // stands in for the file. + if asks_for_robots_txt(&state.settings, &method, &path) { + return answer_robots_txt(state, services, req, &method).await; + } + + // Identity could not be established (for example the selected Edge + // Cookie module is unavailable). Answer with an error rather than + // serving the request with no identity. + let mut ec_context = match build_ec_context(&state.settings, services, &req).await { + Ok(context) => context, + Err(report) => return http_error(&report), + }; + let effects = match run_pre_route_filters(state, services, &mut req, &ec_context).await { + PreRoute::ShortCircuit { response, effects } => { + return attach_request_filter_effects(response, effects); + } + PreRoute::Continue { effects } => effects, + }; + + // tsjs assets are served for `GET` only. + let result = if method == Method::GET && path.starts_with("/static/tsjs=") { + handle_tsjs_dynamic(&req, &state.registry, EDGE_CACHE_HEADER) + } else if state.registry.has_route(&method, &path) { + // A module route is handed what was resolved for its request, the + // permissions among them, as the publisher path is. + state + .registry + .handle_proxy(ProxyDispatchInput { + method: &method, + path: &path, + settings: &state.settings, + kv: None, + ec_context: &mut ec_context, + services, + req, + }) + .await + .unwrap_or_else(|| { + Err(Report::new(TrustedServerError::BadRequest { + message: format!("Unknown integration route: {path}"), + })) + }) + } else { + let auction = AuctionDispatch { + orchestrator: &state.orchestrator, + slots: state.settings.creative_opportunity_slots(), + registry: None, + }; + match handle_publisher_request( + AppContext { + settings: &state.settings, + integration_registry: &state.registry, + }, + services, + None, + &mut ec_context, + auction, + req, + EDGE_CACHE_HEADER, + ) + .await + { + Ok(publisher_response) => { + resolve_publisher_response( + publisher_response, + &method, + &state.settings, + &state.registry, + &state.orchestrator, + services, + ) + .await + } + Err(report) => Err(report), + } + }; + + attach_request_filter_effects(result.unwrap_or_else(|error| http_error(&error)), effects) +} + +/// Whether the request is a crawler's `GET` or `HEAD` of the robots.txt that +/// `[robots-txt]` makes. +fn asks_for_robots_txt(settings: &Settings, method: &Method, path: &str) -> bool { + settings.robots_txt.is_some() + && path == trusted_server_core::robots_txt::ROBOTS_TXT_PATH + && matches!(*method, Method::GET | Method::HEAD) +} + +/// Answers robots.txt through the publisher path with an empty Edge Cookie +/// context and no auction, so none of the crawler's cookies or consent +/// signals are read. +async fn answer_robots_txt( + state: &AppState, + services: &RuntimeServices, + req: Request, + method: &Method, +) -> Response { + let answer = handle_publisher_request( + AppContext { + settings: &state.settings, + integration_registry: &state.registry, + }, + services, + None, + &mut EcContext::default(), + AuctionDispatch { + orchestrator: &state.orchestrator, + slots: &[], + registry: None, + }, + req, + EDGE_CACHE_HEADER, + ) + .await; + let response = match answer { + Ok(answer) => { + resolve_publisher_response( + answer, + method, + &state.settings, + &state.registry, + &state.orchestrator, + services, + ) + .await + } + Err(report) => Err(report), + }; + response.unwrap_or_else(|report| http_error(&report)) +} + // --------------------------------------------------------------------------- // Publisher response helper // --------------------------------------------------------------------------- @@ -772,104 +1179,16 @@ fn build_router(state: &Arc) -> RouterService { { let state = Arc::clone(state); - // Shared fallback dispatch: routes to tsjs (GET only), integration proxy, or publisher. - async fn dispatch( - state: Arc, - ctx: RequestContext, - ) -> Result { - let services = state.services_for_request(&ctx); - let mut req = ctx.into_request(); - if let Some(response) = closed_path_response(&req) { - return Ok(response); - } - if let Err(error) = state.registry.prepare_request(&state.settings, &mut req) { - return Ok(http_error(&error)); - } - let path = req.uri().path().to_owned(); - let method = req.method().clone(); - // tsjs assets are served for GET only, matching the Axum/Fastly adapters. - let allow_tsjs = method == Method::GET; - - let result = if allow_tsjs && path.starts_with("/static/tsjs=") { - handle_tsjs_dynamic(&req, &state.registry, EDGE_CACHE_HEADER) - } else if state.registry.has_route(&method, &path) { - // A module route is handed what was resolved for its request, - // the permissions among them, as the publisher path is. - let mut ec_context = match build_ec_context(&state.settings, &services, &req).await - { - Ok(context) => context, - Err(report) => return Ok(http_error(&report)), - }; - state - .registry - .handle_proxy(ProxyDispatchInput { - method: &method, - path: &path, - settings: &state.settings, - kv: None, - ec_context: &mut ec_context, - services: &services, - req, - }) - .await - .unwrap_or_else(|| { - Err(Report::new(TrustedServerError::BadRequest { - message: format!("Unknown integration route: {path}"), - })) - }) - } else { - // Identity could not be established (for example the selected - // Edge Cookie module is unavailable). Answer with an error - // rather than serving the page with no identity. - let mut ec_context = match build_ec_context(&state.settings, &services, &req).await - { - Ok(context) => context, - Err(report) => return Ok(http_error(&report)), - }; - let auction = AuctionDispatch { - orchestrator: &state.orchestrator, - slots: state.settings.creative_opportunity_slots(), - registry: None, - }; - match handle_publisher_request( - AppContext { - settings: &state.settings, - integration_registry: &state.registry, - }, - &services, - None, - &mut ec_context, - auction, - req, - EDGE_CACHE_HEADER, - ) - .await - { - Ok(pr) => { - resolve_publisher_response( - pr, - &method, - &state.settings, - &state.registry, - &state.orchestrator, - &services, - ) - .await - } - Err(e) => Err(e), - } - }; - - Ok(result.unwrap_or_else(|e| http_error(&e))) - } - + // Shared fallback dispatch: routes to robots.txt, tsjs (GET only), + // integration proxy, or publisher. let fallback = { let s = Arc::clone(&state); move |ctx: RequestContext| { let s = Arc::clone(&s); - dispatch(s, ctx) + execute_fallback(s, ctx) } }; + let named = |handler: NamedRouteHandler| named_route_handler(Arc::clone(&state), handler); let mut router = RouterService::builder() // Outermost middleware: strips the configured trusted-client-IP @@ -880,58 +1199,28 @@ fn build_router(state: &Arc) -> RouterService { .middleware(FinalizeResponseMiddleware::new(Arc::clone(&state.settings))) .get( "/.well-known/trusted-server.json", - make_handler(Arc::clone(&state), |s, services, req| async move { - handle_trusted_server_discovery(&s.settings, &services, req) - }), + named(NamedRouteHandler::TrustedServerDiscovery), ) .post( "/verify-signature", - make_handler(Arc::clone(&state), |s, services, req| async move { - handle_verify_signature(&s.settings, &services, req) - }), - ) - .post( - "/auction", - make_handler(Arc::clone(&state), |s, services, req| async move { - // Build the geo-aware EC context so the auction consent gate - // sees the caller's jurisdiction — `EcContext::default()` - // fails it closed for consented users. - let mut ec_context = build_ec_context(&s.settings, &services, &req).await?; - handle_auction( - &s.settings, - &s.orchestrator, - None, - None, - &mut ec_context, - &services, - req, - ) - .await - }), + named(NamedRouteHandler::VerifySignature), ) + .post("/auction", named(NamedRouteHandler::Auction)) .get( "/first-party/proxy", - make_handler(Arc::clone(&state), |s, services, req| async move { - handle_first_party_proxy(&s.settings, &services, req).await - }), + named(NamedRouteHandler::FirstPartyProxy), ) .get( "/first-party/click", - make_handler(Arc::clone(&state), |s, services, req| async move { - handle_first_party_click(&s.settings, &services, req).await - }), + named(NamedRouteHandler::FirstPartyClick), ) .get( "/first-party/sign", - make_handler(Arc::clone(&state), |s, services, req| async move { - handle_first_party_proxy_sign(&s.settings, &services, req).await - }), + named(NamedRouteHandler::FirstPartySign), ) .post( "/first-party/sign", - make_handler(Arc::clone(&state), |s, services, req| async move { - handle_first_party_proxy_sign(&s.settings, &services, req).await - }), + named(NamedRouteHandler::FirstPartySign), ) // GET serves the click guard's navigation fallback: the creative // iframe is an opaque origin (sandbox without `allow-same-origin`), @@ -939,15 +1228,11 @@ fn build_router(state: &Arc) -> RouterService { // for a 302 instead. .get( "/first-party/proxy-rebuild", - make_handler(Arc::clone(&state), |s, services, req| async move { - handle_first_party_proxy_rebuild(&s.settings, &services, req).await - }), + named(NamedRouteHandler::FirstPartyProxyRebuild), ) .post( "/first-party/proxy-rebuild", - make_handler(Arc::clone(&state), |s, services, req| async move { - handle_first_party_proxy_rebuild(&s.settings, &services, req).await - }), + named(NamedRouteHandler::FirstPartyProxyRebuild), ); // SPA re-auction endpoint, registered on the canonical path and on the @@ -959,22 +1244,10 @@ fn build_router(state: &Arc) -> RouterService { // `X-TSJS-Page-Bids` gate stays trustworthy — an alias that let the // preflight fall through to a permissive origin would reopen exactly // the cross-site hole the canonical path closes. - let page_bids = make_handler(Arc::clone(&state), |s, services, req| async move { - let mut ec_context = build_ec_context(&s.settings, &services, &req).await?; - let auction = AuctionDispatch { - orchestrator: &s.orchestrator, - slots: s.settings.creative_opportunity_slots(), - registry: None, - }; - handle_page_bids(&s.settings, &services, None, auction, &mut ec_context, req).await - }); - let page_bids_preflight = - make_handler(Arc::clone(&state), |_s, _services, _req| async move { - Ok(page_bids_preflight_denied()) - }); + let page_bids = named(NamedRouteHandler::PageBids); for path in [PAGE_BIDS_PATH, PAGE_BIDS_LEGACY_PATH] { router = router.route(path, Method::GET, page_bids.clone()); - router = router.route(path, Method::OPTIONS, page_bids_preflight.clone()); + router = router.route(path, Method::OPTIONS, page_bids.clone()); } // What the deployment decided for the asking request, shown to anyone @@ -1027,6 +1300,9 @@ fn build_router(state: &Arc) -> RouterService { } } +#[cfg(test)] +mod request_cycle_tests; + #[cfg(test)] mod tests { use edgezero_core::http::request_builder; diff --git a/crates/trusted-server-adapter-cloudflare/src/app/request_cycle_tests.rs b/crates/trusted-server-adapter-cloudflare/src/app/request_cycle_tests.rs new file mode 100644 index 0000000000..5f5af49602 --- /dev/null +++ b/crates/trusted-server-adapter-cloudflare/src/app/request_cycle_tests.rs @@ -0,0 +1,405 @@ +//! The request cycle this adapter runs for a routed request, checked through +//! its router: the request filters run before the route and their response +//! effects after the finalize headers, the route's module call carries the +//! device classification, and robots.txt is answered ahead of both. + +use std::sync::Arc; + +use edgezero_core::body::Body; +use edgezero_core::http::{Request, Response, StatusCode, request_builder}; +use edgezero_core::router::RouterService; +use error_stack::Report; +use trusted_server_core::ec::device::{DeviceModule, DeviceSignals}; +use trusted_server_core::error::TrustedServerError; +use trusted_server_core::evidence::RequestInfo; +use trusted_server_core::integrations::{ + HeaderMutation, IntegrationBuilder, IntegrationEndpoint, IntegrationProxy, + IntegrationRegistration, IntegrationRequestFilter, RequestFilterDecision, RequestFilterEffects, + RequestFilterInput, +}; +use trusted_server_core::module_context::ModuleCall; +use trusted_server_core::platform::PlatformHttpClient; +use trusted_server_core::platform::test_support::{ + StubHttpClient, build_services_with_http_client, +}; +use trusted_server_core::settings::Settings; + +use super::{build_router, build_state_with_registrations_and_services}; + +/// The id and module name of the probe, which registers a filter that lets +/// the request continue, a device module and a route reporting what its +/// module call was handed. +const PROBE_ID: &str = "cycle_probe"; +const PROBE_MODULE: &str = "testing.cycle-probe"; + +/// The id and module name of a filter that answers every request. +const ANSWERING_ID: &str = "answering_filter"; +const ANSWERING_MODULE: &str = "testing.answering-filter"; + +/// The response header both filters write as an effect. +const EFFECT_HEADER: &str = "x-cycle-probe"; +/// The request header the continuing filter adds. +const FILTERED_HEADER: &str = "x-cycle-probe-filtered"; +/// What the answering filter answers. +const ANSWER: &str = "answered by the filter"; + +struct AnsweringFilter; + +#[async_trait::async_trait(?Send)] +impl IntegrationRequestFilter for AnsweringFilter { + fn integration_id(&self) -> &'static str { + ANSWERING_ID + } + + async fn filter_request( + &self, + _input: RequestFilterInput<'_>, + ) -> Result> { + let mut response = Response::new(Body::from(ANSWER)); + *response.status_mut() = StatusCode::FORBIDDEN; + Ok(RequestFilterDecision::Respond { + response: Box::new(response), + effects: RequestFilterEffects { + request_headers: Vec::new(), + response_headers: vec![HeaderMutation::set(EFFECT_HEADER, "answered")], + }, + }) + } +} + +struct ContinuingFilter; + +#[async_trait::async_trait(?Send)] +impl IntegrationRequestFilter for ContinuingFilter { + fn integration_id(&self) -> &'static str { + PROBE_ID + } + + async fn filter_request( + &self, + _input: RequestFilterInput<'_>, + ) -> Result> { + Ok(RequestFilterDecision::Continue(RequestFilterEffects { + request_headers: vec![HeaderMutation::set(FILTERED_HEADER, "1")], + response_headers: vec![HeaderMutation::set(EFFECT_HEADER, "continued")], + })) + } +} + +/// A device module that marks its classification with the cookies it was +/// shown, so a test can tell that it ran and what it saw. +struct MarkingDevice; + +impl MarkingDevice { + fn mark(&self, request_info: &dyn RequestInfo) -> DeviceSignals { + let mut signals = DeviceSignals::derive_ua_only(request_info.user_agent()); + signals.platform_class = Some(format!( + "marked:{}", + request_info.header("cookie").unwrap_or_default() + )); + signals + } +} + +#[async_trait::async_trait(?Send)] +impl DeviceModule for MarkingDevice { + fn id(&self) -> &'static str { + PROBE_MODULE + } + + async fn detect(&self, call: ModuleCall<'_>) -> DeviceSignals { + call.inject(self, Self::mark) + .unwrap_or_else(|_| DeviceSignals::unknown()) + } +} + +/// A route answering with the device classification its module call was +/// handed and whether the continuing filter's request header reached it. +struct ReportRoute; + +impl ReportRoute { + fn report( + &self, + req: Request, + device: Option<&DeviceSignals>, + ) -> Result> { + let (parts, _body) = req.into_parts(); + let report = serde_json::json!({ + "device": device.and_then(|signals| signals.platform_class.clone()), + "filtered": parts + .headers + .get(FILTERED_HEADER) + .and_then(|value| value.to_str().ok()), + }); + Ok(Response::new(Body::from(report.to_string()))) + } +} + +#[async_trait::async_trait(?Send)] +impl IntegrationProxy for ReportRoute { + fn integration_name(&self) -> &'static str { + PROBE_ID + } + + fn routes(&self) -> Vec { + vec![self.get("/report")] + } + + async fn handle( + &self, + call: ModuleCall<'_>, + req: Request, + ) -> Result> { + call.inject_with(self, req, Self::report)? + } +} + +fn register_probe( + _settings: &Settings, +) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder(PROBE_ID) + .without_js() + .with_request_filter(Arc::new(ContinuingFilter)) + .with_device_module(PROBE_MODULE, Arc::new(MarkingDevice)) + .with_proxy(Arc::new(ReportRoute)) + .build(), + )) +} + +fn register_answering( + _settings: &Settings, +) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder(ANSWERING_ID) + .without_js() + .with_request_filter(Arc::new(AnsweringFilter)) + .build(), + )) +} + +fn always(_settings: &Settings) -> Result> { + Ok(true) +} + +fn probe_builder() -> IntegrationBuilder { + IntegrationBuilder::new(PROBE_ID, "adapter-tests", register_probe, always) + .with_module_name(PROBE_MODULE) +} + +fn answering_builder() -> IntegrationBuilder { + IntegrationBuilder::new(ANSWERING_ID, "adapter-tests", register_answering, always) + .with_module_name(ANSWERING_MODULE) +} + +/// Settings for the publisher the requests are for, with `extra` appended. +fn settings_with(extra: &str) -> Settings { + Settings::from_toml(&format!( + r#" + [source] + module = "origin" + + [source.origin] + url = "https://origin.test-publisher.example.com" + + [publisher] + domain = "test-publisher.example.com" + cookie_domain = ".test-publisher.example.com" + proxy_secret = "request-cycle-test-proxy-secret" + + [ec] + module = "hmac" + + [ec.hmac] + passphrase = "test-secret-key-32-bytes-minimum" + + [geo] + assume_single_jurisdiction = true + + {extra} + "# + )) + .expect("should parse the request cycle test settings") +} + +/// The router built through the composition root with `builders`, every +/// request sent to `origin`. +fn router_with( + settings: Settings, + builders: &[IntegrationBuilder], + origin: &Arc, +) -> RouterService { + let state = build_state_with_registrations_and_services( + settings, + builders, + Some(build_services_with_http_client( + Arc::clone(origin) as Arc + )), + ) + .expect("should build the state"); + build_router(&state) +} + +fn get(path: &str, cookie: Option<&str>) -> Request { + let mut builder = request_builder() + .method("GET") + .uri(format!("https://test-publisher.example.com{path}")) + .header("host", "test-publisher.example.com"); + if let Some(cookie) = cookie { + builder = builder.header("cookie", cookie); + } + builder + .body(Body::empty()) + .expect("should build the request") +} + +async fn route(router: &RouterService, request: Request) -> Response { + router + .oneshot(request) + .await + .expect("should route the request") +} + +fn header<'a>(response: &'a Response, name: &str) -> Option<&'a str> { + response + .headers() + .get(name) + .and_then(|value| value.to_str().ok()) +} + +fn text(response: Response) -> String { + let bytes = response + .into_body() + .into_bytes() + .expect("should hold the body in memory"); + String::from_utf8(bytes.to_vec()).expect("the body should be text") +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_filter_that_answers_stands_in_for_the_page() { + let origin = Arc::new(StubHttpClient::new()); + let router = router_with( + settings_with("[testing]\nmodules = [\"answering-filter\"]"), + &[answering_builder()], + &origin, + ); + + let response = route(&router, get("/article", None)).await; + + assert_eq!( + response.status(), + StatusCode::FORBIDDEN, + "the filter's answer should stand in for the page" + ); + assert_eq!( + header(&response, EFFECT_HEADER), + Some("answered"), + "the effects the answering filter left should be applied" + ); + assert_eq!(text(response), ANSWER); + assert!( + origin.recorded_request_uris().is_empty(), + "the publisher's origin should not be asked" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_filter_s_request_header_reaches_the_route_and_its_effect_is_written_last() { + let origin = Arc::new(StubHttpClient::new()); + let mut settings = settings_with("[testing]\nmodules = [\"cycle-probe\"]"); + settings + .response_headers + .insert(EFFECT_HEADER.to_owned(), "operator".to_owned()); + let router = router_with(settings, &[probe_builder()], &origin); + + let response = route(&router, get("/integrations/cycle_probe/report", None)).await; + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + header(&response, EFFECT_HEADER), + Some("continued"), + "the filter's response effect should be written after the operator's headers" + ); + let report: serde_json::Value = + serde_json::from_str(&text(response)).expect("the report should be JSON"); + assert_eq!( + report["filtered"], "1", + "the filter's request header should reach the route" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn the_device_module_s_classification_reaches_the_module_call() { + let origin = Arc::new(StubHttpClient::new()); + let router = router_with( + settings_with(&format!( + "[device]\nmodule = \"{PROBE_MODULE}\"\n\n[testing]\nmodules = [\"cycle-probe\"]" + )), + &[probe_builder()], + &origin, + ); + + let response = route( + &router, + get( + "/integrations/cycle_probe/report", + Some("example_width=1280"), + ), + ) + .await; + + assert_eq!(response.status(), StatusCode::OK); + let report: serde_json::Value = + serde_json::from_str(&text(response)).expect("the report should be JSON"); + assert_eq!( + report["device"], "marked:example_width=1280", + "the selected device module should classify the request, shown its cookies" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn robots_txt_is_answered_before_the_request_filters() { + let origin = Arc::new(StubHttpClient::new()); + origin.push_response_with_headers( + 200, + b"User-agent: *\nAllow: /\n".to_vec(), + vec![("content-type", "text/plain")], + ); + let router = router_with( + settings_with( + r#" + [testing] + modules = ["answering-filter"] + + [robots-txt] + modules = "refuse_all" + always_allow = ["/ads.txt"] + + [[fetch]] + media_type = "text/plain" + path = "/robots.txt" + middleware = ["robots_txt"] + "#, + ), + &[answering_builder()], + &origin, + ); + + let response = route(&router, get("/robots.txt", None)).await; + + assert_eq!( + response.status(), + StatusCode::OK, + "no filter's answer should stand in for robots.txt" + ); + assert_eq!( + header(&response, EFFECT_HEADER), + None, + "the filters should not run for robots.txt" + ); + assert_eq!( + text(response), + "User-agent: *\nAllow: /ads.txt\nDisallow: /\n" + ); + assert_eq!(origin.recorded_request_methods(), ["GET"]); +} diff --git a/crates/trusted-server-adapter-cloudflare/src/middleware.rs b/crates/trusted-server-adapter-cloudflare/src/middleware.rs index 49f77502f0..85a12160c3 100644 --- a/crates/trusted-server-adapter-cloudflare/src/middleware.rs +++ b/crates/trusted-server-adapter-cloudflare/src/middleware.rs @@ -8,6 +8,11 @@ use edgezero_core::middleware::{Middleware, Next}; use trusted_server_core::http_util::{ ROUTE_NAMESPACE, sanitize_trusted_client_ip_headers, strip_deployment_headers, }; +use trusted_server_core::integrations::RequestFilterEffects; +use trusted_server_core::response_privacy::{ + TerminalPrivateResponse, enforce_private_no_store, enforce_set_cookie_cache_privacy, + enforce_uncacheable_cache_privacy, +}; use trusted_server_core::settings::Settings; // --------------------------------------------------------------------------- @@ -53,7 +58,8 @@ impl Middleware for SanitizeRequestMiddleware { /// Response-finalization middleware: injects all standard TS response headers. /// /// Registered directly inside [`SanitizeRequestMiddleware`], so that every -/// outgoing response carries a consistent set of headers. +/// outgoing response carries a consistent set of headers. Once the response +/// is finalized it applies the response effects the request filters left. pub struct FinalizeResponseMiddleware { settings: Arc, } @@ -71,13 +77,37 @@ impl Middleware for FinalizeResponseMiddleware { async fn handle(&self, ctx: RequestContext, next: Next<'_>) -> Result { let path = ctx.request().uri().path().to_owned(); let mut response = next.run(ctx).await?; + let request_filter_effects = response.extensions_mut().remove::(); // Before finalizing, so a header finalizing writes is kept. strip_deployment_headers(&path, ROUTE_NAMESPACE, &mut response); apply_finalize_headers(&self.settings, &mut response); + apply_terminal_response_effects(&mut response, request_filter_effects.as_ref()); Ok(response) } } +/// Applies the request filters' response effects to a finalized response, +/// then restores the privacy guards a late header could loosen. +fn apply_terminal_response_effects( + response: &mut Response, + request_filter_effects: Option<&RequestFilterEffects>, +) { + let must_remain_private = response + .extensions() + .get::() + .is_some(); + if let Some(effects) = request_filter_effects { + effects.apply_to_response(response); + } + if must_remain_private { + enforce_private_no_store(response); + } + // A filter's header can add a per-user `Set-Cookie`, or a `private` or + // `no-store` directive, after the finalize headers ran. + enforce_set_cookie_cache_privacy(response); + enforce_uncacheable_cache_privacy(response); +} + // --------------------------------------------------------------------------- // apply_finalize_headers — extracted for unit testing // --------------------------------------------------------------------------- diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index e07091bf78..8d8cf97cd7 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -10,7 +10,9 @@ use edgezero_core::app::Hooks; use edgezero_core::config_store::ConfigStoreHandle; use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; -use edgezero_core::http::{HeaderValue, Method, Request, Response, StatusCode, header}; +use edgezero_core::http::{ + HandlerFuture, HeaderMap, HeaderValue, Method, Request, Response, StatusCode, header, +}; use edgezero_core::router::RouterService; use error_stack::Report; use trusted_server_core::addressing::{RequestAddressing, addressed_router}; @@ -25,8 +27,12 @@ use trusted_server_core::closed_paths::closed_path_response; #[cfg(all(feature = "spin", target_arch = "wasm32"))] use trusted_server_core::config_payload::settings_from_config_blob_with; use trusted_server_core::ec::EcContext; +use trusted_server_core::ec::device::{ + BuiltinDeviceModule, DeviceModule, DeviceSignals, build_device_module, +}; use trusted_server_core::ec::module::{EdgeCookieModule, build_reusable_module}; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; +use trusted_server_core::evidence::BorrowedRequestInfo; use trusted_server_core::http_util::sanitize_forwarded_headers; use trusted_server_core::inspect::config::{CONFIG_JSON_PATH, CONFIG_PAGE_PATH, handle_config}; use trusted_server_core::inspect::data::{DATA_PAGE_PATH, handle_data}; @@ -34,8 +40,10 @@ use trusted_server_core::inspect::permissions::{ PERMISSIONS_JSON_PATH, PERMISSIONS_PAGE_PATH, handle_permissions, }; use trusted_server_core::integrations::{ - IntegrationBuilder, IntegrationRegistry, ProxyDispatchInput, + IntegrationBuilder, IntegrationRegistry, ProxyDispatchInput, RequestFilterEffects, + RequestFilterRegistryInput, RequestFilterRegistryOutcome, }; +use trusted_server_core::module_context::{ModuleContext, ModuleRequest}; use trusted_server_core::platform::RuntimeServices; #[cfg(all(feature = "spin", target_arch = "wasm32"))] use trusted_server_core::platform::{PlatformConfigStore, StoreName}; @@ -643,8 +651,61 @@ fn health_response() -> Response { resp } -/// Builds the geo-aware [`EcContext`] for consent-gated endpoints (`/auction`, -/// `/_ts/page-bids`, and the publisher fallback). +// --------------------------------------------------------------------------- +// Request cycle +// --------------------------------------------------------------------------- + +/// Classifies the request with the device module `[device]` selects. +/// +/// A module a registration supplies, which the request's services carry, is +/// shown the `User-Agent` and the cookies, which carry what a page gathers in +/// the browser. Otherwise [`build_device_module`] gives the built-in module, +/// which reads only the `User-Agent`. This adapter supplies no TLS or HTTP/2 +/// signals, so `platform` classifies as the built-in module does. The module +/// is handed a module context carrying the request, that evidence, the +/// settings and the services. +async fn derive_device_signals( + settings: &Settings, + services: &RuntimeServices, + req: &Request, +) -> DeviceSignals { + let registered = services.device_module(); + let mut headers = HeaderMap::new(); + for (name, shown) in [ + (header::USER_AGENT, true), + (header::COOKIE, registered.is_some()), + ] { + if shown && let Some(value) = req.headers().get(&name) { + headers.insert(name, value.clone()); + } + } + let client_ip = services + .client_info() + .client_ip + .map(|ip| ip.to_string()) + .unwrap_or_default(); + let request_info = BorrowedRequestInfo::new(&client_ip, None).with_headers(&headers); + let built; + let module: &dyn DeviceModule = match registered.as_deref() { + Some(module) => module, + None => { + built = build_device_module(settings, || { + Box::new(BuiltinDeviceModule::new()) as Box + }); + built.as_ref() + } + }; + let context = ModuleContext::new(ModuleRequest::of(req, services.client_info())) + .with_evidence(&request_info) + .with_settings(settings) + .with_services(services); + module + .detect(context.call(module.id(), module.required_permissions())) + .await +} + +/// Builds the geo-aware [`EcContext`] every route in the request cycle is +/// handed, carrying the request's device classification. /// /// The geo lookup runs inside /// [`EcContext::read_from_request_resolving_geo`], so every adapter reports the @@ -670,7 +731,73 @@ async fn build_ec_context( services: &RuntimeServices, req: &Request, ) -> Result> { - EcContext::read_from_request_resolving_geo(settings, req, services).await + let device_signals = derive_device_signals(settings, services, req).await; + let mut context = EcContext::read_from_request_resolving_geo(settings, req, services).await?; + context.set_device_signals(device_signals); + Ok(context) +} + +/// Result of the request-filter step that runs before a route. +enum PreRoute { + /// A filter answered, or failed, and that answer stands in for the + /// route's. Carries the response effects the filters left. + ShortCircuit { + response: Response, + effects: RequestFilterEffects, + }, + /// The route runs, and these response effects are applied afterwards. + Continue { effects: RequestFilterEffects }, +} + +/// Runs the integration request filters before the route. +/// +/// Request header changes are applied to `req`, so the route sees them, and +/// the response effects are returned for [`FinalizeResponseMiddleware`] to +/// apply once the response is finalized. A filter that answers, such as a +/// bot challenge, stands in for the route. Every filter reads the location +/// and the permissions resolved for the request's Edge Cookie context. +async fn run_pre_route_filters( + state: &AppState, + services: &RuntimeServices, + req: &mut Request, + ec_context: &EcContext, +) -> PreRoute { + match state + .registry + .filter_request(RequestFilterRegistryInput { + settings: &state.settings, + services, + req, + geo_info: ec_context.geo_info(), + permissions: Some(ec_context.permissions()), + }) + .await + { + Ok(RequestFilterRegistryOutcome::Continue(effects)) => PreRoute::Continue { effects }, + Ok(RequestFilterRegistryOutcome::Respond { response, effects }) => PreRoute::ShortCircuit { + response: *response, + effects, + }, + Err(report) => { + log::error!("Failed to run integration request filters: {report:?}"); + PreRoute::ShortCircuit { + response: http_error(&report), + effects: RequestFilterEffects::default(), + } + } + } +} + +/// Leaves non-empty request-filter response effects on `response`, for +/// [`FinalizeResponseMiddleware`] to apply after it finalizes the response. +fn attach_request_filter_effects( + mut response: Response, + effects: RequestFilterEffects, +) -> Response { + if !effects.response_headers.is_empty() { + response.extensions_mut().insert(effects); + } + response } // --------------------------------------------------------------------------- @@ -692,6 +819,293 @@ pub(crate) fn http_error(report: &Report) -> Response { response } +// --------------------------------------------------------------------------- +// Named routes +// --------------------------------------------------------------------------- + +/// The routes that run in the request cycle, each registered by +/// [`build_router`] on the paths and methods it answers. +#[derive(Clone, Copy)] +enum NamedRouteHandler { + TrustedServerDiscovery, + VerifySignature, + Auction, + PageBids, + FirstPartyProxy, + FirstPartyClick, + FirstPartySign, + FirstPartyProxyRebuild, +} + +fn named_route_handler( + state: Arc, + handler: NamedRouteHandler, +) -> impl Fn(RequestContext) -> HandlerFuture + Clone + 'static { + move |ctx: RequestContext| { + let state = Arc::clone(&state); + Box::pin(execute_named(state, ctx, handler)) as HandlerFuture + } +} + +/// Runs a named route's request through the request cycle. +/// +/// The modules' preparers run first. The route is then handed the Edge +/// Cookie context and runs once the request filters let the request +/// continue. +async fn execute_named( + state: Arc, + ctx: RequestContext, + handler: NamedRouteHandler, +) -> Result { + let services = state.services_for_request(&ctx); + let mut req = ctx.into_request(); + if let Err(error) = state.registry.prepare_request(&state.settings, &mut req) { + return Ok(http_error(&error)); + } + + let mut ec_context = match build_ec_context(&state.settings, &services, &req).await { + Ok(context) => context, + Err(report) => return Ok(http_error(&report)), + }; + let effects = match run_pre_route_filters(&state, &services, &mut req, &ec_context).await { + PreRoute::ShortCircuit { response, effects } => { + return Ok(attach_request_filter_effects(response, effects)); + } + PreRoute::Continue { effects } => effects, + }; + + let response = run_named_route(&state, &services, req, handler, &mut ec_context) + .await + .unwrap_or_else(|error| http_error(&error)); + Ok(attach_request_filter_effects(response, effects)) +} + +async fn run_named_route( + state: &AppState, + services: &RuntimeServices, + req: Request, + handler: NamedRouteHandler, + ec_context: &mut EcContext, +) -> Result> { + match handler { + NamedRouteHandler::TrustedServerDiscovery => { + handle_trusted_server_discovery(&state.settings, services, req) + } + NamedRouteHandler::VerifySignature => { + handle_verify_signature(&state.settings, services, req) + } + NamedRouteHandler::Auction => { + // The auction consent gate reads the caller's jurisdiction from + // the geo-aware context, where `EcContext::default()` would fail it + // closed for consented users. `NormalizeMiddleware` has already + // derived the trusted Host, scheme and client IP, so the signed + // OpenRTB metadata that auction signing derives from + // `RequestInfo::from_request` uses the trusted runtime authority. + handle_auction( + &state.settings, + &state.orchestrator, + None, + None, + ec_context, + services, + req, + ) + .await + } + NamedRouteHandler::PageBids => { + // The OPTIONS preflight is denied so the GET handler's + // `X-TSJS-Page-Bids` gate stays trustworthy. + if req.method() == Method::OPTIONS { + return Ok(page_bids_preflight_denied()); + } + let auction = AuctionDispatch { + orchestrator: &state.orchestrator, + slots: state.settings.creative_opportunity_slots(), + registry: None, + }; + handle_page_bids(&state.settings, services, None, auction, ec_context, req).await + } + NamedRouteHandler::FirstPartyProxy => { + handle_first_party_proxy(&state.settings, services, req).await + } + NamedRouteHandler::FirstPartyClick => { + handle_first_party_click(&state.settings, services, req).await + } + NamedRouteHandler::FirstPartySign => { + handle_first_party_proxy_sign(&state.settings, services, req).await + } + NamedRouteHandler::FirstPartyProxyRebuild => { + handle_first_party_proxy_rebuild(&state.settings, services, req).await + } + } +} + +// --------------------------------------------------------------------------- +// Fallback dispatcher (robots.txt / tsjs / integration proxy / publisher) +// --------------------------------------------------------------------------- + +async fn execute_fallback( + state: Arc, + ctx: RequestContext, +) -> Result { + let services = state.services_for_request(&ctx); + let mut req = ctx.into_request(); + if let Some(response) = closed_path_response(&req) { + return Ok(response); + } + if let Err(error) = state.registry.prepare_request(&state.settings, &mut req) { + return Ok(http_error(&error)); + } + Ok(dispatch_fallback(&state, &services, req).await) +} + +/// Dispatches the fallback routes: robots.txt, tsjs assets (`GET` only), +/// integration proxy routes, and the publisher origin. +/// +/// The request arrives prepared, and is not to a closed path, because +/// [`execute_fallback`] answers a closed path and runs the preparers first. +async fn dispatch_fallback( + state: &AppState, + services: &RuntimeServices, + mut req: Request, +) -> Response { + let path = req.uri().path().to_owned(); + let method = req.method().clone(); + + // A crawler asking for robots.txt is answered once the modules have + // prepared the request, before the Edge Cookie steps and the request + // filters, so nothing is read from its cookies and no filter's answer + // stands in for the file. + if asks_for_robots_txt(&state.settings, &method, &path) { + return answer_robots_txt(state, services, req, &method).await; + } + + // Identity could not be established (for example the selected Edge + // Cookie module is unavailable). Answer with an error rather than + // serving the request with no identity. + let mut ec_context = match build_ec_context(&state.settings, services, &req).await { + Ok(context) => context, + Err(report) => return http_error(&report), + }; + let effects = match run_pre_route_filters(state, services, &mut req, &ec_context).await { + PreRoute::ShortCircuit { response, effects } => { + return attach_request_filter_effects(response, effects); + } + PreRoute::Continue { effects } => effects, + }; + + // tsjs assets are served for `GET` only. + let result = if method == Method::GET && path.starts_with("/static/tsjs=") { + handle_tsjs_dynamic(&req, &state.registry, EDGE_CACHE_HEADER) + } else if state.registry.has_route(&method, &path) { + // A module route is handed what was resolved for its request, the + // permissions among them, as the publisher path is. + state + .registry + .handle_proxy(ProxyDispatchInput { + method: &method, + path: &path, + settings: &state.settings, + kv: None, + ec_context: &mut ec_context, + services, + req, + }) + .await + .unwrap_or_else(|| { + Err(Report::new(TrustedServerError::BadRequest { + message: format!("Unknown integration route: {path}"), + })) + }) + } else { + let auction = AuctionDispatch { + orchestrator: &state.orchestrator, + slots: state.settings.creative_opportunity_slots(), + registry: None, + }; + match handle_publisher_request( + AppContext { + settings: &state.settings, + integration_registry: &state.registry, + }, + services, + None, + &mut ec_context, + auction, + req, + EDGE_CACHE_HEADER, + ) + .await + { + Ok(publisher_response) => { + resolve_publisher_response( + publisher_response, + &method, + &state.settings, + &state.registry, + &state.orchestrator, + services, + ) + .await + } + Err(report) => Err(report), + } + }; + + attach_request_filter_effects(result.unwrap_or_else(|error| http_error(&error)), effects) +} + +/// Whether the request is a crawler's `GET` or `HEAD` of the robots.txt that +/// `[robots-txt]` makes. +fn asks_for_robots_txt(settings: &Settings, method: &Method, path: &str) -> bool { + settings.robots_txt.is_some() + && path == trusted_server_core::robots_txt::ROBOTS_TXT_PATH + && matches!(*method, Method::GET | Method::HEAD) +} + +/// Answers robots.txt through the publisher path with an empty Edge Cookie +/// context and no auction, so none of the crawler's cookies or consent +/// signals are read. +async fn answer_robots_txt( + state: &AppState, + services: &RuntimeServices, + req: Request, + method: &Method, +) -> Response { + let answer = handle_publisher_request( + AppContext { + settings: &state.settings, + integration_registry: &state.registry, + }, + services, + None, + &mut EcContext::default(), + AuctionDispatch { + orchestrator: &state.orchestrator, + slots: &[], + registry: None, + }, + req, + EDGE_CACHE_HEADER, + ) + .await; + let response = match answer { + Ok(answer) => { + resolve_publisher_response( + answer, + method, + &state.settings, + &state.registry, + &state.orchestrator, + services, + ) + .await + } + Err(report) => Err(report), + }; + response.unwrap_or_else(|report| http_error(&report)) +} + // --------------------------------------------------------------------------- // Startup error fallback // --------------------------------------------------------------------------- @@ -849,18 +1263,6 @@ fn build_router(state: &Arc) -> RouterService { { let state = Arc::clone(state); - // /.well-known/trusted-server.json - let s = Arc::clone(&state); - let discovery_handler = move |ctx: RequestContext| { - let s = Arc::clone(&s); - async move { - let services = s.services_for_request(&ctx); - let req = ctx.into_request(); - Ok(handle_trusted_server_discovery(&s.settings, &services, req) - .unwrap_or_else(|e| http_error(&e))) - } - }; - // /_ts/permissions and /_ts/permissions.json, what the deployment // decided for the asking request, shown to anyone. Read only, so // nothing is written for the reader. @@ -900,251 +1302,17 @@ fn build_router(state: &Arc) -> RouterService { } }; - // /verify-signature - let s = Arc::clone(&state); - let verify_handler = move |ctx: RequestContext| { - let s = Arc::clone(&s); - async move { - let services = s.services_for_request(&ctx); - let req = ctx.into_request(); - Ok(handle_verify_signature(&s.settings, &services, req) - .unwrap_or_else(|e| http_error(&e))) - } - }; - - // /auction - let s = Arc::clone(&state); - let auction_handler = move |ctx: RequestContext| { - let s = Arc::clone(&s); - async move { - let services = s.services_for_request(&ctx); - // Request normalization (forwarded-header stripping, trusted - // Host/scheme/client-IP derivation) is applied centrally by - // `NormalizeMiddleware` before this handler runs, so the signed - // OpenRTB metadata that auction signing derives from - // `RequestInfo::from_request` uses the trusted runtime authority. - let mut req = ctx.into_request(); - if let Err(error) = s.registry.prepare_request(&s.settings, &mut req) { - return Ok(http_error(&error)); - } - // Build the geo-aware EC context so the auction consent gate sees - // the caller's jurisdiction — `EcContext::default()` fails it - // closed for consented users. When identity cannot be - // established at all, answer with an error rather than running - // the auction with no identity. - let mut ec_context = match build_ec_context(&s.settings, &services, &req).await { - Ok(context) => context, - Err(report) => return Ok(http_error(&report)), - }; - Ok(handle_auction( - &s.settings, - &s.orchestrator, - None, - None, - &mut ec_context, - &services, - req, - ) - .await - .unwrap_or_else(|e| http_error(&e))) - } - }; - - // GET /_ts/page-bids — SPA re-auction endpoint. - let s = Arc::clone(&state); - let page_bids_handler = move |ctx: RequestContext| { - let s = Arc::clone(&s); - async move { - let services = s.services_for_request(&ctx); - let mut req = ctx.into_request(); - if let Err(error) = s.registry.prepare_request(&s.settings, &mut req) { - return Ok(http_error(&error)); - } - // Identity could not be established (for example the selected - // Edge Cookie module is unavailable). Answer with an error - // rather than re-running the auction with no identity. - let mut ec_context = match build_ec_context(&s.settings, &services, &req).await { - Ok(context) => context, - Err(report) => return Ok(http_error(&report)), - }; - let auction = AuctionDispatch { - orchestrator: &s.orchestrator, - slots: s.settings.creative_opportunity_slots(), - registry: None, - }; - Ok( - handle_page_bids(&s.settings, &services, None, auction, &mut ec_context, req) - .await - .unwrap_or_else(|e| http_error(&e)), - ) - } - }; - - // OPTIONS /_ts/page-bids — deny the CORS preflight for this - // side-effecting GET so the `X-TSJS-Page-Bids` gate stays trustworthy. - let page_bids_options_handler = |_ctx: RequestContext| async { - Ok::(page_bids_preflight_denied()) - }; - - // GET /first-party/proxy - let s = Arc::clone(&state); - let fp_proxy_handler = move |ctx: RequestContext| { - let s = Arc::clone(&s); - async move { - let services = s.services_for_request(&ctx); - let req = ctx.into_request(); - Ok(handle_first_party_proxy(&s.settings, &services, req) - .await - .unwrap_or_else(|e| http_error(&e))) - } - }; - - // /first-party/click - let s = Arc::clone(&state); - let fp_click_handler = move |ctx: RequestContext| { - let s = Arc::clone(&s); - async move { - let services = s.services_for_request(&ctx); - let req = ctx.into_request(); - Ok(handle_first_party_click(&s.settings, &services, req) - .await - .unwrap_or_else(|e| http_error(&e))) - } - }; - - // GET + POST /first-party/sign — identical handler, cloned for both bindings - let s = Arc::clone(&state); - let fp_sign_handler = move |ctx: RequestContext| { - let s = Arc::clone(&s); - async move { - let services = s.services_for_request(&ctx); - let req = ctx.into_request(); - Ok(handle_first_party_proxy_sign(&s.settings, &services, req) - .await - .unwrap_or_else(|e| http_error(&e))) - } - }; - let fp_sign_post_handler = fp_sign_handler.clone(); - - // GET + POST /first-party/proxy-rebuild — GET serves the click guard's - // navigation fallback: the creative iframe is an opaque origin (sandbox - // without `allow-same-origin`), so its JSON POST is blocked by CORS and - // the guard navigates here for a 302 instead. - let s = Arc::clone(&state); - let fp_rebuild_handler = move |ctx: RequestContext| { - let s = Arc::clone(&s); - async move { - let services = s.services_for_request(&ctx); - let req = ctx.into_request(); - Ok( - handle_first_party_proxy_rebuild(&s.settings, &services, req) - .await - .unwrap_or_else(|e| http_error(&e)), - ) - } - }; - let fp_rebuild_post_handler = fp_rebuild_handler.clone(); - - // Shared fallback dispatch: routes to tsjs (GET only), integration proxy, or publisher. - async fn dispatch( - state: Arc, - ctx: RequestContext, - ) -> Result { - let services = state.services_for_request(&ctx); - let mut req = ctx.into_request(); - if let Some(response) = closed_path_response(&req) { - return Ok(response); - } - if let Err(error) = state.registry.prepare_request(&state.settings, &mut req) { - return Ok(http_error(&error)); - } - - let path = req.uri().path().to_owned(); - let method = req.method().clone(); - - // Dynamic tsjs serving is GET-only; other methods fall through to the - // integration/publisher fallback. - let result = if method == Method::GET && path.starts_with("/static/tsjs=") { - handle_tsjs_dynamic(&req, &state.registry, EDGE_CACHE_HEADER) - } else if state.registry.has_route(&method, &path) { - // A module route is handed what was resolved for its request, - // the permissions among them, as the publisher path is. - let mut ec_context = match build_ec_context(&state.settings, &services, &req).await - { - Ok(context) => context, - Err(report) => return Ok(http_error(&report)), - }; - state - .registry - .handle_proxy(ProxyDispatchInput { - method: &method, - path: &path, - settings: &state.settings, - kv: None, - ec_context: &mut ec_context, - services: &services, - req, - }) - .await - .unwrap_or_else(|| { - Err(Report::new(TrustedServerError::BadRequest { - message: format!("Unknown integration route: {path}"), - })) - }) - } else { - // Identity could not be established (for example the selected - // Edge Cookie module is unavailable). Answer with an error - // rather than serving the page with no identity. - let mut ec_context = match build_ec_context(&state.settings, &services, &req).await - { - Ok(context) => context, - Err(report) => return Ok(http_error(&report)), - }; - let auction = AuctionDispatch { - orchestrator: &state.orchestrator, - slots: state.settings.creative_opportunity_slots(), - registry: None, - }; - match handle_publisher_request( - AppContext { - settings: &state.settings, - integration_registry: &state.registry, - }, - &services, - None, - &mut ec_context, - auction, - req, - EDGE_CACHE_HEADER, - ) - .await - { - Ok(pr) => { - resolve_publisher_response( - pr, - &method, - &state.settings, - &state.registry, - &state.orchestrator, - &services, - ) - .await - } - Err(e) => Err(e), - } - }; - - Ok(result.unwrap_or_else(|e| http_error(&e))) - } - // Single publisher/integration fallback used for every method. The method - // is read inside `dispatch`, so the same closure serves GET (tsjs/publisher) - // and the other supported methods (integration proxy / publisher origin). + // is read inside `dispatch_fallback`, so the same closure serves GET + // (robots.txt/tsjs/publisher) and the other supported methods + // (integration proxy / publisher origin). let s = Arc::clone(&state); let fallback = move |ctx: RequestContext| { let s = Arc::clone(&s); - dispatch(s, ctx) + execute_fallback(s, ctx) }; + let named = |handler: NamedRouteHandler| named_route_handler(Arc::clone(&state), handler); + let page_bids_handler = named(NamedRouteHandler::PageBids); let mut builder = RouterService::builder() // Outermost middleware: strips the configured trusted-client-IP @@ -1163,37 +1331,64 @@ fn build_router(state: &Arc) -> RouterService { .get("/health", |_ctx: RequestContext| async { Ok::(health_response()) }) - .get("/.well-known/trusted-server.json", discovery_handler) - .post("/verify-signature", verify_handler) + .get( + "/.well-known/trusted-server.json", + named(NamedRouteHandler::TrustedServerDiscovery), + ) + .post( + "/verify-signature", + named(NamedRouteHandler::VerifySignature), + ) .get(PERMISSIONS_PAGE_PATH, permissions_handler.clone()) .get(PERMISSIONS_JSON_PATH, permissions_handler) .get(CONFIG_PAGE_PATH, config_handler.clone()) .get(CONFIG_JSON_PATH, config_handler) .get(DATA_PAGE_PATH, data_handler) - .post("/auction", auction_handler) + .post("/auction", named(NamedRouteHandler::Auction)) + // SPA re-auction endpoint. Its OPTIONS preflight is denied so the + // GET handler's `X-TSJS-Page-Bids` gate stays trustworthy. .get(PAGE_BIDS_PATH, page_bids_handler.clone()) - .route(PAGE_BIDS_PATH, Method::OPTIONS, page_bids_options_handler) + .route(PAGE_BIDS_PATH, Method::OPTIONS, page_bids_handler.clone()) // Deprecated double-underscore alias, kept so tsjs bundles served // before the `/_ts/page-bids` rename keep getting ads on SPA // navigations until they age out of browser caches. See // `PAGE_BIDS_LEGACY_PATH`. - .get(PAGE_BIDS_LEGACY_PATH, page_bids_handler) - .route( - PAGE_BIDS_LEGACY_PATH, - Method::OPTIONS, - page_bids_options_handler, + .get(PAGE_BIDS_LEGACY_PATH, page_bids_handler.clone()) + .route(PAGE_BIDS_LEGACY_PATH, Method::OPTIONS, page_bids_handler) + .get( + "/first-party/proxy", + named(NamedRouteHandler::FirstPartyProxy), ) - .get("/first-party/proxy", fp_proxy_handler) - .get("/first-party/click", fp_click_handler) - .get("/first-party/sign", fp_sign_handler) - .post("/first-party/sign", fp_sign_post_handler) - .get("/first-party/proxy-rebuild", fp_rebuild_handler) - .post("/first-party/proxy-rebuild", fp_rebuild_post_handler); + .get( + "/first-party/click", + named(NamedRouteHandler::FirstPartyClick), + ) + .get( + "/first-party/sign", + named(NamedRouteHandler::FirstPartySign), + ) + .post( + "/first-party/sign", + named(NamedRouteHandler::FirstPartySign), + ) + // GET serves the click guard's navigation fallback: the creative + // iframe is an opaque origin (sandbox without `allow-same-origin`), + // so its JSON POST is blocked by CORS and the guard navigates here + // for a 302 instead. + .get( + "/first-party/proxy-rebuild", + named(NamedRouteHandler::FirstPartyProxyRebuild), + ) + .post( + "/first-party/proxy-rebuild", + named(NamedRouteHandler::FirstPartyProxyRebuild), + ); // Mirror the Fastly/Axum publisher fallback: every supported method that is // not a named route's primary method falls through to the publisher origin // (e.g. HEAD /, OPTIONS /page preflight, HEAD /first-party/proxy) instead of - // returning a router-level 405. tsjs handling stays GET-only (see dispatch). + // returning a router-level 405. tsjs handling stays GET-only (see + // `dispatch_fallback`). for (path, primary_methods) in named_fallback_paths() { for method in publisher_fallback_methods() { if !primary_methods.contains(&method) { @@ -1235,6 +1430,9 @@ fn build_router(state: &Arc) -> RouterService { } } +#[cfg(test)] +mod request_cycle_tests; + #[cfg(test)] mod tests { use edgezero_core::http::request_builder; diff --git a/crates/trusted-server-adapter-spin/src/app/request_cycle_tests.rs b/crates/trusted-server-adapter-spin/src/app/request_cycle_tests.rs new file mode 100644 index 0000000000..5f5af49602 --- /dev/null +++ b/crates/trusted-server-adapter-spin/src/app/request_cycle_tests.rs @@ -0,0 +1,405 @@ +//! The request cycle this adapter runs for a routed request, checked through +//! its router: the request filters run before the route and their response +//! effects after the finalize headers, the route's module call carries the +//! device classification, and robots.txt is answered ahead of both. + +use std::sync::Arc; + +use edgezero_core::body::Body; +use edgezero_core::http::{Request, Response, StatusCode, request_builder}; +use edgezero_core::router::RouterService; +use error_stack::Report; +use trusted_server_core::ec::device::{DeviceModule, DeviceSignals}; +use trusted_server_core::error::TrustedServerError; +use trusted_server_core::evidence::RequestInfo; +use trusted_server_core::integrations::{ + HeaderMutation, IntegrationBuilder, IntegrationEndpoint, IntegrationProxy, + IntegrationRegistration, IntegrationRequestFilter, RequestFilterDecision, RequestFilterEffects, + RequestFilterInput, +}; +use trusted_server_core::module_context::ModuleCall; +use trusted_server_core::platform::PlatformHttpClient; +use trusted_server_core::platform::test_support::{ + StubHttpClient, build_services_with_http_client, +}; +use trusted_server_core::settings::Settings; + +use super::{build_router, build_state_with_registrations_and_services}; + +/// The id and module name of the probe, which registers a filter that lets +/// the request continue, a device module and a route reporting what its +/// module call was handed. +const PROBE_ID: &str = "cycle_probe"; +const PROBE_MODULE: &str = "testing.cycle-probe"; + +/// The id and module name of a filter that answers every request. +const ANSWERING_ID: &str = "answering_filter"; +const ANSWERING_MODULE: &str = "testing.answering-filter"; + +/// The response header both filters write as an effect. +const EFFECT_HEADER: &str = "x-cycle-probe"; +/// The request header the continuing filter adds. +const FILTERED_HEADER: &str = "x-cycle-probe-filtered"; +/// What the answering filter answers. +const ANSWER: &str = "answered by the filter"; + +struct AnsweringFilter; + +#[async_trait::async_trait(?Send)] +impl IntegrationRequestFilter for AnsweringFilter { + fn integration_id(&self) -> &'static str { + ANSWERING_ID + } + + async fn filter_request( + &self, + _input: RequestFilterInput<'_>, + ) -> Result> { + let mut response = Response::new(Body::from(ANSWER)); + *response.status_mut() = StatusCode::FORBIDDEN; + Ok(RequestFilterDecision::Respond { + response: Box::new(response), + effects: RequestFilterEffects { + request_headers: Vec::new(), + response_headers: vec![HeaderMutation::set(EFFECT_HEADER, "answered")], + }, + }) + } +} + +struct ContinuingFilter; + +#[async_trait::async_trait(?Send)] +impl IntegrationRequestFilter for ContinuingFilter { + fn integration_id(&self) -> &'static str { + PROBE_ID + } + + async fn filter_request( + &self, + _input: RequestFilterInput<'_>, + ) -> Result> { + Ok(RequestFilterDecision::Continue(RequestFilterEffects { + request_headers: vec![HeaderMutation::set(FILTERED_HEADER, "1")], + response_headers: vec![HeaderMutation::set(EFFECT_HEADER, "continued")], + })) + } +} + +/// A device module that marks its classification with the cookies it was +/// shown, so a test can tell that it ran and what it saw. +struct MarkingDevice; + +impl MarkingDevice { + fn mark(&self, request_info: &dyn RequestInfo) -> DeviceSignals { + let mut signals = DeviceSignals::derive_ua_only(request_info.user_agent()); + signals.platform_class = Some(format!( + "marked:{}", + request_info.header("cookie").unwrap_or_default() + )); + signals + } +} + +#[async_trait::async_trait(?Send)] +impl DeviceModule for MarkingDevice { + fn id(&self) -> &'static str { + PROBE_MODULE + } + + async fn detect(&self, call: ModuleCall<'_>) -> DeviceSignals { + call.inject(self, Self::mark) + .unwrap_or_else(|_| DeviceSignals::unknown()) + } +} + +/// A route answering with the device classification its module call was +/// handed and whether the continuing filter's request header reached it. +struct ReportRoute; + +impl ReportRoute { + fn report( + &self, + req: Request, + device: Option<&DeviceSignals>, + ) -> Result> { + let (parts, _body) = req.into_parts(); + let report = serde_json::json!({ + "device": device.and_then(|signals| signals.platform_class.clone()), + "filtered": parts + .headers + .get(FILTERED_HEADER) + .and_then(|value| value.to_str().ok()), + }); + Ok(Response::new(Body::from(report.to_string()))) + } +} + +#[async_trait::async_trait(?Send)] +impl IntegrationProxy for ReportRoute { + fn integration_name(&self) -> &'static str { + PROBE_ID + } + + fn routes(&self) -> Vec { + vec![self.get("/report")] + } + + async fn handle( + &self, + call: ModuleCall<'_>, + req: Request, + ) -> Result> { + call.inject_with(self, req, Self::report)? + } +} + +fn register_probe( + _settings: &Settings, +) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder(PROBE_ID) + .without_js() + .with_request_filter(Arc::new(ContinuingFilter)) + .with_device_module(PROBE_MODULE, Arc::new(MarkingDevice)) + .with_proxy(Arc::new(ReportRoute)) + .build(), + )) +} + +fn register_answering( + _settings: &Settings, +) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder(ANSWERING_ID) + .without_js() + .with_request_filter(Arc::new(AnsweringFilter)) + .build(), + )) +} + +fn always(_settings: &Settings) -> Result> { + Ok(true) +} + +fn probe_builder() -> IntegrationBuilder { + IntegrationBuilder::new(PROBE_ID, "adapter-tests", register_probe, always) + .with_module_name(PROBE_MODULE) +} + +fn answering_builder() -> IntegrationBuilder { + IntegrationBuilder::new(ANSWERING_ID, "adapter-tests", register_answering, always) + .with_module_name(ANSWERING_MODULE) +} + +/// Settings for the publisher the requests are for, with `extra` appended. +fn settings_with(extra: &str) -> Settings { + Settings::from_toml(&format!( + r#" + [source] + module = "origin" + + [source.origin] + url = "https://origin.test-publisher.example.com" + + [publisher] + domain = "test-publisher.example.com" + cookie_domain = ".test-publisher.example.com" + proxy_secret = "request-cycle-test-proxy-secret" + + [ec] + module = "hmac" + + [ec.hmac] + passphrase = "test-secret-key-32-bytes-minimum" + + [geo] + assume_single_jurisdiction = true + + {extra} + "# + )) + .expect("should parse the request cycle test settings") +} + +/// The router built through the composition root with `builders`, every +/// request sent to `origin`. +fn router_with( + settings: Settings, + builders: &[IntegrationBuilder], + origin: &Arc, +) -> RouterService { + let state = build_state_with_registrations_and_services( + settings, + builders, + Some(build_services_with_http_client( + Arc::clone(origin) as Arc + )), + ) + .expect("should build the state"); + build_router(&state) +} + +fn get(path: &str, cookie: Option<&str>) -> Request { + let mut builder = request_builder() + .method("GET") + .uri(format!("https://test-publisher.example.com{path}")) + .header("host", "test-publisher.example.com"); + if let Some(cookie) = cookie { + builder = builder.header("cookie", cookie); + } + builder + .body(Body::empty()) + .expect("should build the request") +} + +async fn route(router: &RouterService, request: Request) -> Response { + router + .oneshot(request) + .await + .expect("should route the request") +} + +fn header<'a>(response: &'a Response, name: &str) -> Option<&'a str> { + response + .headers() + .get(name) + .and_then(|value| value.to_str().ok()) +} + +fn text(response: Response) -> String { + let bytes = response + .into_body() + .into_bytes() + .expect("should hold the body in memory"); + String::from_utf8(bytes.to_vec()).expect("the body should be text") +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_filter_that_answers_stands_in_for_the_page() { + let origin = Arc::new(StubHttpClient::new()); + let router = router_with( + settings_with("[testing]\nmodules = [\"answering-filter\"]"), + &[answering_builder()], + &origin, + ); + + let response = route(&router, get("/article", None)).await; + + assert_eq!( + response.status(), + StatusCode::FORBIDDEN, + "the filter's answer should stand in for the page" + ); + assert_eq!( + header(&response, EFFECT_HEADER), + Some("answered"), + "the effects the answering filter left should be applied" + ); + assert_eq!(text(response), ANSWER); + assert!( + origin.recorded_request_uris().is_empty(), + "the publisher's origin should not be asked" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_filter_s_request_header_reaches_the_route_and_its_effect_is_written_last() { + let origin = Arc::new(StubHttpClient::new()); + let mut settings = settings_with("[testing]\nmodules = [\"cycle-probe\"]"); + settings + .response_headers + .insert(EFFECT_HEADER.to_owned(), "operator".to_owned()); + let router = router_with(settings, &[probe_builder()], &origin); + + let response = route(&router, get("/integrations/cycle_probe/report", None)).await; + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + header(&response, EFFECT_HEADER), + Some("continued"), + "the filter's response effect should be written after the operator's headers" + ); + let report: serde_json::Value = + serde_json::from_str(&text(response)).expect("the report should be JSON"); + assert_eq!( + report["filtered"], "1", + "the filter's request header should reach the route" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn the_device_module_s_classification_reaches_the_module_call() { + let origin = Arc::new(StubHttpClient::new()); + let router = router_with( + settings_with(&format!( + "[device]\nmodule = \"{PROBE_MODULE}\"\n\n[testing]\nmodules = [\"cycle-probe\"]" + )), + &[probe_builder()], + &origin, + ); + + let response = route( + &router, + get( + "/integrations/cycle_probe/report", + Some("example_width=1280"), + ), + ) + .await; + + assert_eq!(response.status(), StatusCode::OK); + let report: serde_json::Value = + serde_json::from_str(&text(response)).expect("the report should be JSON"); + assert_eq!( + report["device"], "marked:example_width=1280", + "the selected device module should classify the request, shown its cookies" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn robots_txt_is_answered_before_the_request_filters() { + let origin = Arc::new(StubHttpClient::new()); + origin.push_response_with_headers( + 200, + b"User-agent: *\nAllow: /\n".to_vec(), + vec![("content-type", "text/plain")], + ); + let router = router_with( + settings_with( + r#" + [testing] + modules = ["answering-filter"] + + [robots-txt] + modules = "refuse_all" + always_allow = ["/ads.txt"] + + [[fetch]] + media_type = "text/plain" + path = "/robots.txt" + middleware = ["robots_txt"] + "#, + ), + &[answering_builder()], + &origin, + ); + + let response = route(&router, get("/robots.txt", None)).await; + + assert_eq!( + response.status(), + StatusCode::OK, + "no filter's answer should stand in for robots.txt" + ); + assert_eq!( + header(&response, EFFECT_HEADER), + None, + "the filters should not run for robots.txt" + ); + assert_eq!( + text(response), + "User-agent: *\nAllow: /ads.txt\nDisallow: /\n" + ); + assert_eq!(origin.recorded_request_methods(), ["GET"]); +} diff --git a/crates/trusted-server-adapter-spin/src/middleware.rs b/crates/trusted-server-adapter-spin/src/middleware.rs index d7a6b5c82f..b25776ae03 100644 --- a/crates/trusted-server-adapter-spin/src/middleware.rs +++ b/crates/trusted-server-adapter-spin/src/middleware.rs @@ -8,6 +8,11 @@ use edgezero_core::middleware::{Middleware, Next}; use trusted_server_core::http_util::{ ROUTE_NAMESPACE, sanitize_trusted_client_ip_headers, strip_deployment_headers, }; +use trusted_server_core::integrations::RequestFilterEffects; +use trusted_server_core::response_privacy::{ + TerminalPrivateResponse, enforce_private_no_store, enforce_set_cookie_cache_privacy, + enforce_uncacheable_cache_privacy, +}; use trusted_server_core::settings::Settings; // --------------------------------------------------------------------------- @@ -53,7 +58,8 @@ impl Middleware for SanitizeRequestMiddleware { /// Response-finalization middleware: injects all standard TS response headers. /// /// Registered directly inside [`SanitizeRequestMiddleware`], so that every -/// outgoing response carries a consistent set of headers. +/// outgoing response carries a consistent set of headers. Once the response +/// is finalized it applies the response effects the request filters left. pub struct FinalizeResponseMiddleware { settings: Arc, } @@ -71,13 +77,37 @@ impl Middleware for FinalizeResponseMiddleware { async fn handle(&self, ctx: RequestContext, next: Next<'_>) -> Result { let path = ctx.request().uri().path().to_owned(); let mut response = next.run(ctx).await?; + let request_filter_effects = response.extensions_mut().remove::(); // Before finalizing, so a header finalizing writes is kept. strip_deployment_headers(&path, ROUTE_NAMESPACE, &mut response); apply_finalize_headers(&self.settings, &mut response); + apply_terminal_response_effects(&mut response, request_filter_effects.as_ref()); Ok(response) } } +/// Applies the request filters' response effects to a finalized response, +/// then restores the privacy guards a late header could loosen. +fn apply_terminal_response_effects( + response: &mut Response, + request_filter_effects: Option<&RequestFilterEffects>, +) { + let must_remain_private = response + .extensions() + .get::() + .is_some(); + if let Some(effects) = request_filter_effects { + effects.apply_to_response(response); + } + if must_remain_private { + enforce_private_no_store(response); + } + // A filter's header can add a per-user `Set-Cookie`, or a `private` or + // `no-store` directive, after the finalize headers ran. + enforce_set_cookie_cache_privacy(response); + enforce_uncacheable_cache_privacy(response); +} + // --------------------------------------------------------------------------- // NormalizeMiddleware // --------------------------------------------------------------------------- From f77093a5d4422fb3a9305b54d6cdaecf55bbf656 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 17 Oct 2026 11:53:22 +0100 Subject: [PATCH 02/18] A design for running the request filters and the device module on every adapter docs/superpowers/specs/2026-10-17-request-filters-on-every-adapter-design.md sets out what the Fastly adapter runs that the Axum, Cloudflare and Spin adapters do not, the request cycle each of them now runs and the routes it covers, how device classification there differs from the Fastly entry point, the timeouts their HTTP clients give a filter's call, and what a deployment sees change. --- ...request-filters-on-every-adapter-design.md | 87 +++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 docs/superpowers/specs/2026-10-17-request-filters-on-every-adapter-design.md diff --git a/docs/superpowers/specs/2026-10-17-request-filters-on-every-adapter-design.md b/docs/superpowers/specs/2026-10-17-request-filters-on-every-adapter-design.md new file mode 100644 index 0000000000..e23e3d92cd --- /dev/null +++ b/docs/superpowers/specs/2026-10-17-request-filters-on-every-adapter-design.md @@ -0,0 +1,87 @@ +# Design Spec: Request filters and device detection run on every adapter + +**Status:** Proposed, 2026-10-17. A pull request above `ts config migrate`, not yet raised, for Tech Lab review. +**Author:** 51Degrees (contributed), for Tech Lab review +**Related specs:** `2026-10-14-51degrees-client-hints-design.md`, and the design documents of the stack in IABTechLab/trusted-server#1342 +**Related PRs:** none yet +**Issues:** none +**Last updated:** 2026-10-17 + +> **Why this spec exists.** Only the Fastly adapter runs the integration request filters and the device module `[device]` selects. On Axum, Cloudflare and Spin a module's request filter never runs, so the 51Degrees client hints middleware writes nothing and DataDome's protection never asks its service, and no module call carries a device classification. This change runs both on those three adapters where the Fastly adapter runs them. + +## 1. Problem + +The Fastly adapter runs three steps that no other adapter runs. + +1. **Request filters.** `run_pre_route_filters` in `crates/trusted-server-adapter-fastly/src/app.rs` calls `IntegrationRegistry::filter_request` for every named route except the inspection endpoints and batch sync, and for the publisher fallback. It runs after the modules' preparers and once the request's Edge Cookie context is read, and it hands every filter the location and the permissions resolved for that context. A filter that answers stands in for the route. A filter that lets the request continue has its request headers applied before the route, and its response headers are applied by the entry point after the response is finalized (`apply_terminal_response_effects` in `src/lib.rs`). +2. **Device detection.** `derive_device_signals` in `src/lib.rs` calls `build_device_module` and `DeviceModule::detect` at the entry point, and `build_ec_request_state` sets the result on the Edge Cookie context. Every module call made from that context carries the classification (`ModuleContext::with_request_state` in core). +3. **robots.txt first.** `dispatch_fallback` answers robots.txt once the preparers have run and before the Edge Cookie steps and the filters, with an empty Edge Cookie context, so no filter's answer stands in for the file and none of the crawler's cookies are read. + +No code in `crates/trusted-server-adapter-axum`, `-cloudflare` or `-spin` calls `filter_request` or `DeviceModule::detect`. So on those adapters the 51Degrees client hints middleware asks its service nothing and writes no header or tag, DataDome's protection filter never runs, `[device] module` changes nothing, and robots.txt is answered through the publisher path with the crawler's own Edge Cookie context. On Spin the discovery, signature verification and first-party routes and the page-bids preflight also skip the modules' preparers, which every other adapter runs for them. + +## 2. Design + +**One request cycle per adapter, in the order the Fastly adapter runs it.** Each of the three adapters gains the steps the Fastly adapter has, under the Fastly adapter's names, in `src/app.rs`. + +| Step | Function | What it does | +| --------------------- | ------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Preparers | `execute_named`, `execute_fallback` | `IntegrationRegistry::prepare_request`, once for each request, as before | +| robots.txt | `asks_for_robots_txt`, `answer_robots_txt` | Answers a `GET` or `HEAD` of `/robots.txt`, when `[robots-txt]` is present, through the publisher path with `EcContext::default()` and no auction | +| Device classification | `derive_device_signals` | Runs the selected device module, or the built-in one, with the request's module context | +| Edge Cookie context | `build_ec_context` | `EcContext::read_from_request_resolving_geo`, then `set_device_signals`. An error answers the request | +| Request filters | `run_pre_route_filters` | `IntegrationRegistry::filter_request` with the context's location and permissions. An answer, or an error, stands in for the route | +| Route | `run_named_route`, `dispatch_fallback` | The handler, given the context the filters saw | +| Response effects | `attach_request_filter_effects` | Leaves the filters' response headers on the response for the finalize middleware | +| Finalize | `FinalizeResponseMiddleware` | Finalizes the response, then `apply_terminal_response_effects` writes the filters' response headers and restores the privacy guards, as the Fastly entry point does | + +**Which routes run the cycle.** The routes the Fastly adapter runs it for, which are every named route but the inspection endpoints, and the publisher fallback, being robots.txt, tsjs, the module routes and the publisher's origin. The inspection endpoints are read only and stay ahead of the Edge Cookie context and the filters, as on Fastly. + +| Adapter | Named routes in the cycle | Inspection endpoints | Registration | +| ---------- | ----------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------- | +| Axum | Every row of `named_routes()` but `/_ts/permissions`, `/_ts/permissions.json`, `/_ts/config`, `/_ts/config.json`, `/_ts/data` | `execute_named` answers them after the preparers | Unchanged. `named_route_handler` calls `execute_named` and `fallback_handler` calls `execute_fallback`, where both called `execute_handler` | +| Cloudflare | Discovery, `/verify-signature`, `/auction`, both page-bids paths (`GET` and `OPTIONS`), the four first-party paths | `make_handler`, after the preparers, as before | The same paths and methods, each handled by `named_route_handler` | +| Spin | The same routes as Cloudflare | Their own handlers, with no preparers, as before | The same paths and methods, each handled by `named_route_handler` | + +On Spin the discovery, signature verification and first-party routes and the page-bids preflight now run the preparers, because the cycle starts with them. Before this change they ran none. + +**Device classification.** `derive_device_signals` follows the Fastly adapter's function of the same name. A device module a registration supplies is shown the `User-Agent` and the `Cookie` header, and the built-in module the `User-Agent` alone, each in a module context carrying the request, that evidence, the settings and the services. Three things differ from the Fastly adapter. + +1. It runs where the adapter builds the Edge Cookie context, after the preparers, where the Fastly adapter runs it at the entry point before routing. A preparer that removes a reserved cookie therefore removes it before a device module sees it. +2. The registered module is the one the request's services carry (`RuntimeServices::device_module`), which `services_for_request` sets from the registry, and the module is handed those same services, so it reads the platform's key-value store when the services carry one. The Fastly entry point builds services of its own for the call. +3. These adapters supply no TLS or HTTP/2 signals, so `[device] module = "platform"` builds the built-in module, which classifies from the `User-Agent`, because `platform` adds the host's signals only where the platform supplies them. + +**Response effects.** A filter that lets the request continue can name response headers. The route's response carries them in its extensions as `RequestFilterEffects`, and `FinalizeResponseMiddleware` applies them after it writes the finalize headers, so a filter's header is written after an operator's `[response_headers]`, as on Fastly. `apply_terminal_response_effects` then enforces `no-store, private` on a response core marked `TerminalPrivateResponse` and runs `enforce_set_cookie_cache_privacy` and `enforce_uncacheable_cache_privacy`, the steps the Fastly entry point runs before it sends every response. + +**Configuration.** None. The filters a deployment's modules register and the module `[device]` selects now run on every adapter. + +**Consent and privacy.** Every filter is handed the permissions resolved for the request, as on Fastly. The 51Degrees filter sends the reader's `User-Agent` and `Sec-CH-UA` hints to the 51Degrees service for each page navigation an entry naming its middleware covers, whatever the permissions say, which is the behavior and the open question that `2026-10-14-51degrees-client-hints-design.md` describes for Fastly. DataDome's protection filter removes `x-ts-datadome-bypass` from every request it runs for, before its other checks. A device module a registration supplies sees the reader's cookies, as on Fastly. robots.txt is answered with an empty Edge Cookie context, so a crawler's cookies and consent signals are no longer read on these adapters. + +**Failure handling.** A filter that fails answers the request with the error, logged as `Failed to run integration request filters`, as on Fastly. An Edge Cookie context that cannot be read answers the request with the error before the filters run, which now covers every route in the cycle, including tsjs and the discovery, signature verification and first-party routes. The errors are a selected Edge Cookie module the request cannot build and a `Cookie` header that is not UTF-8, and startup refuses a selection the adapter can never build. A filter's outbound call is bounded only by the adapter's HTTP client, which applies no module's own timeout on these adapters. Axum's client allows 5 seconds to connect and 30 in all, Cloudflare leaves the call to the Workers runtime's limit, and Spin to the Spin runtime's default (`src/platform.rs` of each adapter). + +**Rollout.** A deployment whose modules register no filter, and that selects no device module, answers as before. It now reads the Edge Cookie context, with its geo lookup where `[geo]` selects a module, and classifies the device with the built-in module, on every route in the cycle, where it read the context for the auction, page-bids, module routes and the publisher alone. A deployment with `[shared.51degrees]` and a `[[serve]]` entry naming `51degrees.client_hints` now has the headers and the tag written on the pages the entry covers, and such a page keeps to the origin path and is sent `private, no-store`, because the filter leaves the hint names in the request's state. A deployment with DataDome's protection on now has its pages checked, and a challenge answers in place of the page. + +**Tests.** In `src/app/request_cycle_tests.rs` of each of the three adapters, through the adapter's router built by its composition root, with a stub origin: + +- `a_filter_that_answers_stands_in_for_the_page`: the answer and its effect header are the response, and the origin is not asked. +- `a_filter_s_request_header_reaches_the_route_and_its_effect_is_written_last`: a module route sees the request header a filter added, and the filter's response header replaces the operator's header of the same name. +- `the_device_module_s_classification_reaches_the_module_call`: the module `[device]` selects classifies the request, shown its cookies, and a module route's call carries that classification. +- `robots_txt_is_answered_before_the_request_filters`: with a filter that answers every request, robots.txt is still the file, and the filter's header is absent. + +**Open questions.** None. + +## 3. What is not in this change + +- Creating and writing the Edge Cookie, and the resolve route, on these adapters, which the next pull request of the stack covers. +- Asset routes on these adapters, which the pull request after that covers. +- A timeout for a filter's outbound call on Axum, Cloudflare and Spin, which their HTTP clients do not offer. +- Moving the inspection endpoints ahead of the preparers on Axum and Cloudflare, where the Fastly adapter answers two of them before the preparers. + +## 4. Acceptance + +The four tests pass on each of the three adapters, and the existing tests of each adapter and the cross-adapter parity tests pass unchanged. + +## Revision record + +| Date | Change | +| ---------- | ------------ | +| 2026-10-17 | First draft. | From f7789840359aa1e543d06631715e5187f9fea5cc Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 17 Oct 2026 11:53:23 +0100 Subject: [PATCH 03/18] The changelog and the guides say the request filters run on every adapter The changelog gains a line under Changed, and the 51Degrees client hints line no longer says only the Fastly adapter runs request filters. The 51Degrees, DataDome, permission model, configuration and attestation guides no longer limit the request filters, the order robots.txt is answered in or the order the attestation endpoint is answered in to Fastly, the 51Degrees guide gives what the other three HTTP clients wait for, and the Axum, Cloudflare and Spin guides say the filters and the device module run there and what platform classifies from. --- CHANGELOG.md | 3 ++- docs/guide/attestation.md | 8 ++++---- docs/guide/axum-dev.md | 5 ++++- docs/guide/cloudflare.md | 5 ++++- docs/guide/configuration.md | 2 +- docs/guide/integrations/51degrees.md | 14 +++++++------- docs/guide/integrations/datadome.md | 2 +- docs/guide/permission-model.md | 9 ++++----- docs/guide/spin.md | 5 ++++- 9 files changed, 31 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d5f4ed4679..1ce5cdd1f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- The Axum, Cloudflare and Spin adapters run the integration request filters and the device module `[device]` selects, where only the Fastly adapter ran them. Every named route but the inspection endpoints, and the publisher fallback, reads the request's Edge Cookie context with the device classification and then runs the filters before the route, so a filter that answers, such as DataDome's challenge, stands in for the route, and the response headers a filter names are written after the finalize headers. The 51Degrees client hints middleware and DataDome's protection therefore work on every adapter, through HTTP clients that apply no module's own timeout on Axum, Cloudflare and Spin. On those adapters `[device] module = "platform"` classifies from the `User-Agent`, robots.txt is answered before the Edge Cookie steps and the filters with an empty Edge Cookie context, and on Spin the discovery, signature verification and first-party routes now run the modules' preparers. - **Breaking:** the platform's own device module is selected with `[device] module = "platform"`, as `[geo] module = "platform"` selects the platform's geo lookup, where it was `module = "fastly"`. A document that still writes `fastly` is refused at startup as naming a device module no running module supplies. **For crates outside core**, `build_device_module` takes a `build_platform` factory. - **Breaking for crates outside core:** an adapter declares the request headers its platform sets. `platform::declare_platform_tls_headers` names a header the platform sets on a request that arrived over TLS, which core reads for the scheme and never forwards to an origin. `platform::declare_platform_spoofable_headers` names the platform's other headers a client could also send, which `sanitize_forwarded_headers` strips at entry with the standard forwarding headers, in an adapter that calls it. `platform::declare_platform_internal_headers` names headers that are never passed to a third party. The Fastly adapter declares `fastly-ssl`, `fastly-client-ip` and `x-debug-fastly-pop` before it serves. `SPOOFABLE_FORWARDED_HEADERS` holds the three standard forwarding headers, `HEADER_X_DEBUG_FASTLY_POP` leaves core, and `constants::is_internal_header` answers for the fixed list and the declared headers together. A `[trusted_client_ip]` `ip_header` may start with `x-` or end with `-client-ip`, so `fastly-client-ip` stays valid. On the Axum, Cloudflare and Spin adapters, which declare none of these headers, a client's `fastly-ssl` is no longer removed from the request to the publisher's origin, and on Spin a client's `fastly-ssl` and `fastly-client-ip` are no longer stripped at entry. On those adapters the `x-debug-fastly-pop` header from a client is also no longer kept from third parties, because `INTERNAL_HEADERS` no longer holds it, and is no longer refused as a `[trusted_client_ip]` header name. On the Axum and Cloudflare adapters a client's `fastly-ssl` also no longer makes the request HTTPS. On Fastly, `ts config validate`, which runs with no adapter's declarations, no longer refuses `x-debug-fastly-pop` as a `[trusted_client_ip]` header name, and the service refuses it when it loads the document. - **Breaking for crates outside core:** an adapter tells core whether its platform runs the service in a staging environment with `platform::declare_platform_staging`, and a module asks `platform::platform_is_staging`. `ENV_FASTLY_IS_STAGING` leaves core, the Fastly adapter reads `FASTLY_IS_STAGING` itself, and the DataDome module's test bypass follows the declaration. On the Axum, Cloudflare and Spin adapters, which declare no staging environment, the bypass no longer follows a `FASTLY_IS_STAGING` variable. @@ -123,7 +124,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - `ts config migrate ` rewrites a settings document written for an earlier release, such as one written before the breaking changes above, into the shape this release reads, and prints the result. On stderr it lists each change it made, each setting it left for the operator because it needs a decision or a value the document does not hold, and each change in behavior no setting expresses, then checks the result with the settings validation `ts config validate` runs. It keeps the comments and the order of the tables, comments out a module that was not enabled under its new name, never prints a value from the document, rewrites the file in place with `--write`, and with `--check` changes nothing and exits 1 when the document needs migrating. Migrating a migrated document changes nothing. -- A 51Degrees client hints middleware, `51degrees.client_hints`, from `crates/middleware/51degrees`, run by `[shared.51degrees]` and placed by a `[[serve]]` entry. On the Fastly adapter, the only one that runs request filters, it asks the 51Degrees service which client hints the reader's browser can send for each page navigation an entry covers, sending the `User-Agent` and `Sec-CH-UA` hints alone, and leaves the hint names in the request's state for the other 51Degrees modules, which keeps a page it names hints for out of the stored templates and sends it `private, no-store`. On each page an entry covers it writes `Accept-CH` and `Critical-CH` on the response unless the browser already sends every one of those hints, `Permissions-Policy` and the `Delegate-CH` tag that delegate those hints to the service's origin, each keeping every line the response already carries. `critical` in `[middleware.51degrees.client_hints]` turns the retry of the first navigation off. The service's answers are held in the cache `51degrees-answers`, selected and sized in `[cache.51degrees-answers]`. What the 51Degrees modules share, being the table, the one call to the service, the evidence it sends and the cache its answer is held in, is the new `crates/shared/51degrees`. +- A 51Degrees client hints middleware, `51degrees.client_hints`, from `crates/middleware/51degrees`, run by `[shared.51degrees]` and placed by a `[[serve]]` entry. It asks the 51Degrees service which client hints the reader's browser can send for each page navigation an entry covers, sending the `User-Agent` and `Sec-CH-UA` hints alone, and leaves the hint names in the request's state for the other 51Degrees modules, which keeps a page it names hints for out of the stored templates and sends it `private, no-store`. On each page an entry covers it writes `Accept-CH` and `Critical-CH` on the response unless the browser already sends every one of those hints, `Permissions-Policy` and the `Delegate-CH` tag that delegate those hints to the service's origin, each keeping every line the response already carries. `critical` in `[middleware.51degrees.client_hints]` turns the retry of the first navigation off. The service's answers are held in the cache `51degrees-answers`, selected and sized in `[cache.51degrees-answers]`. What the 51Degrees modules share, being the table, the one call to the service, the evidence it sends and the cache its answer is held in, is the new `crates/shared/51degrees`. - `[middleware.]`, a middleware's own settings table, written with the middleware's full name and read through `Settings::middleware_config`. A table holds values and arrays, a table within one is the longer name's table and never a setting, and a table at a name no running module supplies is refused at startup with the names that could be written. - A serve middleware may set headers on a response the publisher path sends for a request path its entry covers, through `Middleware::response_headers`, which is given the response's status and headers and runs whatever the response's media type and whether or not it has a body, so a redirect with no body reaches a middleware that rewrites its `Location`. A header it sets replaces one of the same name. The method is not called for a refusal made before the origin is asked, such as the `405` for a method a signed origin does not take, for a page served from the template cache or the refusal of such a page, or for an answer made from an error. - A publisher's site may be served from a folder of object storage through the publisher proxy, so its pages pass through the page pipeline, which an asset route skips. The folder is the path of `origin_url`, and two optional `[publisher]` settings do the rest, each off unless set. `origin_auth` takes the table an asset route's `auth` takes and signs every origin request with it, through the same signer. `directory_index` names a file added to a request path ending in `/`, for the origin request only. A signed origin is only read, so a method other than `GET` or `HEAD` is answered `405`, and a signed request carries none of the reader's cookies, credentials, referrer, conditional validators or range. A malformed `origin_auth` or `directory_index` is refused when the settings load, with the setting named. diff --git a/docs/guide/attestation.md b/docs/guide/attestation.md index 828b15d0f5..bed50bbb79 100644 --- a/docs/guide/attestation.md +++ b/docs/guide/attestation.md @@ -161,10 +161,10 @@ The Fastly adapter fills the platform fields from the service's environment. The Axum, Cloudflare and Spin adapters report only their own name, with `unknown` as the service id, service version and location, and `staging` false. -On Fastly the endpoint answers ahead of the Edge Cookie lifecycle and the -request filters, because the endpoint is read only and a verifier fetching it -from a server must not be turned away as a bot. The endpoint makes no outbound -request and reads no store. +On every adapter the endpoint answers ahead of the Edge Cookie lifecycle and +the request filters, because the endpoint is read only and a verifier fetching +it from a server must not be turned away as a bot. The endpoint makes no +outbound request and reads no store. ## Configuration diff --git a/docs/guide/axum-dev.md b/docs/guide/axum-dev.md index b5eb0e701c..47101a63c9 100644 --- a/docs/guide/axum-dev.md +++ b/docs/guide/axum-dev.md @@ -63,7 +63,10 @@ local debugging sessions. Axum sanitizes forwarded client-IP headers but does not resolve `[trusted_client_ip]`; see the -[adapter matrix](./api-reference#adapter-and-startup-support). +[adapter matrix](./api-reference#adapter-and-startup-support). The modules' +request filters and the device module `[device]` selects run as on Fastly, +and `[device] module = "platform"` classifies from the `User-Agent`, because +the dev server supplies no TLS or HTTP/2 signals. For an edge deployment, use the [Fastly](./fastly), [Cloudflare](./cloudflare), or [Spin](./spin) guide. diff --git a/docs/guide/cloudflare.md b/docs/guide/cloudflare.md index d9ca66a543..d07efd6b2f 100644 --- a/docs/guide/cloudflare.md +++ b/docs/guide/cloudflare.md @@ -57,7 +57,10 @@ A successful `ts config push` alone does not configure this runtime: the Worker currently does not open `TRUSTED_SERVER_KV` during startup. Request-time store registries are also unwired, so EC KV lookup is not available. Forwarded client-IP headers are sanitized, but `[trusted_client_ip]` is not -resolved. See the [API reference](./api-reference) for route-level behavior. +resolved. The modules' request filters and the device module `[device]` +selects run as on Fastly, and `[device] module = "platform"` classifies from +the `User-Agent`, because the Worker supplies no TLS or HTTP/2 signals. See +the [API reference](./api-reference) for route-level behavior. Compare the [Fastly](./fastly), [Spin](./spin), and [Axum](./axum-dev) adapter journeys. diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 16554ff2ab..8ba51cc753 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -1293,7 +1293,7 @@ A module with settings of its own reads them from its own block, `[robots-txt.` on its `RequestFilterInput`, alongside the geo result it already receives, so a filter can skip or narrow what it does -when a permission it depends on is unset. Request filters run on the Fastly -adapter today, which is the only adapter that runs the filter step, and there -the state is built before the filters run. None of the shipped filters changes -its behavior on that input yet, so for now the state is carried and available -rather than acted on. +when a permission it depends on is unset. Every adapter builds the state +before the filters run. None of the shipped filters changes its behavior on +that input yet, so for now the state is carried and available rather than +acted on. **Page side.** The same resolved state reaches the page as `window.tsjs.permissions`, an object listing the Data Use names that are set for diff --git a/docs/guide/spin.md b/docs/guide/spin.md index 8b84421d5c..f5243e031b 100644 --- a/docs/guide/spin.md +++ b/docs/guide/spin.md @@ -53,7 +53,10 @@ empty variable defaults fail closed. Spin's request-time store registry is currently unwired, so EC KV lookup and request-signing key variables are unavailable. Forwarded client-IP headers are sanitized, and Spin derives its runtime authority, scheme, and client-address headers in the innermost -normalization layer. +normalization layer. The modules' request filters and the device module +`[device]` selects run as on Fastly, and `[device] module = "platform"` +classifies from the `User-Agent`, because Spin supplies no TLS or HTTP/2 +signals. Spin is not installed in the integration CI environment. The recurring manual smoke therefore needs a named owner, tested commit, exact tool versions, and an From 7aea7a72b0be65d633b59f4c85a09cd282e94bde Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 18 Oct 2026 03:42:32 +0100 Subject: [PATCH 04/18] The request cycle tests prove the privacy guards on Axum, Cloudflare and Spin Four tests on each adapter fail when one of the guards that apply_terminal_response_effects runs after a filter's response headers is taken away: a response core marks private stays no-store, private, a filter's Set-Cookie keeps the response from shared caches, a shared cache header a filter adds to a private response is removed, and the validators an operator writes on a response core marks private are removed. Four more show that platform classifies with the built-in module, that an Edge Cookie context that cannot be read answers before the filters while a client's unreadable Cookie header is rebuilt by a linked preparer first, that robots.txt shows the device module none of a crawler's cookies, and that the inspection endpoints and the attestation pages skip the filters. --- .../src/app/request_cycle_tests.rs | 464 +++++++++++++++++- .../src/app/request_cycle_tests.rs | 464 +++++++++++++++++- .../src/app/request_cycle_tests.rs | 464 +++++++++++++++++- 3 files changed, 1365 insertions(+), 27 deletions(-) diff --git a/crates/trusted-server-adapter-axum/src/app/request_cycle_tests.rs b/crates/trusted-server-adapter-axum/src/app/request_cycle_tests.rs index dd576ee018..a7bbe60450 100644 --- a/crates/trusted-server-adapter-axum/src/app/request_cycle_tests.rs +++ b/crates/trusted-server-adapter-axum/src/app/request_cycle_tests.rs @@ -1,12 +1,15 @@ //! The request cycle this adapter runs for a routed request, checked through //! its router: the request filters run before the route and their response -//! effects after the finalize headers, the route's module call carries the -//! device classification, and robots.txt is answered ahead of both. +//! effects after the finalize headers, without making the response less +//! private, the route's module call carries the device classification, an +//! Edge Cookie context that cannot be read answers before the filters, and +//! robots.txt, the inspection endpoints and the attestation pages are answered +//! without them. -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use edgezero_core::body::Body; -use edgezero_core::http::{Request, Response, StatusCode, request_builder}; +use edgezero_core::http::{HeaderValue, Request, Response, StatusCode, request_builder}; use edgezero_core::router::RouterService; use error_stack::Report; use trusted_server_core::ec::device::{DeviceModule, DeviceSignals}; @@ -22,6 +25,7 @@ use trusted_server_core::platform::PlatformHttpClient; use trusted_server_core::platform::test_support::{ StubHttpClient, build_services_with_http_client, }; +use trusted_server_core::response_privacy::enforce_terminal_private_cache_privacy; use trusted_server_core::settings::Settings; use super::TrustedServerApp; @@ -43,6 +47,48 @@ const FILTERED_HEADER: &str = "x-cycle-probe-filtered"; /// What the answering filter answers. const ANSWER: &str = "answered by the filter"; +/// The id and module name of the privacy probe, whose routes answer with a +/// cache policy and whose filter names a response header for each of them. +const PRIVACY_ID: &str = "privacy_probe"; +const PRIVACY_MODULE: &str = "testing.privacy-probe"; + +/// The privacy probe's routes. The first and the last answer with core's +/// marker for a response made for one reader, the second is public, and the +/// third is private by its own policy alone. The filter names a header for +/// each but the last. +const PRIVATE_PATH: &str = "/integrations/privacy_probe/private"; +const PUBLIC_PATH: &str = "/integrations/privacy_probe/public"; +const UNCACHEABLE_PATH: &str = "/integrations/privacy_probe/uncacheable"; +const UNFILTERED_PATH: &str = "/integrations/privacy_probe/unfiltered"; + +/// The policy the public route answers with, and the one the filter writes +/// on the route core marks private. +const PUBLIC_POLICY: &str = "public, max-age=600"; +/// The cookie the filter sets on the public route. +const FILTER_COOKIE: &str = "probe=1; Path=/"; + +/// Cookies sent to robots.txt and to a module route, each found nowhere else. +const CRAWLER_COOKIE: &str = "crawler_session=robots"; +const READER_COOKIE: &str = "reader_session=route"; + +/// The id and module name of a module whose preparer leaves the request for +/// [`SPOILED_PATH`] with a `Cookie` header the Edge Cookie context cannot +/// read. +const SPOILING_ID: &str = "spoiling_preparer"; +const SPOILING_MODULE: &str = "testing.spoiling-preparer"; +const SPOILED_PATH: &str = "/spoiled"; + +/// Every `Cookie` header the marking device module was shown, by every test +/// in this file, so a test can tell whether a request's cookies reached it. +static COOKIES_SHOWN: Mutex> = Mutex::new(Vec::new()); + +fn cookies_shown() -> Vec { + COOKIES_SHOWN + .lock() + .expect("the record of the cookies shown should be readable") + .clone() +} + struct AnsweringFilter; #[async_trait::async_trait(?Send)] @@ -87,16 +133,18 @@ impl IntegrationRequestFilter for ContinuingFilter { } /// A device module that marks its classification with the cookies it was -/// shown, so a test can tell that it ran and what it saw. +/// shown, and records them, so a test can tell that it ran and what it saw. struct MarkingDevice; impl MarkingDevice { fn mark(&self, request_info: &dyn RequestInfo) -> DeviceSignals { + let cookie = request_info.header("cookie").unwrap_or_default(); + COOKIES_SHOWN + .lock() + .expect("the record of the cookies shown should be writable") + .push(cookie.to_owned()); let mut signals = DeviceSignals::derive_ua_only(request_info.user_agent()); - signals.platform_class = Some(format!( - "marked:{}", - request_info.header("cookie").unwrap_or_default() - )); + signals.platform_class = Some(format!("marked:{cookie}")); signals } } @@ -154,6 +202,73 @@ impl IntegrationProxy for ReportRoute { } } +/// A filter that names, for each of the privacy probe's routes, a response +/// header that would let a shared cache store the route's answer. +struct LooseningFilter; + +#[async_trait::async_trait(?Send)] +impl IntegrationRequestFilter for LooseningFilter { + fn integration_id(&self) -> &'static str { + PRIVACY_ID + } + + async fn filter_request( + &self, + input: RequestFilterInput<'_>, + ) -> Result> { + let response_headers = match input.request.uri().path() { + PRIVATE_PATH => vec![HeaderMutation::set("cache-control", PUBLIC_POLICY)], + PUBLIC_PATH => vec![HeaderMutation::append("set-cookie", FILTER_COOKIE)], + UNCACHEABLE_PATH => vec![HeaderMutation::set("surrogate-control", "max-age=3600")], + _ => Vec::new(), + }; + Ok(RequestFilterDecision::Continue(RequestFilterEffects { + request_headers: Vec::new(), + response_headers, + })) + } +} + +/// The privacy probe's routes, each answering with its own cache policy. +struct PrivacyRoutes; + +#[async_trait::async_trait(?Send)] +impl IntegrationProxy for PrivacyRoutes { + fn integration_name(&self) -> &'static str { + PRIVACY_ID + } + + fn routes(&self) -> Vec { + vec![ + self.get("/private"), + self.get("/public"), + self.get("/uncacheable"), + self.get("/unfiltered"), + ] + } + + async fn handle( + &self, + _call: ModuleCall<'_>, + req: Request, + ) -> Result> { + let path = req.uri().path(); + let policy = if path == UNCACHEABLE_PATH { + "private, max-age=600" + } else { + PUBLIC_POLICY + }; + let mut response = Response::new(Body::from("probe")); + response + .headers_mut() + .insert("cache-control", HeaderValue::from_static(policy)); + if path == PRIVATE_PATH || path == UNFILTERED_PATH { + enforce_terminal_private_cache_privacy(&mut response); + } + Ok(response) + } +} + fn register_probe( _settings: &Settings, ) -> Result, Report> { @@ -178,10 +293,45 @@ fn register_answering( )) } +fn register_privacy( + _settings: &Settings, +) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder(PRIVACY_ID) + .without_js() + .with_request_filter(Arc::new(LooseningFilter)) + .with_proxy(Arc::new(PrivacyRoutes)) + .build(), + )) +} + +fn register_nothing( + _settings: &Settings, +) -> Result, Report> { + Ok(None) +} + +/// Leaves the request for [`SPOILED_PATH`] with an unreadable `Cookie` +/// header. A deployment's own modules prepare a request after the modules +/// the adapter links, so this runs after every one of them. +fn spoil_cookie( + _settings: &Settings, + request: &mut Request, +) -> Result<(), Report> { + if request.uri().path() == SPOILED_PATH { + request.headers_mut().insert("cookie", unreadable_cookie()); + } + Ok(()) +} + fn always(_settings: &Settings) -> Result> { Ok(true) } +fn never(_settings: &Settings) -> Result> { + Ok(false) +} + fn probe_builder() -> IntegrationBuilder { IntegrationBuilder::new(PROBE_ID, "adapter-tests", register_probe, always) .with_module_name(PROBE_MODULE) @@ -192,6 +342,17 @@ fn answering_builder() -> IntegrationBuilder { .with_module_name(ANSWERING_MODULE) } +fn privacy_builder() -> IntegrationBuilder { + IntegrationBuilder::new(PRIVACY_ID, "adapter-tests", register_privacy, always) + .with_module_name(PRIVACY_MODULE) +} + +fn spoiling_builder() -> IntegrationBuilder { + IntegrationBuilder::new(SPOILING_ID, "adapter-tests", register_nothing, never) + .with_module_name(SPOILING_MODULE) + .with_request_preparer(spoil_cookie) +} + /// Settings for the publisher the requests are for, with `extra` appended. fn settings_with(extra: &str) -> Settings { Settings::from_toml(&format!( @@ -250,6 +411,21 @@ fn get(path: &str, cookie: Option<&str>) -> Request { .expect("should build the request") } +/// A `Cookie` header holding an accented letter in UTF-8, which a header +/// value read as text refuses. +fn unreadable_cookie() -> HeaderValue { + HeaderValue::from_bytes(b"session=caf\xc3\xa9").expect("should build the header") +} + +/// A `GET` of `path` from the browser `user_agent` names. +fn get_from(path: &str, user_agent: &'static str) -> Request { + let mut request = get(path, None); + request + .headers_mut() + .insert("user-agent", HeaderValue::from_static(user_agent)); + request +} + async fn route(router: &RouterService, request: Request) -> Response { router .oneshot(request) @@ -400,3 +576,273 @@ async fn robots_txt_is_answered_before_the_request_filters() { ); assert_eq!(origin.recorded_request_methods(), ["GET"]); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn robots_txt_shows_the_device_module_none_of_the_crawler_s_cookies() { + let origin = Arc::new(StubHttpClient::new()); + origin.push_response_with_headers( + 200, + b"User-agent: *\nAllow: /\n".to_vec(), + vec![("content-type", "text/plain")], + ); + let router = router_with( + settings_with(&format!( + r#" + [device] + module = "{PROBE_MODULE}" + + [testing] + modules = ["cycle-probe"] + + [robots-txt] + modules = "refuse_all" + always_allow = ["/ads.txt"] + + [[fetch]] + media_type = "text/plain" + path = "/robots.txt" + middleware = ["robots_txt"] + "# + )), + &[probe_builder()], + &origin, + ); + + let robots = route(&router, get("/robots.txt", Some(CRAWLER_COOKIE))).await; + let report = route( + &router, + get("/integrations/cycle_probe/report", Some(READER_COOKIE)), + ) + .await; + + assert_eq!(robots.status(), StatusCode::OK); + assert_eq!( + header(&robots, EFFECT_HEADER), + None, + "the filters should not run for robots.txt" + ); + assert_eq!(report.status(), StatusCode::OK); + let shown = cookies_shown(); + assert!( + shown.iter().any(|cookie| cookie == READER_COOKIE), + "the device module should be shown the cookies of a request in the cycle" + ); + assert!( + !shown.iter().any(|cookie| cookie.contains(CRAWLER_COOKIE)), + "the device module should not be shown the crawler's cookies" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_edge_cookie_context_that_cannot_be_read_answers_before_the_filters() { + let origin = Arc::new(StubHttpClient::new()); + let router = router_with( + settings_with("[testing]\nmodules = [\"answering-filter\"]"), + &[answering_builder(), spoiling_builder()], + &origin, + ); + + let mut sent = get("/article", None); + sent.headers_mut().insert("cookie", unreadable_cookie()); + let page = route(&router, sent).await; + let spoiled = route(&router, get(SPOILED_PATH, None)).await; + + // A preparer the adapter links rebuilds the `Cookie` header from the + // values it can read, so a client's unreadable header never reaches the + // context. + assert_eq!( + text(page), + ANSWER, + "a client's unreadable Cookie header should not stop the cycle" + ); + assert_eq!( + spoiled.status(), + StatusCode::BAD_REQUEST, + "the context's error should answer the request" + ); + assert_eq!( + header(&spoiled, EFFECT_HEADER), + None, + "the filters should not run once the context cannot be read" + ); + assert!( + origin.recorded_request_uris().is_empty(), + "the publisher's origin should not be asked" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn platform_classifies_the_request_with_the_built_in_module() { + let origin = Arc::new(StubHttpClient::new()); + let router = router_with( + settings_with( + r#" + [device] + module = "platform" + + [testing] + modules = ["cycle-probe"] + "#, + ), + &[probe_builder()], + &origin, + ); + + let response = route( + &router, + get_from( + "/integrations/cycle_probe/report", + "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X)", + ), + ) + .await; + + assert_eq!(response.status(), StatusCode::OK); + let report: serde_json::Value = + serde_json::from_str(&text(response)).expect("the report should be JSON"); + assert_eq!( + report["device"], "ios", + "the built-in module should classify the request from its User-Agent" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn the_inspection_endpoints_and_the_attestation_pages_skip_the_request_filters() { + let origin = Arc::new(StubHttpClient::new()); + let router = router_with( + settings_with( + r#" + [testing] + modules = ["answering-filter"] + + [attestation] + operator = "Example Operator" + context = "example-attestation:v1" + verify_url = "https://verifier.example/verify?host=test-publisher.example.com" + "#, + ), + &[answering_builder()], + &origin, + ); + + let page = route(&router, get("/article", None)).await; + assert_eq!( + text(page), + ANSWER, + "the filter should answer a request in the cycle" + ); + for path in [ + "/_ts/permissions", + "/_ts/permissions.json", + "/_ts/config", + "/_ts/config.json", + "/_ts/data", + "/_ts/attestation", + "/_ts/attestation.json", + ] { + let response = route(&router, get(path, None)).await; + assert_eq!( + header(&response, EFFECT_HEADER), + None, + "{path} should be answered without the request filters" + ); + assert_ne!( + text(response), + ANSWER, + "{path} should be answered by its own handler" + ); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_filter_cannot_make_a_response_core_made_private_public() { + let origin = Arc::new(StubHttpClient::new()); + let router = router_with( + settings_with("[testing]\nmodules = [\"privacy-probe\"]"), + &[privacy_builder()], + &origin, + ); + + let response = route(&router, get(PRIVATE_PATH, None)).await; + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + header(&response, "cache-control"), + Some("no-store, private"), + "a response core marked private should stay private after a filter's header" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_filter_s_cookie_keeps_the_response_from_shared_caches() { + let origin = Arc::new(StubHttpClient::new()); + let router = router_with( + settings_with("[testing]\nmodules = [\"privacy-probe\"]"), + &[privacy_builder()], + &origin, + ); + + let response = route(&router, get(PUBLIC_PATH, None)).await; + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(header(&response, "set-cookie"), Some(FILTER_COOKIE)); + assert_eq!( + header(&response, "cache-control"), + Some("private, max-age=0"), + "a response a filter sets a cookie on should not be stored by a shared cache" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_filter_cannot_add_a_shared_cache_header_to_a_private_response() { + let origin = Arc::new(StubHttpClient::new()); + let router = router_with( + settings_with("[testing]\nmodules = [\"privacy-probe\"]"), + &[privacy_builder()], + &origin, + ); + + let response = route(&router, get(UNCACHEABLE_PATH, None)).await; + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + header(&response, "cache-control"), + Some("private, max-age=600") + ); + assert_eq!( + header(&response, "surrogate-control"), + None, + "a private response should carry no shared cache header a filter names" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_response_core_made_private_loses_the_validators_an_operator_writes() { + let origin = Arc::new(StubHttpClient::new()); + let mut settings = settings_with("[testing]\nmodules = [\"privacy-probe\"]"); + for (name, value) in [ + ("etag", "\"operator\""), + ("last-modified", "Wed, 21 Oct 2015 07:28:00 GMT"), + ] { + settings + .response_headers + .insert(name.to_owned(), value.to_owned()); + } + let router = router_with(settings, &[privacy_builder()], &origin); + + let response = route(&router, get(UNFILTERED_PATH, None)).await; + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + header(&response, "cache-control"), + Some("no-store, private") + ); + for name in ["etag", "last-modified"] { + assert_eq!( + header(&response, name), + None, + "a response core marked private should lose the {name} an operator writes, \ + with no filter's header on it" + ); + } +} diff --git a/crates/trusted-server-adapter-cloudflare/src/app/request_cycle_tests.rs b/crates/trusted-server-adapter-cloudflare/src/app/request_cycle_tests.rs index 5f5af49602..d710ce99f1 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app/request_cycle_tests.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app/request_cycle_tests.rs @@ -1,12 +1,15 @@ //! The request cycle this adapter runs for a routed request, checked through //! its router: the request filters run before the route and their response -//! effects after the finalize headers, the route's module call carries the -//! device classification, and robots.txt is answered ahead of both. +//! effects after the finalize headers, without making the response less +//! private, the route's module call carries the device classification, an +//! Edge Cookie context that cannot be read answers before the filters, and +//! robots.txt, the inspection endpoints and the attestation pages are answered +//! without them. -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use edgezero_core::body::Body; -use edgezero_core::http::{Request, Response, StatusCode, request_builder}; +use edgezero_core::http::{HeaderValue, Request, Response, StatusCode, request_builder}; use edgezero_core::router::RouterService; use error_stack::Report; use trusted_server_core::ec::device::{DeviceModule, DeviceSignals}; @@ -22,6 +25,7 @@ use trusted_server_core::platform::PlatformHttpClient; use trusted_server_core::platform::test_support::{ StubHttpClient, build_services_with_http_client, }; +use trusted_server_core::response_privacy::enforce_terminal_private_cache_privacy; use trusted_server_core::settings::Settings; use super::{build_router, build_state_with_registrations_and_services}; @@ -43,6 +47,48 @@ const FILTERED_HEADER: &str = "x-cycle-probe-filtered"; /// What the answering filter answers. const ANSWER: &str = "answered by the filter"; +/// The id and module name of the privacy probe, whose routes answer with a +/// cache policy and whose filter names a response header for each of them. +const PRIVACY_ID: &str = "privacy_probe"; +const PRIVACY_MODULE: &str = "testing.privacy-probe"; + +/// The privacy probe's routes. The first and the last answer with core's +/// marker for a response made for one reader, the second is public, and the +/// third is private by its own policy alone. The filter names a header for +/// each but the last. +const PRIVATE_PATH: &str = "/integrations/privacy_probe/private"; +const PUBLIC_PATH: &str = "/integrations/privacy_probe/public"; +const UNCACHEABLE_PATH: &str = "/integrations/privacy_probe/uncacheable"; +const UNFILTERED_PATH: &str = "/integrations/privacy_probe/unfiltered"; + +/// The policy the public route answers with, and the one the filter writes +/// on the route core marks private. +const PUBLIC_POLICY: &str = "public, max-age=600"; +/// The cookie the filter sets on the public route. +const FILTER_COOKIE: &str = "probe=1; Path=/"; + +/// Cookies sent to robots.txt and to a module route, each found nowhere else. +const CRAWLER_COOKIE: &str = "crawler_session=robots"; +const READER_COOKIE: &str = "reader_session=route"; + +/// The id and module name of a module whose preparer leaves the request for +/// [`SPOILED_PATH`] with a `Cookie` header the Edge Cookie context cannot +/// read. +const SPOILING_ID: &str = "spoiling_preparer"; +const SPOILING_MODULE: &str = "testing.spoiling-preparer"; +const SPOILED_PATH: &str = "/spoiled"; + +/// Every `Cookie` header the marking device module was shown, by every test +/// in this file, so a test can tell whether a request's cookies reached it. +static COOKIES_SHOWN: Mutex> = Mutex::new(Vec::new()); + +fn cookies_shown() -> Vec { + COOKIES_SHOWN + .lock() + .expect("the record of the cookies shown should be readable") + .clone() +} + struct AnsweringFilter; #[async_trait::async_trait(?Send)] @@ -87,16 +133,18 @@ impl IntegrationRequestFilter for ContinuingFilter { } /// A device module that marks its classification with the cookies it was -/// shown, so a test can tell that it ran and what it saw. +/// shown, and records them, so a test can tell that it ran and what it saw. struct MarkingDevice; impl MarkingDevice { fn mark(&self, request_info: &dyn RequestInfo) -> DeviceSignals { + let cookie = request_info.header("cookie").unwrap_or_default(); + COOKIES_SHOWN + .lock() + .expect("the record of the cookies shown should be writable") + .push(cookie.to_owned()); let mut signals = DeviceSignals::derive_ua_only(request_info.user_agent()); - signals.platform_class = Some(format!( - "marked:{}", - request_info.header("cookie").unwrap_or_default() - )); + signals.platform_class = Some(format!("marked:{cookie}")); signals } } @@ -154,6 +202,73 @@ impl IntegrationProxy for ReportRoute { } } +/// A filter that names, for each of the privacy probe's routes, a response +/// header that would let a shared cache store the route's answer. +struct LooseningFilter; + +#[async_trait::async_trait(?Send)] +impl IntegrationRequestFilter for LooseningFilter { + fn integration_id(&self) -> &'static str { + PRIVACY_ID + } + + async fn filter_request( + &self, + input: RequestFilterInput<'_>, + ) -> Result> { + let response_headers = match input.request.uri().path() { + PRIVATE_PATH => vec![HeaderMutation::set("cache-control", PUBLIC_POLICY)], + PUBLIC_PATH => vec![HeaderMutation::append("set-cookie", FILTER_COOKIE)], + UNCACHEABLE_PATH => vec![HeaderMutation::set("surrogate-control", "max-age=3600")], + _ => Vec::new(), + }; + Ok(RequestFilterDecision::Continue(RequestFilterEffects { + request_headers: Vec::new(), + response_headers, + })) + } +} + +/// The privacy probe's routes, each answering with its own cache policy. +struct PrivacyRoutes; + +#[async_trait::async_trait(?Send)] +impl IntegrationProxy for PrivacyRoutes { + fn integration_name(&self) -> &'static str { + PRIVACY_ID + } + + fn routes(&self) -> Vec { + vec![ + self.get("/private"), + self.get("/public"), + self.get("/uncacheable"), + self.get("/unfiltered"), + ] + } + + async fn handle( + &self, + _call: ModuleCall<'_>, + req: Request, + ) -> Result> { + let path = req.uri().path(); + let policy = if path == UNCACHEABLE_PATH { + "private, max-age=600" + } else { + PUBLIC_POLICY + }; + let mut response = Response::new(Body::from("probe")); + response + .headers_mut() + .insert("cache-control", HeaderValue::from_static(policy)); + if path == PRIVATE_PATH || path == UNFILTERED_PATH { + enforce_terminal_private_cache_privacy(&mut response); + } + Ok(response) + } +} + fn register_probe( _settings: &Settings, ) -> Result, Report> { @@ -178,10 +293,45 @@ fn register_answering( )) } +fn register_privacy( + _settings: &Settings, +) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder(PRIVACY_ID) + .without_js() + .with_request_filter(Arc::new(LooseningFilter)) + .with_proxy(Arc::new(PrivacyRoutes)) + .build(), + )) +} + +fn register_nothing( + _settings: &Settings, +) -> Result, Report> { + Ok(None) +} + +/// Leaves the request for [`SPOILED_PATH`] with an unreadable `Cookie` +/// header. A deployment's own modules prepare a request after the modules +/// the adapter links, so this runs after every one of them. +fn spoil_cookie( + _settings: &Settings, + request: &mut Request, +) -> Result<(), Report> { + if request.uri().path() == SPOILED_PATH { + request.headers_mut().insert("cookie", unreadable_cookie()); + } + Ok(()) +} + fn always(_settings: &Settings) -> Result> { Ok(true) } +fn never(_settings: &Settings) -> Result> { + Ok(false) +} + fn probe_builder() -> IntegrationBuilder { IntegrationBuilder::new(PROBE_ID, "adapter-tests", register_probe, always) .with_module_name(PROBE_MODULE) @@ -192,6 +342,17 @@ fn answering_builder() -> IntegrationBuilder { .with_module_name(ANSWERING_MODULE) } +fn privacy_builder() -> IntegrationBuilder { + IntegrationBuilder::new(PRIVACY_ID, "adapter-tests", register_privacy, always) + .with_module_name(PRIVACY_MODULE) +} + +fn spoiling_builder() -> IntegrationBuilder { + IntegrationBuilder::new(SPOILING_ID, "adapter-tests", register_nothing, never) + .with_module_name(SPOILING_MODULE) + .with_request_preparer(spoil_cookie) +} + /// Settings for the publisher the requests are for, with `extra` appended. fn settings_with(extra: &str) -> Settings { Settings::from_toml(&format!( @@ -253,6 +414,21 @@ fn get(path: &str, cookie: Option<&str>) -> Request { .expect("should build the request") } +/// A `Cookie` header holding an accented letter in UTF-8, which a header +/// value read as text refuses. +fn unreadable_cookie() -> HeaderValue { + HeaderValue::from_bytes(b"session=caf\xc3\xa9").expect("should build the header") +} + +/// A `GET` of `path` from the browser `user_agent` names. +fn get_from(path: &str, user_agent: &'static str) -> Request { + let mut request = get(path, None); + request + .headers_mut() + .insert("user-agent", HeaderValue::from_static(user_agent)); + request +} + async fn route(router: &RouterService, request: Request) -> Response { router .oneshot(request) @@ -403,3 +579,273 @@ async fn robots_txt_is_answered_before_the_request_filters() { ); assert_eq!(origin.recorded_request_methods(), ["GET"]); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn robots_txt_shows_the_device_module_none_of_the_crawler_s_cookies() { + let origin = Arc::new(StubHttpClient::new()); + origin.push_response_with_headers( + 200, + b"User-agent: *\nAllow: /\n".to_vec(), + vec![("content-type", "text/plain")], + ); + let router = router_with( + settings_with(&format!( + r#" + [device] + module = "{PROBE_MODULE}" + + [testing] + modules = ["cycle-probe"] + + [robots-txt] + modules = "refuse_all" + always_allow = ["/ads.txt"] + + [[fetch]] + media_type = "text/plain" + path = "/robots.txt" + middleware = ["robots_txt"] + "# + )), + &[probe_builder()], + &origin, + ); + + let robots = route(&router, get("/robots.txt", Some(CRAWLER_COOKIE))).await; + let report = route( + &router, + get("/integrations/cycle_probe/report", Some(READER_COOKIE)), + ) + .await; + + assert_eq!(robots.status(), StatusCode::OK); + assert_eq!( + header(&robots, EFFECT_HEADER), + None, + "the filters should not run for robots.txt" + ); + assert_eq!(report.status(), StatusCode::OK); + let shown = cookies_shown(); + assert!( + shown.iter().any(|cookie| cookie == READER_COOKIE), + "the device module should be shown the cookies of a request in the cycle" + ); + assert!( + !shown.iter().any(|cookie| cookie.contains(CRAWLER_COOKIE)), + "the device module should not be shown the crawler's cookies" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_edge_cookie_context_that_cannot_be_read_answers_before_the_filters() { + let origin = Arc::new(StubHttpClient::new()); + let router = router_with( + settings_with("[testing]\nmodules = [\"answering-filter\"]"), + &[answering_builder(), spoiling_builder()], + &origin, + ); + + let mut sent = get("/article", None); + sent.headers_mut().insert("cookie", unreadable_cookie()); + let page = route(&router, sent).await; + let spoiled = route(&router, get(SPOILED_PATH, None)).await; + + // A preparer the adapter links rebuilds the `Cookie` header from the + // values it can read, so a client's unreadable header never reaches the + // context. + assert_eq!( + text(page), + ANSWER, + "a client's unreadable Cookie header should not stop the cycle" + ); + assert_eq!( + spoiled.status(), + StatusCode::BAD_REQUEST, + "the context's error should answer the request" + ); + assert_eq!( + header(&spoiled, EFFECT_HEADER), + None, + "the filters should not run once the context cannot be read" + ); + assert!( + origin.recorded_request_uris().is_empty(), + "the publisher's origin should not be asked" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn platform_classifies_the_request_with_the_built_in_module() { + let origin = Arc::new(StubHttpClient::new()); + let router = router_with( + settings_with( + r#" + [device] + module = "platform" + + [testing] + modules = ["cycle-probe"] + "#, + ), + &[probe_builder()], + &origin, + ); + + let response = route( + &router, + get_from( + "/integrations/cycle_probe/report", + "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X)", + ), + ) + .await; + + assert_eq!(response.status(), StatusCode::OK); + let report: serde_json::Value = + serde_json::from_str(&text(response)).expect("the report should be JSON"); + assert_eq!( + report["device"], "ios", + "the built-in module should classify the request from its User-Agent" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn the_inspection_endpoints_and_the_attestation_pages_skip_the_request_filters() { + let origin = Arc::new(StubHttpClient::new()); + let router = router_with( + settings_with( + r#" + [testing] + modules = ["answering-filter"] + + [attestation] + operator = "Example Operator" + context = "example-attestation:v1" + verify_url = "https://verifier.example/verify?host=test-publisher.example.com" + "#, + ), + &[answering_builder()], + &origin, + ); + + let page = route(&router, get("/article", None)).await; + assert_eq!( + text(page), + ANSWER, + "the filter should answer a request in the cycle" + ); + for path in [ + "/_ts/permissions", + "/_ts/permissions.json", + "/_ts/config", + "/_ts/config.json", + "/_ts/data", + "/_ts/attestation", + "/_ts/attestation.json", + ] { + let response = route(&router, get(path, None)).await; + assert_eq!( + header(&response, EFFECT_HEADER), + None, + "{path} should be answered without the request filters" + ); + assert_ne!( + text(response), + ANSWER, + "{path} should be answered by its own handler" + ); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_filter_cannot_make_a_response_core_made_private_public() { + let origin = Arc::new(StubHttpClient::new()); + let router = router_with( + settings_with("[testing]\nmodules = [\"privacy-probe\"]"), + &[privacy_builder()], + &origin, + ); + + let response = route(&router, get(PRIVATE_PATH, None)).await; + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + header(&response, "cache-control"), + Some("no-store, private"), + "a response core marked private should stay private after a filter's header" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_filter_s_cookie_keeps_the_response_from_shared_caches() { + let origin = Arc::new(StubHttpClient::new()); + let router = router_with( + settings_with("[testing]\nmodules = [\"privacy-probe\"]"), + &[privacy_builder()], + &origin, + ); + + let response = route(&router, get(PUBLIC_PATH, None)).await; + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(header(&response, "set-cookie"), Some(FILTER_COOKIE)); + assert_eq!( + header(&response, "cache-control"), + Some("private, max-age=0"), + "a response a filter sets a cookie on should not be stored by a shared cache" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_filter_cannot_add_a_shared_cache_header_to_a_private_response() { + let origin = Arc::new(StubHttpClient::new()); + let router = router_with( + settings_with("[testing]\nmodules = [\"privacy-probe\"]"), + &[privacy_builder()], + &origin, + ); + + let response = route(&router, get(UNCACHEABLE_PATH, None)).await; + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + header(&response, "cache-control"), + Some("private, max-age=600") + ); + assert_eq!( + header(&response, "surrogate-control"), + None, + "a private response should carry no shared cache header a filter names" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_response_core_made_private_loses_the_validators_an_operator_writes() { + let origin = Arc::new(StubHttpClient::new()); + let mut settings = settings_with("[testing]\nmodules = [\"privacy-probe\"]"); + for (name, value) in [ + ("etag", "\"operator\""), + ("last-modified", "Wed, 21 Oct 2015 07:28:00 GMT"), + ] { + settings + .response_headers + .insert(name.to_owned(), value.to_owned()); + } + let router = router_with(settings, &[privacy_builder()], &origin); + + let response = route(&router, get(UNFILTERED_PATH, None)).await; + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + header(&response, "cache-control"), + Some("no-store, private") + ); + for name in ["etag", "last-modified"] { + assert_eq!( + header(&response, name), + None, + "a response core marked private should lose the {name} an operator writes, \ + with no filter's header on it" + ); + } +} diff --git a/crates/trusted-server-adapter-spin/src/app/request_cycle_tests.rs b/crates/trusted-server-adapter-spin/src/app/request_cycle_tests.rs index 5f5af49602..d710ce99f1 100644 --- a/crates/trusted-server-adapter-spin/src/app/request_cycle_tests.rs +++ b/crates/trusted-server-adapter-spin/src/app/request_cycle_tests.rs @@ -1,12 +1,15 @@ //! The request cycle this adapter runs for a routed request, checked through //! its router: the request filters run before the route and their response -//! effects after the finalize headers, the route's module call carries the -//! device classification, and robots.txt is answered ahead of both. +//! effects after the finalize headers, without making the response less +//! private, the route's module call carries the device classification, an +//! Edge Cookie context that cannot be read answers before the filters, and +//! robots.txt, the inspection endpoints and the attestation pages are answered +//! without them. -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use edgezero_core::body::Body; -use edgezero_core::http::{Request, Response, StatusCode, request_builder}; +use edgezero_core::http::{HeaderValue, Request, Response, StatusCode, request_builder}; use edgezero_core::router::RouterService; use error_stack::Report; use trusted_server_core::ec::device::{DeviceModule, DeviceSignals}; @@ -22,6 +25,7 @@ use trusted_server_core::platform::PlatformHttpClient; use trusted_server_core::platform::test_support::{ StubHttpClient, build_services_with_http_client, }; +use trusted_server_core::response_privacy::enforce_terminal_private_cache_privacy; use trusted_server_core::settings::Settings; use super::{build_router, build_state_with_registrations_and_services}; @@ -43,6 +47,48 @@ const FILTERED_HEADER: &str = "x-cycle-probe-filtered"; /// What the answering filter answers. const ANSWER: &str = "answered by the filter"; +/// The id and module name of the privacy probe, whose routes answer with a +/// cache policy and whose filter names a response header for each of them. +const PRIVACY_ID: &str = "privacy_probe"; +const PRIVACY_MODULE: &str = "testing.privacy-probe"; + +/// The privacy probe's routes. The first and the last answer with core's +/// marker for a response made for one reader, the second is public, and the +/// third is private by its own policy alone. The filter names a header for +/// each but the last. +const PRIVATE_PATH: &str = "/integrations/privacy_probe/private"; +const PUBLIC_PATH: &str = "/integrations/privacy_probe/public"; +const UNCACHEABLE_PATH: &str = "/integrations/privacy_probe/uncacheable"; +const UNFILTERED_PATH: &str = "/integrations/privacy_probe/unfiltered"; + +/// The policy the public route answers with, and the one the filter writes +/// on the route core marks private. +const PUBLIC_POLICY: &str = "public, max-age=600"; +/// The cookie the filter sets on the public route. +const FILTER_COOKIE: &str = "probe=1; Path=/"; + +/// Cookies sent to robots.txt and to a module route, each found nowhere else. +const CRAWLER_COOKIE: &str = "crawler_session=robots"; +const READER_COOKIE: &str = "reader_session=route"; + +/// The id and module name of a module whose preparer leaves the request for +/// [`SPOILED_PATH`] with a `Cookie` header the Edge Cookie context cannot +/// read. +const SPOILING_ID: &str = "spoiling_preparer"; +const SPOILING_MODULE: &str = "testing.spoiling-preparer"; +const SPOILED_PATH: &str = "/spoiled"; + +/// Every `Cookie` header the marking device module was shown, by every test +/// in this file, so a test can tell whether a request's cookies reached it. +static COOKIES_SHOWN: Mutex> = Mutex::new(Vec::new()); + +fn cookies_shown() -> Vec { + COOKIES_SHOWN + .lock() + .expect("the record of the cookies shown should be readable") + .clone() +} + struct AnsweringFilter; #[async_trait::async_trait(?Send)] @@ -87,16 +133,18 @@ impl IntegrationRequestFilter for ContinuingFilter { } /// A device module that marks its classification with the cookies it was -/// shown, so a test can tell that it ran and what it saw. +/// shown, and records them, so a test can tell that it ran and what it saw. struct MarkingDevice; impl MarkingDevice { fn mark(&self, request_info: &dyn RequestInfo) -> DeviceSignals { + let cookie = request_info.header("cookie").unwrap_or_default(); + COOKIES_SHOWN + .lock() + .expect("the record of the cookies shown should be writable") + .push(cookie.to_owned()); let mut signals = DeviceSignals::derive_ua_only(request_info.user_agent()); - signals.platform_class = Some(format!( - "marked:{}", - request_info.header("cookie").unwrap_or_default() - )); + signals.platform_class = Some(format!("marked:{cookie}")); signals } } @@ -154,6 +202,73 @@ impl IntegrationProxy for ReportRoute { } } +/// A filter that names, for each of the privacy probe's routes, a response +/// header that would let a shared cache store the route's answer. +struct LooseningFilter; + +#[async_trait::async_trait(?Send)] +impl IntegrationRequestFilter for LooseningFilter { + fn integration_id(&self) -> &'static str { + PRIVACY_ID + } + + async fn filter_request( + &self, + input: RequestFilterInput<'_>, + ) -> Result> { + let response_headers = match input.request.uri().path() { + PRIVATE_PATH => vec![HeaderMutation::set("cache-control", PUBLIC_POLICY)], + PUBLIC_PATH => vec![HeaderMutation::append("set-cookie", FILTER_COOKIE)], + UNCACHEABLE_PATH => vec![HeaderMutation::set("surrogate-control", "max-age=3600")], + _ => Vec::new(), + }; + Ok(RequestFilterDecision::Continue(RequestFilterEffects { + request_headers: Vec::new(), + response_headers, + })) + } +} + +/// The privacy probe's routes, each answering with its own cache policy. +struct PrivacyRoutes; + +#[async_trait::async_trait(?Send)] +impl IntegrationProxy for PrivacyRoutes { + fn integration_name(&self) -> &'static str { + PRIVACY_ID + } + + fn routes(&self) -> Vec { + vec![ + self.get("/private"), + self.get("/public"), + self.get("/uncacheable"), + self.get("/unfiltered"), + ] + } + + async fn handle( + &self, + _call: ModuleCall<'_>, + req: Request, + ) -> Result> { + let path = req.uri().path(); + let policy = if path == UNCACHEABLE_PATH { + "private, max-age=600" + } else { + PUBLIC_POLICY + }; + let mut response = Response::new(Body::from("probe")); + response + .headers_mut() + .insert("cache-control", HeaderValue::from_static(policy)); + if path == PRIVATE_PATH || path == UNFILTERED_PATH { + enforce_terminal_private_cache_privacy(&mut response); + } + Ok(response) + } +} + fn register_probe( _settings: &Settings, ) -> Result, Report> { @@ -178,10 +293,45 @@ fn register_answering( )) } +fn register_privacy( + _settings: &Settings, +) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder(PRIVACY_ID) + .without_js() + .with_request_filter(Arc::new(LooseningFilter)) + .with_proxy(Arc::new(PrivacyRoutes)) + .build(), + )) +} + +fn register_nothing( + _settings: &Settings, +) -> Result, Report> { + Ok(None) +} + +/// Leaves the request for [`SPOILED_PATH`] with an unreadable `Cookie` +/// header. A deployment's own modules prepare a request after the modules +/// the adapter links, so this runs after every one of them. +fn spoil_cookie( + _settings: &Settings, + request: &mut Request, +) -> Result<(), Report> { + if request.uri().path() == SPOILED_PATH { + request.headers_mut().insert("cookie", unreadable_cookie()); + } + Ok(()) +} + fn always(_settings: &Settings) -> Result> { Ok(true) } +fn never(_settings: &Settings) -> Result> { + Ok(false) +} + fn probe_builder() -> IntegrationBuilder { IntegrationBuilder::new(PROBE_ID, "adapter-tests", register_probe, always) .with_module_name(PROBE_MODULE) @@ -192,6 +342,17 @@ fn answering_builder() -> IntegrationBuilder { .with_module_name(ANSWERING_MODULE) } +fn privacy_builder() -> IntegrationBuilder { + IntegrationBuilder::new(PRIVACY_ID, "adapter-tests", register_privacy, always) + .with_module_name(PRIVACY_MODULE) +} + +fn spoiling_builder() -> IntegrationBuilder { + IntegrationBuilder::new(SPOILING_ID, "adapter-tests", register_nothing, never) + .with_module_name(SPOILING_MODULE) + .with_request_preparer(spoil_cookie) +} + /// Settings for the publisher the requests are for, with `extra` appended. fn settings_with(extra: &str) -> Settings { Settings::from_toml(&format!( @@ -253,6 +414,21 @@ fn get(path: &str, cookie: Option<&str>) -> Request { .expect("should build the request") } +/// A `Cookie` header holding an accented letter in UTF-8, which a header +/// value read as text refuses. +fn unreadable_cookie() -> HeaderValue { + HeaderValue::from_bytes(b"session=caf\xc3\xa9").expect("should build the header") +} + +/// A `GET` of `path` from the browser `user_agent` names. +fn get_from(path: &str, user_agent: &'static str) -> Request { + let mut request = get(path, None); + request + .headers_mut() + .insert("user-agent", HeaderValue::from_static(user_agent)); + request +} + async fn route(router: &RouterService, request: Request) -> Response { router .oneshot(request) @@ -403,3 +579,273 @@ async fn robots_txt_is_answered_before_the_request_filters() { ); assert_eq!(origin.recorded_request_methods(), ["GET"]); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn robots_txt_shows_the_device_module_none_of_the_crawler_s_cookies() { + let origin = Arc::new(StubHttpClient::new()); + origin.push_response_with_headers( + 200, + b"User-agent: *\nAllow: /\n".to_vec(), + vec![("content-type", "text/plain")], + ); + let router = router_with( + settings_with(&format!( + r#" + [device] + module = "{PROBE_MODULE}" + + [testing] + modules = ["cycle-probe"] + + [robots-txt] + modules = "refuse_all" + always_allow = ["/ads.txt"] + + [[fetch]] + media_type = "text/plain" + path = "/robots.txt" + middleware = ["robots_txt"] + "# + )), + &[probe_builder()], + &origin, + ); + + let robots = route(&router, get("/robots.txt", Some(CRAWLER_COOKIE))).await; + let report = route( + &router, + get("/integrations/cycle_probe/report", Some(READER_COOKIE)), + ) + .await; + + assert_eq!(robots.status(), StatusCode::OK); + assert_eq!( + header(&robots, EFFECT_HEADER), + None, + "the filters should not run for robots.txt" + ); + assert_eq!(report.status(), StatusCode::OK); + let shown = cookies_shown(); + assert!( + shown.iter().any(|cookie| cookie == READER_COOKIE), + "the device module should be shown the cookies of a request in the cycle" + ); + assert!( + !shown.iter().any(|cookie| cookie.contains(CRAWLER_COOKIE)), + "the device module should not be shown the crawler's cookies" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_edge_cookie_context_that_cannot_be_read_answers_before_the_filters() { + let origin = Arc::new(StubHttpClient::new()); + let router = router_with( + settings_with("[testing]\nmodules = [\"answering-filter\"]"), + &[answering_builder(), spoiling_builder()], + &origin, + ); + + let mut sent = get("/article", None); + sent.headers_mut().insert("cookie", unreadable_cookie()); + let page = route(&router, sent).await; + let spoiled = route(&router, get(SPOILED_PATH, None)).await; + + // A preparer the adapter links rebuilds the `Cookie` header from the + // values it can read, so a client's unreadable header never reaches the + // context. + assert_eq!( + text(page), + ANSWER, + "a client's unreadable Cookie header should not stop the cycle" + ); + assert_eq!( + spoiled.status(), + StatusCode::BAD_REQUEST, + "the context's error should answer the request" + ); + assert_eq!( + header(&spoiled, EFFECT_HEADER), + None, + "the filters should not run once the context cannot be read" + ); + assert!( + origin.recorded_request_uris().is_empty(), + "the publisher's origin should not be asked" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn platform_classifies_the_request_with_the_built_in_module() { + let origin = Arc::new(StubHttpClient::new()); + let router = router_with( + settings_with( + r#" + [device] + module = "platform" + + [testing] + modules = ["cycle-probe"] + "#, + ), + &[probe_builder()], + &origin, + ); + + let response = route( + &router, + get_from( + "/integrations/cycle_probe/report", + "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X)", + ), + ) + .await; + + assert_eq!(response.status(), StatusCode::OK); + let report: serde_json::Value = + serde_json::from_str(&text(response)).expect("the report should be JSON"); + assert_eq!( + report["device"], "ios", + "the built-in module should classify the request from its User-Agent" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn the_inspection_endpoints_and_the_attestation_pages_skip_the_request_filters() { + let origin = Arc::new(StubHttpClient::new()); + let router = router_with( + settings_with( + r#" + [testing] + modules = ["answering-filter"] + + [attestation] + operator = "Example Operator" + context = "example-attestation:v1" + verify_url = "https://verifier.example/verify?host=test-publisher.example.com" + "#, + ), + &[answering_builder()], + &origin, + ); + + let page = route(&router, get("/article", None)).await; + assert_eq!( + text(page), + ANSWER, + "the filter should answer a request in the cycle" + ); + for path in [ + "/_ts/permissions", + "/_ts/permissions.json", + "/_ts/config", + "/_ts/config.json", + "/_ts/data", + "/_ts/attestation", + "/_ts/attestation.json", + ] { + let response = route(&router, get(path, None)).await; + assert_eq!( + header(&response, EFFECT_HEADER), + None, + "{path} should be answered without the request filters" + ); + assert_ne!( + text(response), + ANSWER, + "{path} should be answered by its own handler" + ); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_filter_cannot_make_a_response_core_made_private_public() { + let origin = Arc::new(StubHttpClient::new()); + let router = router_with( + settings_with("[testing]\nmodules = [\"privacy-probe\"]"), + &[privacy_builder()], + &origin, + ); + + let response = route(&router, get(PRIVATE_PATH, None)).await; + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + header(&response, "cache-control"), + Some("no-store, private"), + "a response core marked private should stay private after a filter's header" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_filter_s_cookie_keeps_the_response_from_shared_caches() { + let origin = Arc::new(StubHttpClient::new()); + let router = router_with( + settings_with("[testing]\nmodules = [\"privacy-probe\"]"), + &[privacy_builder()], + &origin, + ); + + let response = route(&router, get(PUBLIC_PATH, None)).await; + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(header(&response, "set-cookie"), Some(FILTER_COOKIE)); + assert_eq!( + header(&response, "cache-control"), + Some("private, max-age=0"), + "a response a filter sets a cookie on should not be stored by a shared cache" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_filter_cannot_add_a_shared_cache_header_to_a_private_response() { + let origin = Arc::new(StubHttpClient::new()); + let router = router_with( + settings_with("[testing]\nmodules = [\"privacy-probe\"]"), + &[privacy_builder()], + &origin, + ); + + let response = route(&router, get(UNCACHEABLE_PATH, None)).await; + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + header(&response, "cache-control"), + Some("private, max-age=600") + ); + assert_eq!( + header(&response, "surrogate-control"), + None, + "a private response should carry no shared cache header a filter names" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_response_core_made_private_loses_the_validators_an_operator_writes() { + let origin = Arc::new(StubHttpClient::new()); + let mut settings = settings_with("[testing]\nmodules = [\"privacy-probe\"]"); + for (name, value) in [ + ("etag", "\"operator\""), + ("last-modified", "Wed, 21 Oct 2015 07:28:00 GMT"), + ] { + settings + .response_headers + .insert(name.to_owned(), value.to_owned()); + } + let router = router_with(settings, &[privacy_builder()], &origin); + + let response = route(&router, get(UNFILTERED_PATH, None)).await; + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + header(&response, "cache-control"), + Some("no-store, private") + ); + for name in ["etag", "last-modified"] { + assert_eq!( + header(&response, name), + None, + "a response core marked private should lose the {name} an operator writes, \ + with no filter's header on it" + ); + } +} From 4549047dc95f6e0801d3f0d0fc856fff41b4851f Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 18 Oct 2026 03:42:41 +0100 Subject: [PATCH 05/18] The robots.txt comments say which of a crawler's cookies are read robots.txt is answered with an empty Edge Cookie context, so its Edge Cookie and consent are not read and no device module is shown its cookies, but the publisher path still parses its Cookie header, as for every request. --- crates/trusted-server-adapter-axum/src/app.rs | 9 +++++---- crates/trusted-server-adapter-cloudflare/src/app.rs | 9 +++++---- crates/trusted-server-adapter-spin/src/app.rs | 9 +++++---- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 38aff1499c..c6c203ef4b 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -694,8 +694,9 @@ async fn dispatch_fallback( // A crawler asking for robots.txt is answered once the modules have // prepared the request, before the Edge Cookie steps and the request - // filters, so nothing is read from its cookies and no filter's answer - // stands in for the file. + // filters, so its Edge Cookie and consent are not read, no device + // module is shown its cookies and no filter's answer stands in for the + // file. if asks_for_robots_txt(&state.settings, &method, &path) { return answer_robots_txt(state, services, req, &method).await; } @@ -786,8 +787,8 @@ fn asks_for_robots_txt(settings: &Settings, method: &Method, path: &str) -> bool } /// Answers robots.txt through the publisher path with an empty Edge Cookie -/// context and no auction, so none of the crawler's cookies or consent -/// signals are read. +/// context and no auction, so the crawler's Edge Cookie and consent +/// signals are not read. async fn answer_robots_txt( state: &AppState, services: &RuntimeServices, diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index 7e1a4f8c28..a79870087b 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -833,8 +833,9 @@ async fn dispatch_fallback( // A crawler asking for robots.txt is answered once the modules have // prepared the request, before the Edge Cookie steps and the request - // filters, so nothing is read from its cookies and no filter's answer - // stands in for the file. + // filters, so its Edge Cookie and consent are not read, no device + // module is shown its cookies and no filter's answer stands in for the + // file. if asks_for_robots_txt(&state.settings, &method, &path) { return answer_robots_txt(state, services, req, &method).await; } @@ -923,8 +924,8 @@ fn asks_for_robots_txt(settings: &Settings, method: &Method, path: &str) -> bool } /// Answers robots.txt through the publisher path with an empty Edge Cookie -/// context and no auction, so none of the crawler's cookies or consent -/// signals are read. +/// context and no auction, so the crawler's Edge Cookie and consent +/// signals are not read. async fn answer_robots_txt( state: &AppState, services: &RuntimeServices, diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index 5b56a20761..f2b60d99de 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -1002,8 +1002,9 @@ async fn dispatch_fallback( // A crawler asking for robots.txt is answered once the modules have // prepared the request, before the Edge Cookie steps and the request - // filters, so nothing is read from its cookies and no filter's answer - // stands in for the file. + // filters, so its Edge Cookie and consent are not read, no device + // module is shown its cookies and no filter's answer stands in for the + // file. if asks_for_robots_txt(&state.settings, &method, &path) { return answer_robots_txt(state, services, req, &method).await; } @@ -1092,8 +1093,8 @@ fn asks_for_robots_txt(settings: &Settings, method: &Method, path: &str) -> bool } /// Answers robots.txt through the publisher path with an empty Edge Cookie -/// context and no auction, so none of the crawler's cookies or consent -/// signals are read. +/// context and no auction, so the crawler's Edge Cookie and consent +/// signals are not read. async fn answer_robots_txt( state: &AppState, services: &RuntimeServices, From a584b071f04851fff66739c823508a1c24c37da5 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 18 Oct 2026 03:43:12 +0100 Subject: [PATCH 06/18] The guides and the changelog say what filters and device modules do on every adapter Every adapter now classifies a request with the device module [device] selects, where a module was asked only for bid request attributes off Fastly. DataDome's timeout_ms and the 51Degrees timeout_ms apply on the Fastly adapter only, DataDome checks /auction and both page-bids paths, the client tag suppression holds on every adapter, and a request filter's response header is written after X-Robots-Tag. The middleware readme no longer says only the Fastly adapter runs request filters. --- CHANGELOG.md | 2 +- crates/middleware/51degrees/README.md | 17 ++++++++--------- docs/guide/configuration.md | 2 +- docs/guide/integration-guide.md | 7 ++++--- docs/guide/integrations/51degrees.md | 2 +- docs/guide/integrations/datadome.md | 14 +++++++------- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 23e155c8f0..bd5c041eb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- The Axum, Cloudflare and Spin adapters run the integration request filters and the device module `[device]` selects, where only the Fastly adapter ran them. Every named route but the inspection endpoints, and the publisher fallback, reads the request's Edge Cookie context with the device classification and then runs the filters before the route, so a filter that answers, such as DataDome's challenge, stands in for the route, and the response headers a filter names are written after the finalize headers. The 51Degrees client hints middleware and DataDome's protection therefore work on every adapter, through HTTP clients that apply no module's own timeout on Axum, Cloudflare and Spin. On those adapters `[device] module = "platform"` classifies from the `User-Agent`, robots.txt is answered before the Edge Cookie steps and the filters with an empty Edge Cookie context, and on Spin the discovery, signature verification and first-party routes now run the modules' preparers. +- The Axum, Cloudflare and Spin adapters run the integration request filters and classify each request with the device module `[device]` selects, as the Fastly adapter does. Before, they ran no filter and asked a selected device module only for a bid request's device attributes. Every named route but the inspection endpoints, and the publisher fallback, reads the request's Edge Cookie context with the device classification and then runs the filters before the route, so a filter that answers, such as DataDome's challenge, stands in for the route, and the response headers a filter names are written after the finalize headers, with the Fastly adapter's privacy guards applied after them. The 51Degrees client hints middleware and DataDome's protection therefore work on every adapter, through HTTP clients that apply no module's own timeout on Axum, Cloudflare and Spin, and DataDome's protection checks `/auction` and `GET` on both page-bids paths as well as the publisher's pages. On those adapters `[device] module = "platform"` classifies from the `User-Agent`, a response core marks private loses any `ETag`, `Last-Modified`, `Expires` or `Age` that `[response_headers]` writes, robots.txt is answered before the Edge Cookie steps and the filters with an empty Edge Cookie context, and on Spin the discovery, signature verification and first-party routes and the page-bids preflight now run the modules' preparers. - **Breaking:** the platform's own device module is selected with `[device] module = "platform"`, as `[geo] module = "platform"` selects the platform's geo lookup, where it was `module = "fastly"`. A document that still writes `fastly` is refused at startup as naming a device module no running module supplies. **For crates outside core**, `build_device_module` takes a `build_platform` factory. - **Breaking for crates outside core:** an adapter declares the request headers its platform sets. `platform::declare_platform_tls_headers` names a header the platform sets on a request that arrived over TLS, which core reads for the scheme and never forwards to an origin. `platform::declare_platform_spoofable_headers` names the platform's other headers a client could also send, which `sanitize_forwarded_headers` strips at entry with the standard forwarding headers, in an adapter that calls it. `platform::declare_platform_internal_headers` names headers that are never passed to a third party. The Fastly adapter declares `fastly-ssl`, `fastly-client-ip` and `x-debug-fastly-pop` before it serves. `SPOOFABLE_FORWARDED_HEADERS` holds the three standard forwarding headers, `HEADER_X_DEBUG_FASTLY_POP` leaves core, and `constants::is_internal_header` answers for the fixed list and the declared headers together. A `[trusted_client_ip]` `ip_header` may start with `x-` or end with `-client-ip`, so `fastly-client-ip` stays valid. On the Axum, Cloudflare and Spin adapters, which declare none of these headers, a client's `fastly-ssl` is no longer removed from the request to the publisher's origin, and on Spin a client's `fastly-ssl` and `fastly-client-ip` are no longer stripped at entry. On those adapters the `x-debug-fastly-pop` header from a client is also no longer kept from third parties, because `INTERNAL_HEADERS` no longer holds it, and is no longer refused as a `[trusted_client_ip]` header name. On the Axum and Cloudflare adapters a client's `fastly-ssl` also no longer makes the request HTTPS. On Fastly, `ts config validate`, which runs with no adapter's declarations, no longer refuses `x-debug-fastly-pop` as a `[trusted_client_ip]` header name, and the service refuses it when it loads the document. - **Breaking for crates outside core:** an adapter tells core whether its platform runs the service in a staging environment with `platform::declare_platform_staging`, and a module asks `platform::platform_is_staging`. `ENV_FASTLY_IS_STAGING` leaves core, the Fastly adapter reads `FASTLY_IS_STAGING` itself, and the DataDome module's test bypass follows the declaration. On the Axum, Cloudflare and Spin adapters, which declare no staging environment, the bypass no longer follows a `FASTLY_IS_STAGING` variable. diff --git a/crates/middleware/51degrees/README.md b/crates/middleware/51degrees/README.md index b1c396a2f0..47ddaf3d60 100644 --- a/crates/middleware/51degrees/README.md +++ b/crates/middleware/51degrees/README.md @@ -1,15 +1,14 @@ # 51Degrees client hints middleware `trusted-server-middleware-51degrees` supplies the `51degrees.client_hints` -serve middleware. On the Fastly adapter, the only adapter that runs request -filters, it asks the 51Degrees service which client hints the reader's -browser can send for each page navigation a `[[serve]]` entry naming it -covers, sending the `User-Agent` and `Sec-CH-UA` hints alone, and leaves the -hint names in the request's state for the other 51Degrees modules. On each -page a `[[serve]]` entry names it for, it writes `Accept-CH` and `Critical-CH` -on the response unless the browser already sends every one of those hints, -and `Permissions-Policy` and the `Delegate-CH` tag that delegate those hints -to the service. +serve middleware. It asks the 51Degrees service which client hints the +reader's browser can send for each page navigation a `[[serve]]` entry +naming it covers, sending the `User-Agent` and `Sec-CH-UA` hints alone, and +leaves the hint names in the request's state for the other 51Degrees +modules. On each page a `[[serve]]` entry names it for, it writes +`Accept-CH` and `Critical-CH` on the response unless the browser already +sends every one of those hints, and `Permissions-Policy` and the +`Delegate-CH` tag that delegate those hints to the service. `[shared.51degrees]` runs it, and `critical` in `[middleware.51degrees.client_hints]` turns the retry of the first navigation diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 5fc0f0be4b..b64bb32b0b 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -1297,7 +1297,7 @@ A module with settings of its own reads them from its own block, `[robots-txt. { ### Client-side tag suppression behavior -On the Fastly adapter, a request that matches an IP-based DataDome exclusion -or the configured test-bypass credential also omits Trusted Server's -automatically injected client-side DataDome tag from processed HTML. This keeps -the client-side layer consistent with the server-side Protection API skip. +A request that matches an IP-based DataDome exclusion or the configured +test-bypass credential also omits Trusted Server's automatically injected +client-side DataDome tag from processed HTML. This keeps the client-side layer +consistent with the server-side Protection API skip. This behavior applies to: From 6b4eb0b8cc7fc66d7580b27d4f124883ab0abf73 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 18 Oct 2026 03:43:22 +0100 Subject: [PATCH 07/18] The client hints design says every adapter runs its filter The table of parts, the builder, the timeout_ms setting, consent, failure, rollout, section 3 and the tests no longer say only the Fastly adapter runs request filters, and a table gives what bounds the call on each adapter. The review log moves after the revision record, as in the other designs. --- ...026-10-14-51degrees-client-hints-design.md | 83 +++++++++---------- 1 file changed, 41 insertions(+), 42 deletions(-) diff --git a/docs/superpowers/specs/2026-10-14-51degrees-client-hints-design.md b/docs/superpowers/specs/2026-10-14-51degrees-client-hints-design.md index fb9e6e536c..2fcae80f29 100644 --- a/docs/superpowers/specs/2026-10-14-51degrees-client-hints-design.md +++ b/docs/superpowers/specs/2026-10-14-51degrees-client-hints-design.md @@ -2,10 +2,10 @@ **Status:** Proposed, 2026-10-14. Raised as draft pull request IABTechLab/trusted-server#1386 (from jwrosewell/trusted-server#39), for Tech Lab review. **Author:** 51Degrees (contributed), for Tech Lab review -**Related specs:** `2026-10-12-shared-vendor-tables-design.md` (the `[shared.51degrees]` table), `2026-10-12-loading-cache-design.md` (the `51degrees-answers` cache), `2026-10-12-page-middleware-design.md` (the `[[serve]]` entries) and `2026-10-14-source-modules-design.md` (the source a page is fetched from), all in IABTechLab/trusted-server#1342, which holds every design document of the stack +**Related specs:** `2026-10-12-shared-vendor-tables-design.md` (the `[shared.51degrees]` table), `2026-10-12-loading-cache-design.md` (the `51degrees-answers` cache), `2026-10-12-page-middleware-design.md` (the `[[serve]]` entries), `2026-10-14-source-modules-design.md` (the source a page is fetched from) and `2026-10-17-request-filters-on-every-adapter-design.md` (the request filters every adapter runs), all in IABTechLab/trusted-server#1342, which holds every design document of the stack **Related PRs:** jwrosewell/trusted-server#39, IABTechLab/trusted-server#1386 **Issues:** IABTechLab/trusted-server#1333 -**Last updated:** 2026-10-16 +**Last updated:** 2026-10-18 > **Why this spec exists.** A browser sends its high entropy client hints only when the site asks for them and delegates them, and nothing in the stack writes the headers and the tag that do so for a deployment that uses the 51Degrees service, so every first page view of a session is priced on the User-Agent alone. @@ -50,11 +50,11 @@ own, because a browser keeps an origin's set until a response names another. The three parts run in different places. -| Part | Runs | Decided by | -| --------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | -| The filter, which asks the service and leaves names | Before routing, on the Fastly adapter alone, which is the only adapter that runs request filters, and it acts on page navigations only | Any `[[serve]]` entry naming the middleware whose `path` covers the request path, the rule the headers use | -| The `Delegate-CH` tag | In the serve phase, on each page | The first `[[serve]]` entry that covers the page's media type and path | -| `Accept-CH`, `Critical-CH` and `Permissions-Policy` | In the headers step of the serve phase, on each response the publisher path sends, a redirect included | Any `[[serve]]` entry whose `path` covers the request path, whatever its media type | +| Part | Runs | Decided by | +| --------------------------------------------------- | ------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------- | +| The filter, which asks the service and leaves names | Before routing, on every adapter, and it acts on page navigations only | Any `[[serve]]` entry naming the middleware whose `path` covers the request path, the rule the headers use | +| The `Delegate-CH` tag | In the serve phase, on each page | The first `[[serve]]` entry that covers the page's media type and path | +| `Accept-CH`, `Critical-CH` and `Permissions-Policy` | In the headers step of the serve phase, on each response the publisher path sends, a redirect included | Any `[[serve]]` entry whose `path` covers the request path, whatever its media type | A page navigation, for the filter, is a `GET` or `HEAD` request whose `Sec-Fetch-Dest` is `document` or whose `Accept` names `text/html`, so a @@ -111,7 +111,7 @@ first navigation from each such browser waits for the service. the vendor's shared table, declares the license key as a secret setting in use whenever the table is present, and registers the filter and the middleware together. The stock list ships it, so every adapter and the `ts` -command line tool take it, although only the Fastly adapter runs its filter. +command line tool take it, and every adapter runs its filter. ```toml [shared.51degrees] @@ -129,28 +129,25 @@ middleware = ["51degrees.client_hints"] | ---------------------------------------------- | -------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `[shared.51degrees] endpoint` | Required | A URL. The call goes to the endpoint with its last path segment replaced by `json`, so a resource key written in that segment is never sent, and a query or fragment on the endpoint is dropped. The hints are delegated to the endpoint's origin | | `[shared.51degrees] license_key` | Required | The name of the license key's entry in the secret store. A table with none, or an empty one, is refused by the configuration check and when the application state is built | -| `[shared.51degrees] timeout_ms` | `500` | `1` to `10000` milliseconds. Only the Fastly adapter makes the call, and it applies the value, see "Which adapters run the filter" | +| `[shared.51degrees] timeout_ms` | `500` | `1` to `10000` milliseconds. Only the Fastly adapter applies it, see "What bounds the call on each adapter" | | `[middleware.51degrees.client_hints] critical` | `true` | `true` or `false` | | `[cache.51degrees-answers]` | `memory_seconds` 60, `store_seconds` 604800 and `store_idle_seconds` 86400 | The four settings every declared cache takes. An answer ends after an hour whatever `store_seconds` says | `[shared.51degrees]` and `[middleware.51degrees.client_hints]` each refuse a key they do not read. -**Which adapters run the filter.** Only the Fastly adapter runs request filters -(`docs/guide/permission-model.md`), and no other adapter calls -`IntegrationRegistry::filter_request`. On Axum, Cloudflare and Spin the table -is accepted, the license key is required and the middleware is registered, -but nothing runs the filter, so the service is asked nothing and no headers -and no tag are written. The table gives what `timeout_ms` does to the call -where it is made, and what the adapter's HTTP client would do with it where -it is not. - -| Adapter | Runs the filter | What `timeout_ms` does to the call | -| --------------------------- | --------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Fastly | Yes | It sets the first byte and between bytes timeouts of the backend, and the connection timeout is fixed at one second (`crates/trusted-server-adapter-fastly/src/backend.rs`) | -| Axum, a development adapter | No | The call is never made. The adapter's HTTP client would ignore `timeout_ms` and allow 5 seconds to connect and 30 in all (`crates/trusted-server-adapter-axum/src/platform.rs`) | -| Cloudflare | No | The call is never made. The adapter's fetch would set no timeout, and the adapter's own note says only the Workers runtime's limit bounds a call (`crates/trusted-server-adapter-cloudflare/src/platform.rs`) | -| Spin | No | The call is never made. `spin_sdk::http::send` takes no timeout, so `timeout_ms` would be ignored, and the adapter's own note says the Spin runtime's default applies (`crates/trusted-server-adapter-spin/src/platform.rs`) | +**What bounds the call on each adapter.** Every adapter runs request filters +(`2026-10-17-request-filters-on-every-adapter-design.md`), so the filter asks +the service on each of them. A module's time limit reaches an adapter only as +the timeouts of the backend it registers for the call, and only the Fastly +adapter applies them, so `timeout_ms` bounds the call there alone. + +| Adapter | What bounds the call | +| --------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Fastly | `timeout_ms` sets the first byte and between bytes timeouts of the backend, and the connection timeout is fixed at one second (`crates/trusted-server-adapter-fastly/src/backend.rs`) | +| Axum, a development adapter | The adapter's HTTP client ignores `timeout_ms` and allows 5 seconds to connect and 30 in all (`crates/trusted-server-adapter-axum/src/platform.rs`) | +| Cloudflare | The adapter's fetch sets no timeout, and the adapter's own note says only the Workers runtime's limit bounds a call (`crates/trusted-server-adapter-cloudflare/src/platform.rs`) | +| Spin | `spin_sdk::http::send` takes no timeout, so `timeout_ms` is ignored, and the adapter's own note says the Spin runtime's default applies (`crates/trusted-server-adapter-spin/src/platform.rs`) | **What each navigation sends.** For a page navigation an entry covers, the filter asks the cache for the answer to one question, and where none is held @@ -163,7 +160,7 @@ address nor a cookie nor any other header is sent. A request with no `User-Agent` asks nothing. The filter reads only the three hint properties from the answer. -**Consent.** The Fastly adapter hands every request filter the permissions +**Consent.** Every adapter hands every request filter the permissions resolved for the request, so a filter can skip or narrow what it does when a permission it depends on is unset (`docs/guide/permission-model.md`). The 51Degrees filter does not read them and the middleware declares no @@ -214,12 +211,14 @@ hints is held like any other, for up to an hour. On Fastly a call that cannot connect fails after one second, and one that connects waits `timeout_ms` for the first byte and for each gap between bytes, so while the service is down every page navigation an entry covers with no held answer waits for those -limits before its page is built. The other three adapters make no call. - -**Rollout.** A deployment without `[shared.51degrees]` is unchanged. On any -adapter but Fastly, adding the table writes nothing on a page and needs only -the license key. On Fastly, adding the table and an entry naming the -middleware changes the pages that entry covers, served to a browser the service +limits before its page is built. On Axum, Cloudflare and Spin the call waits +until the adapter's HTTP client gives up, which is 30 seconds on Axum and the +runtime's own limit on Cloudflare and Spin, so while the service is down each +such navigation waits that long. + +**Rollout.** A deployment without `[shared.51degrees]` is unchanged. On every +adapter, adding the table and an entry naming the middleware changes the +pages that entry covers, served to a browser the service names hints for. The filter leaves state on the request, so core keeps the request to the origin path (`IntegrationRequestState` in @@ -247,10 +246,10 @@ the middleware covers. - Any change to core. The middleware uses the serve phase, the headers step, the request state and the middleware and shared tables the stack already has. -- **Request filters on Axum, Cloudflare and Spin.** Those adapters run none, - so the middleware asks the service nothing and writes nothing there. A - filter step added to them would call the service through HTTP clients that - apply no `timeout_ms`. +- **`timeout_ms` on Axum, Cloudflare and Spin.** Those adapters' HTTP clients + apply no module's time limit, and + `2026-10-17-request-filters-on-every-adapter-design.md` says why that is + left to a change of its own. - **Asking permission first, or holding answers longer.** The two open questions under "Open questions for the reviewer" decide those. @@ -301,13 +300,7 @@ Six things have no test. state. 6. The two crates' tests run natively under `cargo test-axum`, and their lints under `cargo clippy-axum`. No alias for the Fastly target lists them, so - they do not run for `wasm32-wasip1`, although the Fastly adapter is the only - one that runs the filter. - -## Review log - -1. **Round 1, 2026-10-15.** A review in a fresh session, against `split/44-client-hints` at `1f6b83d0c`. Issues found, four of them blocking. The four were that the request filter runs wherever the table is present and not only where an entry covers the page, that the cost to existing pages was not stated, that failure handling and the timeout of each adapter were not stated, and that nothing said what each navigation sends to the service or what consent it needs. This revision states each from the code, which also shows that only the Fastly adapter runs request filters, and it corrects the claims about page navigations, the headers and the tests, replaces a related spec that exists in no branch, and corrects the guide, the changelog and the middleware readme. -2. **Round 2, 2026-10-15.** A lighter review in a fresh session, against this page at `126347b00`, of the round 1 fixes and the new text. No blocking issues once the pull request number was corrected. This revision names the pull request that now holds every design document, makes the guide's description of the wait for the first call match the one second connection limit and the limit on each gap between bytes, and lists in section 3 the code comments that still say more than the design does. + they do not run for `wasm32-wasip1`, the Fastly adapter's target. ## Revision record @@ -317,3 +310,9 @@ Six things have no test. | 2026-10-15 | Fixes from review round 1. What the filter runs on and that only the Fastly adapter runs it, the cost to pages, what each navigation sends and the open questions on consent, how each adapter bounds the call, the answers on failure, the settings, what the headers keep, the definition of a page navigation, the untested paths, the related specs and the review log. | | 2026-10-15 | Fixes from review round 2. The pull request that holds the design documents, the wait for the first call in the guide, and the code comments left for a later change. | | 2026-10-15 | Code fixes for the call's scope, what it sends and header merging. The permission question stays open, with what was found, and the answer lifetime is a new open question. | +| 2026-10-18 | Every adapter runs the filter, from `2026-10-17-request-filters-on-every-adapter-design.md` and its first review round. The table of parts, the builder, the `timeout_ms` setting, what bounds the call on each adapter, consent, failure, rollout, section 3 and the tests say so. | + +## Review log + +1. **Round 1, 2026-10-15.** A review in a fresh session, against `split/44-client-hints` at `1f6b83d0c`. Issues found, four of them blocking. The four were that the request filter runs wherever the table is present and not only where an entry covers the page, that the cost to existing pages was not stated, that failure handling and the timeout of each adapter were not stated, and that nothing said what each navigation sends to the service or what consent it needs. This revision states each from the code, which also shows that only the Fastly adapter runs request filters, and it corrects the claims about page navigations, the headers and the tests, replaces a related spec that exists in no branch, and corrects the guide, the changelog and the middleware readme. +2. **Round 2, 2026-10-15.** A lighter review in a fresh session, against this page at `126347b00`, of the round 1 fixes and the new text. No blocking issues once the pull request number was corrected. This revision names the pull request that now holds every design document, makes the guide's description of the wait for the first call match the one second connection limit and the limit on each gap between bytes, and lists in section 3 the code comments that still say more than the design does. From 17136d04be90a97cd10e70a5067d3a30c6972e8f Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 18 Oct 2026 03:46:15 +0100 Subject: [PATCH 08/18] The request filters design answers its first review round It says what a selected device module did on Axum, Cloudflare and Spin before, what DataDome's filter sends and sets with no permission gating it, why no time limit is added to a filter's call, that the Fastly adapter answers all three inspection endpoints before the preparers, which routes stay out of the cycle, which of a crawler's cookies robots.txt reads, when the Cookie check is reached, the configuration, the new tests, the open questions, and ends with its review log. --- ...request-filters-on-every-adapter-design.md | 85 +++++++++++++------ 1 file changed, 61 insertions(+), 24 deletions(-) diff --git a/docs/superpowers/specs/2026-10-17-request-filters-on-every-adapter-design.md b/docs/superpowers/specs/2026-10-17-request-filters-on-every-adapter-design.md index e23e3d92cd..1cbbd87c75 100644 --- a/docs/superpowers/specs/2026-10-17-request-filters-on-every-adapter-design.md +++ b/docs/superpowers/specs/2026-10-17-request-filters-on-every-adapter-design.md @@ -2,39 +2,41 @@ **Status:** Proposed, 2026-10-17. A pull request above `ts config migrate`, not yet raised, for Tech Lab review. **Author:** 51Degrees (contributed), for Tech Lab review -**Related specs:** `2026-10-14-51degrees-client-hints-design.md`, and the design documents of the stack in IABTechLab/trusted-server#1342 +**Related specs:** `2026-10-14-51degrees-client-hints-design.md`, whose sections on the adapters this change amends, and the design documents of the stack in IABTechLab/trusted-server#1342 **Related PRs:** none yet **Issues:** none -**Last updated:** 2026-10-17 +**Last updated:** 2026-10-18 -> **Why this spec exists.** Only the Fastly adapter runs the integration request filters and the device module `[device]` selects. On Axum, Cloudflare and Spin a module's request filter never runs, so the 51Degrees client hints middleware writes nothing and DataDome's protection never asks its service, and no module call carries a device classification. This change runs both on those three adapters where the Fastly adapter runs them. +> **Why this spec exists.** Only the Fastly adapter runs the integration request filters and classifies each request with the device module `[device]` selects. On Axum, Cloudflare and Spin a module's request filter never runs, so the 51Degrees client hints middleware writes nothing and DataDome's protection never asks its service, and a device module is asked only for a bid request's device attributes, so no module call carries a device classification. This change runs both steps on those three adapters where the Fastly adapter runs them. ## 1. Problem The Fastly adapter runs three steps that no other adapter runs. 1. **Request filters.** `run_pre_route_filters` in `crates/trusted-server-adapter-fastly/src/app.rs` calls `IntegrationRegistry::filter_request` for every named route except the inspection endpoints and batch sync, and for the publisher fallback. It runs after the modules' preparers and once the request's Edge Cookie context is read, and it hands every filter the location and the permissions resolved for that context. A filter that answers stands in for the route. A filter that lets the request continue has its request headers applied before the route, and its response headers are applied by the entry point after the response is finalized (`apply_terminal_response_effects` in `src/lib.rs`). -2. **Device detection.** `derive_device_signals` in `src/lib.rs` calls `build_device_module` and `DeviceModule::detect` at the entry point, and `build_ec_request_state` sets the result on the Edge Cookie context. Every module call made from that context carries the classification (`ModuleContext::with_request_state` in core). -3. **robots.txt first.** `dispatch_fallback` answers robots.txt once the preparers have run and before the Edge Cookie steps and the filters, with an empty Edge Cookie context, so no filter's answer stands in for the file and none of the crawler's cookies are read. +2. **Device classification.** For every request, before routing, the entry point's `derive_device_signals` in `src/lib.rs` calls `DeviceModule::detect` on the device module a registration supplies, or on the one `build_device_module` builds where none does, and `build_ec_request_state` sets the result on the Edge Cookie context. A module call made with that context's state carries the classification (`ModuleContext::with_request_state` in core), which covers a module route's call and a serve phase preparation. A request filter is never handed it, because `RequestFilterInput` carries only the location and the permissions, and the fetch phase of a document that may be stored is handed services with nothing of the reader in place of the context's state (`preparation_context` in `crates/trusted-server-core/src/publisher.rs`). +3. **robots.txt first.** `dispatch_fallback` answers robots.txt once the preparers have run and before the Edge Cookie steps and the filters, with an empty Edge Cookie context, so no filter's answer stands in for the file and the crawler's Edge Cookie and consent signals are not read. The entry point has already run the device module by then, so a device module a registration supplies is shown the crawler's cookies. -No code in `crates/trusted-server-adapter-axum`, `-cloudflare` or `-spin` calls `filter_request` or `DeviceModule::detect`. So on those adapters the 51Degrees client hints middleware asks its service nothing and writes no header or tag, DataDome's protection filter never runs, `[device] module` changes nothing, and robots.txt is answered through the publisher path with the crawler's own Edge Cookie context. On Spin the discovery, signature verification and first-party routes and the page-bids preflight also skip the modules' preparers, which every other adapter runs for them. +No source file in `crates/trusted-server-adapter-axum`, `-cloudflare` or `-spin` calls `filter_request` or `DeviceModule::detect`. So on those adapters the 51Degrees client hints middleware asks its service nothing and writes no header or tag, DataDome's protection filter never runs, and robots.txt is answered through the publisher path with the crawler's own Edge Cookie context. A device module `[device]` selects from a registration is not idle there, because each adapter's `services_for_request` puts it on the request's services and core asks it for the bid request's device attributes in every auction, showing it the request's headers and permissions (`attach_device_attributes` in `crates/trusted-server-core/src/auction/formats.rs`, called from `auction/endpoints.rs` and `publisher.rs`). It never classifies a request, so no module call carries a classification. On Spin the discovery, signature verification and first-party routes and the page-bids preflight also skip the modules' preparers, which every other adapter runs for them. ## 2. Design -**One request cycle per adapter, in the order the Fastly adapter runs it.** Each of the three adapters gains the steps the Fastly adapter has, under the Fastly adapter's names, in `src/app.rs`. +**One request cycle per adapter, in the order the Fastly adapter runs it.** Each of the three adapters gains the steps the Fastly adapter has, in `src/app.rs`, with the finalize step in `src/middleware.rs`, under the Fastly adapter's names where it has one. The Edge Cookie step keeps the adapter's own name, `build_ec_context`, which the Fastly adapter calls `build_ec_request_state`, and the response effects are written by `FinalizeResponseMiddleware`, where the Fastly adapter writes them from its entry point. The Fastly adapter and core are unaffected, because no file of either changes. + +Each adapter carries its own copy of the steps. `derive_device_signals`, `run_pre_route_filters`, `attach_request_filter_effects`, `asks_for_robots_txt` and `apply_terminal_response_effects` are the same text in all three, and `answer_robots_txt` and `execute_named` the same in Cloudflare and Spin. One request cycle in core that every adapter calls would replace the copies, but it would also move the Fastly adapter's cycle, which this change leaves as it is, so section 3 leaves one request cycle in core to a later change. | Step | Function | What it does | | --------------------- | ------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Preparers | `execute_named`, `execute_fallback` | `IntegrationRegistry::prepare_request`, once for each request, as before | | robots.txt | `asks_for_robots_txt`, `answer_robots_txt` | Answers a `GET` or `HEAD` of `/robots.txt`, when `[robots-txt]` is present, through the publisher path with `EcContext::default()` and no auction | -| Device classification | `derive_device_signals` | Runs the selected device module, or the built-in one, with the request's module context | +| Device classification | `derive_device_signals` | Runs the device module the request's services carry, or the built-in one, with the request's module context | | Edge Cookie context | `build_ec_context` | `EcContext::read_from_request_resolving_geo`, then `set_device_signals`. An error answers the request | | Request filters | `run_pre_route_filters` | `IntegrationRegistry::filter_request` with the context's location and permissions. An answer, or an error, stands in for the route | | Route | `run_named_route`, `dispatch_fallback` | The handler, given the context the filters saw | | Response effects | `attach_request_filter_effects` | Leaves the filters' response headers on the response for the finalize middleware | | Finalize | `FinalizeResponseMiddleware` | Finalizes the response, then `apply_terminal_response_effects` writes the filters' response headers and restores the privacy guards, as the Fastly entry point does | -**Which routes run the cycle.** The routes the Fastly adapter runs it for, which are every named route but the inspection endpoints, and the publisher fallback, being robots.txt, tsjs, the module routes and the publisher's origin. The inspection endpoints are read only and stay ahead of the Edge Cookie context and the filters, as on Fastly. +**Which routes run the cycle.** The routes the Fastly adapter runs it for, being every named route but the inspection endpoints and batch sync, and the publisher fallback, being robots.txt, tsjs, the module routes and the publisher's origin. These three adapters register no batch sync route, so a request for its path reaches the publisher fallback, as before. The inspection endpoints are read only and stay ahead of the Edge Cookie context and the filters, as on Fastly. Every adapter also answers the attestation pages outside the cycle, for `GET` and `HEAD` and for every method under a prefix the endpoint owns, as on Fastly, and Axum and Spin answer `/health` outside it. | Adapter | Named routes in the cycle | Inspection endpoints | Registration | | ---------- | ----------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------- | @@ -44,44 +46,79 @@ No code in `crates/trusted-server-adapter-axum`, `-cloudflare` or `-spin` calls On Spin the discovery, signature verification and first-party routes and the page-bids preflight now run the preparers, because the cycle starts with them. Before this change they ran none. -**Device classification.** `derive_device_signals` follows the Fastly adapter's function of the same name. A device module a registration supplies is shown the `User-Agent` and the `Cookie` header, and the built-in module the `User-Agent` alone, each in a module context carrying the request, that evidence, the settings and the services. Three things differ from the Fastly adapter. +**Device classification.** `derive_device_signals` follows the Fastly adapter's function of the same name. A device module a registration supplies is shown the `User-Agent` and the `Cookie` header, and the built-in module the `User-Agent` alone, each in a module context carrying the request, that evidence, the settings and the services. Four things differ from the Fastly adapter. 1. It runs where the adapter builds the Edge Cookie context, after the preparers, where the Fastly adapter runs it at the entry point before routing. A preparer that removes a reserved cookie therefore removes it before a device module sees it. -2. The registered module is the one the request's services carry (`RuntimeServices::device_module`), which `services_for_request` sets from the registry, and the module is handed those same services, so it reads the platform's key-value store when the services carry one. The Fastly entry point builds services of its own for the call. -3. These adapters supply no TLS or HTTP/2 signals, so `[device] module = "platform"` builds the built-in module, which classifies from the `User-Agent`, because `platform` adds the host's signals only where the platform supplies them. +2. It runs only for the routes in the cycle, so a request for robots.txt, an inspection endpoint or an attestation page is not classified, where the Fastly entry point classifies every request. On Fastly a device module a registration supplies is therefore shown the cookies a crawler sends with a request for robots.txt, and on these adapters it is not. +3. The registered module is the one the request's services carry (`RuntimeServices::device_module`), which `services_for_request` sets from the registry, and the module is handed those same services, so it reads the platform's key-value store when the services carry one. The Fastly entry point builds services of its own for the call. +4. These adapters supply no TLS or HTTP/2 signals, so `[device] module = "platform"` builds the built-in module, which classifies from the `User-Agent`, because `platform` adds the host's signals only where the platform supplies them. + +As on Fastly, a module route's call and a serve phase preparation carry the classification, a request filter is not handed it, and the fetch phase of a document that may be stored is handed services with nothing of the reader. + +**Response effects.** A filter that lets the request continue can name response headers. The route's response carries them in its extensions as `RequestFilterEffects`, and `FinalizeResponseMiddleware` applies them after it writes the finalize headers, so a filter's header is written after an operator's `[response_headers]`, as on Fastly. `apply_terminal_response_effects` then enforces `no-store, private` on a response core marked `TerminalPrivateResponse` and runs `enforce_set_cookie_cache_privacy` and `enforce_uncacheable_cache_privacy`, the steps the Fastly entry point runs before it sends every response. Those steps run on every response, whether or not a filter named a header, so a response core marked private also loses any `ETag`, `Last-Modified`, `Expires` or `Age` that `[response_headers]` writes, as on Fastly. A filter's header is also written after the `X-Robots-Tag` that `[robots-txt]` writes, and `x-robots-tag` is not among the headers a filter is refused (`is_forbidden_filter_header` in `crates/trusted-server-core/src/integrations/registry.rs`), so a filter that names that header replaces the tag, on Fastly as on these adapters. The open questions ask whether a filter should be refused `X-Robots-Tag`. -**Response effects.** A filter that lets the request continue can name response headers. The route's response carries them in its extensions as `RequestFilterEffects`, and `FinalizeResponseMiddleware` applies them after it writes the finalize headers, so a filter's header is written after an operator's `[response_headers]`, as on Fastly. `apply_terminal_response_effects` then enforces `no-store, private` on a response core marked `TerminalPrivateResponse` and runs `enforce_set_cookie_cache_privacy` and `enforce_uncacheable_cache_privacy`, the steps the Fastly entry point runs before it sends every response. +**Configuration.** No setting is added, removed or given a new default. Three settings act on these adapters where they did not, being `[device] module` when it selects a device module a registration supplies, which now classifies each request in the cycle, and DataDome's `enable_protection` and `[shared.51degrees]`, whose filters now run. Two settings these adapters accept are not applied, being DataDome's `timeout_ms`, default `1500`, and `[shared.51degrees] timeout_ms`, default `500`, which only the Fastly adapter applies, as the timeouts of the backend each module registers for its call. No adapter warns about either at startup, the way the Axum adapter warns about a `proxy.certificate_check` it cannot honor, because an adapter names no vendor's setting, and no platform declaration tells core or a module that an adapter does not apply a backend's timeouts. The DataDome and 51Degrees guides say where `timeout_ms` applies. -**Configuration.** None. The filters a deployment's modules register and the module `[device]` selects now run on every adapter. +**Consent and privacy.** Every filter is handed the permissions resolved for the request, as on Fastly, and no shipped filter reads them yet (`docs/guide/permission-model.md`), so no permission gates what the two shipped filters send. -**Consent and privacy.** Every filter is handed the permissions resolved for the request, as on Fastly. The 51Degrees filter sends the reader's `User-Agent` and `Sec-CH-UA` hints to the 51Degrees service for each page navigation an entry naming its middleware covers, whatever the permissions say, which is the behavior and the open question that `2026-10-14-51degrees-client-hints-design.md` describes for Fastly. DataDome's protection filter removes `x-ts-datadome-bypass` from every request it runs for, before its other checks. A device module a registration supplies sees the reader's cookies, as on Fastly. robots.txt is answered with an empty Edge Cookie context, so a crawler's cookies and consent signals are no longer read on these adapters. +- The 51Degrees filter sends the reader's `User-Agent` and `Sec-CH-UA` hints to the 51Degrees service for a page navigation an entry naming its middleware covers. The call goes through the answer cache the 51Degrees modules share, so the service is asked once for each set of those values while an answer is held, and not at all for a request with no `User-Agent` (`crates/shared/51degrees/src/hints_engine.rs`). That is the behavior `2026-10-14-51degrees-client-hints-design.md` describes, and its open question, whether the call needs a permission, stays open, and now concerns four adapters where it concerned one. +- DataDome's protection filter, which DataDome registers only when `enable_protection` is on, removes `x-ts-datadome-bypass` from every request it runs for, before its other checks. For each request it protects, it sends DataDome's Protection API the server side key, the reader's IP address, the request's method, host, path and query, its `User-Agent`, `Accept`, `Accept-Language`, `Accept-Encoding`, `Referer`, `Origin`, `Via`, `X-Forwarded-For`, `True-Client-IP` and `X-Real-IP` headers among others, its `Sec-CH-UA` and `Sec-Fetch` headers, the list of its header names, the lengths of its `Cookie` and `Authorization` headers, and the client id from the `datadome` cookie or the `x-datadome-clientid` header (`build_protection_payload` in `crates/bot-protection/datadome/src/protection.rs`). The filter does not read the permissions. A response the API allows, or a challenge, can carry DataDome's `Set-Cookie`, which sets a cookie on the publisher's domain and keeps the response out of shared caches, as `private, max-age=0` where the response's own policy allowed them (`enforce_set_cookie_cache_privacy`). A reader an IP rule excludes is not checked, and the request is marked personalized, so an HTML page it asks for is served without DataDome's tag and sent `private, no-store`. +- A device module a registration supplies is shown the reader's `User-Agent` and cookies to classify every request in the cycle, with no permission check, because classification runs before the permissions are resolved and startup refuses a supplied device module that declares a permission (`resolve_device_module` in `crates/trusted-server-core/src/integrations/registry.rs`). That is the Fastly adapter's behavior. Before this change these adapters showed the module the request's headers only in an auction, through `advertising_attributes`, which is handed the permissions. +- robots.txt is answered with an empty Edge Cookie context, so the crawler's Edge Cookie and consent signals are not read and no device module is shown its cookies, where before this change these adapters read the crawler's own context. The publisher path still parses the crawler's `Cookie` header, as it does for every request it handles (`handle_publisher_request`), and sends it on to an origin that is not signed. -**Failure handling.** A filter that fails answers the request with the error, logged as `Failed to run integration request filters`, as on Fastly. An Edge Cookie context that cannot be read answers the request with the error before the filters run, which now covers every route in the cycle, including tsjs and the discovery, signature verification and first-party routes. The errors are a selected Edge Cookie module the request cannot build and a `Cookie` header that is not UTF-8, and startup refuses a selection the adapter can never build. A filter's outbound call is bounded only by the adapter's HTTP client, which applies no module's own timeout on these adapters. Axum's client allows 5 seconds to connect and 30 in all, Cloudflare leaves the call to the Workers runtime's limit, and Spin to the Spin runtime's default (`src/platform.rs` of each adapter). +**Failure handling.** A filter that fails answers the request with the error, logged as `Failed to run integration request filters`, as on Fastly. An Edge Cookie context that cannot be read answers the request with the error before the filters run, which now covers every route in the cycle, including tsjs, discovery, signature verification, the first-party routes and the page-bids preflight. The context has two ways to fail, being a selected Edge Cookie module the request cannot build, which startup refuses where the adapter can never build it, and a `Cookie` header it cannot read as text, which answers `400`, because `handle_request_cookies` in `crates/trusted-server-core/src/cookies.rs` reads the header with `HeaderValue::to_str`, which refuses any byte outside visible ASCII, an accented letter in UTF-8 among them. A client's own `Cookie` header never fails that way, because the preparer of the `ad-tag.google.diagnostics` module, which every adapter links and runs for every request the cycle handles, first rebuilds the header from the values it can read (`sanitize_console_cookie` in `crates/ad-tag/google/src/diagnostics.rs`), so only a preparer of a deployment's own module, which runs after it, can leave such a header. A filter's outbound call is bounded only by the adapter's HTTP client, which applies no module's own timeout on these adapters. Axum's client allows 5 seconds to connect and 30 in all, Cloudflare leaves the call to the Workers runtime's limit, and Spin to the Spin runtime's default (`src/platform.rs` of each adapter). A filter that lets the request through when its call fails, as DataDome's and the 51Degrees filter do, therefore delays the request by up to that bound rather than failing it. -**Rollout.** A deployment whose modules register no filter, and that selects no device module, answers as before. It now reads the Edge Cookie context, with its geo lookup where `[geo]` selects a module, and classifies the device with the built-in module, on every route in the cycle, where it read the context for the auction, page-bids, module routes and the publisher alone. A deployment with `[shared.51degrees]` and a `[[serve]]` entry naming `51degrees.client_hints` now has the headers and the tag written on the pages the entry covers, and such a page keeps to the origin path and is sent `private, no-store`, because the filter leaves the hint names in the request's state. A deployment with DataDome's protection on now has its pages checked, and a challenge answers in place of the page. +**Rollout.** + +- A deployment whose modules register no filter, and that selects no device module a registration supplies, answers as before, apart from three things. It reads the Edge Cookie context, with its geo lookup where `[geo]` selects a module, and classifies the device with the built-in module, on every route in the cycle, where it read the context for the auction, page-bids, the module routes and the publisher alone. And a response core marked private loses any `ETag`, `Last-Modified`, `Expires` or `Age` that `[response_headers]` writes, as Response effects says. On Spin, discovery, signature verification, the first-party routes and the page-bids preflight run the modules' preparers, as on the other adapters. A client's `Cookie` header brings no new `400`, as Failure handling says. +- A deployment that selects a device module a registration supplies has that module asked to classify every request in the cycle, being tsjs, discovery, `/verify-signature`, the first-party routes, the auction, page-bids and its preflight, the module routes and the publisher's pages, each time shown the reader's `User-Agent` and cookies, beside the bid request attributes each auction already asks it for. A module that calls a service to classify makes that call for each of those requests. +- A deployment with `[shared.51degrees]` and a `[[serve]]` entry naming `51degrees.client_hints` now has the headers and the tag written on the pages the entry covers, and such a page keeps to the origin path and is sent `private, no-store`, because the filter leaves the hint names in the request's state. +- A deployment with DataDome's protection on now has each request it protects checked by the Protection API before the route, being the publisher's pages and each subresource the static asset rule does not exclude, `/auction` and `GET` on both page-bids paths, none of which is among DataDome's internal paths (`is_internal_path` in `crates/bot-protection/datadome/src/protection.rs`). A challenge then answers in place of a page or an auction call. A response that carries DataDome's cookie is kept out of shared caches, and a reader an IP rule excludes gets pages that keep to the origin path and are sent `private, no-store`. The call has no time limit of DataDome's own on these adapters, so a slow Protection API delays each protected request until the adapter's HTTP client gives up, which is 30 seconds on Axum, before DataDome lets it through. **Tests.** In `src/app/request_cycle_tests.rs` of each of the three adapters, through the adapter's router built by its composition root, with a stub origin: - `a_filter_that_answers_stands_in_for_the_page`: the answer and its effect header are the response, and the origin is not asked. - `a_filter_s_request_header_reaches_the_route_and_its_effect_is_written_last`: a module route sees the request header a filter added, and the filter's response header replaces the operator's header of the same name. - `the_device_module_s_classification_reaches_the_module_call`: the module `[device]` selects classifies the request, shown its cookies, and a module route's call carries that classification. +- `platform_classifies_the_request_with_the_built_in_module`: with `[device] module = "platform"`, a module route's call carries the built-in module's classification of an iPhone `User-Agent`. +- `an_edge_cookie_context_that_cannot_be_read_answers_before_the_filters`: with a filter that answers every request, a request a module's preparer leaves with an unreadable `Cookie` header is answered `400`, with no filter's effect and no request to the origin, and a client's own unreadable `Cookie` header reaches the filter instead. - `robots_txt_is_answered_before_the_request_filters`: with a filter that answers every request, robots.txt is still the file, and the filter's header is absent. +- `robots_txt_shows_the_device_module_none_of_the_crawler_s_cookies`: the module `[device]` selects, which records the cookies it is shown, never sees the cookies sent with robots.txt, and does see those sent with a module route's request. +- `the_inspection_endpoints_and_the_attestation_pages_skip_the_request_filters`: with a filter that answers every request, the five inspection paths and the two attestation pages are answered by their own handlers with no filter's effect, and a page is the filter's answer. +- `a_filter_cannot_make_a_response_core_made_private_public`: a module route's answer core marks private keeps `no-store, private` when a filter's effect sets `Cache-Control: public, max-age=600`. +- `a_filter_s_cookie_keeps_the_response_from_shared_caches`: a public answer that a filter's effect adds a `Set-Cookie` to is sent `private, max-age=0`. +- `a_filter_cannot_add_a_shared_cache_header_to_a_private_response`: a `Surrogate-Control` that a filter's effect adds to a `private` answer is removed. +- `a_response_core_made_private_loses_the_validators_an_operator_writes`: a module route's answer core marks private, which no filter names a header for, loses the `ETag` and `Last-Modified` that `[response_headers]` writes. + +Taking any one of the three privacy guards out of the Axum adapter's `apply_terminal_response_effects` fails that guard's tests alone, and taking all three out of the Cloudflare or the Spin adapter's fails the four privacy tests. Taking out of the Axum adapter the step each of the other new tests covers fails that test alone. -**Open questions.** None. +No test runs a shipped filter, the 51Degrees middleware or DataDome's protection, through one of these adapters, and none covers `/health`, which runs no module. + +**Open questions.** + +1. Whether the 51Degrees call needs a permission, which `2026-10-14-51degrees-client-hints-design.md` leaves to the reviewer and which now arises on four adapters. +2. Whether a request filter should be refused `X-Robots-Tag`, so that no filter can loosen a site `[robots-txt]` takes out of search, on Fastly as on these adapters. Refusing it is a change to `is_forbidden_filter_header` in core, which this change leaves alone. ## 3. What is not in this change - Creating and writing the Edge Cookie, and the resolve route, on these adapters, which the next pull request of the stack covers. - Asset routes on these adapters, which the pull request after that covers. -- A timeout for a filter's outbound call on Axum, Cloudflare and Spin, which their HTTP clients do not offer. -- Moving the inspection endpoints ahead of the preparers on Axum and Cloudflare, where the Fastly adapter answers two of them before the preparers. +- A time limit for a filter's outbound call on Axum, Cloudflare and Spin. A module's limit reaches the platform only as the `first_byte_timeout` and `between_bytes_timeout` of the `PlatformBackendSpec` it registers, and a request names only its backend (`PlatformHttpRequest`). The Axum backend keeps nothing of those timeouts, naming a backend by its scheme, host, port and discriminator, and its client applies its own two limits, although reqwest's `RequestBuilder::timeout` can bound one request. The Cloudflare backend's name carries the first byte timeout and its client does not read it, although a Worker's fetch can be stopped with `Fetch::send_with_signal` and an `AbortController`. Spin's `spin_sdk::http::send` takes no time limit. Applying a module's limit is a change to each adapter's backend and HTTP client, which every outbound call goes through, the auction's calls to its providers among them, so it is left to a change of its own. +- Moving the inspection endpoints ahead of the preparers on Axum and Cloudflare, where the Fastly adapter answers all three of them, being all five paths, before the preparers. +- One request cycle in core that every adapter calls, in place of the copy each adapter carries, which would move the Fastly adapter's cycle as well. +- A test that runs a shipped filter through one of these adapters, which needs a scripted DataDome Protection API or 51Degrees service and the secret store their keys are read from. ## 4. Acceptance -The four tests pass on each of the three adapters, and the existing tests of each adapter and the cross-adapter parity tests pass unchanged. +The twelve tests pass on each of the three adapters, taking out any of the three privacy guards fails a test, and the existing tests of each adapter and the cross-adapter parity tests pass unchanged. ## Revision record -| Date | Change | -| ---------- | ------------ | -| 2026-10-17 | First draft. | +| Date | Change | +| ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 2026-10-17 | First draft. | +| 2026-10-18 | Fixes from review round 1. What a selected device module did on these adapters before, what DataDome's filter sends and sets, how each filter's call is bounded and why no time limit is added, the inspection endpoints the Fastly adapter answers before the preparers, the routes kept out of the cycle, which of a crawler's cookies robots.txt reads, the `Cookie` check and when it is reached, the configuration, eight new tests on each adapter with the three privacy guards among them, the open questions and the review log. | + +## Review log + +1. **Round 1, 2026-10-18.** A review in a fresh session, against `fixpass6/52-request-filters` at `f0edcd72b`. Issues found, five of them blocking. The five were that the spec said a selected device module did nothing on Axum, Cloudflare and Spin, when core asks it for a bid request's device attributes in every auction there, that nothing said what DataDome's filter sends DataDome or sets, or that no permission gates it, that the client hints design still said only the Fastly adapter runs request filters, that no test proved the privacy guards written after a filter's headers, and that the spec said the Fastly adapter answers two of the inspection endpoints before the preparers, when it answers all three, and that the adapters' HTTP clients cannot bound a call, when the Axum and Cloudflare clients can. Fixed in `7aea7a72b`, which adds the four privacy tests and four more on each adapter, `4549047dc`, which corrects the adapters' comments on robots.txt, `a584b071f`, which corrects the guides, the changelog and the middleware readme, `6b4eb0b8c`, which amends the client hints design, and this revision, which states each from the code and gives the reason a time limit is left to a change of its own. One finding was not borne out. The review said a `Cookie` header holding an accented letter now answers `400` on the routes that newly read the context, but a client's header is rebuilt by the `ad-tag.google.diagnostics` preparer first, which the context test shows, so Rollout names no new `400`. From 7ae3235adde0a3710591a3d144cb1f0476e5d1c0 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 18 Oct 2026 07:47:49 +0100 Subject: [PATCH 09/18] The request cycle tests send every named route through the filters on Axum, Cloudflare and Spin A new test on each adapter sends every named route the cycle runs, being discovery, /verify-signature, /auction, GET and OPTIONS on both page-bids paths and the four first-party paths, to a filter that answers every request, and asserts that each is the filter's answer with its effect header. The spoiling preparer now also spoils the discovery route, so the context test shows the preparers and the Edge Cookie context run for a named route as well as for a page. Taking run_pre_route_filters out of any adapter's execute_named now fails the new test alone, and taking build_ec_context out of it, or on Spin the preparers, fails the context test alone. The context test's comment says the linked preparer drops a Cookie header value it cannot read, rather than rebuilding the header from the values it can. --- .../src/app/request_cycle_tests.rs | 121 ++++++++++++++---- .../src/app/request_cycle_tests.rs | 121 ++++++++++++++---- .../src/app/request_cycle_tests.rs | 121 ++++++++++++++---- 3 files changed, 285 insertions(+), 78 deletions(-) diff --git a/crates/trusted-server-adapter-axum/src/app/request_cycle_tests.rs b/crates/trusted-server-adapter-axum/src/app/request_cycle_tests.rs index a7bbe60450..6f2fddee0e 100644 --- a/crates/trusted-server-adapter-axum/src/app/request_cycle_tests.rs +++ b/crates/trusted-server-adapter-axum/src/app/request_cycle_tests.rs @@ -1,15 +1,15 @@ //! The request cycle this adapter runs for a routed request, checked through -//! its router: the request filters run before the route and their response -//! effects after the finalize headers, without making the response less -//! private, the route's module call carries the device classification, an -//! Edge Cookie context that cannot be read answers before the filters, and -//! robots.txt, the inspection endpoints and the attestation pages are answered -//! without them. +//! its router: the request filters run before a page and before every named +//! route in the cycle, and their response effects after the finalize headers, +//! without making the response less private, the route's module call carries +//! the device classification, an Edge Cookie context that cannot be read +//! answers before the filters, and robots.txt, the inspection endpoints and +//! the attestation pages are answered without them. use std::sync::{Arc, Mutex}; use edgezero_core::body::Body; -use edgezero_core::http::{HeaderValue, Request, Response, StatusCode, request_builder}; +use edgezero_core::http::{HeaderValue, Method, Request, Response, StatusCode, request_builder}; use edgezero_core::router::RouterService; use error_stack::Report; use trusted_server_core::ec::device::{DeviceModule, DeviceSignals}; @@ -25,6 +25,7 @@ use trusted_server_core::platform::PlatformHttpClient; use trusted_server_core::platform::test_support::{ StubHttpClient, build_services_with_http_client, }; +use trusted_server_core::publisher::{PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH}; use trusted_server_core::response_privacy::enforce_terminal_private_cache_privacy; use trusted_server_core::settings::Settings; @@ -72,12 +73,15 @@ const CRAWLER_COOKIE: &str = "crawler_session=robots"; const READER_COOKIE: &str = "reader_session=route"; /// The id and module name of a module whose preparer leaves the request for -/// [`SPOILED_PATH`] with a `Cookie` header the Edge Cookie context cannot -/// read. +/// [`SPOILED_PATH`] or [`DISCOVERY_PATH`] with a `Cookie` header the Edge +/// Cookie context cannot read. const SPOILING_ID: &str = "spoiling_preparer"; const SPOILING_MODULE: &str = "testing.spoiling-preparer"; const SPOILED_PATH: &str = "/spoiled"; +/// The discovery route, a named route in the request cycle. +const DISCOVERY_PATH: &str = "/.well-known/trusted-server.json"; + /// Every `Cookie` header the marking device module was shown, by every test /// in this file, so a test can tell whether a request's cookies reached it. static COOKIES_SHOWN: Mutex> = Mutex::new(Vec::new()); @@ -311,14 +315,15 @@ fn register_nothing( Ok(None) } -/// Leaves the request for [`SPOILED_PATH`] with an unreadable `Cookie` -/// header. A deployment's own modules prepare a request after the modules -/// the adapter links, so this runs after every one of them. +/// Leaves the request for [`SPOILED_PATH`], a page, or [`DISCOVERY_PATH`], a +/// named route, with an unreadable `Cookie` header. A deployment's own +/// modules prepare a request after the modules the adapter links, so this +/// runs after every one of them. fn spoil_cookie( _settings: &Settings, request: &mut Request, ) -> Result<(), Report> { - if request.uri().path() == SPOILED_PATH { + if matches!(request.uri().path(), SPOILED_PATH | DISCOVERY_PATH) { request.headers_mut().insert("cookie", unreadable_cookie()); } Ok(()) @@ -411,6 +416,35 @@ fn get(path: &str, cookie: Option<&str>) -> Request { .expect("should build the request") } +/// A request for `path` by `method`, with no cookie. +fn request_for(method: &Method, path: &str) -> Request { + request_builder() + .method(method.clone()) + .uri(format!("https://test-publisher.example.com{path}")) + .header("host", "test-publisher.example.com") + .body(Body::empty()) + .expect("should build the request") +} + +/// Every named route the request cycle runs, by method and path. +fn named_routes_in_the_cycle() -> [(Method, &'static str); 13] { + [ + (Method::GET, DISCOVERY_PATH), + (Method::POST, "/verify-signature"), + (Method::POST, "/auction"), + (Method::GET, PAGE_BIDS_PATH), + (Method::OPTIONS, PAGE_BIDS_PATH), + (Method::GET, PAGE_BIDS_LEGACY_PATH), + (Method::OPTIONS, PAGE_BIDS_LEGACY_PATH), + (Method::GET, "/first-party/proxy"), + (Method::GET, "/first-party/click"), + (Method::GET, "/first-party/sign"), + (Method::POST, "/first-party/sign"), + (Method::GET, "/first-party/proxy-rebuild"), + (Method::POST, "/first-party/proxy-rebuild"), + ] +} + /// A `Cookie` header holding an accented letter in UTF-8, which a header /// value read as text refuses. fn unreadable_cookie() -> HeaderValue { @@ -476,6 +510,36 @@ async fn a_filter_that_answers_stands_in_for_the_page() { ); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_filter_that_answers_stands_in_for_every_named_route_in_the_cycle() { + let origin = Arc::new(StubHttpClient::new()); + let router = router_with( + settings_with("[testing]\nmodules = [\"answering-filter\"]"), + &[answering_builder()], + &origin, + ); + + for (method, path) in named_routes_in_the_cycle() { + let response = route(&router, request_for(&method, path)).await; + + assert_eq!( + response.status(), + StatusCode::FORBIDDEN, + "the filter's answer should stand in for {method} {path}" + ); + assert_eq!( + header(&response, EFFECT_HEADER), + Some("answered"), + "the effects the answering filter left should be applied to {method} {path}" + ); + assert_eq!(text(response), ANSWER, "{method} {path}"); + } + assert!( + origin.recorded_request_uris().is_empty(), + "no request should be sent on, because no route runs" + ); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn a_filter_s_request_header_reaches_the_route_and_its_effect_is_written_last() { let origin = Arc::new(StubHttpClient::new()); @@ -645,26 +709,31 @@ async fn an_edge_cookie_context_that_cannot_be_read_answers_before_the_filters() let mut sent = get("/article", None); sent.headers_mut().insert("cookie", unreadable_cookie()); let page = route(&router, sent).await; - let spoiled = route(&router, get(SPOILED_PATH, None)).await; - // A preparer the adapter links rebuilds the `Cookie` header from the - // values it can read, so a client's unreadable header never reaches the + // A preparer the adapter links drops a `Cookie` header value it cannot + // read as text, so a client's unreadable header never reaches the // context. assert_eq!( text(page), ANSWER, "a client's unreadable Cookie header should not stop the cycle" ); - assert_eq!( - spoiled.status(), - StatusCode::BAD_REQUEST, - "the context's error should answer the request" - ); - assert_eq!( - header(&spoiled, EFFECT_HEADER), - None, - "the filters should not run once the context cannot be read" - ); + // A page reaches the context through the fallback, and discovery through + // the named route's handler, each once the preparers have run. + for path in [SPOILED_PATH, DISCOVERY_PATH] { + let spoiled = route(&router, get(path, None)).await; + + assert_eq!( + spoiled.status(), + StatusCode::BAD_REQUEST, + "the context's error should answer {path}" + ); + assert_eq!( + header(&spoiled, EFFECT_HEADER), + None, + "the filters should not run for {path} once the context cannot be read" + ); + } assert!( origin.recorded_request_uris().is_empty(), "the publisher's origin should not be asked" diff --git a/crates/trusted-server-adapter-cloudflare/src/app/request_cycle_tests.rs b/crates/trusted-server-adapter-cloudflare/src/app/request_cycle_tests.rs index d710ce99f1..27d8b35d6f 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app/request_cycle_tests.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app/request_cycle_tests.rs @@ -1,15 +1,15 @@ //! The request cycle this adapter runs for a routed request, checked through -//! its router: the request filters run before the route and their response -//! effects after the finalize headers, without making the response less -//! private, the route's module call carries the device classification, an -//! Edge Cookie context that cannot be read answers before the filters, and -//! robots.txt, the inspection endpoints and the attestation pages are answered -//! without them. +//! its router: the request filters run before a page and before every named +//! route in the cycle, and their response effects after the finalize headers, +//! without making the response less private, the route's module call carries +//! the device classification, an Edge Cookie context that cannot be read +//! answers before the filters, and robots.txt, the inspection endpoints and +//! the attestation pages are answered without them. use std::sync::{Arc, Mutex}; use edgezero_core::body::Body; -use edgezero_core::http::{HeaderValue, Request, Response, StatusCode, request_builder}; +use edgezero_core::http::{HeaderValue, Method, Request, Response, StatusCode, request_builder}; use edgezero_core::router::RouterService; use error_stack::Report; use trusted_server_core::ec::device::{DeviceModule, DeviceSignals}; @@ -25,6 +25,7 @@ use trusted_server_core::platform::PlatformHttpClient; use trusted_server_core::platform::test_support::{ StubHttpClient, build_services_with_http_client, }; +use trusted_server_core::publisher::{PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH}; use trusted_server_core::response_privacy::enforce_terminal_private_cache_privacy; use trusted_server_core::settings::Settings; @@ -72,12 +73,15 @@ const CRAWLER_COOKIE: &str = "crawler_session=robots"; const READER_COOKIE: &str = "reader_session=route"; /// The id and module name of a module whose preparer leaves the request for -/// [`SPOILED_PATH`] with a `Cookie` header the Edge Cookie context cannot -/// read. +/// [`SPOILED_PATH`] or [`DISCOVERY_PATH`] with a `Cookie` header the Edge +/// Cookie context cannot read. const SPOILING_ID: &str = "spoiling_preparer"; const SPOILING_MODULE: &str = "testing.spoiling-preparer"; const SPOILED_PATH: &str = "/spoiled"; +/// The discovery route, a named route in the request cycle. +const DISCOVERY_PATH: &str = "/.well-known/trusted-server.json"; + /// Every `Cookie` header the marking device module was shown, by every test /// in this file, so a test can tell whether a request's cookies reached it. static COOKIES_SHOWN: Mutex> = Mutex::new(Vec::new()); @@ -311,14 +315,15 @@ fn register_nothing( Ok(None) } -/// Leaves the request for [`SPOILED_PATH`] with an unreadable `Cookie` -/// header. A deployment's own modules prepare a request after the modules -/// the adapter links, so this runs after every one of them. +/// Leaves the request for [`SPOILED_PATH`], a page, or [`DISCOVERY_PATH`], a +/// named route, with an unreadable `Cookie` header. A deployment's own +/// modules prepare a request after the modules the adapter links, so this +/// runs after every one of them. fn spoil_cookie( _settings: &Settings, request: &mut Request, ) -> Result<(), Report> { - if request.uri().path() == SPOILED_PATH { + if matches!(request.uri().path(), SPOILED_PATH | DISCOVERY_PATH) { request.headers_mut().insert("cookie", unreadable_cookie()); } Ok(()) @@ -414,6 +419,35 @@ fn get(path: &str, cookie: Option<&str>) -> Request { .expect("should build the request") } +/// A request for `path` by `method`, with no cookie. +fn request_for(method: &Method, path: &str) -> Request { + request_builder() + .method(method.clone()) + .uri(format!("https://test-publisher.example.com{path}")) + .header("host", "test-publisher.example.com") + .body(Body::empty()) + .expect("should build the request") +} + +/// Every named route the request cycle runs, by method and path. +fn named_routes_in_the_cycle() -> [(Method, &'static str); 13] { + [ + (Method::GET, DISCOVERY_PATH), + (Method::POST, "/verify-signature"), + (Method::POST, "/auction"), + (Method::GET, PAGE_BIDS_PATH), + (Method::OPTIONS, PAGE_BIDS_PATH), + (Method::GET, PAGE_BIDS_LEGACY_PATH), + (Method::OPTIONS, PAGE_BIDS_LEGACY_PATH), + (Method::GET, "/first-party/proxy"), + (Method::GET, "/first-party/click"), + (Method::GET, "/first-party/sign"), + (Method::POST, "/first-party/sign"), + (Method::GET, "/first-party/proxy-rebuild"), + (Method::POST, "/first-party/proxy-rebuild"), + ] +} + /// A `Cookie` header holding an accented letter in UTF-8, which a header /// value read as text refuses. fn unreadable_cookie() -> HeaderValue { @@ -479,6 +513,36 @@ async fn a_filter_that_answers_stands_in_for_the_page() { ); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_filter_that_answers_stands_in_for_every_named_route_in_the_cycle() { + let origin = Arc::new(StubHttpClient::new()); + let router = router_with( + settings_with("[testing]\nmodules = [\"answering-filter\"]"), + &[answering_builder()], + &origin, + ); + + for (method, path) in named_routes_in_the_cycle() { + let response = route(&router, request_for(&method, path)).await; + + assert_eq!( + response.status(), + StatusCode::FORBIDDEN, + "the filter's answer should stand in for {method} {path}" + ); + assert_eq!( + header(&response, EFFECT_HEADER), + Some("answered"), + "the effects the answering filter left should be applied to {method} {path}" + ); + assert_eq!(text(response), ANSWER, "{method} {path}"); + } + assert!( + origin.recorded_request_uris().is_empty(), + "no request should be sent on, because no route runs" + ); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn a_filter_s_request_header_reaches_the_route_and_its_effect_is_written_last() { let origin = Arc::new(StubHttpClient::new()); @@ -648,26 +712,31 @@ async fn an_edge_cookie_context_that_cannot_be_read_answers_before_the_filters() let mut sent = get("/article", None); sent.headers_mut().insert("cookie", unreadable_cookie()); let page = route(&router, sent).await; - let spoiled = route(&router, get(SPOILED_PATH, None)).await; - // A preparer the adapter links rebuilds the `Cookie` header from the - // values it can read, so a client's unreadable header never reaches the + // A preparer the adapter links drops a `Cookie` header value it cannot + // read as text, so a client's unreadable header never reaches the // context. assert_eq!( text(page), ANSWER, "a client's unreadable Cookie header should not stop the cycle" ); - assert_eq!( - spoiled.status(), - StatusCode::BAD_REQUEST, - "the context's error should answer the request" - ); - assert_eq!( - header(&spoiled, EFFECT_HEADER), - None, - "the filters should not run once the context cannot be read" - ); + // A page reaches the context through the fallback, and discovery through + // the named route's handler, each once the preparers have run. + for path in [SPOILED_PATH, DISCOVERY_PATH] { + let spoiled = route(&router, get(path, None)).await; + + assert_eq!( + spoiled.status(), + StatusCode::BAD_REQUEST, + "the context's error should answer {path}" + ); + assert_eq!( + header(&spoiled, EFFECT_HEADER), + None, + "the filters should not run for {path} once the context cannot be read" + ); + } assert!( origin.recorded_request_uris().is_empty(), "the publisher's origin should not be asked" diff --git a/crates/trusted-server-adapter-spin/src/app/request_cycle_tests.rs b/crates/trusted-server-adapter-spin/src/app/request_cycle_tests.rs index d710ce99f1..27d8b35d6f 100644 --- a/crates/trusted-server-adapter-spin/src/app/request_cycle_tests.rs +++ b/crates/trusted-server-adapter-spin/src/app/request_cycle_tests.rs @@ -1,15 +1,15 @@ //! The request cycle this adapter runs for a routed request, checked through -//! its router: the request filters run before the route and their response -//! effects after the finalize headers, without making the response less -//! private, the route's module call carries the device classification, an -//! Edge Cookie context that cannot be read answers before the filters, and -//! robots.txt, the inspection endpoints and the attestation pages are answered -//! without them. +//! its router: the request filters run before a page and before every named +//! route in the cycle, and their response effects after the finalize headers, +//! without making the response less private, the route's module call carries +//! the device classification, an Edge Cookie context that cannot be read +//! answers before the filters, and robots.txt, the inspection endpoints and +//! the attestation pages are answered without them. use std::sync::{Arc, Mutex}; use edgezero_core::body::Body; -use edgezero_core::http::{HeaderValue, Request, Response, StatusCode, request_builder}; +use edgezero_core::http::{HeaderValue, Method, Request, Response, StatusCode, request_builder}; use edgezero_core::router::RouterService; use error_stack::Report; use trusted_server_core::ec::device::{DeviceModule, DeviceSignals}; @@ -25,6 +25,7 @@ use trusted_server_core::platform::PlatformHttpClient; use trusted_server_core::platform::test_support::{ StubHttpClient, build_services_with_http_client, }; +use trusted_server_core::publisher::{PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH}; use trusted_server_core::response_privacy::enforce_terminal_private_cache_privacy; use trusted_server_core::settings::Settings; @@ -72,12 +73,15 @@ const CRAWLER_COOKIE: &str = "crawler_session=robots"; const READER_COOKIE: &str = "reader_session=route"; /// The id and module name of a module whose preparer leaves the request for -/// [`SPOILED_PATH`] with a `Cookie` header the Edge Cookie context cannot -/// read. +/// [`SPOILED_PATH`] or [`DISCOVERY_PATH`] with a `Cookie` header the Edge +/// Cookie context cannot read. const SPOILING_ID: &str = "spoiling_preparer"; const SPOILING_MODULE: &str = "testing.spoiling-preparer"; const SPOILED_PATH: &str = "/spoiled"; +/// The discovery route, a named route in the request cycle. +const DISCOVERY_PATH: &str = "/.well-known/trusted-server.json"; + /// Every `Cookie` header the marking device module was shown, by every test /// in this file, so a test can tell whether a request's cookies reached it. static COOKIES_SHOWN: Mutex> = Mutex::new(Vec::new()); @@ -311,14 +315,15 @@ fn register_nothing( Ok(None) } -/// Leaves the request for [`SPOILED_PATH`] with an unreadable `Cookie` -/// header. A deployment's own modules prepare a request after the modules -/// the adapter links, so this runs after every one of them. +/// Leaves the request for [`SPOILED_PATH`], a page, or [`DISCOVERY_PATH`], a +/// named route, with an unreadable `Cookie` header. A deployment's own +/// modules prepare a request after the modules the adapter links, so this +/// runs after every one of them. fn spoil_cookie( _settings: &Settings, request: &mut Request, ) -> Result<(), Report> { - if request.uri().path() == SPOILED_PATH { + if matches!(request.uri().path(), SPOILED_PATH | DISCOVERY_PATH) { request.headers_mut().insert("cookie", unreadable_cookie()); } Ok(()) @@ -414,6 +419,35 @@ fn get(path: &str, cookie: Option<&str>) -> Request { .expect("should build the request") } +/// A request for `path` by `method`, with no cookie. +fn request_for(method: &Method, path: &str) -> Request { + request_builder() + .method(method.clone()) + .uri(format!("https://test-publisher.example.com{path}")) + .header("host", "test-publisher.example.com") + .body(Body::empty()) + .expect("should build the request") +} + +/// Every named route the request cycle runs, by method and path. +fn named_routes_in_the_cycle() -> [(Method, &'static str); 13] { + [ + (Method::GET, DISCOVERY_PATH), + (Method::POST, "/verify-signature"), + (Method::POST, "/auction"), + (Method::GET, PAGE_BIDS_PATH), + (Method::OPTIONS, PAGE_BIDS_PATH), + (Method::GET, PAGE_BIDS_LEGACY_PATH), + (Method::OPTIONS, PAGE_BIDS_LEGACY_PATH), + (Method::GET, "/first-party/proxy"), + (Method::GET, "/first-party/click"), + (Method::GET, "/first-party/sign"), + (Method::POST, "/first-party/sign"), + (Method::GET, "/first-party/proxy-rebuild"), + (Method::POST, "/first-party/proxy-rebuild"), + ] +} + /// A `Cookie` header holding an accented letter in UTF-8, which a header /// value read as text refuses. fn unreadable_cookie() -> HeaderValue { @@ -479,6 +513,36 @@ async fn a_filter_that_answers_stands_in_for_the_page() { ); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_filter_that_answers_stands_in_for_every_named_route_in_the_cycle() { + let origin = Arc::new(StubHttpClient::new()); + let router = router_with( + settings_with("[testing]\nmodules = [\"answering-filter\"]"), + &[answering_builder()], + &origin, + ); + + for (method, path) in named_routes_in_the_cycle() { + let response = route(&router, request_for(&method, path)).await; + + assert_eq!( + response.status(), + StatusCode::FORBIDDEN, + "the filter's answer should stand in for {method} {path}" + ); + assert_eq!( + header(&response, EFFECT_HEADER), + Some("answered"), + "the effects the answering filter left should be applied to {method} {path}" + ); + assert_eq!(text(response), ANSWER, "{method} {path}"); + } + assert!( + origin.recorded_request_uris().is_empty(), + "no request should be sent on, because no route runs" + ); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn a_filter_s_request_header_reaches_the_route_and_its_effect_is_written_last() { let origin = Arc::new(StubHttpClient::new()); @@ -648,26 +712,31 @@ async fn an_edge_cookie_context_that_cannot_be_read_answers_before_the_filters() let mut sent = get("/article", None); sent.headers_mut().insert("cookie", unreadable_cookie()); let page = route(&router, sent).await; - let spoiled = route(&router, get(SPOILED_PATH, None)).await; - // A preparer the adapter links rebuilds the `Cookie` header from the - // values it can read, so a client's unreadable header never reaches the + // A preparer the adapter links drops a `Cookie` header value it cannot + // read as text, so a client's unreadable header never reaches the // context. assert_eq!( text(page), ANSWER, "a client's unreadable Cookie header should not stop the cycle" ); - assert_eq!( - spoiled.status(), - StatusCode::BAD_REQUEST, - "the context's error should answer the request" - ); - assert_eq!( - header(&spoiled, EFFECT_HEADER), - None, - "the filters should not run once the context cannot be read" - ); + // A page reaches the context through the fallback, and discovery through + // the named route's handler, each once the preparers have run. + for path in [SPOILED_PATH, DISCOVERY_PATH] { + let spoiled = route(&router, get(path, None)).await; + + assert_eq!( + spoiled.status(), + StatusCode::BAD_REQUEST, + "the context's error should answer {path}" + ); + assert_eq!( + header(&spoiled, EFFECT_HEADER), + None, + "the filters should not run for {path} once the context cannot be read" + ); + } assert!( origin.recorded_request_uris().is_empty(), "the publisher's origin should not be asked" From 8c9b914c7f39f24af17b44d009a0bf7e43d4b447 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 18 Oct 2026 07:48:09 +0100 Subject: [PATCH 10/18] The Edge Cookie context comments say which Cookie header it cannot read build_ec_context answers with an error for a Cookie header that HeaderValue::to_str refuses, being one holding any byte outside visible ASCII, an accented letter in UTF-8 among them, and not only one that is not valid UTF-8, as the comment said on Axum, Cloudflare and Spin. --- crates/trusted-server-adapter-axum/src/app.rs | 4 +++- crates/trusted-server-adapter-cloudflare/src/app.rs | 4 +++- crates/trusted-server-adapter-spin/src/app.rs | 4 +++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index c6c203ef4b..b8c3608283 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -477,7 +477,9 @@ async fn derive_device_signals( /// # Errors /// /// Returns an error when the selected Edge Cookie module cannot be built for -/// this request, or when the request's `Cookie` header is not valid UTF-8. +/// this request, or when the request's `Cookie` header cannot be read as +/// text, because it holds a byte outside visible ASCII, as an accented letter +/// in UTF-8 does. async fn build_ec_context( state: &AppState, services: &RuntimeServices, diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index a79870087b..7a82aa0129 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -553,7 +553,9 @@ async fn derive_device_signals( /// # Errors /// /// Returns an error when the selected Edge Cookie module cannot be built for -/// this request, or when the request's `Cookie` header is not valid UTF-8. +/// this request, or when the request's `Cookie` header cannot be read as +/// text, because it holds a byte outside visible ASCII, as an accented letter +/// in UTF-8 does. async fn build_ec_context( settings: &Settings, services: &RuntimeServices, diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index f2b60d99de..fbaab42ccb 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -753,7 +753,9 @@ async fn derive_device_signals( /// # Errors /// /// Returns an error when the selected Edge Cookie module cannot be built for -/// this request, or when the request's `Cookie` header is not valid UTF-8. +/// this request, or when the request's `Cookie` header cannot be read as +/// text, because it holds a byte outside visible ASCII, as an accented letter +/// in UTF-8 does. async fn build_ec_context( settings: &Settings, services: &RuntimeServices, From 233299a06d4238c297c07d520e67368a7a546353 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 18 Oct 2026 07:48:37 +0100 Subject: [PATCH 11/18] The integration guide says what a device module is shown A device module a registration offers is shown the client's IP address as well as the User-Agent and the cookies, in a module context that carries the request's method, scheme, host, path and query. --- docs/guide/integration-guide.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/guide/integration-guide.md b/docs/guide/integration-guide.md index d477243ddd..bdaf91e615 100644 --- a/docs/guide/integration-guide.md +++ b/docs/guide/integration-guide.md @@ -464,8 +464,9 @@ held for other settings is never served. Trusted Server hashes it into the key the answer is held under, so nothing of it is written to the store. Every adapter classifies a request with the device module `[device]` selects, -and a device module a registration offers is shown the User-Agent and the -request's cookies to do so. In an auction it is also asked for the bid +and a device module a registration offers is shown the client's IP address, +the User-Agent and the request's cookies to do so, with the request's method, +scheme, host, path and query. In an auction it is also asked for the bid request's device attributes. The registry builds only the modules a section selects, so the module that From 52cfd9fe9fe30e0a123d6a9e47eeca4de8369fee Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 18 Oct 2026 07:48:38 +0100 Subject: [PATCH 12/18] The geo design says every adapter writes a filter's response headers Section 2.11 lists apply_terminal_response_effects on all four adapters, at the Fastly entry point and in the other adapters' FinalizeResponseMiddleware, and section 1 no longer calls the Fastly adapter the only one that calls detect. --- .../2026-10-15-geo-module-request-design.md | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/docs/superpowers/specs/2026-10-15-geo-module-request-design.md b/docs/superpowers/specs/2026-10-15-geo-module-request-design.md index fe59846821..ad27cb089d 100644 --- a/docs/superpowers/specs/2026-10-15-geo-module-request-design.md +++ b/docs/superpowers/specs/2026-10-15-geo-module-request-design.md @@ -5,7 +5,7 @@ **Related specs:** `2026-10-12-module-context-design.md`, `2026-10-12-device-signals-design.md`, `2026-08-27-integration-provider-seam-design.md`, `2026-09-07-didomi-geo-design.md`. Every design document of the stack is in IABTechLab/trusted-server#1342. **Related PRs:** IABTechLab/trusted-server#1374, reviewed on jwrosewell/trusted-server#28 (closed) **Issues:** IABTechLab/trusted-server#1321 -**Last updated:** 2026-10-17 +**Last updated:** 2026-10-18 > **Why this spec exists.** A geo module was handed the client address and nothing else of the request, so a vendor whose one service answers location and device from the same request could not answer both from one call. @@ -13,7 +13,7 @@ A geo module was called as `PlatformGeo::lookup(client_ip, services)`, so the module was handed the client address and the services and nothing else of the request. -A device module is called as `DeviceModule::detect(call)`, with a module call that carries the request's view (its method, host, scheme, path and query), the request's evidence, the settings and the services. The evidence is narrow. On Fastly, the only adapter that calls `detect`, the evidence is the client address, the `User-Agent` header and, for a device module that a registration supplies, the `Cookie` header, and the evidence holds no path or query (`derive_device_signals` in `crates/trusted-server-adapter-fastly/src/lib.rs`). A device module's `advertising_attributes`, unlike `detect`, is handed every request header (`attach_device_attributes` in `crates/trusted-server-core/src/auction/formats.rs`). +A device module is called as `DeviceModule::detect(call)`, with a module call that carries the request's view (its method, host, scheme, path and query), the request's evidence, the settings and the services. The evidence is narrow. On Fastly the evidence is the client address, the `User-Agent` header and, for a device module that a registration supplies, the `Cookie` header, and the evidence holds no path or query (`derive_device_signals` in `crates/trusted-server-adapter-fastly/src/lib.rs`). A device module's `advertising_attributes`, unlike `detect`, is handed every request header (`attach_device_attributes` in `crates/trusted-server-core/src/auction/formats.rs`). A vendor whose one service answers location and device from the same request, keyed on the whole request, could not make the geo answer come from the same call as the device answer. The geo module never saw the evidence. The vendor therefore asked its service twice, or the geo answer depended on the device module having run first on the same adapter. @@ -244,16 +244,16 @@ A document that selects the module and names it in no `[[serve]]` entry is refus These still change a response outside middleware, and each could move to a serve middleware in the same way later. -| Change | Where | Adapters | -| -------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------- | ----------------------------------------- | -| Removes `X-Served-By`, `X-Cache` and `X-Cache-Hits` outside `/_ts` | `strip_deployment_headers` in `crates/trusted-server-core/src/http_util.rs` | All four, in `FinalizeResponseMiddleware` | -| Writes `[response_headers]`, and keeps a response with `Set-Cookie`, or one that is uncacheable, private to a shared cache | `apply_response_headers_with_cache_privacy` in `crates/trusted-server-core/src/response_privacy.rs` | All four, in `apply_finalize_headers` | -| Writes `X-Robots-Tag: noindex, nofollow` when `[robots-txt]` selects `refuse_all` | `apply_response_robots_tag` in `crates/trusted-server-core/src/robots_txt/mod.rs` | All four, in `apply_finalize_headers` | -| Writes `X-TS-ENV: staging` when `FASTLY_IS_STAGING` is `1` | `apply_finalize_headers` in `crates/trusted-server-adapter-fastly/src/middleware.rs` | Fastly | -| Writes an asset route's cache directives again | `AssetProxyCachePolicy::apply_after_route_finalization` in `crates/trusted-server-core/src/proxy.rs` | Fastly, at the entry point | -| Writes the Edge Cookie's `Set-Cookie` and identity headers | `ec_finalize_response` in `crates/trusted-server-core/src/ec/finalize.rs` | Fastly, at the entry point | -| Writes the request filters' response headers, `private, no-store` on a response marked private, and the late cache guards | `apply_terminal_response_effects` in `crates/trusted-server-adapter-fastly/src/lib.rs` | Fastly, at the entry point | -| Writes the reusable sandbox's counters on a `private, no-store` response | `attach_sandbox_counters` in `crates/trusted-server-adapter-fastly/src/lib.rs` | Fastly, at the entry point | +| Change | Where | Adapters | +| -------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------- | +| Removes `X-Served-By`, `X-Cache` and `X-Cache-Hits` outside `/_ts` | `strip_deployment_headers` in `crates/trusted-server-core/src/http_util.rs` | All four, in `FinalizeResponseMiddleware` | +| Writes `[response_headers]`, and keeps a response with `Set-Cookie`, or one that is uncacheable, private to a shared cache | `apply_response_headers_with_cache_privacy` in `crates/trusted-server-core/src/response_privacy.rs` | All four, in `apply_finalize_headers` | +| Writes `X-Robots-Tag: noindex, nofollow` when `[robots-txt]` selects `refuse_all` | `apply_response_robots_tag` in `crates/trusted-server-core/src/robots_txt/mod.rs` | All four, in `apply_finalize_headers` | +| Writes `X-TS-ENV: staging` when `FASTLY_IS_STAGING` is `1` | `apply_finalize_headers` in `crates/trusted-server-adapter-fastly/src/middleware.rs` | Fastly | +| Writes an asset route's cache directives again | `AssetProxyCachePolicy::apply_after_route_finalization` in `crates/trusted-server-core/src/proxy.rs` | Fastly, at the entry point | +| Writes the Edge Cookie's `Set-Cookie` and identity headers | `ec_finalize_response` in `crates/trusted-server-core/src/ec/finalize.rs` | Fastly, at the entry point | +| Writes the request filters' response headers, `private, no-store` on a response marked private, and the late cache guards | `apply_terminal_response_effects`, in `crates/trusted-server-adapter-fastly/src/lib.rs` and in each other adapter's `src/middleware.rs` | All four, at the Fastly entry point and in the others' `FinalizeResponseMiddleware` | +| Writes the reusable sandbox's counters on a `private, no-store` response | `attach_sandbox_counters` in `crates/trusted-server-adapter-fastly/src/lib.rs` | Fastly, at the entry point | ### 2.12 Open questions @@ -323,3 +323,4 @@ The pull request records the fork's CI green on all 19 checks at 1b8b43858. Its | 2026-10-15 | Corrections from review round 2. Why the entry-point step skips the registry, which responses carry `EcFinalizeState`, what the Fastly device module reads, the type of the registration field, the permissions endpoint in Axum, Cloudflare and Spin, the round 1 log sentence on the code changes, and the follow-ups for the changelog, the integration guide and four stale comments. | | 2026-10-16 | The `x-geo-*` response headers move from core and the adapters to the serve middleware `headers.geo` (section 2.10), so no finalize step calls a geo module, which answers open questions 1, 2, 3 and 5 of the previous revision. Section 2.11 lists the other changes a finalize step makes to a response, and section 4 the tests. | | 2026-10-17 | Axum and Spin refuse `[geo] module = "platform"` at startup, with a test on each, and the integration tests start the Axum dev server without it (sections 2.5, 2.6, 2.9 and 4). Section 2.5 says Cloudflare's region needs the `cf-region-code` header. | +| 2026-10-18 | Every adapter runs the request filters and calls `detect`, from `2026-10-17-request-filters-on-every-adapter-design.md` and its second review round. Section 1 no longer calls the Fastly adapter the only one that calls `detect`, and section 2.11 says every adapter writes a filter's response headers, `private, no-store` and the late cache guards. | From e7417c898e36d833737840793880dba23e33fbd2 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 18 Oct 2026 07:48:47 +0100 Subject: [PATCH 13/18] The client hints design gives the wait on Axum and the targets its tests skip On Axum a service that cannot be reached fails within the client's 5 second connection limit and a refused connection fails at once, so only a service that accepts the connection and then stalls holds the call for up to 30 seconds. No alias runs the two 51Degrees crates' tests for the WebAssembly target of any adapter, not only the Fastly adapter's, because cargo test-cloudflare and cargo test-spin test the adapter crate alone. --- ...026-10-14-51degrees-client-hints-design.md | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/docs/superpowers/specs/2026-10-14-51degrees-client-hints-design.md b/docs/superpowers/specs/2026-10-14-51degrees-client-hints-design.md index 2fcae80f29..99a5a2de97 100644 --- a/docs/superpowers/specs/2026-10-14-51degrees-client-hints-design.md +++ b/docs/superpowers/specs/2026-10-14-51degrees-client-hints-design.md @@ -211,10 +211,13 @@ hints is held like any other, for up to an hour. On Fastly a call that cannot connect fails after one second, and one that connects waits `timeout_ms` for the first byte and for each gap between bytes, so while the service is down every page navigation an entry covers with no held answer waits for those -limits before its page is built. On Axum, Cloudflare and Spin the call waits -until the adapter's HTTP client gives up, which is 30 seconds on Axum and the -runtime's own limit on Cloudflare and Spin, so while the service is down each -such navigation waits that long. +limits before its page is built. On Axum, Cloudflare and Spin only the +adapter's HTTP client bounds the call. On Axum a service that cannot be +reached fails within the client's 5 second connection limit, a refused +connection fails at once, and a service that accepts the connection and then +stalls holds the call for up to 30 seconds in all. On Cloudflare and Spin the +runtime's own limit bounds it. So while the service is slow or cannot be +reached, each such navigation waits up to that bound. **Rollout.** A deployment without `[shared.51degrees]` is unchanged. On every adapter, adding the table and an entry naming the middleware changes the @@ -299,18 +302,21 @@ Six things have no test. `permissions: None`, so nothing shows what the filter does with permission state. 6. The two crates' tests run natively under `cargo test-axum`, and their lints - under `cargo clippy-axum`. No alias for the Fastly target lists them, so - they do not run for `wasm32-wasip1`, the Fastly adapter's target. + under `cargo clippy-axum`. No alias runs their tests for an adapter's + WebAssembly target, being `wasm32-wasip1` for Fastly and Spin and + `wasm32-unknown-unknown` for Cloudflare, because the Fastly aliases do not + list them and `cargo test-cloudflare` and `cargo test-spin` test the + adapter crate alone, on the host. ## Revision record -| Date | Change | -| ---------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 2026-10-14 | First draft. | -| 2026-10-15 | Fixes from review round 1. What the filter runs on and that only the Fastly adapter runs it, the cost to pages, what each navigation sends and the open questions on consent, how each adapter bounds the call, the answers on failure, the settings, what the headers keep, the definition of a page navigation, the untested paths, the related specs and the review log. | -| 2026-10-15 | Fixes from review round 2. The pull request that holds the design documents, the wait for the first call in the guide, and the code comments left for a later change. | -| 2026-10-15 | Code fixes for the call's scope, what it sends and header merging. The permission question stays open, with what was found, and the answer lifetime is a new open question. | -| 2026-10-18 | Every adapter runs the filter, from `2026-10-17-request-filters-on-every-adapter-design.md` and its first review round. The table of parts, the builder, the `timeout_ms` setting, what bounds the call on each adapter, consent, failure, rollout, section 3 and the tests say so. | +| Date | Change | +| ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 2026-10-14 | First draft. | +| 2026-10-15 | Fixes from review round 1. What the filter runs on and that only the Fastly adapter runs it, the cost to pages, what each navigation sends and the open questions on consent, how each adapter bounds the call, the answers on failure, the settings, what the headers keep, the definition of a page navigation, the untested paths, the related specs and the review log. | +| 2026-10-15 | Fixes from review round 2. The pull request that holds the design documents, the wait for the first call in the guide, and the code comments left for a later change. | +| 2026-10-15 | Code fixes for the call's scope, what it sends and header merging. The permission question stays open, with what was found, and the answer lifetime is a new open question. | +| 2026-10-18 | Every adapter runs the filter, from `2026-10-17-request-filters-on-every-adapter-design.md` and its first review round. The table of parts, the builder, the `timeout_ms` setting, what bounds the call on each adapter, consent, failure, rollout, section 3 and the tests say so. Fixes from review round 3, the wait on Axum for a service that cannot be reached and test item 6. | ## Review log From ac04929328c1a718c5c3776a4af2cabf0b643157 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 18 Oct 2026 07:49:08 +0100 Subject: [PATCH 14/18] The client hints design logs its third review round The review of the change that has every adapter run the filter counts as this design's third round, found nothing blocking and was answered in e7417c898. --- .../specs/2026-10-14-51degrees-client-hints-design.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/superpowers/specs/2026-10-14-51degrees-client-hints-design.md b/docs/superpowers/specs/2026-10-14-51degrees-client-hints-design.md index 99a5a2de97..1fc595b5e0 100644 --- a/docs/superpowers/specs/2026-10-14-51degrees-client-hints-design.md +++ b/docs/superpowers/specs/2026-10-14-51degrees-client-hints-design.md @@ -322,3 +322,4 @@ Six things have no test. 1. **Round 1, 2026-10-15.** A review in a fresh session, against `split/44-client-hints` at `1f6b83d0c`. Issues found, four of them blocking. The four were that the request filter runs wherever the table is present and not only where an entry covers the page, that the cost to existing pages was not stated, that failure handling and the timeout of each adapter were not stated, and that nothing said what each navigation sends to the service or what consent it needs. This revision states each from the code, which also shows that only the Fastly adapter runs request filters, and it corrects the claims about page navigations, the headers and the tests, replaces a related spec that exists in no branch, and corrects the guide, the changelog and the middleware readme. 2. **Round 2, 2026-10-15.** A lighter review in a fresh session, against this page at `126347b00`, of the round 1 fixes and the new text. No blocking issues once the pull request number was corrected. This revision names the pull request that now holds every design document, makes the guide's description of the wait for the first call match the one second connection limit and the limit on each gap between bytes, and lists in section 3 the code comments that still say more than the design does. +3. **Round 3, 2026-10-18.** A review in a fresh session, against `split/52-request-filters` at `17136d04b`, of the change that has every adapter run the filter, made with the request filters design's first round. Issues found, none blocking. The two were that When it fails gave 30 seconds on Axum as the wait for a service that cannot be reached, which fails within the client's 5 second connection limit, and that test item 6 named only the Fastly adapter's target, when no alias runs the two crates' tests for the WebAssembly target of any adapter, all of which now run the filter. Fixed in `e7417c898`. From 076f4add0017f2ab0e2639d0359925120b998c00 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 18 Oct 2026 07:49:58 +0100 Subject: [PATCH 15/18] The request filters design answers its second review round It lists the test that sends every named route in the cycle through the filters and the context test's discovery request, names the tests each removal fails, names the IP address and the request's address a device module is shown and the subresources it classifies, says the diagnostics preparer drops a Cookie header value whole, names the two DataDome designs whose scope it lifts, says what the guards do with a platform's own cache header with an open question on it, gives a true reason no shipped filter is tested through an adapter, and logs the round. --- ...request-filters-on-every-adapter-design.md | 39 +++++++++++-------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/docs/superpowers/specs/2026-10-17-request-filters-on-every-adapter-design.md b/docs/superpowers/specs/2026-10-17-request-filters-on-every-adapter-design.md index 1cbbd87c75..f762bd27e4 100644 --- a/docs/superpowers/specs/2026-10-17-request-filters-on-every-adapter-design.md +++ b/docs/superpowers/specs/2026-10-17-request-filters-on-every-adapter-design.md @@ -2,7 +2,7 @@ **Status:** Proposed, 2026-10-17. A pull request above `ts config migrate`, not yet raised, for Tech Lab review. **Author:** 51Degrees (contributed), for Tech Lab review -**Related specs:** `2026-10-14-51degrees-client-hints-design.md`, whose sections on the adapters this change amends, and the design documents of the stack in IABTechLab/trusted-server#1342 +**Related specs:** `2026-10-14-51degrees-client-hints-design.md` and `2026-10-15-geo-module-request-design.md`, whose statements about the adapters this change amends, `2026-06-11-datadome-server-side-protection-design.md` and `2026-08-03-datadome-ip-excluded-client-tag-design.md`, which describe DataDome's protection filter on the Fastly adapter alone, a scope this change lifts (section 1), and the design documents of the stack in IABTechLab/trusted-server#1342 **Related PRs:** none yet **Issues:** none **Last updated:** 2026-10-18 @@ -17,7 +17,9 @@ The Fastly adapter runs three steps that no other adapter runs. 2. **Device classification.** For every request, before routing, the entry point's `derive_device_signals` in `src/lib.rs` calls `DeviceModule::detect` on the device module a registration supplies, or on the one `build_device_module` builds where none does, and `build_ec_request_state` sets the result on the Edge Cookie context. A module call made with that context's state carries the classification (`ModuleContext::with_request_state` in core), which covers a module route's call and a serve phase preparation. A request filter is never handed it, because `RequestFilterInput` carries only the location and the permissions, and the fetch phase of a document that may be stored is handed services with nothing of the reader in place of the context's state (`preparation_context` in `crates/trusted-server-core/src/publisher.rs`). 3. **robots.txt first.** `dispatch_fallback` answers robots.txt once the preparers have run and before the Edge Cookie steps and the filters, with an empty Edge Cookie context, so no filter's answer stands in for the file and the crawler's Edge Cookie and consent signals are not read. The entry point has already run the device module by then, so a device module a registration supplies is shown the crawler's cookies. -No source file in `crates/trusted-server-adapter-axum`, `-cloudflare` or `-spin` calls `filter_request` or `DeviceModule::detect`. So on those adapters the 51Degrees client hints middleware asks its service nothing and writes no header or tag, DataDome's protection filter never runs, and robots.txt is answered through the publisher path with the crawler's own Edge Cookie context. A device module `[device]` selects from a registration is not idle there, because each adapter's `services_for_request` puts it on the request's services and core asks it for the bid request's device attributes in every auction, showing it the request's headers and permissions (`attach_device_attributes` in `crates/trusted-server-core/src/auction/formats.rs`, called from `auction/endpoints.rs` and `publisher.rs`). It never classifies a request, so no module call carries a classification. On Spin the discovery, signature verification and first-party routes and the page-bids preflight also skip the modules' preparers, which every other adapter runs for them. +No source file in `crates/trusted-server-adapter-axum`, `-cloudflare` or `-spin` calls `filter_request` or `DeviceModule::detect`. So on those adapters the 51Degrees client hints middleware asks its service nothing and writes no header or tag, DataDome's protection filter never runs, and robots.txt is answered through the publisher path with the crawler's own Edge Cookie context. A device module `[device]` selects from a registration is not idle there, because each adapter's `services_for_request` puts it on the request's services and core asks it for the bid request's device attributes in every auction, showing it the request's headers, IP address and address and the permissions (`attach_device_attributes` in `crates/trusted-server-core/src/auction/formats.rs`, called from `auction/endpoints.rs` and `publisher.rs`). It never classifies a request, so no module call carries a classification. On Spin the discovery, signature verification and first-party routes and the page-bids preflight also skip the modules' preparers, which every other adapter runs for them. + +The two DataDome designs describe the protection filter as run by the Fastly adapter alone. `2026-06-11-datadome-server-side-protection-design.md` wires request filters into the Fastly adapter, and `2026-08-03-datadome-ip-excluded-client-tag-design.md` keeps its client tag suppression to the Fastly adapter and lists a change to the other adapters' request filter wiring among its non-goals, because only the Fastly adapter ran the filter. This change lifts that scope, so the protection and its tag suppression run on Axum, Cloudflare and Spin as well. ## 2. Design @@ -46,7 +48,7 @@ Each adapter carries its own copy of the steps. `derive_device_signals`, `run_pr On Spin the discovery, signature verification and first-party routes and the page-bids preflight now run the preparers, because the cycle starts with them. Before this change they ran none. -**Device classification.** `derive_device_signals` follows the Fastly adapter's function of the same name. A device module a registration supplies is shown the `User-Agent` and the `Cookie` header, and the built-in module the `User-Agent` alone, each in a module context carrying the request, that evidence, the settings and the services. Four things differ from the Fastly adapter. +**Device classification.** `derive_device_signals` follows the Fastly adapter's function of the same name. The evidence a module is shown holds the client's IP address and the `User-Agent`, with the `Cookie` header as well for a device module a registration supplies, in a module context that also carries the request's method, scheme, host, path and query, the settings and the services, as on Fastly. Four things differ from the Fastly adapter. 1. It runs where the adapter builds the Edge Cookie context, after the preparers, where the Fastly adapter runs it at the entry point before routing. A preparer that removes a reserved cookie therefore removes it before a device module sees it. 2. It runs only for the routes in the cycle, so a request for robots.txt, an inspection endpoint or an attestation page is not classified, where the Fastly entry point classifies every request. On Fastly a device module a registration supplies is therefore shown the cookies a crawler sends with a request for robots.txt, and on these adapters it is not. @@ -55,7 +57,7 @@ On Spin the discovery, signature verification and first-party routes and the pag As on Fastly, a module route's call and a serve phase preparation carry the classification, a request filter is not handed it, and the fetch phase of a document that may be stored is handed services with nothing of the reader. -**Response effects.** A filter that lets the request continue can name response headers. The route's response carries them in its extensions as `RequestFilterEffects`, and `FinalizeResponseMiddleware` applies them after it writes the finalize headers, so a filter's header is written after an operator's `[response_headers]`, as on Fastly. `apply_terminal_response_effects` then enforces `no-store, private` on a response core marked `TerminalPrivateResponse` and runs `enforce_set_cookie_cache_privacy` and `enforce_uncacheable_cache_privacy`, the steps the Fastly entry point runs before it sends every response. Those steps run on every response, whether or not a filter named a header, so a response core marked private also loses any `ETag`, `Last-Modified`, `Expires` or `Age` that `[response_headers]` writes, as on Fastly. A filter's header is also written after the `X-Robots-Tag` that `[robots-txt]` writes, and `x-robots-tag` is not among the headers a filter is refused (`is_forbidden_filter_header` in `crates/trusted-server-core/src/integrations/registry.rs`), so a filter that names that header replaces the tag, on Fastly as on these adapters. The open questions ask whether a filter should be refused `X-Robots-Tag`. +**Response effects.** A filter that lets the request continue can name response headers. The route's response carries them in its extensions as `RequestFilterEffects`, and `FinalizeResponseMiddleware` applies them after it writes the finalize headers, so a filter's header is written after an operator's `[response_headers]`, as on Fastly. `apply_terminal_response_effects` then enforces `no-store, private` on a response core marked `TerminalPrivateResponse` and runs `enforce_set_cookie_cache_privacy` and `enforce_uncacheable_cache_privacy`, the steps the Fastly entry point runs before it sends every response. Those steps run on every response, whether or not a filter named a header, so a response core marked private also loses any `ETag`, `Last-Modified`, `Expires` or `Age` that `[response_headers]` writes, as on Fastly. A filter's header is also written after the `X-Robots-Tag` that `[robots-txt]` writes, and `x-robots-tag` is not among the headers a filter is refused (`is_forbidden_filter_header` in `crates/trusted-server-core/src/integrations/registry.rs`), so a filter that names that header replaces the tag, on Fastly as on these adapters. The open questions ask whether a filter should be refused `X-Robots-Tag`. The guards always remove `Surrogate-Control` and `CDN-Cache-Control` from a private response, but remove a shared cache header an adapter declares for its platform only where core wrote the policy in that header on the same response (`remove_owned_cache_headers` in `crates/trusted-server-core/src/cache_policy.rs`). A filter that names such a header on a private answer therefore keeps it, being `cloudflare-cdn-cache-control` on Cloudflare, where core writes its policy in that header, and `fastly-surrogate-control` on Fastly, while Axum and Spin declare none. This change does not establish whether a platform then stores such an answer, and the open questions ask whether the guards should remove a declared header whoever wrote it. **Configuration.** No setting is added, removed or given a new default. Three settings act on these adapters where they did not, being `[device] module` when it selects a device module a registration supplies, which now classifies each request in the cycle, and DataDome's `enable_protection` and `[shared.51degrees]`, whose filters now run. Two settings these adapters accept are not applied, being DataDome's `timeout_ms`, default `1500`, and `[shared.51degrees] timeout_ms`, default `500`, which only the Fastly adapter applies, as the timeouts of the backend each module registers for its call. No adapter warns about either at startup, the way the Axum adapter warns about a `proxy.certificate_check` it cannot honor, because an adapter names no vendor's setting, and no platform declaration tells core or a module that an adapter does not apply a backend's timeouts. The DataDome and 51Degrees guides say where `timeout_ms` applies. @@ -63,25 +65,26 @@ As on Fastly, a module route's call and a serve phase preparation carry the clas - The 51Degrees filter sends the reader's `User-Agent` and `Sec-CH-UA` hints to the 51Degrees service for a page navigation an entry naming its middleware covers. The call goes through the answer cache the 51Degrees modules share, so the service is asked once for each set of those values while an answer is held, and not at all for a request with no `User-Agent` (`crates/shared/51degrees/src/hints_engine.rs`). That is the behavior `2026-10-14-51degrees-client-hints-design.md` describes, and its open question, whether the call needs a permission, stays open, and now concerns four adapters where it concerned one. - DataDome's protection filter, which DataDome registers only when `enable_protection` is on, removes `x-ts-datadome-bypass` from every request it runs for, before its other checks. For each request it protects, it sends DataDome's Protection API the server side key, the reader's IP address, the request's method, host, path and query, its `User-Agent`, `Accept`, `Accept-Language`, `Accept-Encoding`, `Referer`, `Origin`, `Via`, `X-Forwarded-For`, `True-Client-IP` and `X-Real-IP` headers among others, its `Sec-CH-UA` and `Sec-Fetch` headers, the list of its header names, the lengths of its `Cookie` and `Authorization` headers, and the client id from the `datadome` cookie or the `x-datadome-clientid` header (`build_protection_payload` in `crates/bot-protection/datadome/src/protection.rs`). The filter does not read the permissions. A response the API allows, or a challenge, can carry DataDome's `Set-Cookie`, which sets a cookie on the publisher's domain and keeps the response out of shared caches, as `private, max-age=0` where the response's own policy allowed them (`enforce_set_cookie_cache_privacy`). A reader an IP rule excludes is not checked, and the request is marked personalized, so an HTML page it asks for is served without DataDome's tag and sent `private, no-store`. -- A device module a registration supplies is shown the reader's `User-Agent` and cookies to classify every request in the cycle, with no permission check, because classification runs before the permissions are resolved and startup refuses a supplied device module that declares a permission (`resolve_device_module` in `crates/trusted-server-core/src/integrations/registry.rs`). That is the Fastly adapter's behavior. Before this change these adapters showed the module the request's headers only in an auction, through `advertising_attributes`, which is handed the permissions. +- A device module a registration supplies is shown the reader's IP address, `User-Agent` and cookies, and the request's method, scheme, host, path and query, to classify every request in the cycle, with no permission check, because classification runs before the permissions are resolved and startup refuses a supplied device module that declares a permission (`resolve_device_module` in `crates/trusted-server-core/src/integrations/registry.rs`). That is the Fastly adapter's behavior. Before this change these adapters showed the module the request's headers, IP address and address only in an auction, through `advertising_attributes`, which is handed the permissions. - robots.txt is answered with an empty Edge Cookie context, so the crawler's Edge Cookie and consent signals are not read and no device module is shown its cookies, where before this change these adapters read the crawler's own context. The publisher path still parses the crawler's `Cookie` header, as it does for every request it handles (`handle_publisher_request`), and sends it on to an origin that is not signed. -**Failure handling.** A filter that fails answers the request with the error, logged as `Failed to run integration request filters`, as on Fastly. An Edge Cookie context that cannot be read answers the request with the error before the filters run, which now covers every route in the cycle, including tsjs, discovery, signature verification, the first-party routes and the page-bids preflight. The context has two ways to fail, being a selected Edge Cookie module the request cannot build, which startup refuses where the adapter can never build it, and a `Cookie` header it cannot read as text, which answers `400`, because `handle_request_cookies` in `crates/trusted-server-core/src/cookies.rs` reads the header with `HeaderValue::to_str`, which refuses any byte outside visible ASCII, an accented letter in UTF-8 among them. A client's own `Cookie` header never fails that way, because the preparer of the `ad-tag.google.diagnostics` module, which every adapter links and runs for every request the cycle handles, first rebuilds the header from the values it can read (`sanitize_console_cookie` in `crates/ad-tag/google/src/diagnostics.rs`), so only a preparer of a deployment's own module, which runs after it, can leave such a header. A filter's outbound call is bounded only by the adapter's HTTP client, which applies no module's own timeout on these adapters. Axum's client allows 5 seconds to connect and 30 in all, Cloudflare leaves the call to the Workers runtime's limit, and Spin to the Spin runtime's default (`src/platform.rs` of each adapter). A filter that lets the request through when its call fails, as DataDome's and the 51Degrees filter do, therefore delays the request by up to that bound rather than failing it. +**Failure handling.** A filter that fails answers the request with the error, logged as `Failed to run integration request filters`, as on Fastly. An Edge Cookie context that cannot be read answers the request with the error before the filters run, which now covers every route in the cycle, including tsjs, discovery, signature verification, the first-party routes and the page-bids preflight. The context has two ways to fail, being a selected Edge Cookie module the request cannot build, which startup refuses where the adapter can never build it, and a `Cookie` header it cannot read as text, which answers `400`, because `handle_request_cookies` in `crates/trusted-server-core/src/cookies.rs` reads the header with `HeaderValue::to_str`, which refuses any byte outside visible ASCII, an accented letter in UTF-8 among them. A client's own `Cookie` header never fails that way, because the preparer of the `ad-tag.google.diagnostics` module, which every adapter links and runs for every request the cycle handles, first rebuilds the header from the header values it can read as text (`sanitize_console_cookie` in `crates/ad-tag/google/src/diagnostics.rs`), so only a preparer of a deployment's own module, which runs after it, can leave such a header. The preparer drops a value it cannot read whole rather than repairing it, so one accented cookie removes every cookie sent in the same header value for that request, the Edge Cookie and the consent cookies among them. That held already wherever the preparers ran, and is new on Spin's discovery, signature verification and first-party routes and its page-bids preflight, which ran none before this change. A filter's outbound call is bounded only by the adapter's HTTP client, which applies no module's own timeout on these adapters. Axum's client allows 5 seconds to connect and 30 in all, Cloudflare leaves the call to the Workers runtime's limit, and Spin to the Spin runtime's default (`src/platform.rs` of each adapter). A filter that lets the request through when its call fails, as DataDome's and the 51Degrees filter do, therefore delays the request by up to that bound rather than failing it. **Rollout.** -- A deployment whose modules register no filter, and that selects no device module a registration supplies, answers as before, apart from three things. It reads the Edge Cookie context, with its geo lookup where `[geo]` selects a module, and classifies the device with the built-in module, on every route in the cycle, where it read the context for the auction, page-bids, the module routes and the publisher alone. And a response core marked private loses any `ETag`, `Last-Modified`, `Expires` or `Age` that `[response_headers]` writes, as Response effects says. On Spin, discovery, signature verification, the first-party routes and the page-bids preflight run the modules' preparers, as on the other adapters. A client's `Cookie` header brings no new `400`, as Failure handling says. -- A deployment that selects a device module a registration supplies has that module asked to classify every request in the cycle, being tsjs, discovery, `/verify-signature`, the first-party routes, the auction, page-bids and its preflight, the module routes and the publisher's pages, each time shown the reader's `User-Agent` and cookies, beside the bid request attributes each auction already asks it for. A module that calls a service to classify makes that call for each of those requests. +- A deployment whose modules register no filter, and that selects no device module a registration supplies, answers as before, apart from three things. It reads the Edge Cookie context, with its geo lookup where `[geo]` selects a module, and classifies the device with the built-in module, on every route in the cycle, where it read the context for the auction, page-bids, the module routes and the publisher alone. And a response core marked private loses any `ETag`, `Last-Modified`, `Expires` or `Age` that `[response_headers]` writes, as Response effects says. On Spin, discovery, signature verification, the first-party routes and the page-bids preflight run the modules' preparers, as on the other adapters, so a `Cookie` header value that cannot be read as text is dropped there as well. A client's `Cookie` header brings no new `400`, as Failure handling says. +- A deployment that selects a device module a registration supplies has that module asked to classify every request in the cycle, being tsjs, discovery, `/verify-signature`, the first-party routes, the auction, page-bids and its preflight, the module routes and every request the publisher fallback proxies, the publisher's pages and each image, script, stylesheet and other subresource among them, each time shown the reader's IP address, `User-Agent` and cookies and the request's address, beside the bid request attributes each auction already asks it for. A module that calls a service to classify makes that call for each of those requests. - A deployment with `[shared.51degrees]` and a `[[serve]]` entry naming `51degrees.client_hints` now has the headers and the tag written on the pages the entry covers, and such a page keeps to the origin path and is sent `private, no-store`, because the filter leaves the hint names in the request's state. - A deployment with DataDome's protection on now has each request it protects checked by the Protection API before the route, being the publisher's pages and each subresource the static asset rule does not exclude, `/auction` and `GET` on both page-bids paths, none of which is among DataDome's internal paths (`is_internal_path` in `crates/bot-protection/datadome/src/protection.rs`). A challenge then answers in place of a page or an auction call. A response that carries DataDome's cookie is kept out of shared caches, and a reader an IP rule excludes gets pages that keep to the origin path and are sent `private, no-store`. The call has no time limit of DataDome's own on these adapters, so a slow Protection API delays each protected request until the adapter's HTTP client gives up, which is 30 seconds on Axum, before DataDome lets it through. **Tests.** In `src/app/request_cycle_tests.rs` of each of the three adapters, through the adapter's router built by its composition root, with a stub origin: - `a_filter_that_answers_stands_in_for_the_page`: the answer and its effect header are the response, and the origin is not asked. +- `a_filter_that_answers_stands_in_for_every_named_route_in_the_cycle`: with a filter that answers every request, discovery, `POST /verify-signature`, `POST /auction`, `GET` and `OPTIONS` on both page-bids paths and the four first-party paths on each method they answer are each the filter's answer with its effect header, and nothing is sent on. - `a_filter_s_request_header_reaches_the_route_and_its_effect_is_written_last`: a module route sees the request header a filter added, and the filter's response header replaces the operator's header of the same name. - `the_device_module_s_classification_reaches_the_module_call`: the module `[device]` selects classifies the request, shown its cookies, and a module route's call carries that classification. - `platform_classifies_the_request_with_the_built_in_module`: with `[device] module = "platform"`, a module route's call carries the built-in module's classification of an iPhone `User-Agent`. -- `an_edge_cookie_context_that_cannot_be_read_answers_before_the_filters`: with a filter that answers every request, a request a module's preparer leaves with an unreadable `Cookie` header is answered `400`, with no filter's effect and no request to the origin, and a client's own unreadable `Cookie` header reaches the filter instead. +- `an_edge_cookie_context_that_cannot_be_read_answers_before_the_filters`: with a filter that answers every request, a page and the discovery route, each of which a module's preparer leaves with an unreadable `Cookie` header, are answered `400`, with no filter's effect and no request to the origin, and a request a client sends with an unreadable `Cookie` header of its own reaches the filter instead. - `robots_txt_is_answered_before_the_request_filters`: with a filter that answers every request, robots.txt is still the file, and the filter's header is absent. - `robots_txt_shows_the_device_module_none_of_the_crawler_s_cookies`: the module `[device]` selects, which records the cookies it is shown, never sees the cookies sent with robots.txt, and does see those sent with a module route's request. - `the_inspection_endpoints_and_the_attestation_pages_skip_the_request_filters`: with a filter that answers every request, the five inspection paths and the two attestation pages are answered by their own handlers with no filter's effect, and a page is the filter's answer. @@ -90,7 +93,7 @@ As on Fastly, a module route's call and a serve phase preparation carry the clas - `a_filter_cannot_add_a_shared_cache_header_to_a_private_response`: a `Surrogate-Control` that a filter's effect adds to a `private` answer is removed. - `a_response_core_made_private_loses_the_validators_an_operator_writes`: a module route's answer core marks private, which no filter names a header for, loses the `ETag` and `Last-Modified` that `[response_headers]` writes. -Taking any one of the three privacy guards out of the Axum adapter's `apply_terminal_response_effects` fails that guard's tests alone, and taking all three out of the Cloudflare or the Spin adapter's fails the four privacy tests. Taking out of the Axum adapter the step each of the other new tests covers fails that test alone. +Taking any one of the three privacy guards out of the Axum adapter's `apply_terminal_response_effects` fails that guard's tests alone, and taking all three out of the Cloudflare or the Spin adapter's fails the four privacy tests. On each of the three adapters, taking `run_pre_route_filters` out of `execute_named` fails the named route test alone, and taking `build_ec_context` out of it fails the context test alone, whose discovery request is then the filter's answer and not `400`, as taking the preparers out of Spin's `execute_named` does. In the Axum adapter, taking out the robots.txt early answer fails both robots.txt tests, taking the classification out of `build_ec_context` fails the classification test, the `platform` test and the robots.txt cookie test, which asserts that a request in the cycle shows the module its cookies, and taking out a filter's answer in the fallback fails the answering test, the context test and the inspection test, each of which asserts that a page is the filter's answer. No test runs a shipped filter, the 51Degrees middleware or DataDome's protection, through one of these adapters, and none covers `/health`, which runs no module. @@ -98,6 +101,7 @@ No test runs a shipped filter, the 51Degrees middleware or DataDome's protection 1. Whether the 51Degrees call needs a permission, which `2026-10-14-51degrees-client-hints-design.md` leaves to the reviewer and which now arises on four adapters. 2. Whether a request filter should be refused `X-Robots-Tag`, so that no filter can loosen a site `[robots-txt]` takes out of search, on Fastly as on these adapters. Refusing it is a change to `is_forbidden_filter_header` in core, which this change leaves alone. +3. Whether the guards should remove a shared cache header an adapter declares, `cloudflare-cdn-cache-control` or `fastly-surrogate-control`, from a private response whoever wrote it, or a filter be refused such a header, so that no filter can leave a private answer carrying a policy the platform's own cache reads. Either is a change to core, to `remove_owned_cache_headers` or to `is_forbidden_filter_header`, which this change leaves alone. ## 3. What is not in this change @@ -106,19 +110,20 @@ No test runs a shipped filter, the 51Degrees middleware or DataDome's protection - A time limit for a filter's outbound call on Axum, Cloudflare and Spin. A module's limit reaches the platform only as the `first_byte_timeout` and `between_bytes_timeout` of the `PlatformBackendSpec` it registers, and a request names only its backend (`PlatformHttpRequest`). The Axum backend keeps nothing of those timeouts, naming a backend by its scheme, host, port and discriminator, and its client applies its own two limits, although reqwest's `RequestBuilder::timeout` can bound one request. The Cloudflare backend's name carries the first byte timeout and its client does not read it, although a Worker's fetch can be stopped with `Fetch::send_with_signal` and an `AbortController`. Spin's `spin_sdk::http::send` takes no time limit. Applying a module's limit is a change to each adapter's backend and HTTP client, which every outbound call goes through, the auction's calls to its providers among them, so it is left to a change of its own. - Moving the inspection endpoints ahead of the preparers on Axum and Cloudflare, where the Fastly adapter answers all three of them, being all five paths, before the preparers. - One request cycle in core that every adapter calls, in place of the copy each adapter carries, which would move the Fastly adapter's cycle as well. -- A test that runs a shipped filter through one of these adapters, which needs a scripted DataDome Protection API or 51Degrees service and the secret store their keys are read from. +- A test that runs a shipped filter through one of these adapters. The pieces such a test needs exist, being the stub client these tests use, which can script the Protection API's or the 51Degrees service's answer, DataDome's key, which is a settings value once resolved, and `HashMapSecretStore` in core's test support. This change proves the cycle with filters written for its tests, and each shipped filter's own tests drive it directly against a scripted service, DataDome's with core's stub client and `HashMapSecretStore` and the 51Degrees filter's with `ScriptedHttpClient` in `crates/shared/51degrees/src/test_services.rs`, so a test of one through an adapter is left to a later change. ## 4. Acceptance -The twelve tests pass on each of the three adapters, taking out any of the three privacy guards fails a test, and the existing tests of each adapter and the cross-adapter parity tests pass unchanged. +The thirteen tests pass on each of the three adapters, taking out any of the three privacy guards fails a test, taking `build_ec_context` or `run_pre_route_filters` out of any adapter's `execute_named` fails a test, and the existing tests of each adapter and the cross-adapter parity tests pass unchanged. ## Revision record -| Date | Change | -| ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 2026-10-17 | First draft. | -| 2026-10-18 | Fixes from review round 1. What a selected device module did on these adapters before, what DataDome's filter sends and sets, how each filter's call is bounded and why no time limit is added, the inspection endpoints the Fastly adapter answers before the preparers, the routes kept out of the cycle, which of a crawler's cookies robots.txt reads, the `Cookie` check and when it is reached, the configuration, eight new tests on each adapter with the three privacy guards among them, the open questions and the review log. | +| Date | Change | +| ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 2026-10-17 | First draft. | +| 2026-10-18 | Fixes from review round 1. What a selected device module did on these adapters before, what DataDome's filter sends and sets, how each filter's call is bounded and why no time limit is added, the inspection endpoints the Fastly adapter answers before the preparers, the routes kept out of the cycle, which of a crawler's cookies robots.txt reads, the `Cookie` check and when it is reached, the configuration, eight new tests on each adapter with the three privacy guards among them, the open questions and the review log. Fixes from review round 2. A test on each adapter that sends every named route in the cycle through the filters, the discovery route in the context test, the tests each removal fails, the IP address and the request's address a device module is shown, the subresources it classifies, what the diagnostics preparer drops, the two DataDome designs whose scope this change lifts, what the guards do with a platform's own cache header with an open question on it, and the reason no shipped filter is tested through an adapter. | ## Review log -1. **Round 1, 2026-10-18.** A review in a fresh session, against `fixpass6/52-request-filters` at `f0edcd72b`. Issues found, five of them blocking. The five were that the spec said a selected device module did nothing on Axum, Cloudflare and Spin, when core asks it for a bid request's device attributes in every auction there, that nothing said what DataDome's filter sends DataDome or sets, or that no permission gates it, that the client hints design still said only the Fastly adapter runs request filters, that no test proved the privacy guards written after a filter's headers, and that the spec said the Fastly adapter answers two of the inspection endpoints before the preparers, when it answers all three, and that the adapters' HTTP clients cannot bound a call, when the Axum and Cloudflare clients can. Fixed in `7aea7a72b`, which adds the four privacy tests and four more on each adapter, `4549047dc`, which corrects the adapters' comments on robots.txt, `a584b071f`, which corrects the guides, the changelog and the middleware readme, `6b4eb0b8c`, which amends the client hints design, and this revision, which states each from the code and gives the reason a time limit is left to a change of its own. One finding was not borne out. The review said a `Cookie` header holding an accented letter now answers `400` on the routes that newly read the context, but a client's header is rebuilt by the `ad-tag.google.diagnostics` preparer first, which the context test shows, so Rollout names no new `400`. +1. **Round 1, 2026-10-18.** A review in a fresh session, against `split/52-request-filters` at `f0edcd72b`. Issues found, five of them blocking. The five were that the spec said a selected device module did nothing on Axum, Cloudflare and Spin, when core asks it for a bid request's device attributes in every auction there, that nothing said what DataDome's filter sends DataDome or sets, or that no permission gates it, that the client hints design still said only the Fastly adapter runs request filters, that no test proved the privacy guards written after a filter's headers, and that the spec said the Fastly adapter answers two of the inspection endpoints before the preparers, when it answers all three, and that the adapters' HTTP clients cannot bound a call, when the Axum and Cloudflare clients can. Fixed in `7aea7a72b`, which adds the four privacy tests and four more on each adapter, `4549047dc`, which corrects the adapters' comments on robots.txt, `a584b071f`, which corrects the guides, the changelog and the middleware readme, `6b4eb0b8c`, which amends the client hints design, and this revision, which states each from the code and gives the reason a time limit is left to a change of its own. One finding was not borne out. The review said a `Cookie` header holding an accented letter now answers `400` on the routes that newly read the context, but a client's header is rebuilt by the `ad-tag.google.diagnostics` preparer first, which the context test shows, so Rollout names no new `400`. +2. **Round 2, 2026-10-18.** A review in a fresh session, against `split/52-request-filters` at `17136d04b`, of the round 1 fixes and the new text. Issues found, one of them blocking. The one was that no test sent a named route through the request cycle, so taking `build_ec_context` or `run_pre_route_filters` out of any adapter's `execute_named` failed no test, although on Cloudflare and Spin this change replaced each named route's own handler with `execute_named`. Fixed in `7ae3235ad`, which adds a test on each adapter that sends every named route in the cycle to a filter that answers and has the context test's preparer spoil the discovery route as well, so that taking either step, or on Spin the preparers, out of `execute_named` fails a test, `8c9b914c7`, which corrects the `build_ec_context` comments on the `Cookie` check, `233299a06`, which names in the integration guide the IP address and the request's address a device module is shown, `52cfd9fe9`, which amends the geo module request design's table of finalize steps and its line on `detect`, `e7417c898` and `ac0492932`, which fix the client hints design's wait on Axum and its test item 6 and log its round, and this revision, which names the tests each removal fails, the IP address and the request's address, the subresources a device module classifies, what the diagnostics preparer drops, the two DataDome designs whose scope this change lifts and what the guards do with a platform's own cache header, on Fastly as on Cloudflare, with an open question on it, and gives a true reason no shipped filter is tested through an adapter. From 93134dd5bf6a2efff336c2cbc33c92ccf2381f38 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 28 Oct 2026 11:30:32 +0000 Subject: [PATCH 16/18] Row 52 takes the final request filters on every adapter design --- ...17-request-filters-on-every-adapter-design.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/superpowers/specs/2026-10-17-request-filters-on-every-adapter-design.md b/docs/superpowers/specs/2026-10-17-request-filters-on-every-adapter-design.md index f762bd27e4..7973cfac6d 100644 --- a/docs/superpowers/specs/2026-10-17-request-filters-on-every-adapter-design.md +++ b/docs/superpowers/specs/2026-10-17-request-filters-on-every-adapter-design.md @@ -1,11 +1,11 @@ # Design Spec: Request filters and device detection run on every adapter -**Status:** Proposed, 2026-10-17. A pull request above `ts config migrate`, not yet raised, for Tech Lab review. +**Status:** Proposed, 2026-10-17. Raised as pull request [IABTechLab/trusted-server#1410](https://github.com/IABTechLab/trusted-server/pull/1410), row 52 of the stack, above `ts config migrate` ([IABTechLab/trusted-server#1409](https://github.com/IABTechLab/trusted-server/pull/1409)), for Tech Lab review. **Author:** 51Degrees (contributed), for Tech Lab review **Related specs:** `2026-10-14-51degrees-client-hints-design.md` and `2026-10-15-geo-module-request-design.md`, whose statements about the adapters this change amends, `2026-06-11-datadome-server-side-protection-design.md` and `2026-08-03-datadome-ip-excluded-client-tag-design.md`, which describe DataDome's protection filter on the Fastly adapter alone, a scope this change lifts (section 1), and the design documents of the stack in IABTechLab/trusted-server#1342 -**Related PRs:** none yet +**Related PRs:** [IABTechLab/trusted-server#1410](https://github.com/IABTechLab/trusted-server/pull/1410) **Issues:** none -**Last updated:** 2026-10-18 +**Last updated:** 2026-10-26 > **Why this spec exists.** Only the Fastly adapter runs the integration request filters and classifies each request with the device module `[device]` selects. On Axum, Cloudflare and Spin a module's request filter never runs, so the 51Degrees client hints middleware writes nothing and DataDome's protection never asks its service, and a device module is asked only for a bid request's device attributes, so no module call carries a device classification. This change runs both steps on those three adapters where the Fastly adapter runs them. @@ -38,7 +38,7 @@ Each adapter carries its own copy of the steps. `derive_device_signals`, `run_pr | Response effects | `attach_request_filter_effects` | Leaves the filters' response headers on the response for the finalize middleware | | Finalize | `FinalizeResponseMiddleware` | Finalizes the response, then `apply_terminal_response_effects` writes the filters' response headers and restores the privacy guards, as the Fastly entry point does | -**Which routes run the cycle.** The routes the Fastly adapter runs it for, being every named route but the inspection endpoints and batch sync, and the publisher fallback, being robots.txt, tsjs, the module routes and the publisher's origin. These three adapters register no batch sync route, so a request for its path reaches the publisher fallback, as before. The inspection endpoints are read only and stay ahead of the Edge Cookie context and the filters, as on Fastly. Every adapter also answers the attestation pages outside the cycle, for `GET` and `HEAD` and for every method under a prefix the endpoint owns, as on Fastly, and Axum and Spin answer `/health` outside it. +**Which routes run the cycle.** The routes the Fastly adapter runs it for, being every named route but the inspection endpoints and batch sync, and the publisher fallback, being robots.txt, tsjs, the module routes and the publisher's origin. These three adapters register no batch sync route, so a request for its path reaches the publisher fallback, as before. The inspection endpoints are read only and stay ahead of the Edge Cookie context and the filters, as on Fastly, and Axum and Spin answer `/health` outside the cycle. | Adapter | Named routes in the cycle | Inspection endpoints | Registration | | ---------- | ----------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------- | @@ -51,7 +51,7 @@ On Spin the discovery, signature verification and first-party routes and the pag **Device classification.** `derive_device_signals` follows the Fastly adapter's function of the same name. The evidence a module is shown holds the client's IP address and the `User-Agent`, with the `Cookie` header as well for a device module a registration supplies, in a module context that also carries the request's method, scheme, host, path and query, the settings and the services, as on Fastly. Four things differ from the Fastly adapter. 1. It runs where the adapter builds the Edge Cookie context, after the preparers, where the Fastly adapter runs it at the entry point before routing. A preparer that removes a reserved cookie therefore removes it before a device module sees it. -2. It runs only for the routes in the cycle, so a request for robots.txt, an inspection endpoint or an attestation page is not classified, where the Fastly entry point classifies every request. On Fastly a device module a registration supplies is therefore shown the cookies a crawler sends with a request for robots.txt, and on these adapters it is not. +2. It runs only for the routes in the cycle, so a request for robots.txt or an inspection endpoint is not classified, where the Fastly entry point classifies every request. On Fastly a device module a registration supplies is therefore shown the cookies a crawler sends with a request for robots.txt, and on these adapters it is not. 3. The registered module is the one the request's services carry (`RuntimeServices::device_module`), which `services_for_request` sets from the registry, and the module is handed those same services, so it reads the platform's key-value store when the services carry one. The Fastly entry point builds services of its own for the call. 4. These adapters supply no TLS or HTTP/2 signals, so `[device] module = "platform"` builds the built-in module, which classifies from the `User-Agent`, because `platform` adds the host's signals only where the platform supplies them. @@ -87,7 +87,7 @@ As on Fastly, a module route's call and a serve phase preparation carry the clas - `an_edge_cookie_context_that_cannot_be_read_answers_before_the_filters`: with a filter that answers every request, a page and the discovery route, each of which a module's preparer leaves with an unreadable `Cookie` header, are answered `400`, with no filter's effect and no request to the origin, and a request a client sends with an unreadable `Cookie` header of its own reaches the filter instead. - `robots_txt_is_answered_before_the_request_filters`: with a filter that answers every request, robots.txt is still the file, and the filter's header is absent. - `robots_txt_shows_the_device_module_none_of_the_crawler_s_cookies`: the module `[device]` selects, which records the cookies it is shown, never sees the cookies sent with robots.txt, and does see those sent with a module route's request. -- `the_inspection_endpoints_and_the_attestation_pages_skip_the_request_filters`: with a filter that answers every request, the five inspection paths and the two attestation pages are answered by their own handlers with no filter's effect, and a page is the filter's answer. +- `the_inspection_endpoints_skip_the_request_filters`: with a filter that answers every request, the five inspection paths are answered by their own handlers with no filter's effect, and a page is the filter's answer. - `a_filter_cannot_make_a_response_core_made_private_public`: a module route's answer core marks private keeps `no-store, private` when a filter's effect sets `Cache-Control: public, max-age=600`. - `a_filter_s_cookie_keeps_the_response_from_shared_caches`: a public answer that a filter's effect adds a `Set-Cookie` to is sent `private, max-age=0`. - `a_filter_cannot_add_a_shared_cache_header_to_a_private_response`: a `Surrogate-Control` that a filter's effect adds to a `private` answer is removed. @@ -122,8 +122,12 @@ The thirteen tests pass on each of the three adapters, taking out any of the thr | ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | 2026-10-17 | First draft. | | 2026-10-18 | Fixes from review round 1. What a selected device module did on these adapters before, what DataDome's filter sends and sets, how each filter's call is bounded and why no time limit is added, the inspection endpoints the Fastly adapter answers before the preparers, the routes kept out of the cycle, which of a crawler's cookies robots.txt reads, the `Cookie` check and when it is reached, the configuration, eight new tests on each adapter with the three privacy guards among them, the open questions and the review log. Fixes from review round 2. A test on each adapter that sends every named route in the cycle through the filters, the discovery route in the context test, the tests each removal fails, the IP address and the request's address a device module is shown, the subresources it classifies, what the diagnostics preparer drops, the two DataDome designs whose scope this change lifts, what the guards do with a platform's own cache header with an open question on it, and the reason no shipped filter is tested through an adapter. | +| 2026-10-26 | Row 12 adds no route to the adapters any more, so section 2 drops row 12's pages from the routes kept out of the cycle, from the requests a device module does not classify and from the inspection test, which is renamed `the_inspection_endpoints_skip_the_request_filters`. | +| 2026-10-26 | The header names the pull request, IABTechLab/trusted-server#1410, row 52 of the stack, in place of saying that none is raised. | ## Review log 1. **Round 1, 2026-10-18.** A review in a fresh session, against `split/52-request-filters` at `f0edcd72b`. Issues found, five of them blocking. The five were that the spec said a selected device module did nothing on Axum, Cloudflare and Spin, when core asks it for a bid request's device attributes in every auction there, that nothing said what DataDome's filter sends DataDome or sets, or that no permission gates it, that the client hints design still said only the Fastly adapter runs request filters, that no test proved the privacy guards written after a filter's headers, and that the spec said the Fastly adapter answers two of the inspection endpoints before the preparers, when it answers all three, and that the adapters' HTTP clients cannot bound a call, when the Axum and Cloudflare clients can. Fixed in `7aea7a72b`, which adds the four privacy tests and four more on each adapter, `4549047dc`, which corrects the adapters' comments on robots.txt, `a584b071f`, which corrects the guides, the changelog and the middleware readme, `6b4eb0b8c`, which amends the client hints design, and this revision, which states each from the code and gives the reason a time limit is left to a change of its own. One finding was not borne out. The review said a `Cookie` header holding an accented letter now answers `400` on the routes that newly read the context, but a client's header is rebuilt by the `ad-tag.google.diagnostics` preparer first, which the context test shows, so Rollout names no new `400`. 2. **Round 2, 2026-10-18.** A review in a fresh session, against `split/52-request-filters` at `17136d04b`, of the round 1 fixes and the new text. Issues found, one of them blocking. The one was that no test sent a named route through the request cycle, so taking `build_ec_context` or `run_pre_route_filters` out of any adapter's `execute_named` failed no test, although on Cloudflare and Spin this change replaced each named route's own handler with `execute_named`. Fixed in `7ae3235ad`, which adds a test on each adapter that sends every named route in the cycle to a filter that answers and has the context test's preparer spoil the discovery route as well, so that taking either step, or on Spin the preparers, out of `execute_named` fails a test, `8c9b914c7`, which corrects the `build_ec_context` comments on the `Cookie` check, `233299a06`, which names in the integration guide the IP address and the request's address a device module is shown, `52cfd9fe9`, which amends the geo module request design's table of finalize steps and its line on `detect`, `e7417c898` and `ac0492932`, which fix the client hints design's wait on Axum and its test item 6 and log its round, and this revision, which names the tests each removal fails, the IP address and the request's address, the subresources a device module classifies, what the diagnostics preparer drops, the two DataDome designs whose scope this change lifts and what the guards do with a platform's own cache header, on Fastly as on Cloudflare, with an open question on it, and gives a true reason no shipped filter is tested through an adapter. +3. **Round 3.** A review in a fresh session, against `split/52-request-filters`, of the changes that follow attestation leaving row 12. Issues found, none blocking. The round found the renamed inspection test, `the_inspection_endpoints_skip_the_request_filters`, right, and found that the header still said the pull request was not yet raised and named no related pull request, although IABTechLab/trusted-server#1410 is open. It also found, as part of a blocking point on the module endpoints design, that the guide section that design plans would go out of date as this row changes the steps before a module route, which that design settles by having the section state only what holds at every row, so this design needs no note about that section. This revision names IABTechLab/trusted-server#1410, row 52 of the stack, in the header. +4. **Round 4.** A review in a fresh session, checking the round 3 fixes. No issues found. The round found the header right, because IABTechLab/trusted-server#1410 is open and not a draft, and found nothing else to change in this design. From b69ae6fde5c0deef94a0c7bf71c0d3081143f161 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 28 Oct 2026 11:30:33 +0000 Subject: [PATCH 17/18] The request cycle tests' settings carry the sample policy --- .../trusted-server-adapter-axum/src/app/request_cycle_tests.rs | 3 +++ .../src/app/request_cycle_tests.rs | 3 +++ .../trusted-server-adapter-spin/src/app/request_cycle_tests.rs | 3 +++ 3 files changed, 9 insertions(+) diff --git a/crates/trusted-server-adapter-axum/src/app/request_cycle_tests.rs b/crates/trusted-server-adapter-axum/src/app/request_cycle_tests.rs index 6156108174..b09080baf3 100644 --- a/crates/trusted-server-adapter-axum/src/app/request_cycle_tests.rs +++ b/crates/trusted-server-adapter-axum/src/app/request_cycle_tests.rs @@ -386,6 +386,9 @@ fn settings_with(extra: &str) -> Settings { "# )) .expect("should parse the request cycle test settings") + // A loaded domain carries its policy, and these tests decide under the + // repository's sample. + .with_permission_policy(trusted_server_core::permissions::PermissionPolicy::sample()) } /// The router built through the composition root with `builders`, every diff --git a/crates/trusted-server-adapter-cloudflare/src/app/request_cycle_tests.rs b/crates/trusted-server-adapter-cloudflare/src/app/request_cycle_tests.rs index 150d0297c0..7eb8a9c850 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app/request_cycle_tests.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app/request_cycle_tests.rs @@ -386,6 +386,9 @@ fn settings_with(extra: &str) -> Settings { "# )) .expect("should parse the request cycle test settings") + // A loaded domain carries its policy, and these tests decide under the + // repository's sample. + .with_permission_policy(trusted_server_core::permissions::PermissionPolicy::sample()) } /// The router built through the composition root with `builders`, every diff --git a/crates/trusted-server-adapter-spin/src/app/request_cycle_tests.rs b/crates/trusted-server-adapter-spin/src/app/request_cycle_tests.rs index 150d0297c0..7eb8a9c850 100644 --- a/crates/trusted-server-adapter-spin/src/app/request_cycle_tests.rs +++ b/crates/trusted-server-adapter-spin/src/app/request_cycle_tests.rs @@ -386,6 +386,9 @@ fn settings_with(extra: &str) -> Settings { "# )) .expect("should parse the request cycle test settings") + // A loaded domain carries its policy, and these tests decide under the + // repository's sample. + .with_permission_policy(trusted_server_core::permissions::PermissionPolicy::sample()) } /// The router built through the composition root with `builders`, every From a47542644cb6552b16b85a82da799cec7654ed4b Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 28 Oct 2026 11:33:14 +0000 Subject: [PATCH 18/18] Row 52's design copy carries the date it is proposed, 2026-10-27 --- .../2026-10-17-request-filters-on-every-adapter-design.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/superpowers/specs/2026-10-17-request-filters-on-every-adapter-design.md b/docs/superpowers/specs/2026-10-17-request-filters-on-every-adapter-design.md index 7973cfac6d..3bd9f396e5 100644 --- a/docs/superpowers/specs/2026-10-17-request-filters-on-every-adapter-design.md +++ b/docs/superpowers/specs/2026-10-17-request-filters-on-every-adapter-design.md @@ -5,7 +5,7 @@ **Related specs:** `2026-10-14-51degrees-client-hints-design.md` and `2026-10-15-geo-module-request-design.md`, whose statements about the adapters this change amends, `2026-06-11-datadome-server-side-protection-design.md` and `2026-08-03-datadome-ip-excluded-client-tag-design.md`, which describe DataDome's protection filter on the Fastly adapter alone, a scope this change lifts (section 1), and the design documents of the stack in IABTechLab/trusted-server#1342 **Related PRs:** [IABTechLab/trusted-server#1410](https://github.com/IABTechLab/trusted-server/pull/1410) **Issues:** none -**Last updated:** 2026-10-26 +**Last updated:** 2026-10-27 > **Why this spec exists.** Only the Fastly adapter runs the integration request filters and classifies each request with the device module `[device]` selects. On Axum, Cloudflare and Spin a module's request filter never runs, so the 51Degrees client hints middleware writes nothing and DataDome's protection never asks its service, and a device module is asked only for a bid request's device attributes, so no module call carries a device classification. This change runs both steps on those three adapters where the Fastly adapter runs them. @@ -122,8 +122,8 @@ The thirteen tests pass on each of the three adapters, taking out any of the thr | ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | 2026-10-17 | First draft. | | 2026-10-18 | Fixes from review round 1. What a selected device module did on these adapters before, what DataDome's filter sends and sets, how each filter's call is bounded and why no time limit is added, the inspection endpoints the Fastly adapter answers before the preparers, the routes kept out of the cycle, which of a crawler's cookies robots.txt reads, the `Cookie` check and when it is reached, the configuration, eight new tests on each adapter with the three privacy guards among them, the open questions and the review log. Fixes from review round 2. A test on each adapter that sends every named route in the cycle through the filters, the discovery route in the context test, the tests each removal fails, the IP address and the request's address a device module is shown, the subresources it classifies, what the diagnostics preparer drops, the two DataDome designs whose scope this change lifts, what the guards do with a platform's own cache header with an open question on it, and the reason no shipped filter is tested through an adapter. | -| 2026-10-26 | Row 12 adds no route to the adapters any more, so section 2 drops row 12's pages from the routes kept out of the cycle, from the requests a device module does not classify and from the inspection test, which is renamed `the_inspection_endpoints_skip_the_request_filters`. | -| 2026-10-26 | The header names the pull request, IABTechLab/trusted-server#1410, row 52 of the stack, in place of saying that none is raised. | +| 2026-10-27 | Row 12 adds no route to the adapters any more, so section 2 drops row 12's pages from the routes kept out of the cycle, from the requests a device module does not classify and from the inspection test, which is renamed `the_inspection_endpoints_skip_the_request_filters`. | +| 2026-10-27 | The header names the pull request, IABTechLab/trusted-server#1410, row 52 of the stack, in place of saying that none is raised. | ## Review log