From 85276bdec79072bd667b652ed47012d673660601 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 28 Aug 2026 11:14:11 +0100 Subject: [PATCH 001/124] Open the integration registry to externally supplied builders --- crates/trusted-server-core/src/config.rs | 6 +- .../src/integrations/aps.rs | 13 + .../src/integrations/datadome.rs | 17 ++ .../src/integrations/didomi.rs | 13 + .../src/integrations/google_tag_manager.rs | 13 + .../src/integrations/gpt.rs | 13 + .../src/integrations/gpt_diagnostics.rs | 13 + .../src/integrations/lockr.rs | 13 + .../src/integrations/mod.rs | 245 +++++++++++++----- .../src/integrations/nextjs/mod.rs | 13 + .../src/integrations/osano.rs | 13 + .../src/integrations/permutive.rs | 13 + .../src/integrations/prebid.rs | 11 + .../src/integrations/registry.rs | 141 +++++++++- .../src/integrations/sourcepoint.rs | 13 + .../src/integrations/testlight.rs | 13 + 16 files changed, 498 insertions(+), 65 deletions(-) diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index 2c9624b027..1843efc388 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -697,7 +697,11 @@ password = "production-admin-password-32-bytes" fn deploy_validation_covers_registered_integration_builders() { let validated_ids: HashSet<&'static str> = DEPLOY_VALIDATED_INTEGRATION_IDS.iter().copied().collect(); - let missing_ids = crate::integrations::registered_builder_ids() + let registry = crate::integrations::IntegrationRegistry::new(&valid_settings()) + .expect("should build registry from valid settings"); + let missing_ids = registry + .registered_builder_ids() + .into_iter() .filter(|id| !validated_ids.contains(id)) .collect::>(); diff --git a/crates/trusted-server-core/src/integrations/aps.rs b/crates/trusted-server-core/src/integrations/aps.rs index 581e5c2001..a7df12e106 100644 --- a/crates/trusted-server-core/src/integrations/aps.rs +++ b/crates/trusted-server-core/src/integrations/aps.rs @@ -1264,6 +1264,19 @@ impl IntegrationHeadInjector for ApsRendererIntegration { } } +/// Validates the APS configuration for deployment and reports whether +/// the integration is enabled. +/// +/// # Errors +/// +/// Returns an error when the APS configuration cannot be parsed or fails +/// validation. +pub(crate) fn validate(settings: &Settings) -> Result> { + settings + .integration_config::(APS_INTEGRATION_ID) + .map(|config| config.is_some()) +} + /// Register the APS static renderer endpoint when APS is enabled. /// /// # Errors diff --git a/crates/trusted-server-core/src/integrations/datadome.rs b/crates/trusted-server-core/src/integrations/datadome.rs index d95ee35ee1..4f00917728 100644 --- a/crates/trusted-server-core/src/integrations/datadome.rs +++ b/crates/trusted-server-core/src/integrations/datadome.rs @@ -973,6 +973,23 @@ fn build( Ok(Some(integration)) } +/// Validates the `DataDome` configuration for deployment and reports whether +/// the integration is enabled. +/// +/// # Errors +/// +/// Returns an error when the `DataDome` configuration cannot be parsed, fails +/// validation, or fails the startup checks on protection, bypass, or +/// client-tag settings. +pub(crate) fn validate(settings: &Settings) -> Result> { + let Some(config) = settings.integration_config::(DATADOME_INTEGRATION_ID)? + else { + return Ok(false); + }; + DataDomeIntegration::validate_config_for_startup(config)?; + Ok(true) +} + /// Register the `DataDome` integration with Trusted Server. /// /// # Errors diff --git a/crates/trusted-server-core/src/integrations/didomi.rs b/crates/trusted-server-core/src/integrations/didomi.rs index f8472b7962..802b59e7e6 100644 --- a/crates/trusted-server-core/src/integrations/didomi.rs +++ b/crates/trusted-server-core/src/integrations/didomi.rs @@ -240,6 +240,19 @@ fn build( Ok(Some(DidomiIntegration::new(Arc::new(config)))) } +/// Validates the Didomi configuration for deployment and reports whether +/// the integration is enabled. +/// +/// # Errors +/// +/// Returns an error when the Didomi configuration cannot be parsed or fails +/// validation. +pub(crate) fn validate(settings: &Settings) -> Result> { + settings + .integration_config::(DIDOMI_INTEGRATION_ID) + .map(|config| config.is_some()) +} + /// Register the Didomi consent notice integration when enabled. /// /// # Errors diff --git a/crates/trusted-server-core/src/integrations/google_tag_manager.rs b/crates/trusted-server-core/src/integrations/google_tag_manager.rs index 0980a76c91..85d382501c 100644 --- a/crates/trusted-server-core/src/integrations/google_tag_manager.rs +++ b/crates/trusted-server-core/src/integrations/google_tag_manager.rs @@ -400,6 +400,19 @@ fn build( Ok(Some(GoogleTagManagerIntegration::new(config))) } +/// Validates the Google Tag Manager configuration for deployment and reports whether +/// the integration is enabled. +/// +/// # Errors +/// +/// Returns an error when the Google Tag Manager configuration cannot be parsed or fails +/// validation. +pub(crate) fn validate(settings: &Settings) -> Result> { + settings + .integration_config::(GTM_INTEGRATION_ID) + .map(|config| config.is_some()) +} + /// Register the Google Tag Manager integration when enabled. /// /// # Errors diff --git a/crates/trusted-server-core/src/integrations/gpt.rs b/crates/trusted-server-core/src/integrations/gpt.rs index 84158c27ec..aaa10f1713 100644 --- a/crates/trusted-server-core/src/integrations/gpt.rs +++ b/crates/trusted-server-core/src/integrations/gpt.rs @@ -377,6 +377,19 @@ fn build(settings: &Settings) -> Result>, Report Result> { + settings + .integration_config::(GPT_INTEGRATION_ID) + .map(|config| config.is_some()) +} + /// Register the GPT integration. /// /// # Errors diff --git a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs b/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs index b4a188f2ab..413e5ee62a 100644 --- a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs +++ b/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs @@ -209,6 +209,19 @@ struct ConsoleCookieState { canonical: bool, } +/// Validates the GPT diagnostics configuration for deployment and reports whether +/// the integration is enabled. +/// +/// # Errors +/// +/// Returns an error when the GPT diagnostics configuration cannot be parsed or fails +/// validation. +pub(crate) fn validate(settings: &Settings) -> Result> { + settings + .integration_config::(GPT_DIAGNOSTICS_INTEGRATION_ID) + .map(|config| config.is_some()) +} + /// Register GPT diagnostics when explicitly enabled. /// /// # Errors diff --git a/crates/trusted-server-core/src/integrations/lockr.rs b/crates/trusted-server-core/src/integrations/lockr.rs index 1f4f04b738..337dc53600 100644 --- a/crates/trusted-server-core/src/integrations/lockr.rs +++ b/crates/trusted-server-core/src/integrations/lockr.rs @@ -312,6 +312,19 @@ fn build(settings: &Settings) -> Result>, Report Result> { + settings + .integration_config::(LOCKR_INTEGRATION_ID) + .map(|config| config.is_some()) +} + /// Register the Lockr integration. /// /// # Errors diff --git a/crates/trusted-server-core/src/integrations/mod.rs b/crates/trusted-server-core/src/integrations/mod.rs index 90d688693f..c1e10c2e9e 100644 --- a/crates/trusted-server-core/src/integrations/mod.rs +++ b/crates/trusted-server-core/src/integrations/mod.rs @@ -5,6 +5,7 @@ use std::time::Duration; use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt}; use futures::StreamExt as _; +use http::Request; use url::Url; use crate::error::TrustedServerError; @@ -279,72 +280,196 @@ pub(crate) async fn collect_response_bounded( } } -type IntegrationBuilderFn = +/// Builds an integration's registration from settings, or `None` when the +/// integration is not enabled. +pub type IntegrationBuilderFn = fn(&Settings) -> Result, Report>; -pub(crate) struct IntegrationBuilder { +/// Validates an integration's configuration for deployment and reports +/// whether the integration is enabled. +/// +/// Runs for every builder, enabled or not, so a typo in a disabled block is +/// still caught. +pub type IntegrationValidateFn = fn(&Settings) -> Result>; + +/// Prepares a request before routing, for every request the adapter sees. +/// +/// Runs whether or not the integration is enabled, so an integration can +/// strip its reserved query or cookie even when it is switched off. +pub type IntegrationPrepareRequestFn = + fn(&Settings, &mut Request) -> Result<(), Report>; + +/// Source label for the built-in integrations. +pub const CORE_SOURCE: &str = "trusted-server-core"; + +/// A named factory for one integration, the unit an adapter or a vendor crate +/// hands to [`IntegrationRegistry::with_registrations`]. +/// +/// # Examples +/// +/// ``` +/// use error_stack::Report; +/// use trusted_server_core::error::TrustedServerError; +/// use trusted_server_core::integrations::{ +/// IntegrationBuilder, IntegrationRegistration, IntegrationRegistry, +/// }; +/// use trusted_server_core::settings::Settings; +/// +/// fn build( +/// _settings: &Settings, +/// ) -> Result, Report> { +/// Ok(Some(IntegrationRegistration::builder("example").build())) +/// } +/// +/// fn validate(_settings: &Settings) -> Result> { +/// Ok(true) +/// } +/// +/// # fn demo(settings: &Settings) -> Result<(), Report> { +/// let builder = IntegrationBuilder::new("example", "example-crate", build, validate); +/// let registry = IntegrationRegistry::with_registrations(settings, &[builder])?; +/// assert!(registry.integration_enabled("example")); +/// # Ok(()) +/// # } +/// ``` +#[derive(Clone, Copy)] +pub struct IntegrationBuilder { id: &'static str, + source: &'static str, build: IntegrationBuilderFn, + validate: IntegrationValidateFn, + prepare_request: Option, } -pub(crate) fn builders() -> &'static [IntegrationBuilder] { - &[ - IntegrationBuilder { - id: "aps", - build: aps::register, - }, - IntegrationBuilder { - id: "prebid", - build: prebid::register, - }, - IntegrationBuilder { - id: "testlight", - build: testlight::register, - }, - IntegrationBuilder { - id: "nextjs", - build: nextjs::register, - }, - IntegrationBuilder { - id: "permutive", - build: permutive::register, - }, - IntegrationBuilder { - id: "lockr", - build: lockr::register, - }, - IntegrationBuilder { - id: "didomi", - build: didomi::register, - }, - IntegrationBuilder { - id: "sourcepoint", - build: sourcepoint::register, - }, - IntegrationBuilder { - id: "osano", - build: osano::register, - }, - IntegrationBuilder { - id: "google_tag_manager", - build: google_tag_manager::register, - }, - IntegrationBuilder { - id: "datadome", - build: datadome::register, - }, - IntegrationBuilder { - id: "gpt", - build: gpt::register, - }, - IntegrationBuilder { - id: "gpt_diagnostics", - build: gpt_diagnostics::register, - }, - ] +impl IntegrationBuilder { + /// Creates a builder for the integration `id`, attributed to `source` + /// (a crate or package name used in duplicate-id errors). + #[must_use] + pub const fn new( + id: &'static str, + source: &'static str, + build: IntegrationBuilderFn, + validate: IntegrationValidateFn, + ) -> Self { + Self { + id, + source, + build, + validate, + prepare_request: None, + } + } + + /// Attaches a request preparation function that runs before routing on + /// every request, enabled or not. + #[must_use] + pub const fn with_request_preparer(mut self, prepare: IntegrationPrepareRequestFn) -> Self { + self.prepare_request = Some(prepare); + self + } + + /// The integration id this builder produces. + #[must_use] + pub const fn id(&self) -> &'static str { + self.id + } + + /// The source label used in diagnostics. + #[must_use] + pub const fn source(&self) -> &'static str { + self.source + } + + /// Builds the registration, or `None` when the integration is not enabled. + /// + /// # Errors + /// + /// Returns an error when the integration is enabled with invalid + /// configuration. + pub(crate) fn build( + &self, + settings: &Settings, + ) -> Result, Report> { + (self.build)(settings) + } + + /// Validates the integration's configuration for deployment and reports + /// whether the integration is enabled. + /// + /// # Errors + /// + /// Returns an error when the configuration cannot be parsed or fails + /// validation. + // Deploy validation does not yet enumerate builders, so nothing in the + // crate calls this until that is wired up. + #[allow(dead_code)] + pub(crate) fn validate(&self, settings: &Settings) -> Result> { + (self.validate)(settings) + } + + /// The request preparation function, when one is attached. + // The adapters do not yet run request preparers, so nothing in the crate + // calls this until that is wired up. + #[allow(dead_code)] + pub(crate) fn prepare_request(&self) -> Option { + self.prepare_request + } } -#[cfg(test)] -pub(crate) fn registered_builder_ids() -> impl Iterator { - builders().iter().map(|builder| builder.id) +/// The built-in integrations, in hook order. +const BUILT_IN_BUILDERS: &[IntegrationBuilder] = &[ + IntegrationBuilder::new("aps", CORE_SOURCE, aps::register, aps::validate), + IntegrationBuilder::new("prebid", CORE_SOURCE, prebid::register, prebid::validate), + IntegrationBuilder::new( + "testlight", + CORE_SOURCE, + testlight::register, + testlight::validate, + ), + IntegrationBuilder::new("nextjs", CORE_SOURCE, nextjs::register, nextjs::validate), + IntegrationBuilder::new( + "permutive", + CORE_SOURCE, + permutive::register, + permutive::validate, + ), + IntegrationBuilder::new("lockr", CORE_SOURCE, lockr::register, lockr::validate), + IntegrationBuilder::new("didomi", CORE_SOURCE, didomi::register, didomi::validate), + IntegrationBuilder::new( + "sourcepoint", + CORE_SOURCE, + sourcepoint::register, + sourcepoint::validate, + ), + IntegrationBuilder::new("osano", CORE_SOURCE, osano::register, osano::validate), + IntegrationBuilder::new( + "google_tag_manager", + CORE_SOURCE, + google_tag_manager::register, + google_tag_manager::validate, + ), + IntegrationBuilder::new( + "datadome", + CORE_SOURCE, + datadome::register, + datadome::validate, + ), + IntegrationBuilder::new("gpt", CORE_SOURCE, gpt::register, gpt::validate), + IntegrationBuilder::new( + "gpt_diagnostics", + CORE_SOURCE, + gpt_diagnostics::register, + gpt_diagnostics::validate, + ), +]; + +/// Every builder the registry will consider: the built-in set followed by +/// `extra`, in that order, so hook order for the built-ins never changes. +pub(crate) fn all_builders( + extra: &[IntegrationBuilder], +) -> impl Iterator + '_ { + BUILT_IN_BUILDERS + .iter() + .copied() + .chain(extra.iter().copied()) } diff --git a/crates/trusted-server-core/src/integrations/nextjs/mod.rs b/crates/trusted-server-core/src/integrations/nextjs/mod.rs index 5452260e70..b048b3b312 100644 --- a/crates/trusted-server-core/src/integrations/nextjs/mod.rs +++ b/crates/trusted-server-core/src/integrations/nextjs/mod.rs @@ -70,6 +70,19 @@ pub(super) fn configuration_error(message: impl Into) -> Report Result> { + settings + .integration_config::(NEXTJS_INTEGRATION_ID) + .map(|config| config.is_some()) +} + /// Register the Next.js integration when enabled. /// /// # Errors diff --git a/crates/trusted-server-core/src/integrations/osano.rs b/crates/trusted-server-core/src/integrations/osano.rs index 9fc3aa66c1..da96779758 100644 --- a/crates/trusted-server-core/src/integrations/osano.rs +++ b/crates/trusted-server-core/src/integrations/osano.rs @@ -31,6 +31,19 @@ impl IntegrationConfig for OsanoConfig { } } +/// Validates the Osano configuration for deployment and reports whether +/// the integration is enabled. +/// +/// # Errors +/// +/// Returns an error when the Osano configuration cannot be parsed or fails +/// validation. +pub(crate) fn validate(settings: &Settings) -> Result> { + settings + .integration_config::(OSANO_INTEGRATION_ID) + .map(|config| config.is_some()) +} + /// Register the Osano JS integration when enabled. /// /// # Errors diff --git a/crates/trusted-server-core/src/integrations/permutive.rs b/crates/trusted-server-core/src/integrations/permutive.rs index aa684c620c..d0e14cadaa 100644 --- a/crates/trusted-server-core/src/integrations/permutive.rs +++ b/crates/trusted-server-core/src/integrations/permutive.rs @@ -303,6 +303,19 @@ fn build( Ok(Some(PermutiveIntegration::new(config))) } +/// Validates the Permutive configuration for deployment and reports whether +/// the integration is enabled. +/// +/// # Errors +/// +/// Returns an error when the Permutive configuration cannot be parsed or fails +/// validation. +pub(crate) fn validate(settings: &Settings) -> Result> { + settings + .integration_config::(PERMUTIVE_INTEGRATION_ID) + .map(|config| config.is_some()) +} + /// Register the Permutive integration. /// /// # Errors diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index e60cbb973a..4843d213f2 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -977,6 +977,17 @@ fn build( Ok(Some(PrebidIntegration::try_new(config)?)) } +/// Validates the Prebid configuration for deployment and reports whether +/// the integration is enabled. +/// +/// # Errors +/// +/// Returns an error when the Prebid configuration cannot be parsed or fails +/// validation. +pub(crate) fn validate(settings: &Settings) -> Result> { + validate_config_for_startup(settings).map(|config| config.is_some()) +} + /// Register the Prebid integration when enabled. /// /// # Errors diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index 70005b210b..0e1de64ee5 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -710,6 +710,8 @@ struct IntegrationRegistryInner { // Metadata for introspection routes: Vec<(IntegrationEndpoint, &'static str)>, + // Every builder considered at construction, enabled or not, in order. + builder_ids: Vec<(&'static str, &'static str)>, enabled_integration_ids: Vec<&'static str>, deferred_js_ids: Vec<&'static str>, disabled_js_ids: Vec<&'static str>, @@ -735,6 +737,7 @@ impl Default for IntegrationRegistryInner { head_router: Router::new(), options_router: Router::new(), routes: Vec::new(), + builder_ids: Vec::new(), enabled_integration_ids: Vec::new(), deferred_js_ids: Vec::new(), disabled_js_ids: Vec::new(), @@ -794,7 +797,7 @@ pub struct IntegrationRegistry { } impl IntegrationRegistry { - /// Build a registry from the provided settings. + /// Build a registry from the built-in integrations. /// /// # Errors /// @@ -804,12 +807,45 @@ impl IntegrationRegistry { /// /// Panics if a route path ends with `/*` but `strip_suffix` unexpectedly fails (invariant violation). pub fn new(settings: &Settings) -> Result> { + Self::with_registrations(settings, &[]) + } + + /// Build a registry from the built-in integrations followed by `extra`, + /// the builders an adapter or a vendor crate supplies. + /// + /// # Errors + /// + /// Returns an error when two builders claim the same integration id, when + /// route registration fails due to duplicate routes or invalid paths, or + /// when a builder fails. + /// + /// # Panics + /// + /// Panics if a route path ends with `/*` but `strip_suffix` unexpectedly fails (invariant violation). + pub fn with_registrations( + settings: &Settings, + extra: &[crate::integrations::IntegrationBuilder], + ) -> Result> { let mut inner = IntegrationRegistryInner::default(); - for builder in crate::integrations::builders() { - if let Some(registration) = (builder.build)(settings)? { + for builder in crate::integrations::all_builders(extra) { + if let Some((_, first_source)) = + inner.builder_ids.iter().find(|(id, _)| *id == builder.id()) + { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "integration id `{}` is registered twice, by `{first_source}` and by `{}`", + builder.id(), + builder.source() + ), + })); + } + inner.builder_ids.push((builder.id(), builder.source())); + + if let Some(registration) = builder.build(settings)? { debug_assert_eq!( - registration.integration_id, builder.id, + registration.integration_id, + builder.id(), "integration builder ID should match registration ID" ); inner @@ -897,6 +933,14 @@ impl IntegrationRegistry { }) } + /// Every integration id the registry was built from, enabled or not, in + /// registration order. The deploy validation and registry tests enumerate + /// this to check every builder was considered. + #[must_use] + pub fn registered_builder_ids(&self) -> Vec<&'static str> { + self.inner.builder_ids.iter().map(|(id, _)| *id).collect() + } + fn find_route(&self, method: &Method, path: &str) -> Option<&RouteValue> { let router = match *method { Method::GET => &self.inner.get_router, @@ -1246,6 +1290,7 @@ impl IntegrationRegistry { head_router: Router::new(), options_router: Router::new(), routes: Vec::new(), + builder_ids: Vec::new(), enabled_integration_ids: Vec::new(), html_rewriters: attribute_rewriters, script_rewriters, @@ -1276,6 +1321,7 @@ impl IntegrationRegistry { head_router: Router::new(), options_router: Router::new(), routes: Vec::new(), + builder_ids: Vec::new(), enabled_integration_ids: Vec::new(), html_rewriters: attribute_rewriters, script_rewriters, @@ -1302,6 +1348,7 @@ impl IntegrationRegistry { head_router: Router::new(), options_router: Router::new(), routes: Vec::new(), + builder_ids: Vec::new(), enabled_integration_ids: Vec::new(), html_rewriters: Vec::new(), script_rewriters: Vec::new(), @@ -1368,6 +1415,7 @@ impl IntegrationRegistry { head_router, options_router, routes: Vec::new(), + builder_ids: Vec::new(), enabled_integration_ids: Vec::new(), html_rewriters: Vec::new(), script_rewriters: Vec::new(), @@ -1382,8 +1430,32 @@ impl IntegrationRegistry { } } +#[cfg(test)] +pub(crate) mod test_support { + use error_stack::Report; + + use super::IntegrationRegistration; + use crate::error::TrustedServerError; + use crate::settings::Settings; + + /// Builds a registration for the `probe` integration on every call. + pub(crate) fn probe_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some(IntegrationRegistration::builder("probe").build())) + } + + /// Validates nothing and reports the integration as enabled. + pub(crate) fn validate_nothing( + _settings: &Settings, + ) -> Result> { + Ok(true) + } +} + #[cfg(test)] mod tests { + use super::test_support::{probe_registration, validate_nothing}; use super::*; use crate::constants::COOKIE_TS_EC; use crate::permissions::{Permission, PermissionSet, PermissionState}; @@ -2451,4 +2523,65 @@ mod tests { "should reconstruct full module list from immediate + deferred" ); } + + fn duplicate_lockr_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some(IntegrationRegistration::builder("lockr").build())) + } + + #[test] + fn with_registrations_adds_an_external_builder_after_the_built_ins() { + let settings = crate::test_support::tests::create_test_settings(); + let extra = [crate::integrations::IntegrationBuilder::new( + "probe", + "seam-probe", + probe_registration, + validate_nothing, + )]; + + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should build registry with an external builder"); + + assert!( + registry.integration_enabled("probe"), + "should register the external integration" + ); + let ids = registry.registered_builder_ids(); + assert_eq!( + ids.last().copied(), + Some("probe"), + "should order the external builder after every built-in" + ); + } + + #[test] + fn with_registrations_rejects_a_duplicate_integration_id_naming_both_sources() { + let mut settings = crate::test_support::tests::create_test_settings(); + settings + .integrations + .insert_config( + "lockr", + &serde_json::json!({ "enabled": true, "app_id": "test-app-id" }), + ) + .expect("should insert lockr config"); + let extra = [crate::integrations::IntegrationBuilder::new( + "lockr", + "seam-probe", + duplicate_lockr_registration, + validate_nothing, + )]; + + let error = IntegrationRegistry::with_registrations(&settings, &extra) + .err() + .expect("should reject a duplicate integration id"); + + let message = error.to_string(); + assert!( + message.contains("lockr") + && message.contains("trusted-server-core") + && message.contains("seam-probe"), + "error should name the id and both sources: {message}" + ); + } } diff --git a/crates/trusted-server-core/src/integrations/sourcepoint.rs b/crates/trusted-server-core/src/integrations/sourcepoint.rs index 3caaadeef6..ee5a5f0555 100644 --- a/crates/trusted-server-core/src/integrations/sourcepoint.rs +++ b/crates/trusted-server-core/src/integrations/sourcepoint.rs @@ -751,6 +751,19 @@ fn build( Ok(Some(SourcepointIntegration::new(Arc::new(config)))) } +/// Validates the Sourcepoint configuration for deployment and reports whether +/// the integration is enabled. +/// +/// # Errors +/// +/// Returns an error when the Sourcepoint configuration cannot be parsed or fails +/// validation. +pub(crate) fn validate(settings: &Settings) -> Result> { + settings + .integration_config::(SOURCEPOINT_INTEGRATION_ID) + .map(|config| config.is_some()) +} + /// Register the Sourcepoint integration when enabled. /// /// # Errors diff --git a/crates/trusted-server-core/src/integrations/testlight.rs b/crates/trusted-server-core/src/integrations/testlight.rs index 51c404a3d8..d6a619f78a 100644 --- a/crates/trusted-server-core/src/integrations/testlight.rs +++ b/crates/trusted-server-core/src/integrations/testlight.rs @@ -141,6 +141,19 @@ fn build( Ok(Some(TestlightIntegration::new(config))) } +/// Validates the Testlight configuration for deployment and reports whether +/// the integration is enabled. +/// +/// # Errors +/// +/// Returns an error when the Testlight configuration cannot be parsed or fails +/// validation. +pub(crate) fn validate(settings: &Settings) -> Result> { + settings + .integration_config::(TESTLIGHT_INTEGRATION_ID) + .map(|config| config.is_some()) +} + /// Register the Testlight integration when enabled. /// /// # Errors From b67518968db58e3577c23f5788408eccfd5daf15 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 28 Aug 2026 11:31:16 +0100 Subject: [PATCH 002/124] Keep builders() as the built-in set and derive Debug for the builder --- crates/trusted-server-core/src/integrations/mod.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/mod.rs b/crates/trusted-server-core/src/integrations/mod.rs index c1e10c2e9e..92ee5e765a 100644 --- a/crates/trusted-server-core/src/integrations/mod.rs +++ b/crates/trusted-server-core/src/integrations/mod.rs @@ -332,7 +332,7 @@ pub const CORE_SOURCE: &str = "trusted-server-core"; /// # Ok(()) /// # } /// ``` -#[derive(Clone, Copy)] +#[derive(Clone, Copy, Debug)] pub struct IntegrationBuilder { id: &'static str, source: &'static str, @@ -463,13 +463,15 @@ const BUILT_IN_BUILDERS: &[IntegrationBuilder] = &[ ), ]; +/// The built-in integration builders, in hook order. +pub(crate) fn builders() -> &'static [IntegrationBuilder] { + BUILT_IN_BUILDERS +} + /// Every builder the registry will consider: the built-in set followed by /// `extra`, in that order, so hook order for the built-ins never changes. pub(crate) fn all_builders( extra: &[IntegrationBuilder], ) -> impl Iterator + '_ { - BUILT_IN_BUILDERS - .iter() - .copied() - .chain(extra.iter().copied()) + builders().iter().copied().chain(extra.iter().copied()) } From 1ef46fdb724a5180fd71df6f0e2f482773dbe9de Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 28 Aug 2026 11:40:03 +0100 Subject: [PATCH 003/124] Compose the served tsjs script in core from content-addressed parts --- crates/trusted-server-core/src/lib.rs | 1 + crates/trusted-server-core/src/tsjs_bundle.rs | 365 ++++++++++++++++++ 2 files changed, 366 insertions(+) create mode 100644 crates/trusted-server-core/src/tsjs_bundle.rs diff --git a/crates/trusted-server-core/src/lib.rs b/crates/trusted-server-core/src/lib.rs index 31bc119aec..932cc9692d 100644 --- a/crates/trusted-server-core/src/lib.rs +++ b/crates/trusted-server-core/src/lib.rs @@ -74,6 +74,7 @@ pub mod streaming_replacer; pub mod test_support; pub mod tester_cookie; pub mod tsjs; +pub mod tsjs_bundle; #[cfg(test)] mod migration_guards; diff --git a/crates/trusted-server-core/src/tsjs_bundle.rs b/crates/trusted-server-core/src/tsjs_bundle.rs new file mode 100644 index 0000000000..d787efcb63 --- /dev/null +++ b/crates/trusted-server-core/src/tsjs_bundle.rs @@ -0,0 +1,365 @@ +//! Composition of the served tsjs script from module parts. +//! +//! `trusted-server-js` knows only the modules compiled into it. A module a +//! vendor crate carries on its registration is not in that map, so the +//! composition of the served script and its cache-busting hash live here, +//! keyed on content rather than on ids. The byte rule is unchanged from +//! `trusted_server_js::concatenate_modules`: core first, then each part in +//! order, joined by `;\n`, so every existing `?v=` hash is preserved. + +use std::collections::HashMap; +use std::sync::{Mutex, MutexGuard, OnceLock}; + +use sha2::{Digest as _, Sha256}; + +/// One module of the served script. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct JsModulePart { + /// Module id, for example `core` or `lockr`. + pub id: &'static str, + /// The built IIFE. + pub source: &'static str, + /// SHA-256 of `source`, hex encoded. Identifies the content in the memo. + pub sha256: &'static str, +} + +impl JsModulePart { + /// Looks up a compile-time module of `trusted-server-js` by id. + /// + /// Returns `None` when no module with that id was compiled in. + #[must_use] + pub fn compile_time(id: &str) -> Option { + let source = trusted_server_js::module_bundle(id)?; + let sha256 = trusted_server_js::single_module_hash(id)?; + let id = trusted_server_js::all_module_ids() + .into_iter() + .find(|candidate| *candidate == id)?; + Some(Self { id, source, sha256 }) + } +} + +/// Resolves compile-time parts for a list of ids, dropping unknown ids, as +/// `trusted_server_js` does today. +#[must_use] +pub fn compile_time_parts(ids: &[&str]) -> Vec { + ids.iter() + .filter_map(|id| JsModulePart::compile_time(id)) + .collect() +} + +/// Concatenates the parts into the served script. +/// +/// Core comes first (the given `core` part, or the compile-time core when +/// none is given), then the remaining parts in input order. Each id appears +/// once, keeping its first occurrence, and parts are joined by `;\n` so one +/// IIFE cannot run into the next. +/// +/// # Examples +/// +/// ``` +/// use trusted_server_core::tsjs_bundle::{JsModulePart, compose}; +/// +/// let parts = [ +/// JsModulePart { +/// id: "example", +/// source: "(() => { window.example = true; })()", +/// sha256: "e1", +/// }, +/// JsModulePart { +/// id: "core", +/// source: "(() => { window.tsjs = {}; })()", +/// sha256: "c1", +/// }, +/// ]; +/// +/// assert_eq!( +/// compose(&parts), +/// "(() => { window.tsjs = {}; })();\n(() => { window.example = true; })()" +/// ); +/// ``` +#[must_use] +pub fn compose(parts: &[JsModulePart]) -> String { + let ordered = ordered(parts); + let mut body = String::new(); + visit_parts(&ordered, |part| body.push_str(part)); + body +} + +/// SHA-256 of [`compose`]'s output, hex encoded, without materializing it. +/// +/// The result is memoized per ordered set of `(id, sha256)` pairs, so a +/// per-request caller hashes a given module set once per process or isolate. +/// Because the key carries each part's content hash, a carried module that +/// keeps its id but changes its source gets a new hash. +/// +/// # Examples +/// +/// ``` +/// use trusted_server_core::tsjs_bundle::{JsModulePart, compose_hash}; +/// +/// let core = JsModulePart { +/// id: "core", +/// source: "(() => { window.tsjs = {}; })()", +/// sha256: "c1", +/// }; +/// let before = [ +/// core, +/// JsModulePart { +/// id: "example", +/// source: "(() => { window.example = 1; })()", +/// sha256: "e1", +/// }, +/// ]; +/// let after = [ +/// core, +/// JsModulePart { +/// id: "example", +/// source: "(() => { window.example = 2; })()", +/// sha256: "e2", +/// }, +/// ]; +/// +/// assert_eq!(compose_hash(&before).len(), 64); +/// assert_eq!(compose_hash(&before), compose_hash(&before)); +/// assert_ne!(compose_hash(&before), compose_hash(&after)); +/// ``` +#[must_use] +pub fn compose_hash(parts: &[JsModulePart]) -> String { + let ordered = ordered(parts); + let key = ordered + .iter() + .map(|part| (part.id, part.sha256)) + .collect::>(); + if let Some(hash) = lock_cache().get(&key).cloned() { + return hash; + } + + let mut hasher = Sha256::new(); + visit_parts(&ordered, |part| hasher.update(part.as_bytes())); + let hash = hex::encode(hasher.finalize()); + lock_cache().insert(key, hash.clone()); + hash +} + +/// Orders the parts for the served script: core first, then every non-core +/// part in input order, keeping the first occurrence of each id. +fn ordered(parts: &[JsModulePart]) -> Vec { + let mut result = Vec::with_capacity(parts.len() + 1); + + let core = parts + .iter() + .find(|part| part.id == "core") + .copied() + .or_else(|| JsModulePart::compile_time("core")); + if let Some(core) = core { + result.push(core); + } + + for part in parts { + if part.id == "core" { + continue; + } + if result.iter().any(|taken| taken.id == part.id) { + continue; + } + result.push(*part); + } + + result +} + +/// Visits the byte pieces of the served script in order, with `;\n` between +/// consecutive parts. +fn visit_parts(parts: &[JsModulePart], mut visit: F) { + let mut first = true; + for part in parts { + if first { + first = false; + } else { + visit(";\n"); + } + visit(part.source); + } +} + +type HashCache = HashMap, String>; + +fn lock_cache() -> MutexGuard<'static, HashCache> { + static CACHE: OnceLock> = OnceLock::new(); + match CACHE.get_or_init(|| Mutex::new(HashMap::new())).lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn part(id: &'static str, source: &'static str) -> JsModulePart { + JsModulePart { + id, + source, + sha256: "0000000000000000000000000000000000000000000000000000000000000000", + } + } + + fn sha256_hex(bytes: &[u8]) -> String { + hex::encode(Sha256::digest(bytes)) + } + + #[test] + fn compose_puts_core_first_and_joins_with_a_semicolon_and_newline() { + let parts = [ + part("lockr", "L"), + part("core", "C"), + part("permutive", "P"), + ]; + + assert_eq!( + compose(&parts), + "C;\nL;\nP", + "should order core first and join parts" + ); + } + + #[test] + fn compose_hash_matches_the_compile_time_hash_for_built_in_modules() { + let ids = ["lockr", "permutive"]; + let parts = compile_time_parts(&ids); + + assert_eq!( + compose_hash(&parts), + trusted_server_js::concatenated_hash(&ids), + "should reproduce today's hash for a built-in module set" + ); + assert_eq!( + compose(&parts), + trusted_server_js::concatenate_modules(&ids), + "should reproduce today's bytes for a built-in module set" + ); + } + + #[test] + fn compose_of_no_parts_is_the_compile_time_core_alone() { + assert_eq!( + compose(&[]), + trusted_server_js::concatenate_modules(&[]), + "should serve core alone when no parts are given" + ); + assert_eq!( + compose_hash(&[]), + trusted_server_js::concatenated_hash(&[]), + "should hash core alone when no parts are given" + ); + } + + #[test] + fn compose_matches_every_compile_time_module_set_the_old_api_produces() { + let all = trusted_server_js::all_module_ids(); + let non_core = all + .iter() + .copied() + .filter(|id| *id != "core") + .collect::>(); + let mut cases = vec![all.clone(), non_core.clone()]; + cases.push(non_core.iter().rev().copied().collect()); + cases.push(vec!["core"]); + + for ids in cases { + let parts = compile_time_parts(&ids); + assert_eq!( + compose(&parts), + trusted_server_js::concatenate_modules(&ids), + "should reproduce today's bytes for {ids:?}" + ); + assert_eq!( + compose_hash(&parts), + trusted_server_js::concatenated_hash(&ids), + "should reproduce today's hash for {ids:?}" + ); + } + } + + #[test] + fn compose_hash_changes_when_a_carried_module_changes() { + let before = [ + part("core", "C"), + JsModulePart { + id: "probe", + source: "A", + sha256: "a", + }, + ]; + let after = [ + part("core", "C"), + JsModulePart { + id: "probe", + source: "B", + sha256: "b", + }, + ]; + + assert_ne!( + compose_hash(&before), + compose_hash(&after), + "should hash carried content" + ); + } + + #[test] + fn compose_keeps_a_duplicate_id_once_using_its_first_occurrence() { + let parts = [ + part("core", "C"), + part("lockr", "L1"), + part("permutive", "P"), + part("lockr", "L2"), + part("core", "C2"), + ]; + + assert_eq!( + compose(&parts), + "C;\nL1;\nP", + "should keep the first occurrence of each id and drop later ones" + ); + } + + #[test] + fn compose_hash_is_the_hex_sha256_of_compose() { + let parts = [ + part("core", "(() => {})()"), + JsModulePart { + id: "carried", + source: "(() => { window.carried = true; })()", + sha256: "carried-1", + }, + part("lockr", "L"), + ]; + + assert_eq!( + compose_hash(&parts), + sha256_hex(compose(&parts).as_bytes()), + "should hash the exact bytes compose produces" + ); + } + + #[test] + fn compile_time_parts_drops_unknown_ids() { + let parts = compile_time_parts(&["lockr", "not-a-module", "permutive"]); + let ids = parts.iter().map(|part| part.id).collect::>(); + + assert_eq!( + ids, + ["lockr", "permutive"], + "should keep known ids in order and drop unknown ids" + ); + for part in parts { + assert_eq!( + part.sha256, + sha256_hex(part.source.as_bytes()), + "should carry the compile-time hash of {}", + part.id + ); + } + } +} From e7ac3321ea3e939bff0d0c0cfd8d61f4e531f10b Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 28 Aug 2026 11:55:42 +0100 Subject: [PATCH 004/124] Tighten the tsjs part contract on hashes and ids --- crates/trusted-server-core/src/tsjs_bundle.rs | 152 ++++++++++-------- 1 file changed, 86 insertions(+), 66 deletions(-) diff --git a/crates/trusted-server-core/src/tsjs_bundle.rs b/crates/trusted-server-core/src/tsjs_bundle.rs index d787efcb63..d39baa997e 100644 --- a/crates/trusted-server-core/src/tsjs_bundle.rs +++ b/crates/trusted-server-core/src/tsjs_bundle.rs @@ -20,6 +20,11 @@ pub struct JsModulePart { /// The built IIFE. pub source: &'static str, /// SHA-256 of `source`, hex encoded. Identifies the content in the memo. + /// + /// The memo in [`compose_hash`] trusts this value rather than hashing + /// `source` again, so a part that declares the wrong hash serves a stale + /// `?v=` under a valid-looking URL. Debug builds and tests check the + /// value against `source`; release builds do not. pub sha256: &'static str, } @@ -27,21 +32,38 @@ impl JsModulePart { /// Looks up a compile-time module of `trusted-server-js` by id. /// /// Returns `None` when no module with that id was compiled in. + /// + /// # Examples + /// + /// ``` + /// use trusted_server_core::tsjs_bundle::JsModulePart; + /// + /// assert!(JsModulePart::compile_time("core").is_some()); + /// assert!(JsModulePart::compile_time("not-a-module").is_none()); + /// ``` #[must_use] - pub fn compile_time(id: &str) -> Option { + pub fn compile_time(id: &'static str) -> Option { let source = trusted_server_js::module_bundle(id)?; let sha256 = trusted_server_js::single_module_hash(id)?; - let id = trusted_server_js::all_module_ids() - .into_iter() - .find(|candidate| *candidate == id)?; Some(Self { id, source, sha256 }) } } /// Resolves compile-time parts for a list of ids, dropping unknown ids, as /// `trusted_server_js` does today. +/// +/// # Examples +/// +/// ``` +/// use trusted_server_core::tsjs_bundle::compile_time_parts; +/// +/// let parts = compile_time_parts(&["core", "not-a-module"]); +/// +/// assert_eq!(parts.len(), 1); +/// assert_eq!(parts[0].id, "core"); +/// ``` #[must_use] -pub fn compile_time_parts(ids: &[&str]) -> Vec { +pub fn compile_time_parts(ids: &[&'static str]) -> Vec { ids.iter() .filter_map(|id| JsModulePart::compile_time(id)) .collect() @@ -57,19 +79,17 @@ pub fn compile_time_parts(ids: &[&str]) -> Vec { /// # Examples /// /// ``` +/// use sha2::{Digest as _, Sha256}; /// use trusted_server_core::tsjs_bundle::{JsModulePart, compose}; /// +/// fn part(id: &'static str, source: &'static str) -> JsModulePart { +/// let sha256 = Box::leak(hex::encode(Sha256::digest(source)).into_boxed_str()); +/// JsModulePart { id, source, sha256 } +/// } +/// /// let parts = [ -/// JsModulePart { -/// id: "example", -/// source: "(() => { window.example = true; })()", -/// sha256: "e1", -/// }, -/// JsModulePart { -/// id: "core", -/// source: "(() => { window.tsjs = {}; })()", -/// sha256: "c1", -/// }, +/// part("example", "(() => { window.example = true; })()"), +/// part("core", "(() => { window.tsjs = {}; })()"), /// ]; /// /// assert_eq!( @@ -90,34 +110,29 @@ pub fn compose(parts: &[JsModulePart]) -> String { /// The result is memoized per ordered set of `(id, sha256)` pairs, so a /// per-request caller hashes a given module set once per process or isolate. /// Because the key carries each part's content hash, a carried module that -/// keeps its id but changes its source gets a new hash. +/// keeps its id but changes its source gets a new hash. The memo never +/// evicts, so feed it only sets derived from configuration, never sets +/// derived from request input. +/// +/// # Panics +/// +/// In debug builds, panics when a part's `sha256` is not the SHA-256 of its +/// `source`. Release builds trust the declared hash. /// /// # Examples /// /// ``` +/// use sha2::{Digest as _, Sha256}; /// use trusted_server_core::tsjs_bundle::{JsModulePart, compose_hash}; /// -/// let core = JsModulePart { -/// id: "core", -/// source: "(() => { window.tsjs = {}; })()", -/// sha256: "c1", -/// }; -/// let before = [ -/// core, -/// JsModulePart { -/// id: "example", -/// source: "(() => { window.example = 1; })()", -/// sha256: "e1", -/// }, -/// ]; -/// let after = [ -/// core, -/// JsModulePart { -/// id: "example", -/// source: "(() => { window.example = 2; })()", -/// sha256: "e2", -/// }, -/// ]; +/// fn part(id: &'static str, source: &'static str) -> JsModulePart { +/// let sha256 = Box::leak(hex::encode(Sha256::digest(source)).into_boxed_str()); +/// JsModulePart { id, source, sha256 } +/// } +/// +/// let core = JsModulePart::compile_time("core").expect("should have compiled core in"); +/// let before = [core, part("example", "(() => { window.example = 1; })()")]; +/// let after = [core, part("example", "(() => { window.example = 2; })()")]; /// /// assert_eq!(compose_hash(&before).len(), 64); /// assert_eq!(compose_hash(&before), compose_hash(&before)); @@ -125,6 +140,15 @@ pub fn compose(parts: &[JsModulePart]) -> String { /// ``` #[must_use] pub fn compose_hash(parts: &[JsModulePart]) -> String { + for part in parts { + debug_assert_eq!( + part.sha256, + hex::encode(Sha256::digest(part.source)), + "should declare the SHA-256 of its source for part `{}`", + part.id + ); + } + let ordered = ordered(parts); let key = ordered .iter() @@ -196,18 +220,17 @@ fn lock_cache() -> MutexGuard<'static, HashCache> { mod tests { use super::*; - fn part(id: &'static str, source: &'static str) -> JsModulePart { - JsModulePart { - id, - source, - sha256: "0000000000000000000000000000000000000000000000000000000000000000", - } - } - fn sha256_hex(bytes: &[u8]) -> String { hex::encode(Sha256::digest(bytes)) } + /// Builds a part whose `sha256` really is the hash of `source`, so no two + /// parts with different content ever share a memo key. + fn part(id: &'static str, source: &'static str) -> JsModulePart { + let sha256 = Box::leak(sha256_hex(source.as_bytes()).into_boxed_str()); + JsModulePart { id, source, sha256 } + } + #[test] fn compose_puts_core_first_and_joins_with_a_semicolon_and_newline() { let parts = [ @@ -283,22 +306,8 @@ mod tests { #[test] fn compose_hash_changes_when_a_carried_module_changes() { - let before = [ - part("core", "C"), - JsModulePart { - id: "probe", - source: "A", - sha256: "a", - }, - ]; - let after = [ - part("core", "C"), - JsModulePart { - id: "probe", - source: "B", - sha256: "b", - }, - ]; + let before = [part("core", "C"), part("probe", "A")]; + let after = [part("core", "C"), part("probe", "B")]; assert_ne!( compose_hash(&before), @@ -328,11 +337,7 @@ mod tests { fn compose_hash_is_the_hex_sha256_of_compose() { let parts = [ part("core", "(() => {})()"), - JsModulePart { - id: "carried", - source: "(() => { window.carried = true; })()", - sha256: "carried-1", - }, + part("carried", "(() => { window.carried = true; })()"), part("lockr", "L"), ]; @@ -343,6 +348,21 @@ mod tests { ); } + #[test] + #[should_panic(expected = "should declare the SHA-256 of its source for part `lying`")] + fn compose_hash_rejects_a_part_whose_declared_hash_is_wrong() { + let parts = [ + part("core", "C"), + JsModulePart { + id: "lying", + source: "A", + sha256: "not-the-hash-of-a", + }, + ]; + + let _ = compose_hash(&parts); + } + #[test] fn compile_time_parts_drops_unknown_ids() { let parts = compile_time_parts(&["lockr", "not-a-module", "permutive"]); From fd18ab220b93257757ce7b6a2c32214515cabd17 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 28 Aug 2026 12:10:59 +0100 Subject: [PATCH 005/124] Carry a browser module on the integration registration --- .../src/integrations/gpt_diagnostics.rs | 6 +- .../src/integrations/mod.rs | 6 +- .../src/integrations/registry.rs | 346 +++++++++++++++++- 3 files changed, 349 insertions(+), 9 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs b/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs index 413e5ee62a..7284289131 100644 --- a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs +++ b/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs @@ -239,7 +239,7 @@ pub fn register( Ok(Some( IntegrationRegistration::builder(GPT_DIAGNOSTICS_INTEGRATION_ID) - .without_js() + .with_standalone_js() .build(), )) } @@ -493,6 +493,10 @@ mod tests { .js_module_ids_deferred() .contains(&GPT_DIAGNOSTICS_INTEGRATION_ID) ); + assert!( + registry.js_part(GPT_DIAGNOSTICS_INTEGRATION_ID).is_some(), + "should serve the diagnostics module standalone" + ); } #[test] diff --git a/crates/trusted-server-core/src/integrations/mod.rs b/crates/trusted-server-core/src/integrations/mod.rs index 92ee5e765a..a88f6187fb 100644 --- a/crates/trusted-server-core/src/integrations/mod.rs +++ b/crates/trusted-server-core/src/integrations/mod.rs @@ -29,9 +29,9 @@ pub mod sourcepoint; pub mod testlight; pub use registry::{ - AttributeRewriteAction, AttributeRewriteOutcome, HeaderMutation, HeaderMutationMode, - IntegrationAttributeContext, IntegrationAttributeRewriter, IntegrationDocumentState, - IntegrationEndpoint, IntegrationHeadInjector, IntegrationHtmlContext, + AttributeRewriteAction, AttributeRewriteOutcome, CarriedJsModule, HeaderMutation, + HeaderMutationMode, IntegrationAttributeContext, IntegrationAttributeRewriter, + IntegrationDocumentState, IntegrationEndpoint, IntegrationHeadInjector, IntegrationHtmlContext, IntegrationHtmlPostProcessor, IntegrationMetadata, IntegrationProxy, IntegrationRegistration, IntegrationRegistrationBuilder, IntegrationRegistry, IntegrationRequestFilter, IntegrationScriptContext, IntegrationScriptRewriter, ProxyDispatchInput, RequestFilterDecision, diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index 0e1de64ee5..49f495ea9a 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -7,6 +7,7 @@ use edgezero_core::body::Body as EdgeBody; use error_stack::Report; use http::{Method, Request, Response}; use matchit::Router; +use sha2::{Digest as _, Sha256}; use crate::constants::HEADER_X_TS_EC; use crate::ec::EcContext; @@ -591,11 +592,30 @@ pub trait IntegrationHeadInjector: Send + Sync { } } +/// A browser module a registration carries, for a module built outside +/// `trusted-server-js`. The crate embeds its built IIFE with `include_str!` +/// and states its SHA-256 as a literal next to it; the registry verifies the +/// two agree when it is built, and the served `?v=` hash is derived from it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CarriedJsModule { + /// The built IIFE. + pub source: &'static str, + /// SHA-256 of `source`, hex encoded, lower case. + pub sha256: &'static str, +} + /// Registration payload returned by integration builders. pub struct IntegrationRegistration { pub integration_id: &'static str, pub js_deferred: bool, pub js_disabled: bool, + /// Browser module carried by the registration, when the module is not + /// compiled into `trusted-server-js`. + pub js_module: Option, + /// Serve the module only on its own `/static/tsjs=tsjs-.min.js` path, + /// never in the unified bundle and never as a deferred tag; the + /// integration injects the tag itself when it decides to. + pub js_standalone: bool, pub proxies: Vec>, pub attribute_rewriters: Vec>, pub script_rewriters: Vec>, @@ -622,6 +642,8 @@ impl IntegrationRegistrationBuilder { integration_id, js_deferred: false, js_disabled: false, + js_module: None, + js_standalone: false, proxies: Vec::new(), attribute_rewriters: Vec::new(), script_rewriters: Vec::new(), @@ -690,6 +712,23 @@ impl IntegrationRegistrationBuilder { self } + /// Carry a browser module built outside `trusted-server-js`. + #[must_use] + pub fn with_js_module(mut self, module: CarriedJsModule) -> Self { + self.registration.js_module = Some(module); + self + } + + /// Serve the module standalone only; see + /// [`IntegrationRegistration::js_standalone`]. + #[must_use] + pub fn with_standalone_js(mut self) -> Self { + self.registration.js_standalone = true; + self.registration.js_disabled = false; + self.registration.js_deferred = false; + self + } + #[must_use] pub fn build(self) -> IntegrationRegistration { self.registration @@ -715,6 +754,11 @@ struct IntegrationRegistryInner { enabled_integration_ids: Vec<&'static str>, deferred_js_ids: Vec<&'static str>, disabled_js_ids: Vec<&'static str>, + // Enabled modules served only on their own path, never in the bundle. + standalone_js_ids: Vec<&'static str>, + // Modules carried by their registrations, verified against their + // declared hash at construction. + carried_js: Vec<(&'static str, CarriedJsModule)>, html_rewriters: Vec>, script_rewriters: Vec>, html_post_processors: Vec>, @@ -741,6 +785,8 @@ impl Default for IntegrationRegistryInner { enabled_integration_ids: Vec::new(), deferred_js_ids: Vec::new(), disabled_js_ids: Vec::new(), + standalone_js_ids: Vec::new(), + carried_js: Vec::new(), html_rewriters: Vec::new(), script_rewriters: Vec::new(), html_post_processors: Vec::new(), @@ -914,6 +960,23 @@ impl IntegrationRegistry { } else if registration.js_deferred { inner.deferred_js_ids.push(registration.integration_id); } + if registration.js_standalone { + inner.standalone_js_ids.push(registration.integration_id); + } + if let Some(module) = registration.js_module { + // The served `?v=` hash and its memo trust this value, so a + // stale literal is a startup error rather than a stale script. + let actual = hex::encode(Sha256::digest(module.source.as_bytes())); + if actual != module.sha256 { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "integration `{}` carries a browser module whose declared SHA-256 does not match its source (declared {}, actual {actual})", + registration.integration_id, module.sha256 + ), + })); + } + inner.carried_js.push((registration.integration_id, module)); + } } } @@ -1213,8 +1276,9 @@ impl IntegrationRegistry { /// Return JS module IDs that should be included in the tsjs bundle. /// /// Always includes JS-only modules with no Rust-side registration. - /// Includes enabled integrations only when the generated TSJS registry has a - /// corresponding browser module. + /// Includes enabled integrations only when a browser module serves their + /// id, either compiled into `trusted-server-js` or carried by the + /// registration, and excludes modules served standalone only. #[must_use] pub fn js_module_ids(&self) -> Vec<&'static str> { // Core JS-only modules that do not have a Rust-side registration. @@ -1223,8 +1287,9 @@ impl IntegrationRegistry { let mut ids: Vec<&'static str> = JS_ALWAYS.to_vec(); for id in &self.inner.enabled_integration_ids { - if trusted_server_js::module_bundle(id).is_some() + if self.js_part(id).is_some() && !self.inner.disabled_js_ids.contains(id) + && !self.inner.standalone_js_ids.contains(id) && !ids.contains(id) { ids.push(*id); @@ -1242,6 +1307,90 @@ impl IntegrationRegistry { ids } + /// The module part for one id: the registration that carries it, else + /// the compile-time module, for an enabled integration or an always-on + /// core module (`core`, `creative`). `None` when nothing serves that id. + /// + /// # Examples + /// + /// ``` + /// use trusted_server_core::integrations::IntegrationRegistry; + /// + /// let registry = IntegrationRegistry::default(); + /// + /// assert!(registry.js_part("core").is_some()); + /// assert!(registry.js_part("lockr").is_none()); + /// ``` + #[must_use] + pub fn js_part(&self, id: &'static str) -> Option { + if let Some((carried_id, module)) = self + .inner + .carried_js + .iter() + .find(|(carried_id, _)| *carried_id == id) + { + return Some(crate::tsjs_bundle::JsModulePart { + id: carried_id, + source: module.source, + sha256: module.sha256, + }); + } + if id != "core" && id != "creative" && !self.integration_enabled(id) { + return None; + } + crate::tsjs_bundle::JsModulePart::compile_time(id) + } + + /// Ids of enabled modules served standalone only. + #[must_use] + pub fn js_standalone_ids(&self) -> Vec<&'static str> { + self.inner + .standalone_js_ids + .iter() + .copied() + .filter(|id| self.integration_enabled(id)) + .collect() + } + + /// Parts of the unified bundle: core, then every immediate module. + #[must_use] + pub fn js_parts_immediate(&self) -> Vec { + self.parts_for(&self.js_module_ids_immediate()) + } + + /// Parts served with `", - tsjs::tsjs_single_module_script_src(GPT_DIAGNOSTICS_INTEGRATION_ID) + tsjs::tsjs_single_module_script_src(module) ) }) } @@ -131,6 +135,12 @@ impl GptDiagnosticsRequestDecision { mod head_seam_invariant_tests { use super::*; + /// The compile-time diagnostics module, as the registry would serve it. + fn diagnostics_part() -> JsModulePart { + JsModulePart::compile_time(GPT_DIAGNOSTICS_INTEGRATION_ID) + .expect("should have compiled the diagnostics module in") + } + /// Every combination of the three fields the decision carries. fn all_decisions() -> Vec { let mut out = Vec::new(); @@ -162,9 +172,10 @@ mod head_seam_invariant_tests { // // If a future change makes a script emit without also requiring the stamp, // this fails here rather than silently in a cached template. + let module = diagnostics_part(); for decision in all_decisions() { - let injects = - decision.bootstrap_script().is_some() || decision.module_script_tag().is_some(); + let injects = decision.bootstrap_script().is_some() + || decision.module_script_tag(&module).is_some(); if injects { assert!( decision.requires_private_no_store(), @@ -184,7 +195,7 @@ mod head_seam_invariant_tests { "should not inject a bootstrap for an inert decision" ); assert_eq!( - decision.module_script_tag(), + decision.module_script_tag(&diagnostics_part()), None, "should not inject a module for an inert decision" ); diff --git a/crates/trusted-server-core/src/integrations/mod.rs b/crates/trusted-server-core/src/integrations/mod.rs index a88f6187fb..36876786a8 100644 --- a/crates/trusted-server-core/src/integrations/mod.rs +++ b/crates/trusted-server-core/src/integrations/mod.rs @@ -25,6 +25,8 @@ pub mod osano; pub mod permutive; pub mod prebid; mod registry; +#[cfg(test)] +pub(crate) use registry::test_support as registry_test_support; pub mod sourcepoint; pub mod testlight; diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index be4873a392..b7382a4fe4 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -1364,6 +1364,33 @@ impl IntegrationRegistry { self.inner.standalone_js_ids.clone() } + /// Resolves a module id given as request text (the stem of a + /// `tsjs-.min.js` filename) to this registry's own `&'static str` + /// id, covering bundle, deferred and standalone modules. + /// + /// `trusted_server_js::all_module_ids` knows only compile-time modules, + /// so a carried module could never be looked up through it. Returns + /// `None` for an id this registry serves nowhere, which includes a + /// disabled or unknown integration. + /// + /// # Examples + /// + /// ``` + /// use trusted_server_core::integrations::IntegrationRegistry; + /// + /// let registry = IntegrationRegistry::default(); + /// + /// assert_eq!(registry.js_module_id("creative"), Some("creative")); + /// assert_eq!(registry.js_module_id("not-a-module"), None); + /// ``` + #[must_use] + pub fn js_module_id(&self, stem: &str) -> Option<&'static str> { + self.js_module_ids() + .into_iter() + .chain(self.js_standalone_ids()) + .find(|id| *id == stem) + } + /// Parts of the unified bundle: core, then every immediate module. #[must_use] pub fn js_parts_immediate(&self) -> Vec { diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index a9adcf41df..0fe435f79e 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -518,8 +518,11 @@ fn encode_complete_body( /// Serves two types of bundles: /// - **Unified bundle** (`tsjs-unified.min.js`): core + immediate (non-deferred) /// integration modules. -/// - **Deferred module** (`tsjs-{id}.min.js`): a single self-contained IIFE for -/// modules loaded with `defer` (e.g., prebid). +/// - **Single module** (`tsjs-{id}.min.js`): a single self-contained IIFE for +/// modules loaded with `defer` (e.g., prebid) or registered as standalone. +/// +/// Every module comes from the registry's parts, so a module a registration +/// carries is served and versioned like a compile-time one. /// /// # Errors /// @@ -540,28 +543,28 @@ pub fn handle_tsjs_dynamic( if UNIFIED_FILENAMES.contains(&filename) { // Serve core + immediate modules (excludes deferred like prebid). - let module_ids = integration_registry.js_module_ids_immediate(); - let body = trusted_server_js::concatenate_modules(&module_ids); - let hash = trusted_server_js::concatenated_hash(&module_ids); + let parts = integration_registry.js_parts_immediate(); + let body = crate::tsjs_bundle::compose(&parts); + let hash = crate::tsjs_bundle::compose_hash(&parts); return Ok(serve_tsjs_static(req, &body, &hash, edge_header)); } - if let Some(module_id) = parse_single_module_filename(filename) { - // Deferred modules and the conditionally injected diagnostics module - // are served as content-addressed standalone assets. Delivery remains - // cookie-independent so the static response can stay publicly cached. - let deferred_ids = integration_registry.js_module_ids_deferred(); - let diagnostics_standalone = module_id - == crate::integrations::gpt_diagnostics::GPT_DIAGNOSTICS_INTEGRATION_ID - && integration_registry.integration_enabled(module_id); - if !deferred_ids.contains(&module_id) && !diagnostics_standalone { + if let Some(module_id) = parse_single_module_filename(filename, integration_registry) { + // Deferred modules and standalone modules are served as + // content-addressed single files. Delivery remains cookie-independent + // so the static response can stay publicly cached. + let deferred = integration_registry.js_module_ids_deferred(); + let standalone = integration_registry.js_standalone_ids(); + if !deferred.contains(&module_id) && !standalone.contains(&module_id) { return Ok(not_found_response()); } - if let (Some(content), Some(hash)) = ( - trusted_server_js::module_bundle(module_id), - trusted_server_js::single_module_hash(module_id), - ) { - return Ok(serve_tsjs_static(req, content, hash, edge_header)); + if let Some(part) = integration_registry.js_part(module_id) { + return Ok(serve_tsjs_static( + req, + part.source, + part.sha256, + edge_header, + )); } } @@ -597,20 +600,22 @@ fn request_version_hash(req: &Request) -> Option<&str> { }) } -/// Extract a module ID from a deferred-module filename like `tsjs-sourcepoint.min.js`. +/// Extract a module ID from a single-module filename like `tsjs-sourcepoint.min.js`. /// -/// Returns `Some(&'static str)` if the filename matches a known JS module ID, -/// `None` otherwise. The caller must additionally verify that the module is -/// both deferred and enabled via the [`IntegrationRegistry`]. +/// Returns `Some(&'static str)` if the filename names a module the registry +/// serves somewhere (bundle, deferred or standalone), resolved through the +/// registry so a carried module is found. `None` otherwise. The caller must +/// additionally verify that the module is deferred or standalone. #[must_use] -fn parse_single_module_filename(filename: &str) -> Option<&'static str> { +fn parse_single_module_filename( + filename: &str, + registry: &IntegrationRegistry, +) -> Option<&'static str> { let stem = filename .strip_prefix("tsjs-") .and_then(|s| s.strip_suffix(".min.js").or_else(|| s.strip_suffix(".js")))?; - trusted_server_js::all_module_ids() - .into_iter() - .find(|&id| id == stem) + registry.js_module_id(stem) } /// Parameters for processing response streaming. @@ -6797,7 +6802,12 @@ mod tests { use crate::auction::orchestrator::OrchestrationResult; use crate::auction::types::AuctionResponse; use crate::auction::types::{AdFormat, AdSlot, MediaType}; - use crate::integrations::IntegrationRegistry; + use crate::integrations::registry_test_support::{ + PROBE_JS, PROBE_JS_SHA256, carried_probe_registration, validate_nothing, + }; + use crate::integrations::{ + CarriedJsModule, IntegrationBuilder, IntegrationRegistration, IntegrationRegistry, + }; use crate::permissions::{Permission, PermissionSet}; use crate::platform::test_support::{ NoopSecretStore, StubHttpClient, build_services_with_http_client, @@ -15601,37 +15611,64 @@ mod tests { #[test] fn parse_single_module_filename_extracts_known_id() { + let settings = create_test_settings(); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + assert_eq!( - parse_single_module_filename("tsjs-sourcepoint.min.js"), - Some("sourcepoint"), - "should extract sourcepoint from minified filename" + parse_single_module_filename("tsjs-prebid.min.js", ®istry), + Some("prebid"), + "should extract prebid from minified filename" ); assert_eq!( - parse_single_module_filename("tsjs-sourcepoint.js"), - Some("sourcepoint"), - "should extract sourcepoint from unminified filename" + parse_single_module_filename("tsjs-prebid.js", ®istry), + Some("prebid"), + "should extract prebid from unminified filename" + ); + } + + #[test] + fn parse_single_module_filename_resolves_a_carried_module_id() { + let settings = create_test_settings(); + let extra = [IntegrationBuilder::new( + "probe", + "seam-probe", + carried_probe_registration, + validate_nothing, + )]; + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should build a registry with a carried module"); + + assert_eq!( + parse_single_module_filename("tsjs-probe.min.js", ®istry), + Some("probe"), + "should resolve a module id trusted-server-js has never heard of" ); } #[test] fn parse_single_module_filename_rejects_unknown_ids() { + let settings = create_test_settings(); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + assert_eq!( - parse_single_module_filename("tsjs-evil.min.js"), + parse_single_module_filename("tsjs-evil.min.js", ®istry), None, "should reject unknown module names" ); assert_eq!( - parse_single_module_filename("tsjs-core.min.js"), - Some("core"), - "should accept any known module ID (deferred check happens in caller)" + parse_single_module_filename("tsjs-core.min.js", ®istry), + None, + "should not resolve core, which is only ever served inside the bundle" ); assert_eq!( - parse_single_module_filename("prebid.min.js"), + parse_single_module_filename("prebid.min.js", ®istry), None, "should reject without tsjs- prefix" ); assert_eq!( - parse_single_module_filename("tsjs-sourcepoint.txt"), + parse_single_module_filename("tsjs-prebid.txt", ®istry), None, "should reject non-js extension" ); @@ -15772,6 +15809,163 @@ mod tests { ); } + fn body_text(response: http::Response) -> String { + let bytes = response + .into_body() + .into_bytes() + .expect("should read the tsjs response body") + .to_vec(); + String::from_utf8(bytes).expect("should serve UTF-8 JavaScript") + } + + fn cache_control_text(response: &http::Response) -> String { + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()) + .unwrap_or_default() + .to_string() + } + + fn carried_deferred_probe_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("probe") + .with_js_module(CarriedJsModule { + source: PROBE_JS, + sha256: PROBE_JS_SHA256, + }) + .with_deferred_js() + .build(), + )) + } + + #[test] + fn tsjs_dynamic_serves_a_carried_module_in_the_unified_bundle_under_the_composed_hash() { + let settings = create_test_settings(); + let extra = [IntegrationBuilder::new( + "probe", + "seam-probe", + carried_probe_registration, + validate_nothing, + )]; + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should build a registry with a carried module"); + let parts = registry.js_parts_immediate(); + let expected_hash = crate::tsjs_bundle::compose_hash(&parts); + let request = build_request( + Method::GET, + &format!("https://publisher.example/static/tsjs=tsjs-unified.min.js?v={expected_hash}"), + ); + + let response = handle_tsjs_dynamic(&request, ®istry, EdgeCacheHeader::SMaxageFallback) + .expect("should handle tsjs request"); + + assert_eq!(response.status(), StatusCode::OK, "should serve the bundle"); + assert!( + cache_control_text(&response).contains("immutable"), + "should treat the composed hash as the matching version" + ); + let body = body_text(response); + let core = trusted_server_js::module_bundle("core").expect("should have compiled core in"); + assert!( + body.starts_with(core), + "should put the compile-time core first in the bundle" + ); + assert!( + body.contains("window.__probe=1"), + "should serve the carried module inside the unified bundle" + ); + } + + #[test] + fn tsjs_dynamic_serves_a_carried_deferred_module_standalone() { + let settings = create_test_settings(); + let extra = [IntegrationBuilder::new( + "probe", + "seam-probe", + carried_deferred_probe_registration, + validate_nothing, + )]; + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should build a registry with a carried deferred module"); + let request = build_request( + Method::GET, + &format!("https://publisher.example/static/tsjs=tsjs-probe.min.js?v={PROBE_JS_SHA256}"), + ); + + let response = handle_tsjs_dynamic(&request, ®istry, EdgeCacheHeader::SMaxageFallback) + .expect("should handle tsjs request"); + + assert_eq!( + response.status(), + StatusCode::OK, + "should serve a carried deferred module as its own file" + ); + assert!( + cache_control_text(&response).contains("immutable"), + "should treat the carried module's own hash as the matching version" + ); + assert_eq!( + body_text(response), + PROBE_JS, + "should serve the carried source verbatim" + ); + } + + #[test] + fn tsjs_dynamic_serves_a_standalone_module_by_registration_flag_not_by_name() { + let mut settings = create_test_settings(); + settings + .integrations + .insert_config("gpt_diagnostics", &serde_json::json!({ "enabled": true })) + .expect("should insert gpt_diagnostics config"); + settings + .integrations + .insert_config( + "lockr", + &serde_json::json!({ "enabled": true, "app_id": "test-app-id" }), + ) + .expect("should insert lockr config"); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + assert!( + registry.js_module_ids_immediate().contains(&"lockr"), + "fixture should put lockr in the unified bundle" + ); + + let standalone = handle_tsjs_dynamic( + &build_request( + Method::GET, + "https://publisher.example/static/tsjs=tsjs-gpt_diagnostics.min.js", + ), + ®istry, + EdgeCacheHeader::SMaxageFallback, + ) + .expect("should handle tsjs request"); + assert_eq!( + standalone.status(), + StatusCode::OK, + "should serve a module its registration marks standalone" + ); + + let bundled_only = handle_tsjs_dynamic( + &build_request( + Method::GET, + "https://publisher.example/static/tsjs=tsjs-lockr.min.js", + ), + ®istry, + EdgeCacheHeader::SMaxageFallback, + ) + .expect("should handle tsjs request"); + assert_eq!( + bundled_only.status(), + StatusCode::NOT_FOUND, + "should not serve a bundle-only module as a single file" + ); + } + #[test] fn tsjs_dynamic_keeps_short_cache_for_mismatched_hash() { let settings = create_test_settings(); diff --git a/crates/trusted-server-core/src/tsjs.rs b/crates/trusted-server-core/src/tsjs.rs index e8275b8144..8cb2216a2e 100644 --- a/crates/trusted-server-core/src/tsjs.rs +++ b/crates/trusted-server-core/src/tsjs.rs @@ -1,23 +1,29 @@ -use trusted_server_js::{concatenated_hash, single_module_hash}; +//! URLs and `", - tsjs_script_src(module_ids), + tsjs_script_src(parts), ) } -/// `/static` URL for the unified bundle when exact module IDs are unavailable. +/// `/static` URL for the unified bundle when the exact parts are unavailable. /// /// This intentionally omits `?v=` because the serving path can only mark a URL /// immutable when the hash matches the exact enabled module set. Use -/// [`tsjs_script_src`] with exact module IDs when [`IntegrationRegistry`] is -/// available. +/// [`tsjs_script_src`] with the registry's parts when [`IntegrationRegistry`] +/// is available. /// /// [`IntegrationRegistry`]: crate::integrations::IntegrationRegistry #[must_use] @@ -59,7 +65,7 @@ pub fn tsjs_unified_script_src() -> String { "/static/tsjs=tsjs-unified.min.js".to_string() } -/// `", - tsjs_deferred_script_src(module_id) + tsjs_deferred_script_src(part) ) } -/// Generate all deferred `"), "should generate exactly one trusted server script tag" ); @@ -204,18 +251,18 @@ mod tests { #[test] fn publisher_tsjs_script_tag_renders_static_attributes() { - let module_ids = ["gpt"]; - let src = tsjs_script_src(&module_ids); + let parts = compile_time_parts(&["gpt"]); + let src = tsjs_script_src(&parts); assert_eq!( - tsjs_script_tag_with_attributes(&module_ids, &[("data-ts-gam-attribution", "true")]), + tsjs_script_tag_with_attributes(&parts, &[("data-ts-gam-attribution", "true")]), format!( "" ), "should render trusted static attributes on the publisher bundle tag" ); assert_eq!( - tsjs_script_tag(&module_ids), + tsjs_script_tag(&parts), format!(""), "should keep the generic tag byte-for-byte unmarked" ); @@ -226,7 +273,10 @@ mod tests { expected = "attribute name should contain only lowercase ASCII letters, digits, and hyphens" )] fn publisher_tsjs_script_tag_rejects_invalid_attribute_name() { - let _ = tsjs_script_tag_with_attributes(&["gpt"], &[("data-bad_name", "true")]); + let _ = tsjs_script_tag_with_attributes( + &compile_time_parts(&["gpt"]), + &[("data-bad_name", "true")], + ); } #[test] @@ -234,31 +284,43 @@ mod tests { expected = "attribute name should contain only lowercase ASCII letters, digits, and hyphens" )] fn publisher_tsjs_script_tag_rejects_empty_attribute_name() { - let _ = tsjs_script_tag_with_attributes(&["gpt"], &[("", "true")]); + let _ = tsjs_script_tag_with_attributes(&compile_time_parts(&["gpt"]), &[("", "true")]); } #[test] #[should_panic(expected = "attribute value should not contain HTML-sensitive characters")] fn publisher_tsjs_script_tag_rejects_double_quote_in_attribute_value() { - let _ = tsjs_script_tag_with_attributes(&["gpt"], &[("data-safe-name", "bad\"value")]); + let _ = tsjs_script_tag_with_attributes( + &compile_time_parts(&["gpt"]), + &[("data-safe-name", "bad\"value")], + ); } #[test] #[should_panic(expected = "attribute value should not contain HTML-sensitive characters")] fn publisher_tsjs_script_tag_rejects_ampersand_in_attribute_value() { - let _ = tsjs_script_tag_with_attributes(&["gpt"], &[("data-safe-name", "bad&value")]); + let _ = tsjs_script_tag_with_attributes( + &compile_time_parts(&["gpt"]), + &[("data-safe-name", "bad&value")], + ); } #[test] #[should_panic(expected = "attribute value should not contain HTML-sensitive characters")] fn publisher_tsjs_script_tag_rejects_less_than_in_attribute_value() { - let _ = tsjs_script_tag_with_attributes(&["gpt"], &[("data-safe-name", "badvalue")]); + let _ = tsjs_script_tag_with_attributes( + &compile_time_parts(&["gpt"]), + &[("data-safe-name", "bad>value")], + ); } #[test] @@ -278,36 +340,58 @@ mod tests { #[test] fn tsjs_single_module_script_src_formats_known_module_url_with_hash() { - let src = tsjs_single_module_script_src("creative"); + let parts = compile_time_parts(&["creative"]); + let src = tsjs_single_module_script_src(&parts[0]); assert!( src.starts_with("/static/tsjs=tsjs-creative.min.js?v="), "should use per-module static bundle path" ); assert_sha256_hex_hash(hash_query_value(&src)); + assert_eq!( + src, + format!( + "/static/tsjs=tsjs-creative.min.js?v={}", + trusted_server_js::single_module_hash("creative") + .expect("should have compiled creative in") + ), + "should keep today's URL for a compile-time module" + ); } #[test] - fn tsjs_deferred_script_src_hashes_prebid_shim_and_empties_unknown_module() { - let prebid_src = tsjs_deferred_script_src("prebid"); + fn tsjs_deferred_script_src_hashes_prebid_shim() { + let parts = compile_time_parts(&["prebid"]); + let prebid_src = tsjs_deferred_script_src(&parts[0]); + assert!( prebid_src.starts_with("/static/tsjs=tsjs-prebid.min.js?v="), "prebid shim should be served from the deferred tsjs route" ); assert_sha256_hex_hash(hash_query_value(&prebid_src)); + } + + #[test] + fn tsjs_deferred_script_tag_carries_a_carried_parts_own_hash() { + let part = carried_part("probe", "(() => { window.probe = 1; })()"); + assert_eq!( - tsjs_deferred_script_src("unknown-module"), - "/static/tsjs=tsjs-unknown-module.min.js?v=", - "should document current unknown-module hash behavior" + tsjs_deferred_script_tag(&part), + format!( + "", + part.sha256 + ), + "should version a carried deferred module by its own content hash" ); } #[test] fn tsjs_deferred_script_tag_marks_script_defer() { - let src = tsjs_deferred_script_src("prebid"); + let parts = compile_time_parts(&["prebid"]); + let src = tsjs_deferred_script_src(&parts[0]); assert_eq!( - tsjs_deferred_script_tag("prebid"), + tsjs_deferred_script_tag(&parts[0]), format!(""), "should generate a deferred script tag" ); @@ -324,12 +408,14 @@ mod tests { #[test] fn tsjs_deferred_script_tags_preserves_input_order() { + let parts = compile_time_parts(&["prebid", "creative"]); + assert_eq!( - tsjs_deferred_script_tags(&["prebid", "creative"]), + tsjs_deferred_script_tags(&parts), format!( "{}{}", - tsjs_deferred_script_tag("prebid"), - tsjs_deferred_script_tag("creative") + tsjs_deferred_script_tag(&parts[0]), + tsjs_deferred_script_tag(&parts[1]) ), "should preserve caller-provided deferred module order" ); @@ -353,18 +439,9 @@ mod tests { #[test] fn tsjs_script_src_differs_for_different_module_sets() { assert_ne!( - tsjs_script_src(&["lockr"]), - tsjs_script_src(&["lockr", "permutive"]), + tsjs_script_src(&compile_time_parts(&["lockr"])), + tsjs_script_src(&compile_time_parts(&["lockr", "permutive"])), "should bust the cache when the module set content changes" ); } - - #[test] - fn tsjs_deferred_script_src_has_empty_hash_for_unknown_module() { - assert_eq!( - tsjs_deferred_script_src("does-not-exist"), - "/static/tsjs=tsjs-does-not-exist.min.js?v=", - "should fall back to an empty cache-busting hash for an unknown module" - ); - } } From 521fb51f074f50e2912133c988c6c4dee3313ade Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 28 Aug 2026 17:14:49 +0100 Subject: [PATCH 008/124] Cover carried modules in the template cache fingerprint The template cache key digested `concatenated_hash(all_module_ids())`, every module compiled into `trusted-server-js`. A vendor crate that carries its own browser module through `CarriedJsModule` was outside that map, so rebuilding the module left the fingerprint where it was and a cached template kept pointing at the old `?v=` hash. `template_fingerprint` now digests `compose_hash(registry.js_parts_all())`, the parts the registry can actually serve, carried modules included, so a vendor rebuild invalidates the templates cached under the old hash. It also narrows the digest from every compiled module to the enabled set, which is a tightening rather than a loosening, because a module no deployment serves cannot change what a template renders. That is why `handle_publisher_request` now needs the integration registry: the fingerprint is built inside it and the registry is the only thing that knows the served parts. The settings and the registry are both built once at startup and always travel together, so they arrive as one `AppContext` argument rather than two, which keeps the call at seven arguments. The function destructures the context on its first line, leaving the body unchanged. The four adapters already hold both on their app state and pass them through. `a_change_to_a_carried_module_changes_the_fingerprint` builds two registries from the same settings differing only in a carried module's source and asserts the fingerprints differ. Checked against the old digest, where it fails. --- crates/trusted-server-adapter-axum/src/app.rs | 10 +- .../src/app.rs | 7 +- .../trusted-server-adapter-fastly/src/app.rs | 16 +- crates/trusted-server-adapter-spin/src/app.rs | 7 +- .../trusted-server-core/src/auction/README.md | 6 +- crates/trusted-server-core/src/publisher.rs | 203 +++++++++++++++--- 6 files changed, 211 insertions(+), 38 deletions(-) diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 90bcac37eb..d446a7c013 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -25,8 +25,9 @@ use trusted_server_core::proxy::{ handle_first_party_proxy_sign, }; use trusted_server_core::publisher::{ - AuctionDispatch, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, buffer_publisher_response_async, - handle_page_bids, handle_publisher_request, handle_tsjs_dynamic, page_bids_preflight_denied, + AppContext, AuctionDispatch, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, + buffer_publisher_response_async, handle_page_bids, handle_publisher_request, + handle_tsjs_dynamic, page_bids_preflight_denied, }; use trusted_server_core::request_signing::{ handle_trusted_server_discovery, handle_verify_signature, @@ -247,7 +248,10 @@ async fn dispatch_fallback( registry: None, }; let publisher_response = handle_publisher_request( - &state.settings, + AppContext { + settings: &state.settings, + integrations: &state.registry, + }, services, None, &mut ec_context, diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index fad5e65a4c..3417d6b157 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -28,7 +28,7 @@ use trusted_server_core::proxy::{ handle_first_party_proxy_sign, }; use trusted_server_core::publisher::{ - AuctionDispatch, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, PublisherResponse, + AppContext, AuctionDispatch, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, PublisherResponse, buffer_publisher_response_async, handle_page_bids, handle_publisher_request, handle_tsjs_dynamic, page_bids_preflight_denied, }; @@ -463,7 +463,10 @@ fn build_router(state: &Arc) -> RouterService { registry: None, }; match handle_publisher_request( - &state.settings, + AppContext { + settings: &state.settings, + integrations: &state.registry, + }, &services, None, &mut ec_context, diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 5fcaaac7cf..8482359219 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -132,7 +132,7 @@ use trusted_server_core::proxy::{ handle_first_party_proxy, handle_first_party_proxy_rebuild, handle_first_party_proxy_sign, }; use trusted_server_core::publisher::{ - AuctionDispatch, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, handle_page_bids, + AppContext, AuctionDispatch, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, handle_page_bids, handle_publisher_request, handle_tsjs_dynamic, page_bids_preflight_denied, publisher_response_into_streaming_response, }; @@ -951,7 +951,10 @@ async fn dispatch_fallback( registry: Some(&partner_registry), }; match handle_publisher_request( - &state.settings, + AppContext { + settings: &state.settings, + integrations: state.registry.as_ref(), + }, &publisher_services, ec.kv_graph.as_ref(), &mut ec.ec_context, @@ -1428,8 +1431,8 @@ mod tests { use std::time::Duration; use super::{ - AppState, AuctionDispatch, EcContext, EdgeCacheHeader, HandlerFuture, NAMED_ROUTES, - NamedRouteHandler, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, TrustedServerApp, + AppContext, AppState, AuctionDispatch, EcContext, EdgeCacheHeader, HandlerFuture, + NAMED_ROUTES, NamedRouteHandler, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, TrustedServerApp, build_per_request_services, build_state_from_settings, handle_publisher_request, publisher_response_into_streaming_response, startup_error_router, }; @@ -3022,7 +3025,10 @@ mod tests { Err(report) => return Ok(super::http_error(&report)), }; let response = match handle_publisher_request( - &settings, + AppContext { + settings: &settings, + integration_registry: registry.as_ref(), + }, &services, None, &mut ec_context, diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index 8936256f88..8695cb5e0e 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -27,7 +27,7 @@ use trusted_server_core::proxy::{ handle_first_party_proxy_sign, }; use trusted_server_core::publisher::{ - AuctionDispatch, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, PublisherResponse, + AppContext, AuctionDispatch, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, PublisherResponse, buffer_publisher_response_async, handle_page_bids, handle_publisher_request, handle_tsjs_dynamic, page_bids_preflight_denied, }; @@ -795,7 +795,10 @@ fn build_router(state: &Arc) -> RouterService { registry: None, }; match handle_publisher_request( - &state.settings, + AppContext { + settings: &state.settings, + integrations: &state.registry, + }, &services, None, &mut ec_context, diff --git a/crates/trusted-server-core/src/auction/README.md b/crates/trusted-server-core/src/auction/README.md index 69c19475be..0e5ca41e06 100644 --- a/crates/trusted-server-core/src/auction/README.md +++ b/crates/trusted-server-core/src/auction/README.md @@ -304,7 +304,11 @@ let result = match (method, path.as_str()) { }, // Fallback to publisher origin - _ => handle_publisher_request(&settings, &integration_registry, &runtime_services, req), + _ => handle_publisher_request( + AppContext { settings: &settings, integrations: &integration_registry }, + &runtime_services, + req, + ), } ``` diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 0fe435f79e..f3b45b72d5 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1987,7 +1987,7 @@ fn assemble_if_shared( Ok((out, Some(AssemblyResponseState::ByteSeamFallback))) } -/// Fingerprint of every configuration input plus the compiled browser bundle. +/// Fingerprint of every configuration input plus the browser modules this registry serves. /// /// This intentionally over-invalidates. Trying to maintain a hand-written list already /// omitted publisher origin identity and creative-opportunity shaping fields. A digest of @@ -1995,17 +1995,23 @@ fn assemble_if_shared( /// safe by default: a change misses until someone proves it irrelevant, never cross-serves /// an old template under new behavior. /// +/// The module digest covers the parts this registry can actually serve, carried modules +/// included, so a vendor crate that rebuilds its browser module moves the fingerprint and +/// leaves behind the templates cached under its old `?v=` hash. It also narrows the module +/// set from every module compiled into the binary to the enabled set, which is a tightening +/// rather than a loosening, because a module no deployment serves cannot change what a +/// template renders. +/// /// # Panics /// /// Does not panic: serializing the already-deserialized typed settings to a JSON value is /// infallible for this schema. -fn template_fingerprint(settings: &Settings) -> String { +fn template_fingerprint(settings: &Settings, integration_registry: &IntegrationRegistry) -> String { use sha2::Digest as _; let mut hasher = sha2::Sha256::new(); - hasher.update( - trusted_server_js::concatenated_hash(&trusted_server_js::all_module_ids()).as_bytes(), - ); + hasher + .update(crate::tsjs_bundle::compose_hash(&integration_registry.js_parts_all()).as_bytes()); // `serde_json::Value` uses a sorted object map without `preserve_order`, making // independently deserialized HashMaps canonical before they are serialized again. let canonical = serde_json::to_value(settings) @@ -4068,6 +4074,18 @@ pub struct AuctionDispatch<'a> { pub registry: Option<&'a PartnerRegistry>, } +/// The operator configuration and the integrations built from it. +/// +/// Both are constructed once at startup and always travel together, so they +/// pass as one argument rather than two. [`AuctionDispatch`] groups the auction +/// side of the same call for the same reason. +pub struct AppContext<'a> { + /// Operator configuration for this deployment. + pub settings: &'a Settings, + /// Integrations registered for this deployment. + pub integrations: &'a IntegrationRegistry, +} + /// Proxies requests to the publisher's origin server. /// /// Returns a [`PublisherResponse`] indicating how the response should be sent: @@ -4083,7 +4101,7 @@ pub struct AuctionDispatch<'a> { /// Returns a [`TrustedServerError`] if the proxy request fails or the /// origin backend is unreachable. pub async fn handle_publisher_request( - settings: &Settings, + app: AppContext<'_>, services: &RuntimeServices, kv: Option<&KvIdentityGraph>, ec_context: &mut EcContext, @@ -4091,6 +4109,11 @@ pub async fn handle_publisher_request( mut req: Request, edge_header: EdgeCacheHeader, ) -> Result> { + let AppContext { + settings, + integrations: integration_registry, + } = app; + log::debug!("Proxying request to publisher_origin"); // Adapter fallbacks prepare this before EC/cookie handling. Keep this @@ -4512,7 +4535,7 @@ pub async fn handle_publisher_request( .map(CreativeOpportunitiesConfig::template_cache_vary) .unwrap_or_else(|| VarySpec::new([])) .values_from(req.headers()), - template_fingerprint: template_fingerprint(settings), + template_fingerprint: template_fingerprint(settings, integration_registry), schema_version: crate::platform::TEMPLATE_SCHEMA_VERSION, }); let mut template_cache_response_state = matches!(assembly_mode, AssemblyMode::Esi) @@ -6833,6 +6856,20 @@ mod tests { .page_json() } + /// Integration registry for a test's `settings`. + /// + /// Fixtures that parse `crate_test_settings_str()` directly leave + /// `proxy.allowed_domains` empty, and the Prebid integration refuses to build + /// while the external bundle host is missing from that list, so fill it in the + /// same way `create_test_settings` does. Only the allowed-domain list differs + /// from the configuration under test, and the registry reaches + /// `handle_publisher_request` here solely as a fingerprint input. + fn test_registry(settings: &Settings) -> IntegrationRegistry { + let mut settings = settings.clone(); + settings.proxy.allowed_domains = vec!["*.example".to_string(), "*.example.com".to_string()]; + IntegrationRegistry::new(&settings).expect("should create integration registry") + } + fn make_test_bid_with_creative(creative: &str) -> Bid { Bid { slot_id: "slot".to_string(), @@ -7895,8 +7932,12 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let mut ec_context = EcContext::read_from_request(settings, &req, services).expect("should read EC context"); + let registry = test_registry(settings); handle_publisher_request( - settings, + AppContext { + settings, + integrations: ®istry, + }, services, None, &mut ec_context, @@ -8163,6 +8204,75 @@ mod tests { mod template_fingerprint_tests { use super::*; + use crate::integrations::IntegrationBuilderFn; + use sha2::Digest as _; + + /// Fingerprint for a configuration, through a registry built from it. + fn fingerprint(settings: &Settings) -> String { + let registry = + IntegrationRegistry::new(settings).expect("should create integration registry"); + template_fingerprint(settings, ®istry) + } + + /// A browser module a vendor crate carries, before its rebuild. + const CARRIED_BEFORE: &str = "(function(){window.__carried=1;})();"; + + /// The same module after the vendor rebuilt it. + const CARRIED_AFTER: &str = "(function(){window.__carried=2;})();"; + + /// Hex SHA-256 of `source`, leaked so it can be stated as the + /// `&'static str` [`CarriedJsModule`] declares. Two short strings for the + /// life of the test binary. + fn leaked_hash(source: &str) -> &'static str { + Box::leak(hex::encode(sha2::Sha256::digest(source.as_bytes())).into_boxed_str()) + } + + /// Registration carrying [`CARRIED_BEFORE`]. A `fn` pointer cannot + /// capture, so the two sources need a function each. + fn carrying_before( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("probe") + .with_js_module(CarriedJsModule { + source: CARRIED_BEFORE, + sha256: leaked_hash(CARRIED_BEFORE), + }) + .build(), + )) + } + + /// Registration carrying [`CARRIED_AFTER`], the rebuilt module. + fn carrying_after( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("probe") + .with_js_module(CarriedJsModule { + source: CARRIED_AFTER, + sha256: leaked_hash(CARRIED_AFTER), + }) + .build(), + )) + } + + /// Fingerprint for `settings` through a registry whose `probe` + /// integration carries the module `registration` supplies. + fn fingerprint_with_carried( + settings: &Settings, + registration: IntegrationBuilderFn, + ) -> String { + let extra = [IntegrationBuilder::new( + "probe", + "fingerprint-probe", + registration, + validate_nothing, + )]; + let registry = IntegrationRegistry::with_registrations(settings, &extra) + .expect("should build a registry with a carried module"); + template_fingerprint(settings, ®istry) + } + /// Base settings with one integration's config replaced. /// /// Edits the parsed `[integrations]` map rather than appending TOML, so the two @@ -8189,8 +8299,8 @@ mod tests { // integration off changed the injected `", diff --git a/crates/trusted-server-core/src/integrations/mod.rs b/crates/trusted-server-core/src/integrations/mod.rs index 36876786a8..056d674b46 100644 --- a/crates/trusted-server-core/src/integrations/mod.rs +++ b/crates/trusted-server-core/src/integrations/mod.rs @@ -25,11 +25,11 @@ pub mod osano; pub mod permutive; pub mod prebid; mod registry; -#[cfg(test)] -pub(crate) use registry::test_support as registry_test_support; pub mod sourcepoint; pub mod testlight; +#[cfg(test)] +pub(crate) use registry::test_support as registry_test_support; pub use registry::{ AttributeRewriteAction, AttributeRewriteOutcome, CarriedJsModule, HeaderMutation, HeaderMutationMode, IntegrationAttributeContext, IntegrationAttributeRewriter, diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index f3b45b72d5..458b598b1e 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1996,11 +1996,11 @@ fn assemble_if_shared( /// an old template under new behavior. /// /// The module digest covers the parts this registry can actually serve, carried modules -/// included, so a vendor crate that rebuilds its browser module moves the fingerprint and -/// leaves behind the templates cached under its old `?v=` hash. It also narrows the module -/// set from every module compiled into the binary to the enabled set, which is a tightening -/// rather than a loosening, because a module no deployment serves cannot change what a -/// template renders. +/// included, so a vendor crate that rebuilds its browser module moves the fingerprint, and +/// templates keyed under the previous fingerprint are no longer read. It also narrows the +/// module set from every module compiled into the binary to the ones this deployment can +/// serve. That is safe because a module this deployment never serves appears in no bundle +/// hash and no injected URL, so it cannot change what a template renders. /// /// # Panics /// @@ -4083,7 +4083,7 @@ pub struct AppContext<'a> { /// Operator configuration for this deployment. pub settings: &'a Settings, /// Integrations registered for this deployment. - pub integrations: &'a IntegrationRegistry, + pub integration_registry: &'a IntegrationRegistry, } /// Proxies requests to the publisher's origin server. @@ -4111,7 +4111,7 @@ pub async fn handle_publisher_request( ) -> Result> { let AppContext { settings, - integrations: integration_registry, + integration_registry, } = app; log::debug!("Proxying request to publisher_origin"); @@ -6856,18 +6856,24 @@ mod tests { .page_json() } - /// Integration registry for a test's `settings`. + /// Settings parsed from `toml`, with the allowed-domain list + /// [`create_test_settings`] sets. /// - /// Fixtures that parse `crate_test_settings_str()` directly leave - /// `proxy.allowed_domains` empty, and the Prebid integration refuses to build - /// while the external bundle host is missing from that list, so fill it in the - /// same way `create_test_settings` does. Only the allowed-domain list differs - /// from the configuration under test, and the registry reaches - /// `handle_publisher_request` here solely as a fingerprint input. + /// The Prebid integration refuses to build while the external bundle host of + /// the shared fixture is missing from `proxy.allowed_domains`, so a fixture + /// that extends `crate_test_settings_str()` with its own sections and parses + /// the result itself has to carry that list over. Otherwise it describes a + /// configuration no deployment could run, and no registry could be built from + /// it. + fn settings_from_toml(toml: &str) -> Settings { + let mut settings = Settings::from_toml(toml).expect("should parse test settings"); + settings.proxy.allowed_domains = create_test_settings().proxy.allowed_domains; + settings + } + + /// Integration registry for a test's `settings`. fn test_registry(settings: &Settings) -> IntegrationRegistry { - let mut settings = settings.clone(); - settings.proxy.allowed_domains = vec!["*.example".to_string(), "*.example.com".to_string()]; - IntegrationRegistry::new(&settings).expect("should create integration registry") + IntegrationRegistry::new(settings).expect("should create integration registry") } fn make_test_bid_with_creative(creative: &str) -> Bid { @@ -7936,7 +7942,7 @@ mod tests { handle_publisher_request( AppContext { settings, - integrations: ®istry, + integration_registry: ®istry, }, services, None, @@ -9276,7 +9282,7 @@ mod tests { let publisher_response = handle_publisher_request( AppContext { settings, - integrations: ®istry, + integration_registry: ®istry, }, services, None, @@ -10717,7 +10723,7 @@ mod tests { let publisher_response = handle_publisher_request( AppContext { settings: &settings, - integrations: ®istry, + integration_registry: ®istry, }, &services, None, @@ -10888,7 +10894,7 @@ mod tests { let publisher_response = handle_publisher_request( AppContext { settings: &settings, - integrations: ®istry, + integration_registry: ®istry, }, &services, None, @@ -11957,7 +11963,7 @@ mod tests { let _ = handle_publisher_request( AppContext { settings: &settings, - integrations: ®istry, + integration_registry: ®istry, }, &services, None, @@ -12998,7 +13004,7 @@ mod tests { #[tokio::test] async fn publisher_asset_cache_policy_applies_to_non_html_response() { - let settings = Settings::from_toml(&format!( + let settings = settings_from_toml(&format!( r#"{} [[cache.asset_rules]] @@ -13012,8 +13018,7 @@ mod tests { immutable = true "#, crate_test_settings_str() - )) - .expect("should parse settings with cache rule"); + )); let stub = Arc::new(StubHttpClient::new()); stub.push_response_with_headers( 200, @@ -13058,7 +13063,7 @@ mod tests { #[tokio::test] async fn publisher_asset_policy_response_with_cookie_is_private_after_finalization() { - let settings = Settings::from_toml(&format!( + let settings = settings_from_toml(&format!( r#"{} [[cache.asset_rules]] @@ -13072,8 +13077,7 @@ mod tests { immutable = true "#, crate_test_settings_str() - )) - .expect("should parse settings with cache rule"); + )); let stub = Arc::new(StubHttpClient::new()); stub.push_response_with_headers( 200, @@ -13119,7 +13123,7 @@ mod tests { #[tokio::test] async fn publisher_asset_cache_policy_skips_html_response() { - let settings = Settings::from_toml(&format!( + let settings = settings_from_toml(&format!( r#"{} [[cache.asset_rules]] @@ -13133,8 +13137,7 @@ mod tests { fingerprint_style = "hex" "#, crate_test_settings_str() - )) - .expect("should parse settings with cache rule"); + )); let stub = Arc::new(StubHttpClient::new()); stub.push_response_with_headers( 200, @@ -13328,8 +13331,7 @@ mod tests { [creative_opportunities]\ngam_network_id = \"12345\"\n", crate_test_settings_str() ); - Settings::from_toml(&toml) - .expect("should parse settings with auction and creative opportunities enabled") + settings_from_toml(&toml) } fn settings_with_disabled_ad_templates() -> Settings { @@ -13338,7 +13340,7 @@ mod tests { [creative_opportunities]\nenabled = false\ngam_network_id = \"12345\"\n", crate_test_settings_str() ); - Settings::from_toml(&toml).expect("should parse settings with disabled ad templates") + settings_from_toml(&toml) } fn settings_with_disabled_auction() -> Settings { @@ -13347,12 +13349,11 @@ mod tests { [creative_opportunities]\ngam_network_id = \"12345\"\n", crate_test_settings_str() ); - Settings::from_toml(&toml).expect("should parse settings with disabled auction") + settings_from_toml(&toml) } fn settings_without_creative_opportunities() -> Settings { - Settings::from_toml(&crate_test_settings_str()) - .expect("should parse settings without creative opportunities") + settings_from_toml(&crate_test_settings_str()) } fn settings_with_dispatching_provider() -> Settings { @@ -13361,8 +13362,7 @@ mod tests { [creative_opportunities]\ngam_network_id = \"12345\"\n", crate_test_settings_str() ); - Settings::from_toml(&toml) - .expect("should parse settings with the dispatching test provider") + settings_from_toml(&toml) } fn services_with_telemetry( @@ -13509,7 +13509,7 @@ mod tests { handle_publisher_request( AppContext { settings, - integrations: ®istry, + integration_registry: ®istry, }, services, None, @@ -14631,7 +14631,7 @@ mod tests { let _ = handle_publisher_request( AppContext { settings: &settings, - integrations: ®istry, + integration_registry: ®istry, }, &services, None, @@ -16059,13 +16059,27 @@ mod tests { ); } + fn carried_standalone_probe_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("probe") + .with_js_module(CarriedJsModule { + source: PROBE_JS, + sha256: PROBE_JS_SHA256, + }) + .with_standalone_js() + .build(), + )) + } + #[test] fn tsjs_dynamic_serves_a_standalone_module_by_registration_flag_not_by_name() { + // The module the registration marks standalone is one a vendor crate + // carries, so nothing in the binary knows its name. A branch that named + // an integration instead of reading the flag could not serve it, and a + // reintroduced constant would fail this test rather than pass it. let mut settings = create_test_settings(); - settings - .integrations - .insert_config("gpt_diagnostics", &serde_json::json!({ "enabled": true })) - .expect("should insert gpt_diagnostics config"); settings .integrations .insert_config( @@ -16073,17 +16087,29 @@ mod tests { &serde_json::json!({ "enabled": true, "app_id": "test-app-id" }), ) .expect("should insert lockr config"); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); + let extra = [IntegrationBuilder::new( + "probe", + "standalone-probe", + carried_standalone_probe_registration, + validate_nothing, + )]; + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should build a registry with a carried standalone module"); assert!( registry.js_module_ids_immediate().contains(&"lockr"), "fixture should put lockr in the unified bundle" ); + assert!( + !registry.js_module_ids().contains(&"probe"), + "a standalone module should stay out of the bundled module ids" + ); let standalone = handle_tsjs_dynamic( &build_request( Method::GET, - "https://publisher.example/static/tsjs=tsjs-gpt_diagnostics.min.js", + &format!( + "https://publisher.example/static/tsjs=tsjs-probe.min.js?v={PROBE_JS_SHA256}" + ), ), ®istry, EdgeCacheHeader::SMaxageFallback, @@ -16094,6 +16120,29 @@ mod tests { StatusCode::OK, "should serve a module its registration marks standalone" ); + assert!( + cache_control_text(&standalone).contains("immutable"), + "should treat the carried module's own hash as the matching version" + ); + assert_eq!( + body_text(standalone), + PROBE_JS, + "should serve the standalone module as its own file" + ); + + let unified = handle_tsjs_dynamic( + &build_request( + Method::GET, + "https://publisher.example/static/tsjs=tsjs-unified.min.js", + ), + ®istry, + EdgeCacheHeader::SMaxageFallback, + ) + .expect("should handle tsjs request"); + assert!( + !body_text(unified).contains(PROBE_JS), + "a standalone module should stay out of the unified bundle" + ); let bundled_only = handle_tsjs_dynamic( &build_request( @@ -20921,7 +20970,7 @@ mod tests { [creative_opportunities]\ngam_network_id = \"12345\"\n", crate_test_settings_str() ); - Settings::from_toml(&toml).expect("should parse settings with a capturing provider") + settings_from_toml(&toml) } fn article_slot() -> Vec { @@ -21094,7 +21143,7 @@ mod tests { let _ = handle_publisher_request( AppContext { settings: &settings, - integrations: ®istry, + integration_registry: ®istry, }, &services, None, @@ -21182,7 +21231,7 @@ mod tests { let _ = handle_publisher_request( AppContext { settings: &settings, - integrations: ®istry, + integration_registry: ®istry, }, &services, None, From 9c141e265d68a33eb8cedd9faa53caed7b8e35e5 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 28 Aug 2026 18:22:27 +0100 Subject: [PATCH 010/124] Read a neutral personalized-response marker instead of DataDome's Core decided two things about a response by reading a DataDome-named type out of the request extensions, being whether the request must go to origin rather than be served from a shared template, and whether the response must be privately cached rather than shared. Neither decision is about DataDome, so neither should name a vendor. `response_privacy` now carries a `PersonalizedResponse` request marker that any integration may set, and `publisher` reads it through a `request_requires_personalized_delivery` helper. The derived locals and `apply_datadome_client_tag_cache_privacy` are renamed to match, with the behavior unchanged, including the HTML-document gate on buffering the full body. DataDome's protection filter sets both markers wherever it used to set one, so the running behavior is the same. The tag-suppression flag itself stays DataDome's own and is still read from the request at the one site that builds `HtmlProcessorConfig`, because it drives DataDome's head injection and moves out of core with the DataDome integration. Four existing tests inserted only the DataDome marker and relied on the fused behavior. They now insert both markers exactly as the production filter does. Three of them fail without the neutral marker, which was checked by removing it. --- .../src/integrations/datadome/protection.rs | 15 ++ crates/trusted-server-core/src/publisher.rs | 142 +++++++++++++++--- .../src/response_privacy.rs | 10 ++ 3 files changed, 147 insertions(+), 20 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/datadome/protection.rs b/crates/trusted-server-core/src/integrations/datadome/protection.rs index 291d834106..aa5706a3ef 100644 --- a/crates/trusted-server-core/src/integrations/datadome/protection.rs +++ b/crates/trusted-server-core/src/integrations/datadome/protection.rs @@ -46,10 +46,17 @@ impl DataDomeIntegration { let test_bypass_matched = self.take_protection_test_bypass_header(input.request, input.services); if test_bypass_matched { + // The first marker is DataDome's own tag-suppression signal, read by its + // head injector. The second tells core the response is personalized to + // this request and cannot be shared through a cache or a template. input .request .extensions_mut() .insert(super::DataDomeClientTagSuppressed); + input + .request + .extensions_mut() + .insert(crate::response_privacy::PersonalizedResponse); log_protection_test_bypass(&input); return RequestFilterDecision::Continue(RequestFilterEffects::default()); } @@ -162,10 +169,18 @@ impl DataDomeIntegration { suppress_client_tag, } => { if suppress_client_tag { + // The first marker is DataDome's own tag-suppression signal, read + // by its head injector. The second tells core the response is + // personalized to this request and cannot be shared through a + // cache or a template. input .request .extensions_mut() .insert(super::DataDomeClientTagSuppressed); + input + .request + .extensions_mut() + .insert(crate::response_privacy::PersonalizedResponse); } log_protection_skip(input, &rule_id, reason, suppress_client_tag); return false; diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 458b598b1e..123394d1a9 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -2678,6 +2678,13 @@ fn is_html_document_request(req: &Request) -> bool { is_navigation_request(req) } +/// Whether an integration marked this request's response as personalized. +fn request_requires_personalized_delivery(req: &Request) -> bool { + req.extensions() + .get::() + .is_some() +} + /// Removes request headers that can produce a bodyless or partial origin response. fn strip_conditional_and_range_headers(req: &mut Request) { req.headers_mut().remove(header::IF_NONE_MATCH); @@ -2701,14 +2708,14 @@ fn response_carries_body(method: &Method, status: StatusCode) -> bool { && status != StatusCode::NOT_MODIFIED } -/// Prevent shared caches from replaying tag-suppressed HTML to other clients. -fn apply_datadome_client_tag_cache_privacy( +/// Prevent shared caches from replaying personalized HTML to other clients. +fn apply_personalized_response_cache_privacy( response: &mut Response, method: &Method, - suppress_datadome_client_side_tag: bool, + response_is_personalized: bool, content_type: &str, ) { - if suppress_datadome_client_side_tag + if response_is_personalized && response_carries_body(method, response.status()) && is_html_content_type(content_type) { @@ -4449,19 +4456,25 @@ pub async fn handle_publisher_request( .creative_opportunities .as_ref() .is_some_and(CreativeOpportunitiesConfig::origin_is_cookie_independent); + let response_is_personalized = request_requires_personalized_delivery(&req); + // A personalized response is request-scoped (for example, an IP exclusion), while a + // template cache template is shared across readers. A shared template can represent + // neither the personalized nor the ordinary variant safely for the other population. + let personalization_requires_origin = response_is_personalized; + let personalization_requires_full_body = + response_is_personalized && is_html_document_request(&req); + let request_requires_origin = request_bypasses_template_cache(req.headers()) + || gpt_diagnostics.requires_private_no_store() + || personalization_requires_origin; + // The only vendor-named signal core still reads. It decides nothing about caching + // or the origin path, because it travels to `HtmlProcessorConfig` so DataDome's own + // head injection can leave its client-side tag out, and it moves out of core with + // the DataDome integration. Read from the request rather than derived from the + // neutral personalization marker above, so the two concerns stay separate. let suppress_datadome_client_side_tag = req .extensions() .get::() .is_some(); - // Tag suppression is request-scoped (for example, an IP exclusion), while a template cache - // template is shared across readers. A shared template can represent neither the - // suppressed nor unsuppressed variant safely for the other population. - let datadome_suppression_requires_origin = suppress_datadome_client_side_tag; - let datadome_suppression_requires_full_body = - suppress_datadome_client_side_tag && is_html_document_request(&req); - let request_requires_origin = request_bypasses_template_cache(req.headers()) - || gpt_diagnostics.requires_private_no_store() - || datadome_suppression_requires_origin; let reader_compression = negotiate_reader_compression(req.headers()); let reader_supports_assembly = reader_compression.is_ok(); // A failed negotiation bypasses template cache below, so this value is used only on an @@ -4469,7 +4482,7 @@ pub async fn handle_publisher_request( // a panic-prone invariant in the public request handler. let reader_compression = reader_compression.unwrap_or(Compression::None); - if should_run_ad_stack || datadome_suppression_requires_full_body { + if should_run_ad_stack || personalization_requires_full_body { // HTML document contexts whose output may be synthesized must not // receive a cached 304 or partial 206. Non-document subresources contain // no executable injected tag, so retain their validators and ranges. @@ -4850,10 +4863,10 @@ pub async fn handle_publisher_request( } } } - apply_datadome_client_tag_cache_privacy( + apply_personalized_response_cache_privacy( &mut response, &request_method, - suppress_datadome_client_side_tag, + response_is_personalized, &origin_content_type, ); apply_publisher_asset_cache_policy( @@ -11280,6 +11293,9 @@ mod tests { suppressed_request .extensions_mut() .insert(crate::integrations::datadome::DataDomeClientTagSuppressed); + suppressed_request + .extensions_mut() + .insert(crate::response_privacy::PersonalizedResponse); let suppressed = run(&settings, &services, suppressed_request).await; assert_eq!( suppressed @@ -14413,6 +14429,8 @@ mod tests { .expect("should build conditional request"); req.extensions_mut() .insert(crate::integrations::datadome::DataDomeClientTagSuppressed); + req.extensions_mut() + .insert(crate::response_privacy::PersonalizedResponse); let _response = run_publisher_proxy(&settings, &services, req).await; @@ -14461,6 +14479,8 @@ mod tests { .expect("should build conditional iframe request"); req.extensions_mut() .insert(crate::integrations::datadome::DataDomeClientTagSuppressed); + req.extensions_mut() + .insert(crate::response_privacy::PersonalizedResponse); let _response = run_publisher_proxy(&settings, &services, req).await; @@ -14509,6 +14529,8 @@ mod tests { .expect("should build conditional subresource request"); req.extensions_mut() .insert(crate::integrations::datadome::DataDomeClientTagSuppressed); + req.extensions_mut() + .insert(crate::response_privacy::PersonalizedResponse); let _response = run_publisher_proxy(&settings, &services, req).await; @@ -14780,7 +14802,7 @@ mod tests { .body(EdgeBody::empty()) .expect("should build cacheable HTML response"); - super::apply_datadome_client_tag_cache_privacy( + super::apply_personalized_response_cache_privacy( &mut response, &Method::GET, true, @@ -14826,7 +14848,7 @@ mod tests { .header(header::CACHE_CONTROL, "no-store") .body(EdgeBody::empty()) .expect("should build no-store HTML response"); - super::apply_datadome_client_tag_cache_privacy( + super::apply_personalized_response_cache_privacy( &mut no_store_response, &Method::GET, true, @@ -14851,7 +14873,7 @@ mod tests { .body(EdgeBody::empty()) .expect("should build cacheable response"); - super::apply_datadome_client_tag_cache_privacy( + super::apply_personalized_response_cache_privacy( &mut response, &Method::GET, false, @@ -14866,7 +14888,7 @@ mod tests { "unsuppressed HTML should retain its existing cache policy" ); - super::apply_datadome_client_tag_cache_privacy( + super::apply_personalized_response_cache_privacy( &mut response, &Method::GET, true, @@ -14882,6 +14904,86 @@ mod tests { ); } + #[test] + fn a_personalized_response_marker_from_any_integration_forces_private_caching() { + let mut response = Response::builder() + .status(StatusCode::OK) + .header(header::CACHE_CONTROL, "public, max-age=600") + .header("surrogate-control", "max-age=600") + .header("fastly-surrogate-control", "max-age=600") + .header("cloudflare-cdn-cache-control", "max-age=600") + .header("cdn-cache-control", "max-age=600") + .header(header::ETAG, "\"origin-tag\"") + .header(header::LAST_MODIFIED, "Wed, 21 Oct 2015 07:28:00 GMT") + .body(EdgeBody::empty()) + .expect("should build cacheable HTML response"); + + super::apply_personalized_response_cache_privacy( + &mut response, + &Method::GET, + true, + "text/html; charset=utf-8", + ); + + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("no-store, private"), + "a personalized response should be private and non-storable" + ); + for header_name in [ + "surrogate-control", + "fastly-surrogate-control", + "cloudflare-cdn-cache-control", + "cdn-cache-control", + ] { + assert!( + response.headers().get(header_name).is_none(), + "a personalized response should not retain {header_name}" + ); + } + for header_name in [header::ETAG, header::LAST_MODIFIED] { + assert!( + !response.headers().contains_key(&header_name), + "a personalized response should not retain {header_name}" + ); + } + } + + #[test] + fn core_reads_the_neutral_marker_not_the_datadome_type() { + let mut personalized = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/page") + .header(header::HOST, "publisher.example") + .header("sec-fetch-dest", "document") + .body(EdgeBody::empty()) + .expect("should build personalized request"); + personalized + .extensions_mut() + .insert(crate::response_privacy::PersonalizedResponse); + + assert!( + super::request_requires_personalized_delivery(&personalized), + "the neutral marker alone should mark a response personalized" + ); + + let ordinary = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/page") + .header(header::HOST, "publisher.example") + .header("sec-fetch-dest", "document") + .body(EdgeBody::empty()) + .expect("should build ordinary request"); + + assert!( + !super::request_requires_personalized_delivery(&ordinary), + "a request carrying no marker should not be personalized" + ); + } + #[test] fn response_carries_body_preserves_bodiless_metadata() { // A processable GET 200 buffers a body and recomputes Content-Length. diff --git a/crates/trusted-server-core/src/response_privacy.rs b/crates/trusted-server-core/src/response_privacy.rs index bfdb97f864..97cb6598e4 100644 --- a/crates/trusted-server-core/src/response_privacy.rs +++ b/crates/trusted-server-core/src/response_privacy.rs @@ -24,6 +24,16 @@ use crate::settings::Settings; #[derive(Clone, Copy, Debug)] pub struct TerminalPrivateResponse; +/// Request marker meaning the response is personalized to this request and +/// must not be shared through any cache or template. +/// +/// Any integration may set it from a request filter. Core buffers the full +/// document, keeps the request on the origin path rather than a shared +/// template, and enforces private caching whenever it is present, without +/// knowing which integration asked. +#[derive(Debug, Clone, Copy)] +pub struct PersonalizedResponse; + const INACTIVE_AD_STACK_BROWSER_CACHE_CONTROL: &str = "private, max-age=60"; #[derive(Debug, Clone, Copy)] From 8373d72c3d1e55551aaf10b39b9877f462fffd8e Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 28 Aug 2026 18:42:06 +0100 Subject: [PATCH 011/124] Open the auction orchestrator to externally supplied providers `auction/mod.rs` carried a private `ProviderBuilder` fn-pointer alias and a closed `provider_builders()` table, so an auction provider could only be contributed by a module inside `trusted-server-core`. Replace both with a public `AuctionProviderBuilder` that mirrors `IntegrationBuilder`: a name, a source label for diagnostics, a build function and a validate function, with `pub(crate)` `build` and `validate` accessors. The three built-ins (prebid, aps, adserver_mock) move into `BUILT_IN_PROVIDER_BUILDERS` in the same order, `provider_builders()` returns that constant, and `all_provider_builders(extra)` chains the built-ins then the extras. `build_orchestrator` now delegates to a new public `build_orchestrator_with_providers(settings, extra)`, which is the seam an adapter or a vendor crate registers through. `AuctionOrchestrator::register_provider` silently overwrote a provider that shared a name with one already registered, which would let a vendor crate displace a built-in without any signal. It now takes the registering source, returns `Result`, and rejects a duplicate name in the same wording the integration registry uses for a duplicate id, naming both sources. A `provider_sources` field keeps insertion order so the first source can be named. Every caller passes a source; the test callers add `.expect("should register")`. `adserver_mock` had no `validate` function because, unlike the other vendor modules, it is an auction provider only and never an integration builder. Add one that reproduces what `config.rs` does for it today, so its auction builder can carry it. `AuctionProviderBuilder::validate` keeps a temporary `#[allow(dead_code)]` until deploy validation calls it in the next commit. --- .../trusted-server-core/src/auction/README.md | 9 +- .../src/auction/endpoints.rs | 29 +- crates/trusted-server-core/src/auction/mod.rs | 267 +++++++++++- .../src/auction/orchestrator.rs | 382 +++++++++++++----- .../src/auction/test_support.rs | 59 ++- .../src/integrations/adserver_mock.rs | 18 +- crates/trusted-server-core/src/publisher.rs | 38 +- 7 files changed, 654 insertions(+), 148 deletions(-) diff --git a/crates/trusted-server-core/src/auction/README.md b/crates/trusted-server-core/src/auction/README.md index 435bd832f5..fd62bcac0c 100644 --- a/crates/trusted-server-core/src/auction/README.md +++ b/crates/trusted-server-core/src/auction/README.md @@ -556,9 +556,12 @@ Set `mock = false` in APS config when real APS integration is ready. ### Example Test Flow ```rust -let orchestrator = AuctionOrchestrator::new(config); -orchestrator.register_provider(Arc::new(PrebidAuctionProvider::try_new(prebid_config)?)); -orchestrator.register_provider(Arc::new(ApsAuctionProvider::new(aps_config))); +let mut orchestrator = AuctionOrchestrator::new(config); +orchestrator.register_provider( + Arc::new(PrebidAuctionProvider::try_new(prebid_config)?), + "example-crate", +)?; +orchestrator.register_provider(Arc::new(ApsAuctionProvider::new(aps_config)), "example-crate")?; let result = orchestrator.run_auction(&request, &context, &services).await?; diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index 86bd7b981d..d821eb926a 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -728,9 +728,14 @@ mod tests { .expect("should parse settings with disabled templates"); let calls = Arc::new(Mutex::new(0)); let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - orchestrator.register_provider(Arc::new(TemplateSwitchProbeProvider { - calls: Arc::clone(&calls), - })); + orchestrator + .register_provider( + Arc::new(TemplateSwitchProbeProvider { + calls: Arc::clone(&calls), + }), + crate::integrations::CORE_SOURCE, + ) + .expect("should register"); let stub = Arc::new(StubHttpClient::new()); stub.push_response(200, b"probe response".to_vec()); @@ -797,7 +802,12 @@ mod tests { ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); - orchestrator.register_provider(Arc::new(PanicOnBidProvider)); + orchestrator + .register_provider( + Arc::new(PanicOnBidProvider), + crate::integrations::CORE_SOURCE, + ) + .expect("should register"); let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); let services = services_with_telemetry(Arc::clone(&telemetry_sink)); let ec_id = format!("{}.ABC123", "a".repeat(64)); @@ -928,9 +938,14 @@ mod tests { }; let mut orchestrator = AuctionOrchestrator::new(config); let had_eids = Arc::new(std::sync::Mutex::new(None)); - orchestrator.register_provider(Arc::new(EidCapturingProvider { - had_eids: Arc::clone(&had_eids), - })); + orchestrator + .register_provider( + Arc::new(EidCapturingProvider { + had_eids: Arc::clone(&had_eids), + }), + crate::integrations::CORE_SOURCE, + ) + .expect("should register"); let services = noop_services(); // US-state jurisdiction with an explicit GPC opt-out: auction allowed, diff --git a/crates/trusted-server-core/src/auction/mod.rs b/crates/trusted-server-core/src/auction/mod.rs index 986beb984f..669c2a329c 100644 --- a/crates/trusted-server-core/src/auction/mod.rs +++ b/crates/trusted-server-core/src/auction/mod.rs @@ -37,27 +37,160 @@ pub use types::{ AdFormat, AuctionContext, AuctionRequest, AuctionResponse, Bid, BidStatus, MediaType, }; -/// Type alias for provider builder functions. -type ProviderBuilder = +/// Builds the auction providers an integration contributes from settings. +pub type AuctionProviderBuilderFn = fn(&Settings) -> Result>, Report>; -/// Returns the list of all available provider builder functions. +/// Validates the provider's configuration for deployment and reports whether +/// it is enabled. /// -/// This list is used to auto-discover and register auction providers from settings. -/// Each builder function checks the settings for its specific provider configuration -/// and returns any enabled providers. -fn provider_builders() -> &'static [ProviderBuilder] { - &[ +/// Runs for every builder, enabled or not, so a typo in a disabled block is +/// still caught. +pub type AuctionProviderValidateFn = fn(&Settings) -> Result>; + +/// A named factory for one auction provider, the unit an adapter or a vendor +/// crate hands to [`build_orchestrator_with_providers`]. +/// +/// # Examples +/// +/// ``` +/// use std::sync::Arc; +/// +/// use error_stack::Report; +/// use trusted_server_core::auction::{ +/// AuctionProvider, AuctionProviderBuilder, build_orchestrator_with_providers, +/// }; +/// use trusted_server_core::error::TrustedServerError; +/// use trusted_server_core::settings::Settings; +/// +/// fn build( +/// _settings: &Settings, +/// ) -> Result>, Report> { +/// Ok(Vec::new()) +/// } +/// +/// fn validate(_settings: &Settings) -> Result> { +/// Ok(false) +/// } +/// +/// # fn demo(settings: &Settings) -> Result<(), Report> { +/// let builder = AuctionProviderBuilder::new("example", "example-crate", build, validate); +/// let orchestrator = build_orchestrator_with_providers(settings, &[builder])?; +/// assert_eq!(orchestrator.provider_count(), 0); +/// # Ok(()) +/// # } +/// ``` +#[derive(Clone, Copy, Debug)] +pub struct AuctionProviderBuilder { + name: &'static str, + source: &'static str, + build: AuctionProviderBuilderFn, + validate: AuctionProviderValidateFn, +} + +impl AuctionProviderBuilder { + /// Creates a builder for the auction provider `name`, attributed to + /// `source` (a crate or package name used in duplicate-name errors). + #[must_use] + pub const fn new( + name: &'static str, + source: &'static str, + build: AuctionProviderBuilderFn, + validate: AuctionProviderValidateFn, + ) -> Self { + Self { + name, + source, + build, + validate, + } + } + + /// The auction provider name this builder produces. + #[must_use] + pub const fn name(&self) -> &'static str { + self.name + } + + /// The source label used in diagnostics. + #[must_use] + pub const fn source(&self) -> &'static str { + self.source + } + + /// Builds the providers this builder contributes, empty when the provider + /// is not enabled. + /// + /// # Errors + /// + /// Returns an error when the provider is enabled with invalid + /// configuration. + pub(crate) fn build( + &self, + settings: &Settings, + ) -> Result>, Report> { + (self.build)(settings) + } + + /// Validates the provider's configuration for deployment and reports + /// whether the provider is enabled. + /// + /// # Errors + /// + /// Returns an error when the configuration cannot be parsed or fails + /// validation. + // Deploy validation does not yet enumerate auction provider builders, so + // nothing in the crate calls this until that is wired up. + #[allow(dead_code)] + pub(crate) fn validate(&self, settings: &Settings) -> Result> { + (self.validate)(settings) + } +} + +/// The built-in auction providers, in registration order. +const BUILT_IN_PROVIDER_BUILDERS: &[AuctionProviderBuilder] = &[ + AuctionProviderBuilder::new( + "prebid", + crate::integrations::CORE_SOURCE, crate::integrations::prebid::register_auction_provider, + crate::integrations::prebid::validate, + ), + AuctionProviderBuilder::new( + "aps", + crate::integrations::CORE_SOURCE, crate::integrations::aps::register_providers, + crate::integrations::aps::validate, + ), + AuctionProviderBuilder::new( + "adserver_mock", + crate::integrations::CORE_SOURCE, crate::integrations::adserver_mock::register_providers, - ] + crate::integrations::adserver_mock::validate, + ), +]; + +/// The built-in auction provider builders, in registration order. +pub(crate) fn provider_builders() -> &'static [AuctionProviderBuilder] { + BUILT_IN_PROVIDER_BUILDERS +} + +/// Every auction provider builder the orchestrator will consider: the built-in +/// set followed by `extra`, in that order, so registration order for the +/// built-ins never changes. +pub(crate) fn all_provider_builders( + extra: &[AuctionProviderBuilder], +) -> impl Iterator + '_ { + provider_builders() + .iter() + .copied() + .chain(extra.iter().copied()) } /// Build a new auction orchestrator for the current settings. /// -/// This constructor registers all auction providers discovered from the provided settings. -/// Callers can reuse the returned [`AuctionOrchestrator`] across requests. +/// This constructor registers all built-in auction providers discovered from +/// the provided settings. Callers can reuse the returned +/// [`AuctionOrchestrator`] across requests. /// /// # Arguments /// * `settings` - Application settings used to configure the orchestrator and providers @@ -67,15 +200,34 @@ fn provider_builders() -> &'static [ProviderBuilder] { /// Returns an error when an enabled auction provider has invalid configuration. pub fn build_orchestrator( settings: &Settings, +) -> Result> { + build_orchestrator_with_providers(settings, &[]) +} + +/// Build a new auction orchestrator from the built-in provider builders +/// followed by the externally supplied builders an adapter registers. +/// +/// # Arguments +/// * `settings` - Application settings used to configure the orchestrator and providers +/// * `extra` - Provider builders supplied by an adapter or a vendor crate +/// +/// # Errors +/// +/// Returns an error when an enabled auction provider has invalid +/// configuration, when two builders produce the same provider name, or when a +/// configured provider name has no registered provider. +pub fn build_orchestrator_with_providers( + settings: &Settings, + extra: &[AuctionProviderBuilder], ) -> Result> { log::info!("Building auction orchestrator"); let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); // Auto-discover and register all auction providers from settings - for builder in provider_builders() { - for provider in builder(settings)? { - orchestrator.register_provider(provider); + for builder in all_provider_builders(extra) { + for provider in builder.build(settings)? { + orchestrator.register_provider(provider, builder.source())?; } } @@ -94,7 +246,16 @@ mod tests { use crate::settings::Settings; use crate::test_support::tests::crate_test_settings_str; - use super::build_orchestrator; + use std::sync::Arc; + + use error_stack::Report; + + use super::test_support::NamedTestProvider; + use super::{ + AuctionProvider, AuctionProviderBuilder, build_orchestrator, + build_orchestrator_with_providers, + }; + use crate::error::TrustedServerError; fn settings_with_auction_config(auction_config: &str) -> Settings { let settings_str = format!("{}\n{auction_config}", crate_test_settings_str()); @@ -165,4 +326,80 @@ mod tests { "Auction provider `missing-mediator` is listed in [auction] but no enabled integration provides it", ); } + + /// Builds one provider named `seam-probe`, standing in for a provider a + /// vendor crate contributes. + fn build_probe_provider( + _settings: &Settings, + ) -> Result>, Report> { + Ok(vec![Arc::new(NamedTestProvider::new("seam-probe"))]) + } + + /// Builds one provider that claims the built-in `prebid` name. + fn build_conflicting_prebid_provider( + _settings: &Settings, + ) -> Result>, Report> { + Ok(vec![Arc::new(NamedTestProvider::new("prebid"))]) + } + + fn probe_enabled(_settings: &Settings) -> Result> { + Ok(true) + } + + #[test] + fn build_orchestrator_with_providers_registers_an_external_provider() { + let settings = settings_with_auction_config( + r#" + [auction] + enabled = true + providers = ["prebid", "seam-probe"] + timeout_ms = 2000 + "#, + ); + let extra = [AuctionProviderBuilder::new( + "seam-probe", + "seam-probe-crate", + build_probe_provider, + probe_enabled, + )]; + + let orchestrator = build_orchestrator_with_providers(&settings, &extra) + .expect("should register the external auction provider"); + + assert_eq!( + orchestrator.provider_count(), + 2, + "should register the built-in prebid provider and the external provider" + ); + } + + #[test] + fn build_orchestrator_with_providers_rejects_a_duplicate_provider_name() { + let settings = settings_with_auction_config( + r#" + [auction] + enabled = true + providers = ["prebid"] + timeout_ms = 2000 + "#, + ); + let extra = [AuctionProviderBuilder::new( + "prebid", + "seam-probe-crate", + build_conflicting_prebid_provider, + probe_enabled, + )]; + + let error = build_orchestrator_with_providers(&settings, &extra) + .err() + .expect("should reject a duplicate auction provider name"); + + let message = error.to_string(); + assert!( + message.contains("prebid") + && message.contains(crate::integrations::CORE_SOURCE) + && message.contains("seam-probe-crate"), + "error should name the provider and both sources: {message}" + ); + } } diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 728cc1efe8..7932a187bf 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -205,6 +205,9 @@ fn snapshot_context_request(request: &Request) -> Request { pub struct AuctionOrchestrator { config: AuctionConfig, providers: HashMap>, + /// Provider names in registration order, each paired with the source that + /// registered it, so a duplicate can name the source that came first. + provider_sources: Vec<(String, &'static str)>, } impl AuctionOrchestrator { @@ -214,14 +217,36 @@ impl AuctionOrchestrator { Self { config, providers: HashMap::new(), + provider_sources: Vec::new(), } } - /// Register an auction provider. - pub fn register_provider(&mut self, provider: Arc) { + /// Register an auction provider attributed to `source`. + /// + /// # Errors + /// + /// Returns an error when a provider with the same name is already registered. + pub fn register_provider( + &mut self, + provider: Arc, + source: &'static str, + ) -> Result<(), Report> { let name = provider.provider_name().to_string(); - log::info!("Registering auction provider: {}", name); + if let Some((_, first_source)) = self + .provider_sources + .iter() + .find(|(registered, _)| *registered == name) + { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "auction provider `{name}` is registered twice, by `{first_source}` and by `{source}`" + ), + })); + } + log::info!("Registering auction provider: {name}"); + self.provider_sources.push((name.clone(), source)); self.providers.insert(name, provider); + Ok(()) } /// Get the number of registered providers. @@ -1971,11 +1996,21 @@ mod tests { ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); - orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "bidder", - backend: "bidder-backend", - })); - orchestrator.register_provider(Arc::new(CacheRestoringMediator)); + orchestrator + .register_provider( + Arc::new(StubAuctionProvider { + name: "bidder", + backend: "bidder-backend", + }), + crate::integrations::CORE_SOURCE, + ) + .expect("should register"); + orchestrator + .register_provider( + Arc::new(CacheRestoringMediator), + crate::integrations::CORE_SOURCE, + ) + .expect("should register"); let request = create_test_auction_request(); let settings = create_test_settings(); @@ -2027,11 +2062,21 @@ mod tests { ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); - orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "bidder", - backend: "bidder-backend", - })); - orchestrator.register_provider(Arc::new(ImmediateMediator)); + orchestrator + .register_provider( + Arc::new(StubAuctionProvider { + name: "bidder", + backend: "bidder-backend", + }), + crate::integrations::CORE_SOURCE, + ) + .expect("should register"); + orchestrator + .register_provider( + Arc::new(ImmediateMediator), + crate::integrations::CORE_SOURCE, + ) + .expect("should register"); let request = create_test_auction_request(); let settings = create_test_settings(); let downstream = http::Request::new(edgezero_core::body::Body::empty()); @@ -2215,7 +2260,12 @@ mod tests { ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); - orchestrator.register_provider(Arc::new(ImmediateNoBidProvider)); + orchestrator + .register_provider( + Arc::new(ImmediateNoBidProvider), + crate::integrations::CORE_SOURCE, + ) + .expect("should register"); let settings = create_test_settings(); let services = noop_services(); let downstream = http::Request::new(edgezero_core::body::Body::empty()); @@ -2240,7 +2290,12 @@ mod tests { ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); - orchestrator.register_provider(Arc::new(ImmediateNoBidProvider)); + orchestrator + .register_provider( + Arc::new(ImmediateNoBidProvider), + crate::integrations::CORE_SOURCE, + ) + .expect("should register"); let settings = create_test_settings(); let services = noop_services(); let downstream = http::Request::new(edgezero_core::body::Body::empty()); @@ -2271,11 +2326,21 @@ mod tests { ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); - orchestrator.register_provider(Arc::new(ImmediateNoBidProvider)); - orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "pending", - backend: "pending-backend", - })); + orchestrator + .register_provider( + Arc::new(ImmediateNoBidProvider), + crate::integrations::CORE_SOURCE, + ) + .expect("should register"); + orchestrator + .register_provider( + Arc::new(StubAuctionProvider { + name: "pending", + backend: "pending-backend", + }), + crate::integrations::CORE_SOURCE, + ) + .expect("should register"); let stub = Arc::new(StubHttpClient::new()); stub.push_response(200, b"{}".to_vec()); let services = build_services_with_http_client(stub); @@ -2528,7 +2593,12 @@ mod tests { ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); - orchestrator.register_provider(Arc::new(LaunchFailingProvider)); + orchestrator + .register_provider( + Arc::new(LaunchFailingProvider), + crate::integrations::CORE_SOURCE, + ) + .expect("should register"); let request = create_test_auction_request(); let settings = create_test_settings(); @@ -2592,14 +2662,24 @@ mod tests { ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); - orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-a", - backend: "shared-backend", - })); - orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-b", - backend: "shared-backend", - })); + orchestrator + .register_provider( + Arc::new(StubAuctionProvider { + name: "provider-a", + backend: "shared-backend", + }), + crate::integrations::CORE_SOURCE, + ) + .expect("should register"); + orchestrator + .register_provider( + Arc::new(StubAuctionProvider { + name: "provider-b", + backend: "shared-backend", + }), + crate::integrations::CORE_SOURCE, + ) + .expect("should register"); let stub = Arc::new(StubHttpClient::new()); stub.push_response(200, b"{}".to_vec()); let services = build_services_with_http_client(stub); @@ -2706,13 +2786,18 @@ mod tests { timeout_ms: 2000, ..Default::default() }); - orchestrator.register_provider(Arc::new(recording_provider( - "bidder", - "bidder-backend", - 1000, - &predicted, - &requested, - ))); + orchestrator + .register_provider( + Arc::new(recording_provider( + "bidder", + "bidder-backend", + 1000, + &predicted, + &requested, + )), + crate::integrations::CORE_SOURCE, + ) + .expect("should register"); let settings = create_test_settings(); let downstream = http::Request::new(edgezero_core::body::Body::empty()); let context = immediate_test_context(&settings, &downstream, &services); @@ -2750,13 +2835,18 @@ mod tests { timeout_ms: 2000, ..Default::default() }); - orchestrator.register_provider(Arc::new(recording_provider( - "bidder", - "bidder-backend", - 1000, - &predicted, - &requested, - ))); + orchestrator + .register_provider( + Arc::new(recording_provider( + "bidder", + "bidder-backend", + 1000, + &predicted, + &requested, + )), + crate::integrations::CORE_SOURCE, + ) + .expect("should register"); let settings = create_test_settings(); let downstream = http::Request::new(edgezero_core::body::Body::empty()); let context = immediate_test_context(&settings, &downstream, &services); @@ -2794,17 +2884,27 @@ mod tests { timeout_ms: 2000, ..Default::default() }); - orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "bidder", - backend: "bidder-backend", - })); - orchestrator.register_provider(Arc::new(recording_provider( - "mediator", - "mediator-backend", - 2000, - &predicted, - &requested, - ))); + orchestrator + .register_provider( + Arc::new(StubAuctionProvider { + name: "bidder", + backend: "bidder-backend", + }), + crate::integrations::CORE_SOURCE, + ) + .expect("should register"); + orchestrator + .register_provider( + Arc::new(recording_provider( + "mediator", + "mediator-backend", + 2000, + &predicted, + &requested, + )), + crate::integrations::CORE_SOURCE, + ) + .expect("should register"); let settings = create_test_settings(); let downstream = http::Request::new(edgezero_core::body::Body::empty()); let context = immediate_test_context(&settings, &downstream, &services); @@ -2844,20 +2944,30 @@ mod tests { timeout_ms: 2000, ..Default::default() }); - orchestrator.register_provider(Arc::new(recording_provider( - "bidder", - "bidder-backend", - 2000, - &bidder_predicted, - &bidder_requested, - ))); - orchestrator.register_provider(Arc::new(recording_provider( - "mediator", - "mediator-backend", - 2000, - &mediator_predicted, - &mediator_requested, - ))); + orchestrator + .register_provider( + Arc::new(recording_provider( + "bidder", + "bidder-backend", + 2000, + &bidder_predicted, + &bidder_requested, + )), + crate::integrations::CORE_SOURCE, + ) + .expect("should register"); + orchestrator + .register_provider( + Arc::new(recording_provider( + "mediator", + "mediator-backend", + 2000, + &mediator_predicted, + &mediator_requested, + )), + crate::integrations::CORE_SOURCE, + ) + .expect("should register"); let settings = create_test_settings(); let downstream = http::Request::new(edgezero_core::body::Body::empty()); let context = immediate_test_context(&settings, &downstream, &services); @@ -2905,11 +3015,16 @@ mod tests { timeout_ms: 2000, ..Default::default() }); - orchestrator.register_provider(Arc::new(DivergentBackendProvider { - name: "provider-a", - predicted: "predicted-backend", - resolved: "resolved-backend", - })); + orchestrator + .register_provider( + Arc::new(DivergentBackendProvider { + name: "provider-a", + predicted: "predicted-backend", + resolved: "resolved-backend", + }), + crate::integrations::CORE_SOURCE, + ) + .expect("should register"); let settings = create_test_settings(); let downstream = http::Request::new(edgezero_core::body::Body::empty()); let context = immediate_test_context(&settings, &downstream, &services); @@ -2943,16 +3058,26 @@ mod tests { timeout_ms: 2000, ..Default::default() }); - orchestrator.register_provider(Arc::new(DivergentBackendProvider { - name: "provider-a", - predicted: "predicted-a", - resolved: "shared-resolved", - })); - orchestrator.register_provider(Arc::new(DivergentBackendProvider { - name: "provider-b", - predicted: "predicted-b", - resolved: "shared-resolved", - })); + orchestrator + .register_provider( + Arc::new(DivergentBackendProvider { + name: "provider-a", + predicted: "predicted-a", + resolved: "shared-resolved", + }), + crate::integrations::CORE_SOURCE, + ) + .expect("should register"); + orchestrator + .register_provider( + Arc::new(DivergentBackendProvider { + name: "provider-b", + predicted: "predicted-b", + resolved: "shared-resolved", + }), + crate::integrations::CORE_SOURCE, + ) + .expect("should register"); let settings = create_test_settings(); let downstream = http::Request::new(edgezero_core::body::Body::empty()); let context = immediate_test_context(&settings, &downstream, &services); @@ -3000,14 +3125,24 @@ mod tests { ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); - orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-a", - backend: "backend-a", - })); - orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-b", - backend: "backend-b", - })); + orchestrator + .register_provider( + Arc::new(StubAuctionProvider { + name: "provider-a", + backend: "backend-a", + }), + crate::integrations::CORE_SOURCE, + ) + .expect("should register"); + orchestrator + .register_provider( + Arc::new(StubAuctionProvider { + name: "provider-b", + backend: "backend-b", + }), + crate::integrations::CORE_SOURCE, + ) + .expect("should register"); let request = create_test_auction_request(); let settings = create_test_settings(); @@ -3075,10 +3210,15 @@ mod tests { ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); - orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-a", - backend: "backend-a", - })); + orchestrator + .register_provider( + Arc::new(StubAuctionProvider { + name: "provider-a", + backend: "backend-a", + }), + crate::integrations::CORE_SOURCE, + ) + .expect("should register"); let request = create_test_auction_request(); let settings = create_test_settings(); let downstream = http::Request::builder() @@ -3153,14 +3293,24 @@ mod tests { ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); - orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-a", - backend: "backend-a", - })); - orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-b", - backend: "backend-b", - })); + orchestrator + .register_provider( + Arc::new(StubAuctionProvider { + name: "provider-a", + backend: "backend-a", + }), + crate::integrations::CORE_SOURCE, + ) + .expect("should register"); + orchestrator + .register_provider( + Arc::new(StubAuctionProvider { + name: "provider-b", + backend: "backend-b", + }), + crate::integrations::CORE_SOURCE, + ) + .expect("should register"); let request = create_test_auction_request(); let settings = create_test_settings(); @@ -3218,14 +3368,24 @@ mod tests { ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); - orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-a", - backend: "backend-a", - })); - orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-b", - backend: "backend-b", - })); + orchestrator + .register_provider( + Arc::new(StubAuctionProvider { + name: "provider-a", + backend: "backend-a", + }), + crate::integrations::CORE_SOURCE, + ) + .expect("should register"); + orchestrator + .register_provider( + Arc::new(StubAuctionProvider { + name: "provider-b", + backend: "backend-b", + }), + crate::integrations::CORE_SOURCE, + ) + .expect("should register"); let request = create_test_auction_request(); let settings = create_test_settings(); diff --git a/crates/trusted-server-core/src/auction/test_support.rs b/crates/trusted-server-core/src/auction/test_support.rs index e4b953e050..cc8e687d7c 100644 --- a/crates/trusted-server-core/src/auction/test_support.rs +++ b/crates/trusted-server-core/src/auction/test_support.rs @@ -3,10 +3,17 @@ use std::sync::LazyLock; use edgezero_core::body::Body as EdgeBody; use http::Request; -use super::AuctionContext; -use crate::platform::{RuntimeServices, test_support::noop_services}; +use error_stack::Report; + +use super::provider::{AuctionProvider, ProviderRequestOutcome}; +use super::types::{AuctionContext, AuctionRequest, AuctionResponse}; +use crate::error::TrustedServerError; +use crate::platform::{PlatformResponse, RuntimeServices, test_support::noop_services}; use crate::settings::Settings; +/// Timeout the named test provider reports; no test asserts on the value. +const NAMED_TEST_PROVIDER_TIMEOUT_MS: u32 = 2000; + static TEST_SERVICES: LazyLock = LazyLock::new(noop_services); pub(crate) fn create_test_auction_context<'a>( @@ -23,3 +30,51 @@ pub(crate) fn create_test_auction_context<'a>( services, } } + +/// A provider that reports the name it was constructed with and answers every +/// bid request immediately, for tests that only need a named registration. +pub(crate) struct NamedTestProvider { + name: &'static str, +} + +impl NamedTestProvider { + /// Creates a provider that reports `name` as its provider name. + pub(crate) const fn new(name: &'static str) -> Self { + Self { name } + } +} + +#[async_trait::async_trait(?Send)] +impl AuctionProvider for NamedTestProvider { + fn provider_name(&self) -> &'static str { + self.name + } + + async fn request_bids( + &self, + _request: &AuctionRequest, + _context: &AuctionContext<'_>, + ) -> Result> { + Ok(ProviderRequestOutcome::Immediate(AuctionResponse::success( + self.name, + vec![], + 0, + ))) + } + + async fn parse_response( + &self, + _response: PlatformResponse, + response_time_ms: u64, + ) -> Result> { + Ok(AuctionResponse::success( + self.name, + vec![], + response_time_ms, + )) + } + + fn timeout_ms(&self) -> u32 { + NAMED_TEST_PROVIDER_TIMEOUT_MS + } +} diff --git a/crates/trusted-server-core/src/integrations/adserver_mock.rs b/crates/trusted-server-core/src/integrations/adserver_mock.rs index 1d6527934a..1b4cf403f5 100644 --- a/crates/trusted-server-core/src/integrations/adserver_mock.rs +++ b/crates/trusted-server-core/src/integrations/adserver_mock.rs @@ -32,6 +32,9 @@ use crate::settings::{IntegrationConfig, Settings}; // Configuration // ============================================================================ +/// Integration id the ad server mock provider is configured under. +const ADSERVER_MOCK_INTEGRATION_ID: &str = "adserver_mock"; + /// Configuration for mock ad server integration. #[derive(Debug, Clone, Deserialize, Serialize, Validate)] pub struct AdServerMockConfig { @@ -556,6 +559,19 @@ impl AuctionProvider for AdServerMockProvider { // Auto-Registration // ============================================================================ +/// Validates the ad server mock configuration for deployment and reports +/// whether the provider is enabled. +/// +/// # Errors +/// +/// Returns an error when the ad server mock configuration cannot be parsed or +/// fails validation. +pub(crate) fn validate(settings: &Settings) -> Result> { + settings + .integration_config::(ADSERVER_MOCK_INTEGRATION_ID) + .map(|config| config.is_some()) +} + /// Auto-register ad server mock provider based on settings configuration. /// /// # Errors @@ -567,7 +583,7 @@ pub fn register_providers( ) -> Result>, Report> { let mut providers: Vec> = Vec::new(); - match settings.integration_config::("adserver_mock") { + match settings.integration_config::(ADSERVER_MOCK_INTEGRATION_ID) { Ok(Some(config)) => { log::info!( "Registering AdServer Mock mediator (endpoint: {})", diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 123394d1a9..1fb150bef0 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -9266,7 +9266,12 @@ mod tests { request: Request, ) -> Response { let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - orchestrator.register_provider(Arc::new(WinningBidProvider)); + orchestrator + .register_provider( + Arc::new(WinningBidProvider), + crate::integrations::CORE_SOURCE, + ) + .expect("should register"); run_with_orchestrator( settings, services, @@ -14145,7 +14150,12 @@ mod tests { // Arrange let settings = settings_with_dispatching_provider(); let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - orchestrator.register_provider(Arc::new(DispatchingTestProvider)); + orchestrator + .register_provider( + Arc::new(DispatchingTestProvider), + crate::integrations::CORE_SOURCE, + ) + .expect("should register"); let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); let stub = Arc::new(StubHttpClient::new()); @@ -20248,10 +20258,15 @@ mod tests { winning_bid: bool, ) -> AuctionOrchestrator { let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - orchestrator.register_provider(Arc::new(AuctionIdTestProvider { - captured_request, - winning_bid, - })); + orchestrator + .register_provider( + Arc::new(AuctionIdTestProvider { + captured_request, + winning_bid, + }), + crate::integrations::CORE_SOURCE, + ) + .expect("should register"); orchestrator } @@ -21157,9 +21172,14 @@ mod tests { captured: &Arc>>, ) -> AuctionOrchestrator { let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - orchestrator.register_provider(Arc::new(RequestCapturingProvider { - captured: Arc::clone(captured), - })); + orchestrator + .register_provider( + Arc::new(RequestCapturingProvider { + captured: Arc::clone(captured), + }), + crate::integrations::CORE_SOURCE, + ) + .expect("should register"); orchestrator } From ab2121d951311a50ba12be6db968190a2132f2c6 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 28 Aug 2026 18:53:44 +0100 Subject: [PATCH 012/124] Drive deploy validation from the integration and auction builders `config.rs` named every vendor by hand: a per-vendor import block, a `#[cfg(test)] DEPLOY_VALIDATED_INTEGRATION_IDS` list a test compared the registry against, and a `validate_enabled_integrations` body that called `validate_integration::` once per integration. A vendor integration living in its own crate could not be reached by any of it, and the list was a second place to keep in step by hand. Deploy validation now walks the builders. `validate_settings_for_deploy` delegates to a new public `validate_settings_for_deploy_with(settings, extra_integrations, extra_auction_providers)`, which runs `crate::integrations::all_builders(...)` for the integrations and `crate::auction::all_provider_builders(...)` for the auction providers, collecting the enabled provider names the auction builders report. Every builder validates whether or not it is enabled, so a typo in a disabled block is still caught, as before. `validate_auction_provider_names` and `report_to_validation_errors` are unchanged; the per-vendor imports, `DEPLOY_VALIDATED_INTEGRATION_IDS`, `validate_enabled_integrations`, `validate_prebid` and `validate_integration` are gone. `IntegrationBuilder::validate` and `AuctionProviderBuilder::validate` now have a caller, so both lose their temporary `#[allow(dead_code)]`. The enabled auction provider set is unchanged. Each vendor `validate` is the same expression the deleted code used (prebid via `validate_config_for_startup(...).is_some()`, aps and adserver_mock via `integration_config::(id).is_some()`, with the id consts holding the same literals), and a temporary parity test compared the deleted shape against the builder shape over all 27 enabled/disabled/absent permutations of prebid, aps and adserver_mock plus three invalid blocks, matching on both the resulting set and the rejection text. `adserver_mock` is the one id that is not an integration builder; it is validated through its auction provider builder. `deploy_validation_covers_registered_integration_builders` compared the hand list against the registry, which no longer exists. `deploy_validation_runs_every_registered_builder` replaces it, asserting the set deploy validation walks equals the set the registry registers. Three new tests cover the seam: an external integration builder's rejection and an external auction provider builder's rejection both reach the caller with the message intact, and an external auction provider satisfies a configured `[auction] providers` name that fails without it. --- crates/trusted-server-core/src/auction/mod.rs | 3 - crates/trusted-server-core/src/config.rs | 224 +++++++++++------- .../src/integrations/mod.rs | 3 - 3 files changed, 139 insertions(+), 91 deletions(-) diff --git a/crates/trusted-server-core/src/auction/mod.rs b/crates/trusted-server-core/src/auction/mod.rs index 669c2a329c..bfe4b2974e 100644 --- a/crates/trusted-server-core/src/auction/mod.rs +++ b/crates/trusted-server-core/src/auction/mod.rs @@ -139,9 +139,6 @@ impl AuctionProviderBuilder { /// /// Returns an error when the configuration cannot be parsed or fails /// validation. - // Deploy validation does not yet enumerate auction provider builders, so - // nothing in the crate calls this until that is wired up. - #[allow(dead_code)] pub(crate) fn validate(&self, settings: &Settings) -> Result> { (self.validate)(settings) } diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index 1843efc388..79aa98c629 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -13,35 +13,13 @@ use error_stack::Report; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use validator::{Validate, ValidationError, ValidationErrors}; +use crate::auction::AuctionProviderBuilder; use crate::ec::registry::PartnerRegistry; use crate::error::TrustedServerError; -use crate::integrations::{ - adserver_mock::AdServerMockConfig, aps::ApsConfig, datadome::DataDomeConfig, - didomi::DidomiIntegrationConfig, google_tag_manager::GoogleTagManagerConfig, gpt::GptConfig, - gpt_diagnostics::GptDiagnosticsConfig, lockr::LockrConfig, nextjs::NextJsIntegrationConfig, - osano::OsanoConfig, permutive::PermutiveConfig, prebid, sourcepoint::SourcepointConfig, - testlight::TestlightConfig, -}; -use crate::settings::{IntegrationConfig, Settings}; +use crate::integrations::IntegrationBuilder; +use crate::settings::Settings; const DEPLOY_VALIDATION_FIELD: &str = "trusted_server"; -#[cfg(test)] -const DEPLOY_VALIDATED_INTEGRATION_IDS: &[&str] = &[ - "prebid", - "aps", - "adserver_mock", - "testlight", - "nextjs", - "permutive", - "lockr", - "didomi", - "sourcepoint", - "osano", - "google_tag_manager", - "datadome", - "gpt", - "gpt_diagnostics", -]; /// Typed app-config root used by the `ts` CLI. /// @@ -118,69 +96,47 @@ impl edgezero_core::app_config::AppConfigMeta for TrustedServerAppConfig { } } -/// Runs Trusted Server deploy-time validation for pushed app config. +/// Runs Trusted Server deploy-time validation for pushed app config with the +/// built-in integrations and auction providers only. /// /// This supplements [`Settings`] structural validation with checks that should /// fail before an operator publishes a config blob: placeholder secrets, -/// enabled integration startup checks, auction provider references, and EC -/// partner registry construction. +/// integration startup checks, auction provider references, and EC partner +/// registry construction. /// /// # Errors /// /// Returns [`TrustedServerError`] when the config should not be deployed. pub fn validate_settings_for_deploy(settings: &Settings) -> Result<(), Report> { - settings.reject_placeholder_secrets()?; - let enabled_auction_providers = validate_enabled_integrations(settings)?; - validate_auction_provider_names(settings, &enabled_auction_providers)?; - PartnerRegistry::from_config(&settings.ec.partners).map(|_| ())?; - Ok(()) + validate_settings_for_deploy_with(settings, &[], &[]) } -fn validate_enabled_integrations( +/// Validates settings for deployment with the built-in integrations and +/// auction providers followed by the externally supplied builders an adapter +/// registers. Every builder validates, enabled or not, so a typo in a +/// disabled block is still caught. +/// +/// # Errors +/// +/// Returns [`TrustedServerError`] when the config should not be deployed. +pub fn validate_settings_for_deploy_with( settings: &Settings, -) -> Result, Report> { - let mut enabled_auction_providers = HashSet::new(); - - if validate_prebid(settings)? { - enabled_auction_providers.insert("prebid"); - } - if validate_integration::(settings, "aps")? { - enabled_auction_providers.insert("aps"); - } - if validate_integration::(settings, "adserver_mock")? { - enabled_auction_providers.insert("adserver_mock"); + extra_integrations: &[IntegrationBuilder], + extra_auction_providers: &[AuctionProviderBuilder], +) -> Result<(), Report> { + settings.reject_placeholder_secrets()?; + for builder in crate::integrations::all_builders(extra_integrations) { + builder.validate(settings)?; } - validate_integration::(settings, "testlight")?; - validate_integration::(settings, "nextjs")?; - validate_integration::(settings, "permutive")?; - validate_integration::(settings, "lockr")?; - validate_integration::(settings, "didomi")?; - validate_integration::(settings, "sourcepoint")?; - validate_integration::(settings, "osano")?; - validate_integration::(settings, "google_tag_manager")?; - if let Some(config) = settings.integration_config::("datadome")? { - crate::integrations::datadome::DataDomeIntegration::validate_config_for_startup(config)?; + let mut enabled_auction_providers = HashSet::new(); + for builder in crate::auction::all_provider_builders(extra_auction_providers) { + if builder.validate(settings)? { + enabled_auction_providers.insert(builder.name()); + } } - validate_integration::(settings, "gpt")?; - validate_integration::(settings, "gpt_diagnostics")?; - - Ok(enabled_auction_providers) -} - -fn validate_prebid(settings: &Settings) -> Result> { - prebid::validate_config_for_startup(settings).map(|config| config.is_some()) -} - -fn validate_integration( - settings: &Settings, - integration_id: &str, -) -> Result> -where - T: IntegrationConfig, -{ - settings - .integration_config::(integration_id) - .map(|config| config.is_some()) + validate_auction_provider_names(settings, &enabled_auction_providers)?; + PartnerRegistry::from_config(&settings.ec.partners).map(|_| ())?; + Ok(()) } fn validate_auction_provider_names( @@ -220,9 +176,44 @@ fn report_to_validation_errors(report: &Report) -> Validatio #[cfg(test)] mod tests { + use std::sync::Arc; + use super::*; + use crate::auction::AuctionProvider; + use crate::integrations::{ + IntegrationRegistration, lockr::LockrConfig, permutive::PermutiveConfig, + sourcepoint::SourcepointConfig, + }; use crate::test_support::tests::crate_test_settings_str; + /// Message an external builder rejects with, so the test can prove the + /// rejection reached the caller intact. + const EXTERNAL_REJECTION_MESSAGE: &str = "seam probe refuses to deploy"; + + /// Stands in for a vendor integration builder that never enables. + fn build_nothing( + _settings: &Settings, + ) -> Result, Report> { + Ok(None) + } + + /// Stands in for a vendor auction provider builder that never registers. + fn build_no_providers( + _settings: &Settings, + ) -> Result>, Report> { + Ok(Vec::new()) + } + + fn reject_deploy(_settings: &Settings) -> Result> { + Err(Report::new(TrustedServerError::Configuration { + message: EXTERNAL_REJECTION_MESSAGE.to_string(), + })) + } + + fn report_enabled(_settings: &Settings) -> Result> { + Ok(true) + } + #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] #[allow(dead_code)] @@ -693,21 +684,84 @@ password = "production-admin-password-32-bytes" ); } + /// Deploy validation walks `crate::integrations::builders()`, so that set + /// has to be the same set the registry registers, or a builder would be + /// deployed without ever being validated. #[test] - fn deploy_validation_covers_registered_integration_builders() { - let validated_ids: HashSet<&'static str> = - DEPLOY_VALIDATED_INTEGRATION_IDS.iter().copied().collect(); + fn deploy_validation_runs_every_registered_builder() { + let validated_ids: HashSet<&'static str> = crate::integrations::builders() + .iter() + .map(IntegrationBuilder::id) + .collect(); let registry = crate::integrations::IntegrationRegistry::new(&valid_settings()) .expect("should build registry from valid settings"); - let missing_ids = registry - .registered_builder_ids() - .into_iter() - .filter(|id| !validated_ids.contains(id)) - .collect::>(); + let registered_ids: HashSet<&'static str> = + registry.registered_builder_ids().into_iter().collect(); + + assert_eq!( + validated_ids, registered_ids, + "deploy validation should run every builder the registry registers" + ); + } + + #[test] + fn deploy_validation_surfaces_an_external_integration_builders_rejection() { + let extra = [IntegrationBuilder::new( + "seam-probe", + "seam-probe-crate", + build_nothing, + reject_deploy, + )]; + + let err = validate_settings_for_deploy_with(&valid_settings(), &extra, &[]) + .expect_err("should surface the external integration builder's rejection"); + + assert!( + err.to_string().contains(EXTERNAL_REJECTION_MESSAGE), + "should keep the external builder's message intact: {err:?}" + ); + } + + #[test] + fn deploy_validation_surfaces_an_external_auction_providers_rejection() { + let extra = [AuctionProviderBuilder::new( + "seam-probe", + "seam-probe-crate", + build_no_providers, + reject_deploy, + )]; + + let err = validate_settings_for_deploy_with(&valid_settings(), &[], &extra) + .expect_err("should surface the external auction provider builder's rejection"); + + assert!( + err.to_string().contains(EXTERNAL_REJECTION_MESSAGE), + "should keep the external builder's message intact: {err:?}" + ); + } + + #[test] + fn an_external_auction_provider_satisfies_a_configured_provider_name() { + let mut settings = valid_settings(); + settings.auction.enabled = true; + settings.auction.providers = vec!["probe".to_string()]; + let extra = [AuctionProviderBuilder::new( + "probe", + "seam-probe-crate", + build_no_providers, + report_enabled, + )]; + + validate_settings_for_deploy_with(&settings, &[], &extra) + .expect("an external auction provider should satisfy its configured name"); + + let err = validate_settings_for_deploy(&settings) + .expect_err("should reject the configured name without the external builder"); assert!( - missing_ids.is_empty(), - "deploy validation should cover all registered integration builders: {missing_ids:?}" + err.to_string() + .contains("no enabled integration provides it"), + "should report the unprovided auction provider name: {err:?}" ); } diff --git a/crates/trusted-server-core/src/integrations/mod.rs b/crates/trusted-server-core/src/integrations/mod.rs index 056d674b46..5fb7545884 100644 --- a/crates/trusted-server-core/src/integrations/mod.rs +++ b/crates/trusted-server-core/src/integrations/mod.rs @@ -402,9 +402,6 @@ impl IntegrationBuilder { /// /// Returns an error when the configuration cannot be parsed or fails /// validation. - // Deploy validation does not yet enumerate builders, so nothing in the - // crate calls this until that is wired up. - #[allow(dead_code)] pub(crate) fn validate(&self, settings: &Settings) -> Result> { (self.validate)(settings) } From 5079bc0c02407dc0dff476a5115cece3b868c61d Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 28 Aug 2026 19:23:55 +0100 Subject: [PATCH 013/124] Bind the DataDome filter to the neutral personalized-response marker in tests The filter now inserts `PersonalizedResponse` alongside `DataDomeClientTagSuppressed` at two sites, but every assertion in `protection.rs` went through `has_client_tag_suppression_marker`, which reads only the DataDome type, and the publisher tests insert both markers by hand. Deleting either `PersonalizedResponse` insert left the whole suite green while HTML responses that must stay private became shareable through a cache or a template. What used to be structural, one type driving both concerns, had become a convention at two sites with nothing holding it. Add `has_personalized_response_marker` next to the DataDome helper and assert the two together at all fifteen sites, positive and negative, so both markers are checked everywhere the filter is exercised. Verified not vacuous by removing each insert in turn: dropping the test-bypass insert fails three tests, dropping the scope-skip insert fails two. Both restored. Each site also takes one `extensions_mut()` binding instead of two calls, which puts the paired inserts next to each other. `deploy_validation_runs_every_registered_builder` could not fail. Both sides derived from `BUILT_IN_BUILDERS` through different paths and it never called `validate_settings_for_deploy_with`, so its claim to guard against a builder being deployed unvalidated was not one it could make. Replace it with two tests that can fail. `deploy_validation_runs_every_builder_it_is_given` passes recording builders through `validate_settings_for_deploy_with` and counts the calls in a static, since a builder holds fn pointers and cannot capture. `deploy_validation_reaches_every_built_in_builder` plants a block each config type cannot deserialize, a string in the boolean `enabled` field, and asserts deploy validation rejects for every built-in integration and auction provider id. Verified not vacuous: gutting the integration validate loop fails five config tests, skipping the auction validate fails three. Its doc records what it cannot catch, a builder deleted from `BUILT_IN_BUILDERS`, because no independent list of the built-ins exists in the crate. The `PersonalizedResponse` doc overstated the mechanism. Buffering is gated on `is_html_document_request` and private caching on a body-carrying HTML content type, while only the origin-path choice is unconditional. Reword to say so. --- crates/trusted-server-core/src/config.rs | 85 ++++++++++++--- .../src/integrations/datadome/protection.rs | 102 ++++++++++++++---- .../src/response_privacy.rs | 9 +- 3 files changed, 155 insertions(+), 41 deletions(-) diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index 79aa98c629..3e79d2ac66 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -177,6 +177,7 @@ fn report_to_validation_errors(report: &Report) -> Validatio #[cfg(test)] mod tests { use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; use super::*; use crate::auction::AuctionProvider; @@ -684,26 +685,82 @@ password = "production-admin-password-32-bytes" ); } - /// Deploy validation walks `crate::integrations::builders()`, so that set - /// has to be the same set the registry registers, or a builder would be - /// deployed without ever being validated. + /// Counts calls to [`record_validate_call`]. A builder holds plain fn + /// pointers and cannot capture, so the recording has to go through a + /// static. + static RECORDED_VALIDATE_CALLS: AtomicUsize = AtomicUsize::new(0); + + fn record_validate_call(_settings: &Settings) -> Result> { + RECORDED_VALIDATE_CALLS.fetch_add(1, Ordering::SeqCst); + Ok(false) + } + + /// Every builder handed to deploy validation has its `validate` run, and + /// reporting disabled does not excuse a builder from validating. #[test] - fn deploy_validation_runs_every_registered_builder() { - let validated_ids: HashSet<&'static str> = crate::integrations::builders() - .iter() - .map(IntegrationBuilder::id) - .collect(); - let registry = crate::integrations::IntegrationRegistry::new(&valid_settings()) - .expect("should build registry from valid settings"); - let registered_ids: HashSet<&'static str> = - registry.registered_builder_ids().into_iter().collect(); + fn deploy_validation_runs_every_builder_it_is_given() { + RECORDED_VALIDATE_CALLS.store(0, Ordering::SeqCst); + let extra_integrations = [IntegrationBuilder::new( + "seam-probe-integration", + "seam-probe-crate", + build_nothing, + record_validate_call, + )]; + let extra_auction_providers = [AuctionProviderBuilder::new( + "seam-probe-provider", + "seam-probe-crate", + build_no_providers, + record_validate_call, + )]; + + validate_settings_for_deploy_with( + &valid_settings(), + &extra_integrations, + &extra_auction_providers, + ) + .expect("should accept settings whose external builders report disabled"); assert_eq!( - validated_ids, registered_ids, - "deploy validation should run every builder the registry registers" + RECORDED_VALIDATE_CALLS.load(Ordering::SeqCst), + 2, + "both external builders should validate even though each reports disabled" ); } + /// Deploy validation reaches each built-in builder's own config type, one + /// id at a time, by planting a block that type cannot deserialize. Every + /// integration config carries a boolean `enabled`, so a string there fails + /// for all of them. + /// + /// This catches deploy validation ceasing to validate the built-ins, or + /// validating only the enabled ones. It cannot catch a builder deleted + /// from `BUILT_IN_BUILDERS`, because the loop below reads the same + /// constant the validation walks; no independent list of the built-ins + /// exists in the crate. + #[test] + fn deploy_validation_reaches_every_built_in_builder() { + for id in crate::integrations::builders() + .iter() + .map(IntegrationBuilder::id) + .chain( + crate::auction::provider_builders() + .iter() + .map(AuctionProviderBuilder::name), + ) + { + let mut settings = valid_settings(); + settings + .integrations + .insert_config(id, &serde_json::json!({ "enabled": "not-a-boolean" })) + .expect("should insert the probe config"); + + assert!( + validate_settings_for_deploy(&settings).is_err(), + "deploy validation should reach the `{id}` builder and reject its planted config" + ); + } + } + #[test] fn deploy_validation_surfaces_an_external_integration_builders_rejection() { let extra = [IntegrationBuilder::new( diff --git a/crates/trusted-server-core/src/integrations/datadome/protection.rs b/crates/trusted-server-core/src/integrations/datadome/protection.rs index aa5706a3ef..31e1c1e5f5 100644 --- a/crates/trusted-server-core/src/integrations/datadome/protection.rs +++ b/crates/trusted-server-core/src/integrations/datadome/protection.rs @@ -46,17 +46,13 @@ impl DataDomeIntegration { let test_bypass_matched = self.take_protection_test_bypass_header(input.request, input.services); if test_bypass_matched { - // The first marker is DataDome's own tag-suppression signal, read by its - // head injector. The second tells core the response is personalized to - // this request and cannot be shared through a cache or a template. - input - .request - .extensions_mut() - .insert(super::DataDomeClientTagSuppressed); - input - .request - .extensions_mut() - .insert(crate::response_privacy::PersonalizedResponse); + // Both markers travel together. The first is DataDome's own + // tag-suppression signal, read by its head injector. The second tells + // core the response is personalized to this request and cannot be + // shared through a cache or a template. + let extensions = input.request.extensions_mut(); + extensions.insert(super::DataDomeClientTagSuppressed); + extensions.insert(crate::response_privacy::PersonalizedResponse); log_protection_test_bypass(&input); return RequestFilterDecision::Continue(RequestFilterEffects::default()); } @@ -169,18 +165,13 @@ impl DataDomeIntegration { suppress_client_tag, } => { if suppress_client_tag { - // The first marker is DataDome's own tag-suppression signal, read - // by its head injector. The second tells core the response is - // personalized to this request and cannot be shared through a - // cache or a template. - input - .request - .extensions_mut() - .insert(super::DataDomeClientTagSuppressed); - input - .request - .extensions_mut() - .insert(crate::response_privacy::PersonalizedResponse); + // Both markers travel together. The first is DataDome's own + // tag-suppression signal, read by its head injector. The second + // tells core the response is personalized to this request and + // cannot be shared through a cache or a template. + let extensions = input.request.extensions_mut(); + extensions.insert(super::DataDomeClientTagSuppressed); + extensions.insert(crate::response_privacy::PersonalizedResponse); } log_protection_skip(input, &rule_id, reason, suppress_client_tag); return false; @@ -960,6 +951,17 @@ mod tests { .is_some() } + /// The filter sets this alongside the `DataDome` marker at every site. The + /// two are asserted together everywhere below, so dropping either insert + /// fails a test rather than silently making a personalized HTML response + /// shareable through a cache or a template. + fn has_personalized_response_marker(request: &Request) -> bool { + request + .extensions() + .get::() + .is_some() + } + #[test] fn protection_test_bypass_skips_api_suppresses_tag_and_strips_header() { let config = DataDomeConfig { @@ -1000,6 +1002,10 @@ mod tests { has_client_tag_suppression_marker(&request), "the bypass should suppress the automatic DataDome client tag" ); + assert!( + has_personalized_response_marker(&request), + "the bypass should mark the response personalized to this request" + ); assert!( request .headers() @@ -1070,6 +1076,10 @@ mod tests { !has_client_tag_suppression_marker(&request), "an inactive bypass must not suppress the DataDome client tag" ); + assert!( + !has_personalized_response_marker(&request), + "an inactive bypass must not mark the response personalized" + ); assert_eq!( http_client.recorded_backend_names().len(), 1, @@ -1152,6 +1162,10 @@ mod tests { !has_client_tag_suppression_marker(&request), "the bypass must not suppress the DataDome client tag outside staging" ); + assert!( + !has_personalized_response_marker(&request), + "the bypass must not mark the response personalized outside staging" + ); assert_eq!( http_client.recorded_backend_names().len(), 1, @@ -1207,6 +1221,10 @@ mod tests { has_client_tag_suppression_marker(&request), "a matching test credential should suppress the tag even on an excluded path" ); + assert!( + has_personalized_response_marker(&request), + "a matching test credential should mark the response personalized even on an excluded path" + ); assert!( http_client.recorded_backend_names().is_empty(), "a matching test credential must not call the Protection API" @@ -1262,6 +1280,10 @@ mod tests { !has_client_tag_suppression_marker(&request), "a non-matching credential must not suppress the DataDome client tag" ); + assert!( + !has_personalized_response_marker(&request), + "a non-matching credential must not mark the response personalized" + ); assert!( request .headers() @@ -1331,6 +1353,7 @@ mod tests { "all duplicate bypass values should be stripped" ); assert!(!has_client_tag_suppression_marker(&request)); + assert!(!has_personalized_response_marker(&request)); assert_eq!(http_client.recorded_backend_names().len(), 1); } @@ -1390,6 +1413,7 @@ mod tests { assert!(matches!(decision, RequestFilterDecision::Continue(_))); assert_eq!(has_client_tag_suppression_marker(&request), should_match); + assert_eq!(has_personalized_response_marker(&request), should_match); assert_eq!( http_client.recorded_backend_names().is_empty(), should_match, @@ -1443,6 +1467,10 @@ mod tests { has_client_tag_suppression_marker(&inline_request), "inline IP exclusions should mark the request" ); + assert!( + has_personalized_response_marker(&inline_request), + "inline IP exclusions should mark the response personalized" + ); inline.protection_excluded_ip_cidrs.clear(); inline.protection_excluded_ip_cidr_sources = @@ -1462,6 +1490,10 @@ mod tests { has_client_tag_suppression_marker(&source_request), "Config Store IP exclusions should mark the request" ); + assert!( + has_personalized_response_marker(&source_request), + "Config Store IP exclusions should mark the response personalized" + ); let structured_ip = DataDomeConfig { enabled: true, @@ -1482,6 +1514,10 @@ mod tests { has_client_tag_suppression_marker(&structured_request), "structured IP exclusions should mark the request" ); + assert!( + has_personalized_response_marker(&structured_request), + "structured IP exclusions should mark the response personalized" + ); let structured_source = DataDomeConfig { enabled: true, @@ -1510,6 +1546,10 @@ mod tests { has_client_tag_suppression_marker(&structured_source_request), "structured Config Store IP exclusions should mark the request" ); + assert!( + has_personalized_response_marker(&structured_source_request), + "structured Config Store IP exclusions should mark the response personalized" + ); } #[test] @@ -1560,6 +1600,10 @@ mod tests { !has_client_tag_suppression_marker(&request), "matching non-IP exclusion should not mark {uri}" ); + assert!( + !has_personalized_response_marker(&request), + "matching non-IP exclusion should not mark {uri} as personalized" + ); } } @@ -1596,6 +1640,10 @@ mod tests { has_client_tag_suppression_marker(&request), "overlapping IP exclusion should suppress even when path remains the primary reason" ); + assert!( + has_personalized_response_marker(&request), + "overlapping IP exclusion should mark the response personalized even when path remains the primary reason" + ); } #[test] @@ -1625,6 +1673,10 @@ mod tests { !has_client_tag_suppression_marker(&request), "ASN exclusions should not mark the request" ); + assert!( + !has_personalized_response_marker(&request), + "ASN exclusions should not mark the response personalized" + ); } #[test] @@ -1643,6 +1695,10 @@ mod tests { !has_client_tag_suppression_marker(&request), "a non-matching IP should not mark the request" ); + assert!( + !has_personalized_response_marker(&request), + "a non-matching IP should not mark the response personalized" + ); } #[test] diff --git a/crates/trusted-server-core/src/response_privacy.rs b/crates/trusted-server-core/src/response_privacy.rs index 97cb6598e4..58c614dac0 100644 --- a/crates/trusted-server-core/src/response_privacy.rs +++ b/crates/trusted-server-core/src/response_privacy.rs @@ -27,10 +27,11 @@ pub struct TerminalPrivateResponse; /// Request marker meaning the response is personalized to this request and /// must not be shared through any cache or template. /// -/// Any integration may set it from a request filter. Core buffers the full -/// document, keeps the request on the origin path rather than a shared -/// template, and enforces private caching whenever it is present, without -/// knowing which integration asked. +/// Any integration may set it from a request filter, and core acts on it +/// without knowing which integration asked. Whenever it is present the request +/// keeps to the origin path rather than a shared template. For an HTML +/// document core also buffers the full body, and it enforces private caching +/// on the HTML response that body produces. #[derive(Debug, Clone, Copy)] pub struct PersonalizedResponse; From 0e5eec533046b04e1272df3c23dbb5c575b17d08 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 28 Aug 2026 19:48:13 +0100 Subject: [PATCH 014/124] Open the bid renderer descriptor while keeping its wire form BidRenderer was a closed enum with a single Aps variant, and the APS descriptor types sat in the neutral auction types, so an APS integration could not move into its own crate without taking core's renderer enum with it. BidRenderer is now a type tag plus a payload the auction provider that produced the bid defines, serialized flat as {"type": "", ...payload}. Providers build one with new() from JSON or from_typed() from their own struct, and read one back with payload_as::(tag), which returns None under a different tag. A payload must be a JSON object and must not carry its own "type" key, since that would collide with the tag. The serialized bytes are unchanged. The page receives the renderer through serde_json::to_value, in the OpenRTB response extension via BidExt::to_ext and in window.tsjs.bids via build_bid_map, and both forms produce the same JSON object with the same keys and values. The exact strings were captured from the enum form before the change and are pinned by a test, and the existing wire-shape assertions in auction/types.rs and auction/formats.rs pass unchanged. ApsRendererV1 and ApsTagType moved from auction/types.rs into integrations/aps.rs, with a shared APS_RENDERER_TYPE tag constant, so they travel with APS when it moves out. adserver_mock builds the neutral descriptor from the wire keys instead, since the mock is not APS. --- .../src/auction/formats.rs | 89 +++-- .../src/auction/orchestrator.rs | 31 +- .../trusted-server-core/src/auction/types.rs | 314 ++++++++++++++---- .../src/integrations/adserver_mock.rs | 79 +++-- .../src/integrations/aps.rs | 65 +++- crates/trusted-server-core/src/publisher.rs | 46 ++- 6 files changed, 458 insertions(+), 166 deletions(-) diff --git a/crates/trusted-server-core/src/auction/formats.rs b/crates/trusted-server-core/src/auction/formats.rs index e09912aab4..2549058932 100644 --- a/crates/trusted-server-core/src/auction/formats.rs +++ b/crates/trusted-server-core/src/auction/formats.rs @@ -552,9 +552,8 @@ pub(crate) fn convert_to_openrtb_response_with_report( #[cfg(test)] mod tests { use super::*; - use crate::auction::types::{ - ApsRendererV1, ApsTagType, AuctionResponse, Bid, BidRenderer, BidStatus, - }; + use crate::auction::types::{AuctionResponse, Bid, BidRenderer, BidStatus}; + use crate::integrations::aps::{APS_RENDERER_TYPE, ApsRendererV1, ApsTagType}; use crate::openrtb::{Eid, Uid}; use crate::platform::test_support::noop_services; use crate::test_support::tests::create_test_settings; @@ -1467,17 +1466,23 @@ mod tests { renderer.creative = Some("".to_string()); renderer.bid_id = Some("upstream-renderer-bid".to_string()); renderer.creative_id = None; - renderer.renderer = Some(BidRenderer::Aps(ApsRendererV1 { - version: 1, - account_id: "example-account".to_string(), - bid_id: "upstream-renderer-bid".to_string(), - creative_id: None, - tag_type: ApsTagType::Iframe, - creative_url: "https://creative.example/render".to_string(), - aax_response: "fictional-base64".to_string(), - width: 300, - height: 250, - })); + renderer.renderer = Some( + BidRenderer::from_typed( + APS_RENDERER_TYPE, + &ApsRendererV1 { + version: 1, + account_id: "example-account".to_string(), + bid_id: "upstream-renderer-bid".to_string(), + creative_id: None, + tag_type: ApsTagType::Iframe, + creative_url: "https://creative.example/render".to_string(), + aax_response: "fictional-base64".to_string(), + width: 300, + height: 250, + }, + ) + .expect("should build APS renderer descriptor"), + ); let result = OrchestrationResult { provider_responses: vec![], mediator_response: None, @@ -1565,17 +1570,23 @@ mod tests { let settings = make_settings(); let auction_request = make_auction_request(); let mut bid = make_bid("div-gpt-top", "aps", Some(2.75)); - bid.renderer = Some(BidRenderer::Aps(ApsRendererV1 { - version: 1, - account_id: "example-account".to_string(), - bid_id: "fictional-bid".to_string(), - creative_id: None, - tag_type: ApsTagType::Iframe, - creative_url: "https://creative.example/render".to_string(), - aax_response: "fictional-base64".to_string(), - width: 300, - height: 250, - })); + bid.renderer = Some( + BidRenderer::from_typed( + APS_RENDERER_TYPE, + &ApsRendererV1 { + version: 1, + account_id: "example-account".to_string(), + bid_id: "fictional-bid".to_string(), + creative_id: None, + tag_type: ApsTagType::Iframe, + creative_url: "https://creative.example/render".to_string(), + aax_response: "fictional-base64".to_string(), + width: 300, + height: 250, + }, + ) + .expect("should build APS renderer descriptor"), + ); let result = make_result(bid); let response = convert_to_openrtb_response(&result, &settings, &auction_request, false) @@ -1605,17 +1616,23 @@ mod tests { bid.bid_id = Some("fictional-bid".to_string()); bid.ad_id = Some("fictional-ad".to_string()); bid.creative_id = Some("fictional-creative".to_string()); - bid.renderer = Some(BidRenderer::Aps(ApsRendererV1 { - version: 1, - account_id: "example-account".to_string(), - bid_id: "fictional-bid".to_string(), - creative_id: Some("fictional-creative".to_string()), - tag_type: ApsTagType::Iframe, - creative_url: "https://creative.example/render".to_string(), - aax_response: "fictional-base64".to_string(), - width: 300, - height: 250, - })); + bid.renderer = Some( + BidRenderer::from_typed( + APS_RENDERER_TYPE, + &ApsRendererV1 { + version: 1, + account_id: "example-account".to_string(), + bid_id: "fictional-bid".to_string(), + creative_id: Some("fictional-creative".to_string()), + tag_type: ApsTagType::Iframe, + creative_url: "https://creative.example/render".to_string(), + aax_response: "fictional-base64".to_string(), + width: 300, + height: 250, + }, + ) + .expect("should build APS renderer descriptor"), + ); let result = make_result(bid); let response = convert_to_openrtb_response(&result, &settings, &auction_request, false) diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 7932a187bf..e63d74d9dd 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -1543,10 +1543,11 @@ mod tests { use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; use crate::auction::test_support::create_test_auction_context; use crate::auction::types::{ - AdFormat, AdSlot, ApsRendererV1, ApsTagType, AuctionContext, AuctionRequest, - AuctionResponse, Bid, BidRenderer, BidStatus, MediaType, PublisherInfo, UserInfo, + AdFormat, AdSlot, AuctionContext, AuctionRequest, AuctionResponse, Bid, BidRenderer, + BidStatus, MediaType, PublisherInfo, UserInfo, }; use crate::error::TrustedServerError; + use crate::integrations::aps::{APS_RENDERER_TYPE, ApsRendererV1, ApsTagType}; use crate::platform::test_support::{ StubHttpClient, build_services_with_backend_and_http_client, build_services_with_http_client, noop_services, @@ -1814,17 +1815,21 @@ mod tests { fn auction_bid(bidder: &str, price: f64) -> Bid { let renderer = (bidder == "aps").then(|| { - BidRenderer::Aps(ApsRendererV1 { - version: 1, - account_id: "example-account".to_string(), - bid_id: "aps-selected-bid".to_string(), - creative_id: None, - tag_type: ApsTagType::Iframe, - creative_url: "https://creative.example/render".to_string(), - aax_response: "fictional-base64".to_string(), - width: 300, - height: 250, - }) + BidRenderer::from_typed( + APS_RENDERER_TYPE, + &ApsRendererV1 { + version: 1, + account_id: "example-account".to_string(), + bid_id: "aps-selected-bid".to_string(), + creative_id: None, + tag_type: ApsTagType::Iframe, + creative_url: "https://creative.example/render".to_string(), + aax_response: "fictional-base64".to_string(), + width: 300, + height: 250, + }, + ) + .expect("should build APS renderer descriptor") }); Bid { slot_id: "slot-1".to_string(), diff --git a/crates/trusted-server-core/src/auction/types.rs b/crates/trusted-server-core/src/auction/types.rs index f61334787f..d9459733e5 100644 --- a/crates/trusted-server-core/src/auction/types.rs +++ b/crates/trusted-server-core/src/auction/types.rs @@ -1,11 +1,14 @@ //! Core types for auction requests and responses. use edgezero_core::body::Body as EdgeBody; +use error_stack::{Report, ResultExt as _, bail, ensure}; use http::Request; +use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, HashMap}; use crate::auction::context::ContextValue; +use crate::error::TrustedServerError; use crate::geo::GeoInfo; use crate::platform::RuntimeServices; use crate::settings::Settings; @@ -175,56 +178,117 @@ pub struct AuctionResponse { pub metadata: HashMap, } -/// APS creative tag type accepted by the Trusted Server renderer. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum ApsTagType { - /// APS loads the creative URL in a nested iframe. - Iframe, - /// APS fetches creative HTML and executes it in its nested renderer frame. - Script, -} - -/// Version 1 APS renderer descriptor shared with browser clients. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ApsRendererV1 { - /// Renderer contract version. - pub version: u8, - /// APS account identifier used to initialize the fixed runner. - pub account_id: String, - /// Selected `OpenRTB` bid identifier. - pub bid_id: String, - /// Optional `OpenRTB` creative identifier. - #[serde(skip_serializing_if = "Option::is_none")] - pub creative_id: Option, - /// APS creative delivery mode. - pub tag_type: ApsTagType, - /// HTTPS creative URL consumed by the fixed APS runner. - pub creative_url: String, - /// Base64-encoded exact one-bid APS response envelope. - pub aax_response: String, - /// Creative width. - pub width: u32, - /// Creative height. - pub height: u32, -} +/// Wire key carrying the renderer type tag. +/// +/// A payload may not use this key, since it would collide with the tag when +/// the descriptor is serialized flat. +const RENDERER_TYPE_KEY: &str = "type"; -/// Typed browser renderer capability carried by a bid. +/// Browser renderer capability carried by a bid: a type tag and the payload +/// the auction provider that produced the bid defines. +/// +/// Serialized flat, as `{"type": "", ...payload}`, so a page receives +/// the same bytes whether the provider lives in core or in its own crate. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "lowercase")] -pub enum BidRenderer { - /// APS renderer version 1. - Aps(ApsRendererV1), +pub struct BidRenderer { + #[serde(rename = "type")] + renderer_type: String, + #[serde(flatten)] + payload: serde_json::Map, } impl BidRenderer { - /// Return the APS renderer descriptor when this is an APS renderer. + /// Build a descriptor from a type tag and the provider's JSON payload. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::Auction`] when `payload` is not a JSON + /// object, or when it carries its own `type` key, which would collide with + /// the tag once the descriptor is serialized flat. + /// + /// # Examples + /// + /// ``` + /// use serde_json::json; + /// use trusted_server_core::auction::types::BidRenderer; + /// + /// let renderer = BidRenderer::new("example", json!({ "version": 1 })) + /// .expect("should accept an object payload"); + /// assert_eq!(renderer.renderer_type(), "example"); + /// ``` + pub fn new( + renderer_type: &str, + payload: serde_json::Value, + ) -> Result> { + let serde_json::Value::Object(payload) = payload else { + bail!(TrustedServerError::Auction { + message: format!("Renderer '{renderer_type}' payload must be a JSON object"), + }); + }; + ensure!( + !payload.contains_key(RENDERER_TYPE_KEY), + TrustedServerError::Auction { + message: format!( + "Renderer '{renderer_type}' payload must not carry a '{RENDERER_TYPE_KEY}' key" + ), + } + ); + Ok(Self { + renderer_type: renderer_type.to_string(), + payload, + }) + } + + /// Build a descriptor by serializing a provider's own payload type. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::Auction`] when `payload` cannot be + /// serialized, and when the serialized form is rejected by + /// [`new`](Self::new). + /// + /// # Examples + /// + /// ``` + /// use serde::Serialize; + /// use trusted_server_core::auction::types::BidRenderer; + /// + /// #[derive(Serialize)] + /// struct ExampleRendererV1 { + /// version: u8, + /// } + /// + /// let renderer = BidRenderer::from_typed("example", &ExampleRendererV1 { version: 1 }) + /// .expect("should accept a struct payload"); + /// assert_eq!(renderer.renderer_type(), "example"); + /// ``` + pub fn from_typed( + renderer_type: &str, + payload: &T, + ) -> Result> { + let payload = + serde_json::to_value(payload).change_context(TrustedServerError::Auction { + message: format!("Failed to serialize renderer '{renderer_type}' payload"), + })?; + Self::new(renderer_type, payload) + } + + /// Return the renderer type tag a page reads to select its renderer. #[must_use] - pub fn as_aps(&self) -> Option<&ApsRendererV1> { - match self { - Self::Aps(renderer) => Some(renderer), + pub fn renderer_type(&self) -> &str { + &self.renderer_type + } + + /// Deserialize the payload into the provider's own descriptor type. + /// + /// Returns `None` when the descriptor carries a different tag, and when the + /// payload does not match `T`. + #[must_use] + pub fn payload_as(&self, renderer_type: &str) -> Option { + if self.renderer_type != renderer_type { + return None; } + serde_json::from_value(serde_json::Value::Object(self.payload.clone())).ok() } } @@ -399,6 +463,7 @@ impl AuctionResponse { #[cfg(test)] mod tests { use super::*; + use crate::integrations::aps::{APS_RENDERER_TYPE, ApsRendererV1, ApsTagType}; use serde_json::json; fn make_bid(bidder: &str) -> Bid { @@ -575,17 +640,21 @@ mod tests { #[test] fn aps_renderer_serializes_to_versioned_camel_case_contract() { - let renderer = BidRenderer::Aps(ApsRendererV1 { - version: 1, - account_id: "example-account-id".to_string(), - bid_id: "fictional-bid-id".to_string(), - creative_id: Some("fictional-creative-id".to_string()), - tag_type: ApsTagType::Iframe, - creative_url: "https://creative.example/render".to_string(), - aax_response: "base64-data".to_string(), - width: 300, - height: 250, - }); + let renderer = BidRenderer::from_typed( + APS_RENDERER_TYPE, + &ApsRendererV1 { + version: 1, + account_id: "example-account-id".to_string(), + bid_id: "fictional-bid-id".to_string(), + creative_id: Some("fictional-creative-id".to_string()), + tag_type: ApsTagType::Iframe, + creative_url: "https://creative.example/render".to_string(), + aax_response: "base64-data".to_string(), + width: 300, + height: 250, + }, + ) + .expect("should build APS renderer descriptor"); let serialized = serde_json::to_value(&renderer).expect("should serialize renderer"); @@ -609,23 +678,148 @@ mod tests { #[test] fn aps_renderer_omits_absent_creative_id() { - let renderer = BidRenderer::Aps(ApsRendererV1 { + let renderer = BidRenderer::from_typed( + APS_RENDERER_TYPE, + &ApsRendererV1 { + version: 1, + account_id: "example-account-id".to_string(), + bid_id: "fictional-bid-id".to_string(), + creative_id: None, + tag_type: ApsTagType::Iframe, + creative_url: "https://creative.example/render".to_string(), + aax_response: "base64-data".to_string(), + width: 300, + height: 250, + }, + ) + .expect("should build APS renderer descriptor"); + + let serialized = serde_json::to_value(&renderer).expect("should serialize renderer"); + + assert!( + serialized.get("creativeId").is_none(), + "should omit absent creative ID" + ); + } + + #[test] + fn the_open_renderer_serializes_to_the_same_bytes_as_the_aps_variant_did() { + // Literal strings captured from the closed-enum form before this + // change, through the same `serde_json::to_value` path production + // uses: `BidExt::to_ext` for the OpenRTB response extension, and + // `build_bid_map` for `window.tsjs.bids`. Both hand the page an object + // whose keys are ordered by `serde_json::Map`, so these are the bytes + // a browser actually receives. + let full = BidRenderer::from_typed( + APS_RENDERER_TYPE, + &ApsRendererV1 { + version: 1, + account_id: "example-account-id".to_string(), + bid_id: "fictional-bid-id".to_string(), + creative_id: Some("fictional-creative-id".to_string()), + tag_type: ApsTagType::Iframe, + creative_url: "https://creative.example/render".to_string(), + aax_response: "base64-data".to_string(), + width: 300, + height: 250, + }, + ) + .expect("should build APS renderer descriptor"); + let absent = BidRenderer::from_typed( + APS_RENDERER_TYPE, + &ApsRendererV1 { + version: 1, + account_id: "example-account-id".to_string(), + bid_id: "fictional-bid-id".to_string(), + creative_id: None, + tag_type: ApsTagType::Script, + creative_url: "https://creative.example/render".to_string(), + aax_response: "base64-data".to_string(), + width: 300, + height: 250, + }, + ) + .expect("should build APS renderer descriptor"); + + let full_bytes = serde_json::to_string( + &serde_json::to_value(&full).expect("should convert renderer to a JSON value"), + ) + .expect("should serialize renderer"); + let absent_bytes = serde_json::to_string( + &serde_json::to_value(&absent).expect("should convert renderer to a JSON value"), + ) + .expect("should serialize renderer"); + + assert_eq!( + full_bytes, + "{\"aaxResponse\":\"base64-data\",\"accountId\":\"example-account-id\",\"bidId\":\"fictional-bid-id\",\"creativeId\":\"fictional-creative-id\",\"creativeUrl\":\"https://creative.example/render\",\"height\":250,\"tagType\":\"iframe\",\"type\":\"aps\",\"version\":1,\"width\":300}", + "should serialize to the bytes the closed enum produced" + ); + assert_eq!( + absent_bytes, + "{\"aaxResponse\":\"base64-data\",\"accountId\":\"example-account-id\",\"bidId\":\"fictional-bid-id\",\"creativeUrl\":\"https://creative.example/render\",\"height\":250,\"tagType\":\"script\",\"type\":\"aps\",\"version\":1,\"width\":300}", + "should serialize to the bytes the closed enum produced with no creative ID" + ); + } + + #[test] + fn renderer_round_trips_through_its_wire_form() { + let descriptor = ApsRendererV1 { version: 1, account_id: "example-account-id".to_string(), bid_id: "fictional-bid-id".to_string(), - creative_id: None, + creative_id: Some("fictional-creative-id".to_string()), tag_type: ApsTagType::Iframe, creative_url: "https://creative.example/render".to_string(), aax_response: "base64-data".to_string(), width: 300, height: 250, - }); + }; + let renderer = BidRenderer::from_typed(APS_RENDERER_TYPE, &descriptor) + .expect("should build APS renderer descriptor"); - let serialized = serde_json::to_value(&renderer).expect("should serialize renderer"); + let serialized = serde_json::to_string(&renderer).expect("should serialize renderer"); + let restored: BidRenderer = + serde_json::from_str(&serialized).expect("should deserialize renderer"); + + assert_eq!( + restored.renderer_type(), + APS_RENDERER_TYPE, + "should round-trip the renderer type tag" + ); + assert_eq!( + restored + .payload_as::(APS_RENDERER_TYPE) + .expect("should deserialize the APS payload"), + descriptor, + "should round-trip the provider payload" + ); + } + + #[test] + fn renderer_payload_is_hidden_from_a_different_type_tag() { + let renderer = BidRenderer::new(APS_RENDERER_TYPE, json!({ "version": 1 })) + .expect("should build renderer descriptor"); assert!( - serialized.get("creativeId").is_none(), - "should omit absent creative ID" + renderer.payload_as::("example").is_none(), + "should refuse a payload requested under a different tag" + ); + } + + #[test] + fn renderer_rejects_a_payload_that_is_not_an_object() { + assert!( + BidRenderer::new(APS_RENDERER_TYPE, json!("not-an-object")).is_err(), + "should reject a payload that is not a JSON object" + ); + } + + #[test] + fn renderer_rejects_a_payload_carrying_its_own_type_key() { + assert!( + BidRenderer::new(APS_RENDERER_TYPE, json!({ "type": "other", "version": 1 })).is_err(), + "should reject a payload that would collide with the type tag" ); } diff --git a/crates/trusted-server-core/src/integrations/adserver_mock.rs b/crates/trusted-server-core/src/integrations/adserver_mock.rs index 1b4cf403f5..a0d13a618a 100644 --- a/crates/trusted-server-core/src/integrations/adserver_mock.rs +++ b/crates/trusted-server-core/src/integrations/adserver_mock.rs @@ -660,17 +660,27 @@ mod tests { bid_id: Some(bid_id.to_string()), ad_id: None, creative_id: Some(format!("creative-{bid_id}")), - renderer: Some(BidRenderer::Aps(ApsRendererV1 { - version: 1, - account_id: "example-account".to_string(), - bid_id: bid_id.to_string(), - creative_id: Some(format!("creative-{bid_id}")), - tag_type: ApsTagType::Iframe, - creative_url: format!("https://creative.example/{bid_id}"), - aax_response: format!("fictional-{bid_id}-base64"), - width: 728, - height: 90, - })), + // The mock is not APS, so it builds the neutral descriptor from + // the wire keys rather than from the APS descriptor type. The "aps" + // tag is used on purpose: these fixtures mimic an APS bid arriving + // at the mediator. + renderer: Some( + BidRenderer::new( + "aps", + json!({ + "version": 1, + "accountId": "example-account", + "bidId": bid_id, + "creativeId": format!("creative-{bid_id}"), + "tagType": "iframe", + "creativeUrl": format!("https://creative.example/{bid_id}"), + "aaxResponse": format!("fictional-{bid_id}-base64"), + "width": 728, + "height": 90 + }), + ) + .expect("should build renderer descriptor"), + ), cache_id: None, cache_host: None, cache_path: None, @@ -857,17 +867,25 @@ mod tests { bid_id: Some("source-bid-id".to_string()), ad_id: Some("bid-impression-id".to_string()), creative_id: Some("source-creative-id".to_string()), - renderer: Some(BidRenderer::Aps(ApsRendererV1 { - version: 1, - account_id: "example-account".to_string(), - bid_id: "source-bid-id".to_string(), - creative_id: Some("source-creative-id".to_string()), - tag_type: ApsTagType::Iframe, - creative_url: "https://creative.example/render".to_string(), - aax_response: "fictional-base64".to_string(), - width: 728, - height: 90, - })), + // Neutral descriptor with the "aps" tag: the mock mimics an + // APS bid here without depending on the APS descriptor type. + renderer: Some( + BidRenderer::new( + "aps", + json!({ + "version": 1, + "accountId": "example-account", + "bidId": "source-bid-id", + "creativeId": "source-creative-id", + "tagType": "iframe", + "creativeUrl": "https://creative.example/render", + "aaxResponse": "fictional-base64", + "width": 728, + "height": 90 + }), + ) + .expect("should build renderer descriptor"), + ), cache_id: Some("cache-uuid".to_string()), cache_host: Some("cache.example".to_string()), cache_path: Some("/cache".to_string()), @@ -1044,16 +1062,13 @@ mod tests { .first() .expect("should restore mediated APS winner"); assert_eq!(winner.bid_id.as_deref(), Some("selected")); - assert_eq!( - winner - .renderer - .as_ref() - .expect("should restore APS renderer") - .as_aps() - .expect("should be APS renderer") - .bid_id, - "selected" - ); + let renderer = winner + .renderer + .as_ref() + .expect("should restore APS renderer") + .payload_as::("aps") + .expect("should carry an APS-tagged payload"); + assert_eq!(renderer["bidId"], "selected"); assert!(winner.creative.is_none()); } diff --git a/crates/trusted-server-core/src/integrations/aps.rs b/crates/trusted-server-core/src/integrations/aps.rs index a7df12e106..6673afc6ad 100644 --- a/crates/trusted-server-core/src/integrations/aps.rs +++ b/crates/trusted-server-core/src/integrations/aps.rs @@ -18,8 +18,7 @@ use validator::{Validate, ValidationError}; use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; use crate::auction::types::{ - AdSlot, ApsRendererV1, ApsTagType, AuctionContext, AuctionRequest, AuctionResponse, Bid, - BidRenderer, MediaType, + AdSlot, AuctionContext, AuctionRequest, AuctionResponse, Bid, BidRenderer, MediaType, }; use crate::error::TrustedServerError; use crate::integrations::{ @@ -35,6 +34,9 @@ use crate::platform::{PlatformHttpRequest, PlatformResponse, RuntimeServices}; use crate::settings::{IntegrationConfig, Settings}; const APS_INTEGRATION_ID: &str = "aps"; +/// Renderer type tag carried on the wire by an APS bid, read by the browser to +/// select the APS renderer. +pub const APS_RENDERER_TYPE: &str = "aps"; const APS_RENDERER_ROUTE: &str = "/integrations/aps/renderer"; const DEFAULT_CURRENCY: &str = "USD"; const APS_SDK_SOURCE: &str = "prebid"; @@ -48,6 +50,45 @@ const MAX_PAGE_URL_BYTES: usize = 8192; const MAX_RENDER_ENVELOPE_BYTES: usize = 256 * 1024; const APS_RENDERER_CSP: &str = "default-src 'none'; sandbox allow-forms allow-pointer-lock allow-popups allow-popups-to-escape-sandbox allow-scripts allow-top-navigation-by-user-activation; script-src 'unsafe-inline' https:; connect-src https:; frame-src https:; img-src https: data:; media-src https: blob:; style-src 'unsafe-inline' https:; font-src https: data:;"; +/// APS creative tag type accepted by the Trusted Server renderer. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum ApsTagType { + /// APS loads the creative URL in a nested iframe. + Iframe, + /// APS fetches creative HTML and executes it in its nested renderer frame. + Script, +} + +/// Version 1 APS renderer descriptor shared with browser clients. +/// +/// Carried by a bid as the payload of a [`BidRenderer`] tagged +/// [`APS_RENDERER_TYPE`], so it travels with the APS integration rather than +/// with the neutral auction types. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ApsRendererV1 { + /// Renderer contract version. + pub version: u8, + /// APS account identifier used to initialize the fixed runner. + pub account_id: String, + /// Selected `OpenRTB` bid identifier. + pub bid_id: String, + /// Optional `OpenRTB` creative identifier. + #[serde(skip_serializing_if = "Option::is_none")] + pub creative_id: Option, + /// APS creative delivery mode. + pub tag_type: ApsTagType, + /// HTTPS creative URL consumed by the fixed APS runner. + pub creative_url: String, + /// Base64-encoded exact one-bid APS response envelope. + pub aax_response: String, + /// Creative width. + pub width: u32, + /// Creative height. + pub height: u32, +} + const APS_RENDERER_DOCUMENT: &str = r#" @@ -719,7 +760,7 @@ impl ApsAuctionProvider { if serialized.len() > MAX_RENDER_ENVELOPE_BYTES { return None; } - Some(BidRenderer::Aps(ApsRendererV1 { + let descriptor = ApsRendererV1 { version: 1, account_id: self.config.account_id.clone(), bid_id: input.bid_id.to_string(), @@ -729,7 +770,17 @@ impl ApsAuctionProvider { aax_response: BASE64_STANDARD.encode(serialized), width: input.width, height: input.height, - })) + }; + match BidRenderer::from_typed(APS_RENDERER_TYPE, &descriptor) { + Ok(renderer) => Some(renderer), + Err(error) => { + log::warn!( + "Dropping APS bid '{}': its renderer descriptor could not be built: {error:?}", + input.bid_id + ); + None + } + } } fn increment_reason(reasons: &mut BTreeMap, reason: &'static str) { @@ -1829,7 +1880,7 @@ mod tests { .renderer .as_ref() .expect("should include renderer") - .as_aps() + .payload_as::(APS_RENDERER_TYPE) .expect("should be APS renderer"); let decoded = BASE64_STANDARD .decode(&renderer.aax_response) @@ -1883,7 +1934,7 @@ mod tests { bid.renderer .as_ref() .expect("should retain renderer") - .as_aps() + .payload_as::(APS_RENDERER_TYPE) .expect("should be APS renderer") .creative_id .is_none() @@ -2295,7 +2346,7 @@ mod tests { .renderer .as_ref() .expect("should keep script renderer") - .as_aps() + .payload_as::(APS_RENDERER_TYPE) .expect("should be APS renderer"); assert_eq!(renderer.tag_type, ApsTagType::Script); } diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 1fb150bef0..1824ae6bd4 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -65,6 +65,7 @@ use crate::error::TrustedServerError; use crate::html_processor::BodyCloseInjection; use crate::http_util::{RequestInfo, is_navigation_request, serve_static_with_etag}; use crate::integrations::IntegrationRegistry; +use crate::integrations::aps::{APS_RENDERER_TYPE, ApsRendererV1}; use crate::permissions::PermissionState; use crate::platform::{ GeoInfo, PlatformBackendSpec, PlatformHttpRequest, RuntimeServices, VarySpec, @@ -5267,13 +5268,15 @@ pub(crate) fn build_bid_map_with_auction_id( // OpenRTB bid ID. The latter is the last resort: it is unique per // bid instance rather than a creative, but GAM echoes it verbatim // so the render bridge can find the exact winning bid. - let renderer_bid_id = bid.renderer.as_ref().and_then(|renderer| { - renderer - .as_aps() - .map(|renderer| renderer.bid_id.as_str()) - }); + let renderer_bid_id = bid + .renderer + .as_ref() + .and_then(|renderer| { + renderer.payload_as::(APS_RENDERER_TYPE) + }) + .map(|renderer| renderer.bid_id); let hb_adid = non_empty(bid.cache_id.as_deref()) - .or_else(|| renderer_bid_id.and_then(|id| non_empty(Some(id)))) + .or_else(|| non_empty(renderer_bid_id.as_deref())) .or_else(|| non_empty(bid.ad_id.as_deref())) .or_else(|| non_empty(bid.bid_id.as_deref())); if let Some(id) = hb_adid { @@ -18359,12 +18362,13 @@ mod tests { build_bid_map, build_bids_script, diagnostics_auction_id, html_escape_for_script, write_bids_to_state, }; - use crate::auction::types::{ApsRendererV1, ApsTagType, Bid, BidRenderer, MediaType}; + use crate::auction::types::{Bid, BidRenderer, MediaType}; use crate::consent::ConsentContext; use crate::creative_opportunities::{ CreativeOpportunitiesConfig, CreativeOpportunityFormat, CreativeOpportunitySlot, }; use crate::http_util::RequestInfo; + use crate::integrations::aps::{APS_RENDERER_TYPE, ApsRendererV1, ApsTagType}; use crate::price_bucket::PriceGranularity; use crate::settings::Settings; use std::collections::HashMap; @@ -19533,17 +19537,23 @@ mod tests { bid.creative = Some("".to_string()); bid.nurl = None; bid.burl = None; - bid.renderer = Some(BidRenderer::Aps(ApsRendererV1 { - version: 1, - account_id: "example-account".to_string(), - bid_id: "selected-bid".to_string(), - creative_id: None, - tag_type: ApsTagType::Iframe, - creative_url: "https://creative.example/render".to_string(), - aax_response: "fictional-base64".to_string(), - width: 300, - height: 250, - })); + bid.renderer = Some( + BidRenderer::from_typed( + APS_RENDERER_TYPE, + &ApsRendererV1 { + version: 1, + account_id: "example-account".to_string(), + bid_id: "selected-bid".to_string(), + creative_id: None, + tag_type: ApsTagType::Iframe, + creative_url: "https://creative.example/render".to_string(), + aax_response: "fictional-base64".to_string(), + width: 300, + height: 250, + }, + ) + .expect("should build APS renderer descriptor"), + ); let winning_bids = HashMap::from([("atf_sidebar_ad".to_string(), bid)]); let map = build_bid_map(&winning_bids, PriceGranularity::Dense, &settings, "", false); From 162bf70a200d397ebe8eacd6b05cca0257e0dfb5 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 28 Aug 2026 20:12:04 +0100 Subject: [PATCH 015/124] Run integration request preparers from the registry Every adapter used to call one named integration, GPT diagnostics, directly on the request path before routing, which kept a vendor-shaped integration wired into all four entry points. The registry now collects the request preparers the integration builders carry and runs them in registration order, so the adapters call `state.registry.prepare_request(...)` and no longer name an integration. Preparers are collected before the enabled check, so they run whether or not their integration is enabled. That is the behavior the adapters had, because an integration must be able to strip its own reserved query or cookie in a deployment that has it switched off, which is exactly what GPT diagnostics does with `ts_console` and `__Host-ts-console`. GPT diagnostics is now attached as a builder hook, `with_request_preparer(gpt_diagnostics::prepare_request_hook)`, in the built-in table. The hook discards the decision, which stays in the request extensions for the publisher path to read, and core still calls `gpt_diagnostics::prepare_request` directly in `publisher.rs` as an idempotent safety net for direct handler tests. --- crates/trusted-server-adapter-axum/src/app.rs | 7 +- .../src/app.rs | 10 +- .../trusted-server-adapter-fastly/src/app.rs | 10 +- crates/trusted-server-adapter-spin/src/app.rs | 19 +- .../src/integrations/gpt_diagnostics.rs | 13 + .../src/integrations/mod.rs | 6 +- .../src/integrations/registry.rs | 228 ++++++++++++++++++ 7 files changed, 252 insertions(+), 41 deletions(-) diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 6864a6b933..5f292c44d7 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -152,10 +152,7 @@ where { let services = build_runtime_services(&ctx, &state.settings); let mut req = ctx.into_request(); - if let Err(error) = trusted_server_core::integrations::gpt_diagnostics::prepare_request( - &state.settings, - &mut req, - ) { + if let Err(error) = state.registry.prepare_request(&state.settings, &mut req) { return Ok(http_error(&error)); } Ok(handler(state, services, req) @@ -210,7 +207,7 @@ async fn dispatch_fallback( return Ok(response); } - trusted_server_core::integrations::gpt_diagnostics::prepare_request(&state.settings, &mut req)?; + state.registry.prepare_request(&state.settings, &mut req)?; let path = req.uri().path().to_string(); let method = req.method().clone(); diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index 93ba63b3fc..474cf23e18 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -212,10 +212,7 @@ where Box::pin(async move { let services = build_per_request_services(&s, &ctx); let mut req = ctx.into_request(); - if let Err(error) = trusted_server_core::integrations::gpt_diagnostics::prepare_request( - &s.settings, - &mut req, - ) { + if let Err(error) = s.registry.prepare_request(&s.settings, &mut req) { return Ok(http_error(&error)); } Ok(f(s, services, req).await.unwrap_or_else(|e| http_error(&e))) @@ -413,10 +410,7 @@ fn build_router(state: &Arc) -> RouterService { if let Some(response) = deny_admin_diagnostic_fallback(&req) { return Ok(response); } - if let Err(error) = trusted_server_core::integrations::gpt_diagnostics::prepare_request( - &state.settings, - &mut req, - ) { + if let Err(error) = state.registry.prepare_request(&state.settings, &mut req) { return Ok(http_error(&error)); } let path = req.uri().path().to_owned(); diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index f1204a8613..c7f3c94f81 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -643,10 +643,7 @@ async fn execute_named( return Ok(response); } - if let Err(report) = trusted_server_core::integrations::gpt_diagnostics::prepare_request( - &state.settings, - &mut req, - ) { + if let Err(report) = state.registry.prepare_request(&state.settings, &mut req) { return Ok(http_error(&report)); } @@ -853,10 +850,7 @@ async fn dispatch_fallback( let path = req.uri().path().to_string(); let method = req.method().clone(); - if let Err(report) = trusted_server_core::integrations::gpt_diagnostics::prepare_request( - &state.settings, - &mut req, - ) { + if let Err(report) = state.registry.prepare_request(&state.settings, &mut req) { return http_error(&report); } diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index de3bc7f9c1..6ee3fd6434 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -607,12 +607,7 @@ fn build_router(state: &Arc) -> RouterService { // OpenRTB metadata that auction signing derives from // `RequestInfo::from_request` uses the trusted runtime authority. let mut req = ctx.into_request(); - if let Err(error) = - trusted_server_core::integrations::gpt_diagnostics::prepare_request( - &s.settings, - &mut req, - ) - { + if let Err(error) = s.registry.prepare_request(&s.settings, &mut req) { return Ok(http_error(&error)); } // Build the geo-aware EC context so the auction consent gate sees @@ -645,12 +640,7 @@ fn build_router(state: &Arc) -> RouterService { async move { let services = build_per_request_services(&s, &ctx); let mut req = ctx.into_request(); - if let Err(error) = - trusted_server_core::integrations::gpt_diagnostics::prepare_request( - &s.settings, - &mut req, - ) - { + if let Err(error) = s.registry.prepare_request(&s.settings, &mut req) { return Ok(http_error(&error)); } // Identity could not be established (for example the selected @@ -748,10 +738,7 @@ fn build_router(state: &Arc) -> RouterService { if let Some(response) = deny_admin_diagnostic_fallback(&req) { return Ok(response); } - if let Err(error) = trusted_server_core::integrations::gpt_diagnostics::prepare_request( - &state.settings, - &mut req, - ) { + if let Err(error) = state.registry.prepare_request(&state.settings, &mut req) { return Ok(http_error(&error)); } diff --git a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs b/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs index 60c2111023..e3f5226182 100644 --- a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs +++ b/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs @@ -333,6 +333,19 @@ pub fn prepare_request( Ok(decision) } +/// Builder hook: prepares the request and discards the decision, which stays +/// in the request extensions for the publisher path to read. +/// +/// # Errors +/// +/// Returns the error from [`prepare_request`]. +pub(crate) fn prepare_request_hook( + settings: &Settings, + request: &mut Request, +) -> Result<(), Report> { + prepare_request(settings, request).map(|_| ()) +} + /// Read the request decision, defaulting to inactive when not prepared. #[must_use] pub fn request_decision(request: &Request) -> GptDiagnosticsRequestDecision { diff --git a/crates/trusted-server-core/src/integrations/mod.rs b/crates/trusted-server-core/src/integrations/mod.rs index 5fb7545884..2a8d7d645a 100644 --- a/crates/trusted-server-core/src/integrations/mod.rs +++ b/crates/trusted-server-core/src/integrations/mod.rs @@ -407,9 +407,6 @@ impl IntegrationBuilder { } /// The request preparation function, when one is attached. - // The adapters do not yet run request preparers, so nothing in the crate - // calls this until that is wired up. - #[allow(dead_code)] pub(crate) fn prepare_request(&self) -> Option { self.prepare_request } @@ -459,7 +456,8 @@ const BUILT_IN_BUILDERS: &[IntegrationBuilder] = &[ CORE_SOURCE, gpt_diagnostics::register, gpt_diagnostics::validate, - ), + ) + .with_request_preparer(gpt_diagnostics::prepare_request_hook), ]; /// The built-in integration builders, in hook order. diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index b7382a4fe4..793b9c0424 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -775,6 +775,8 @@ struct IntegrationRegistryInner { /// a registered integration, for example a module tied to the selected Edge /// Cookie provider. Populated in [`IntegrationRegistry::new`] from settings. extra_js_module_ids: Vec<&'static str>, + // Preparers from every builder, enabled or not, in registration order. + request_preparers: Vec, } impl Default for IntegrationRegistryInner { @@ -800,6 +802,7 @@ impl Default for IntegrationRegistryInner { head_injectors: Vec::new(), request_filters: Vec::new(), extra_js_module_ids: Vec::new(), + request_preparers: Vec::new(), } } } @@ -897,6 +900,12 @@ impl IntegrationRegistry { })); } inner.builder_ids.push((builder.id(), builder.source())); + // Preparers are collected before the enabled check, so an + // integration can sanitize its own reserved query or cookie in a + // deployment that has it switched off. + if let Some(prepare) = builder.prepare_request() { + inner.request_preparers.push(prepare); + } if let Some(registration) = builder.build(settings)? { debug_assert_eq!( @@ -1038,6 +1047,27 @@ impl IntegrationRegistry { self.find_route(method, path).is_some() } + /// Runs every registered integration's request preparer, in registration + /// order, before routing. + /// + /// Preparers run whether or not their integration is enabled, so an + /// integration can sanitize its own reserved query or cookie in a + /// deployment that has it switched off. + /// + /// # Errors + /// + /// Returns the first preparer's error. + pub fn prepare_request( + &self, + settings: &Settings, + request: &mut Request, + ) -> Result<(), Report> { + for prepare in &self.inner.request_preparers { + prepare(settings, request)?; + } + Ok(()) + } + /// Run pre-routing request filters. /// /// Request header mutations are applied immediately so later filters and @@ -1485,6 +1515,7 @@ impl IntegrationRegistry { html_post_processors: Vec::new(), head_injectors: Vec::new(), request_filters: Vec::new(), + request_preparers: Vec::new(), deferred_js_ids: Vec::new(), disabled_js_ids: Vec::new(), extra_js_module_ids: Vec::new(), @@ -1518,6 +1549,7 @@ impl IntegrationRegistry { html_post_processors: Vec::new(), head_injectors, request_filters: Vec::new(), + request_preparers: Vec::new(), deferred_js_ids: Vec::new(), disabled_js_ids: Vec::new(), extra_js_module_ids: Vec::new(), @@ -1547,6 +1579,7 @@ impl IntegrationRegistry { html_post_processors: Vec::new(), head_injectors: Vec::new(), request_filters, + request_preparers: Vec::new(), deferred_js_ids: Vec::new(), disabled_js_ids: Vec::new(), extra_js_module_ids: Vec::new(), @@ -1616,6 +1649,7 @@ impl IntegrationRegistry { html_post_processors: Vec::new(), head_injectors: Vec::new(), request_filters: Vec::new(), + request_preparers: Vec::new(), deferred_js_ids: Vec::new(), disabled_js_ids: Vec::new(), extra_js_module_ids: Vec::new(), @@ -3054,4 +3088,198 @@ mod tests { ); } } + + /// A builder function for an integration that is never enabled, so its + /// preparer is the only thing the registry can take from it. + fn never_enabled_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(None) + } + + /// Preparers cannot capture, because a builder holds plain function + /// pointers, so they record the order they ran in here. + static PREPARER_ORDER: Mutex> = Mutex::new(Vec::new()); + + fn record_first_preparer( + _settings: &Settings, + _request: &mut Request, + ) -> Result<(), Report> { + PREPARER_ORDER + .lock() + .expect("should lock the preparer order") + .push("first"); + Ok(()) + } + + fn record_second_preparer( + _settings: &Settings, + _request: &mut Request, + ) -> Result<(), Report> { + PREPARER_ORDER + .lock() + .expect("should lock the preparer order") + .push("second"); + Ok(()) + } + + /// Records whether the preparer registered after the failing one ran. + static PREPARER_AFTER_FAILURE: Mutex> = Mutex::new(Vec::new()); + + const FAILING_PREPARER_MESSAGE: &str = "probe preparer refused the request"; + + fn failing_preparer( + _settings: &Settings, + _request: &mut Request, + ) -> Result<(), Report> { + Err(Report::new(TrustedServerError::Configuration { + message: FAILING_PREPARER_MESSAGE.to_owned(), + })) + } + + fn record_after_failure_preparer( + _settings: &Settings, + _request: &mut Request, + ) -> Result<(), Report> { + PREPARER_AFTER_FAILURE + .lock() + .expect("should lock the after-failure record") + .push("after"); + Ok(()) + } + + fn plain_request() -> Request { + Request::builder() + .method(Method::GET) + .uri("https://publisher.example.com/article") + .body(EdgeBody::empty()) + .expect("should build request") + } + + #[test] + fn prepare_request_runs_every_preparer_in_registration_order_enabled_or_not() { + let settings = crate::test_support::tests::create_test_settings(); + let extra = [ + crate::integrations::IntegrationBuilder::new( + "probe-first", + "seam-probe", + never_enabled_registration, + validate_nothing, + ) + .with_request_preparer(record_first_preparer), + crate::integrations::IntegrationBuilder::new( + "probe-second", + "seam-probe", + never_enabled_registration, + validate_nothing, + ) + .with_request_preparer(record_second_preparer), + ]; + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should build registry with request preparers"); + assert!( + !registry.integration_enabled("probe-first"), + "should leave the first probe integration disabled" + ); + assert!( + !registry.integration_enabled("probe-second"), + "should leave the second probe integration disabled" + ); + PREPARER_ORDER + .lock() + .expect("should lock the preparer order") + .clear(); + let mut request = plain_request(); + + registry + .prepare_request(&settings, &mut request) + .expect("should run every request preparer"); + + assert_eq!( + *PREPARER_ORDER + .lock() + .expect("should lock the preparer order"), + vec!["first", "second"], + "should run both preparers in registration order even though neither integration is enabled" + ); + } + + #[test] + fn prepare_request_surfaces_the_first_preparer_error() { + let settings = crate::test_support::tests::create_test_settings(); + let extra = [ + crate::integrations::IntegrationBuilder::new( + "probe-failing", + "seam-probe", + never_enabled_registration, + validate_nothing, + ) + .with_request_preparer(failing_preparer), + crate::integrations::IntegrationBuilder::new( + "probe-after-failure", + "seam-probe", + never_enabled_registration, + validate_nothing, + ) + .with_request_preparer(record_after_failure_preparer), + ]; + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should build registry with request preparers"); + PREPARER_AFTER_FAILURE + .lock() + .expect("should lock the after-failure record") + .clear(); + let mut request = plain_request(); + + let error = registry + .prepare_request(&settings, &mut request) + .expect_err("should surface the failing preparer's error"); + + assert!( + error.to_string().contains(FAILING_PREPARER_MESSAGE), + "should keep the preparer's message intact: {error}" + ); + assert!( + PREPARER_AFTER_FAILURE + .lock() + .expect("should lock the after-failure record") + .is_empty(), + "should not run a preparer registered after the failing one" + ); + } + + #[test] + fn prepare_request_runs_the_built_in_gpt_diagnostics_preparer() { + // The adapters call the registry rather than naming GPT diagnostics, so + // the built-in table is what attaches the sanitizing preparer. This is + // asserted here rather than through an adapter route test because the + // stripped query is only visible on the publisher HTML path, which needs + // a live origin the adapter test harnesses do not have. + let settings = crate::test_support::tests::create_test_settings(); + let registry = IntegrationRegistry::new(&settings).expect("should build registry"); + let mut request = Request::builder() + .method(Method::GET) + .uri("https://publisher.example.com/article?ts_console=1&keep=yes") + .header(header::COOKIE, "__Host-ts-console=1; keep-me=yes") + .body(EdgeBody::empty()) + .expect("should build request"); + + registry + .prepare_request(&settings, &mut request) + .expect("should run the built-in preparers"); + + assert_eq!( + request.uri().query(), + Some("keep=yes"), + "should strip the reserved diagnostics query and keep the rest" + ); + assert_eq!( + request + .headers() + .get(header::COOKIE) + .map(|value| value.to_str().expect("cookie should be text")), + Some("keep-me=yes"), + "should strip the reserved diagnostics cookie and keep the rest" + ); + } } From 943c2552edb9de215987ace84dcf5dcc4be74d6b Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 28 Aug 2026 20:43:05 +0100 Subject: [PATCH 016/124] Select a module's geo provider through the registration Geolocation is supplied by vendors as often as by hosts, so it becomes the first capability an integration module can declare on its own registration and a deployment can select by configuration. Identity and device providers follow the same shape once their traits land. An `IntegrationRegistration` now carries an optional `geo_provider`, set with the new `with_geo_provider` builder method. The registry collects every declared provider from the enabled registrations and resolves the new `[geo] provider` setting against them once at construction: - unset leaves the adapter's own host lookup in place, - `none` resolves the new `DisabledGeo`, which returns no location at all, - a module id resolves that module's provider. Anything else is a startup configuration error. Because only an enabled registration reaches the collection loop, the error tells the three cases apart by name, so a selector pointing at a module that is registered but switched off says so rather than reading as a module that never declared the capability. A module that declares a geo provider the selector does not choose is logged as a warning, so an operator can see a capability shipped but unused. Each adapter applies the resolved provider where it builds per-request services, through `RuntimeServices::with_geo`, so the request path is unchanged from the caller's point of view when no module is selected. --- crates/trusted-server-adapter-axum/src/app.rs | 19 +- .../src/app.rs | 19 +- .../trusted-server-adapter-fastly/src/app.rs | 13 +- crates/trusted-server-adapter-spin/src/app.rs | 28 +- .../src/integrations/registry.rs | 304 +++++++++++++++++- .../trusted-server-core/src/platform/geo.rs | 25 ++ .../trusted-server-core/src/platform/mod.rs | 21 +- .../trusted-server-core/src/platform/types.rs | 12 + crates/trusted-server-core/src/settings.rs | 49 ++- trusted-server.example.toml | 7 + 10 files changed, 455 insertions(+), 42 deletions(-) create mode 100644 crates/trusted-server-core/src/platform/geo.rs diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 5f292c44d7..af71abce52 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -101,6 +101,23 @@ fn build_state_with_settings( })) } +// --------------------------------------------------------------------------- +// Per-request RuntimeServices +// --------------------------------------------------------------------------- + +/// Build per-request [`RuntimeServices`], applying the module-supplied geo +/// provider selected by `[geo] provider`. +/// +/// When the selector is unset the registry resolves no provider and this +/// adapter's own host lookup stands, so the request path is unchanged. +fn build_per_request_services(state: &AppState, ctx: &RequestContext) -> RuntimeServices { + let services = build_runtime_services(ctx, &state.settings); + match state.registry.geo_provider() { + Some(provider) => services.with_geo(provider), + None => services, + } +} + // --------------------------------------------------------------------------- // Error helper // --------------------------------------------------------------------------- @@ -150,7 +167,7 @@ where F: FnOnce(Arc, RuntimeServices, Request) -> Fut, Fut: Future>>, { - let services = build_runtime_services(&ctx, &state.settings); + let services = build_per_request_services(&state, &ctx); let mut req = ctx.into_request(); if let Err(error) = state.registry.prepare_request(&state.settings, &mut req) { return Ok(http_error(&error)); diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index 474cf23e18..3435a284ed 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -148,13 +148,20 @@ fn build_state_with_settings( // Per-request RuntimeServices // --------------------------------------------------------------------------- -/// Builds the per-request services, carrying the Edge Cookie provider the -/// composition root already resolved so the request path does not resolve -/// `[ec] provider` a second time. Nothing is carried when the composition root -/// found nothing safe to keep, and the request path resolves for itself. +/// Build per-request [`RuntimeServices`], carrying the Edge Cookie provider the +/// composition root already resolved and applying the module-supplied geo +/// provider selected by `[geo] provider`. +/// +/// No Edge Cookie provider is carried when the composition root found nothing +/// safe to keep, and no geo provider is applied when the selector is unset and +/// the registry resolves none, so in both cases the request path is unchanged. fn build_per_request_services(state: &AppState, ctx: &RequestContext) -> RuntimeServices { - build_runtime_services(ctx, &state.settings) - .with_resolved_ec_provider(state.ec_provider.clone()) + let services = build_runtime_services(ctx, &state.settings) + .with_resolved_ec_provider(state.ec_provider.clone()); + match state.registry.geo_provider() { + Some(provider) => services.with_geo(provider), + None => services, + } } /// Builds the geo-aware [`EcContext`] for consent-gated endpoints (`/auction`, diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index c7f3c94f81..fbb3ae8a2b 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -297,6 +297,10 @@ pub(crate) fn runtime_services_for_consent_route( /// absent (e.g. tests that dispatch without the entry point). Scheme detection /// continues to rely on the trusted `fastly-ssl` header injected by /// `edgezero_main` after sanitization. +/// +/// Applies the module-supplied geo provider selected by `[geo] provider`. When +/// the selector is unset the registry resolves no provider and the Fastly +/// lookup stands, so the request path is unchanged. fn build_per_request_services(state: &AppState, ctx: &RequestContext) -> RuntimeServices { let client_info = ctx .request() @@ -349,9 +353,16 @@ fn build_per_request_services(state: &AppState, ctx: &RequestContext) -> Runtime // again. Nothing is set for a deployment that selects no provider, or one // whose provider is built from this request's own host signals, and both // are resolved on the request path instead. - match state.ec_provider.clone() { + let services = match state.ec_provider.clone() { Some(provider) => builder.resolved_ec_provider(provider).build(), None => builder.build(), + }; + + // A module-supplied geo provider selected by `[geo] provider` replaces the + // Fastly lookup, and an unset selector leaves the host lookup in place. + match state.registry.geo_provider() { + Some(provider) => services.with_geo(provider), + None => services, } } diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index 6ee3fd6434..65ec505339 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -116,6 +116,26 @@ fn build_state_with_settings( })) } +// --------------------------------------------------------------------------- +// Per-request RuntimeServices +// --------------------------------------------------------------------------- + +/// Build per-request [`RuntimeServices`], carrying the Edge Cookie provider the +/// composition root already resolved and applying the module-supplied geo +/// provider selected by `[geo] provider`. +/// +/// No Edge Cookie provider is carried when the composition root found nothing +/// safe to keep, and no geo provider is applied when the selector is unset and +/// the registry resolves none, so in both cases the request path is unchanged. +fn build_per_request_services(state: &AppState, ctx: &RequestContext) -> RuntimeServices { + let services = build_runtime_services(ctx, &state.settings) + .with_resolved_ec_provider(state.ec_provider.clone()); + match state.registry.geo_provider() { + Some(provider) => services.with_geo(provider), + None => services, + } +} + // --------------------------------------------------------------------------- // Publisher response helper // --------------------------------------------------------------------------- @@ -538,14 +558,6 @@ impl TrustedServerApp { } } -/// Builds the per-request services, carrying the Edge Cookie provider the -/// composition root already resolved so the request path does not resolve -/// `[ec] provider` a second time. -fn build_per_request_services(state: &AppState, ctx: &RequestContext) -> RuntimeServices { - build_runtime_services(ctx, &state.settings) - .with_resolved_ec_provider(state.ec_provider.clone()) -} - fn build_router(state: &Arc) -> RouterService { { let state = Arc::clone(state); diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index 793b9c0424..45c92759bd 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -15,7 +15,7 @@ use crate::ec::kv::KvIdentityGraph; use crate::error::TrustedServerError; use crate::geo::GeoInfo; use crate::http_util::is_navigation_request; -use crate::platform::RuntimeServices; +use crate::platform::{DisabledGeo, PlatformGeo, RuntimeServices}; use crate::settings::Settings; /// Action returned by attribute rewriters to describe how the runtime should mutate the element. @@ -622,6 +622,11 @@ pub struct IntegrationRegistration { pub html_post_processors: Vec>, pub head_injectors: Vec>, pub request_filters: Vec>, + /// Geo provider this module supplies, selectable by `[geo] provider`. + /// + /// Declaring one does not make it active: the module is only asked to + /// resolve location when `[geo] provider` names this module's id. + pub geo_provider: Option>, } impl IntegrationRegistration { @@ -650,6 +655,7 @@ impl IntegrationRegistrationBuilder { html_post_processors: Vec::new(), head_injectors: Vec::new(), request_filters: Vec::new(), + geo_provider: None, }, } } @@ -696,6 +702,17 @@ impl IntegrationRegistrationBuilder { self } + /// Declare the geo provider this module supplies. + /// + /// The provider only resolves location when `[geo] provider` names this + /// module's id, and a declared provider the selector does not choose is + /// reported as a startup warning. + #[must_use] + pub fn with_geo_provider(mut self, provider: Arc) -> Self { + self.registration.geo_provider = Some(provider); + self + } + /// Mark this integration's JS module for deferred loading via /// `".to_string()); + bid.renderer = Some( + BidRenderer::from_typed( + APS_RENDERER_TYPE, + &ApsRendererV1 { + version: 1, + account_id: "example-account".to_string(), + bid_id: "renderer-bid-id".to_string(), + creative_id: None, + tag_type: ApsTagType::Iframe, + creative_url: "https://creative.example/render".to_string(), + aax_response: "A".repeat(200 * 1024), + width: 300, + height: 250, + }, + ) + .expect("should build APS renderer descriptor"), + ); + let winning_bids = HashMap::from([("atf_sidebar_ad".to_string(), bid)]); + + let map = build_bid_map(&winning_bids, PriceGranularity::Dense, &settings, "", false); + let obj = map["atf_sidebar_ad"] + .as_object() + .expect("should include APS bid"); + + assert_eq!( + obj["hb_adid"], "renderer-bid-id", + "should read hb_adid from the renderer payload rather than ad_id or bid_id" + ); + } + + #[test] + fn bid_map_ignores_a_renderer_bid_id_carried_under_another_type_tag() { + let mut settings = test_settings(); + settings.auction.sanitize_creatives = true; + let mut bid = make_bid("atf_sidebar_ad", 1.50, "example", "ad-id-value", "", ""); + bid.cache_id = None; + bid.renderer = Some( + BidRenderer::new("example", serde_json::json!({ "bidId": "renderer-bid-id" })) + .expect("should build renderer descriptor"), + ); + let winning_bids = HashMap::from([("atf_sidebar_ad".to_string(), bid)]); + + let map = build_bid_map(&winning_bids, PriceGranularity::Dense, &settings, "", false); + let obj = map["atf_sidebar_ad"] + .as_object() + .expect("should include the bid"); + + assert_eq!( + obj["hb_adid"], "ad-id-value", + "should ignore a bidId carried under a tag that is not the APS renderer" + ); + } + #[test] fn bid_map_omits_creative_rejected_by_processing_without_renderer() { // Sanitization is opt-in, so enable it: script-only markup is what From 36080d4b4d07e6790d26a23b64049909f8f2e64f Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 29 Aug 2026 12:29:58 +0100 Subject: [PATCH 024/124] Correct where the application is actually built once The comment on the carried-module hash check said the SHA-256 costs one per request on Fastly and "once per process on the other adapters". That is wrong on two hosts of three. Reading the pinned EdgeZero adapters (v0.0.4, git checkout 9e661ae): `edgezero_adapter_cloudflare::run_app` calls `A::build_app()` inside the per-request entry point, and its own comment says "every Worker request re-enters this function"; `edgezero_adapter_spin::run_app` does the same, with the matching comment about `#[http_service]`. Fastly's `run_app` also calls `A::build_app()` per request, and this repo's Fastly `main` calls `TrustedServerApp::build_app_with_state()` per request in any case. Only `edgezero_adapter_axum::run_app` calls `A::build_app()` once before serving, and this repo's Axum `main` likewise calls `TrustedServerApp::routes()` before handing the router to the dev server. So the application is built once on Axum and per request everywhere else. Corrected here: - `integrations/registry.rs`, the carried-module hash comment, which is the comment the finding names. - `docs/guide/configuration.md`, which said a provider-selector failure "shows there on every request rather than once" only because "the Fastly adapter builds the application per request". It shows on every request on Cloudflare Workers and Spin too. `tsjs_bundle.rs`'s `compose_hash` doc carried the same claim and was corrected in the previous commit. Also corrected, though pre-existing rather than new on this branch: the `AppState` doc comments in the Cloudflare and Spin adapters both said "built once at startup and shared across all requests", which is false for the same reason, and the Axum one now says why it is the exception. Fastly's already said "once per Wasm instance" and is accurate. --- crates/trusted-server-adapter-axum/src/app.rs | 4 ++++ crates/trusted-server-adapter-cloudflare/src/app.rs | 6 +++++- crates/trusted-server-adapter-spin/src/app.rs | 6 +++++- .../src/integrations/registry.rs | 13 +++++++++---- docs/guide/configuration.md | 2 +- 5 files changed, 24 insertions(+), 7 deletions(-) diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 683079b84b..5408923013 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -51,6 +51,10 @@ use crate::platform::{AxumPlatformConfigStore, build_runtime_services}; // --------------------------------------------------------------------------- /// Application state built once at startup and shared across all requests. +/// +/// Built once here, unlike on the other three adapters, because `main` +/// calls [`TrustedServerApp::routes`] before serving, so the router holds +/// one state for the process's lifetime. pub struct AppState { settings: Arc, orchestrator: Arc, diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index ae7bdbfc0d..2f58be39a0 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -56,7 +56,11 @@ pub fn set_cloudflare_config_json(value: String) { let _ = CLOUDFLARE_CONFIG_JSON.set(value); } -/// Application state built once at startup and shared across all requests. +/// Application state shared by everything that serves one request. +/// +/// Not built once per Worker isolate, because +/// `edgezero_adapter_cloudflare::run_app` calls `build_app` inside the +/// per-request entry point, so this is rebuilt on every request. pub struct AppState { settings: Arc, orchestrator: Arc, diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index 16fa7b9d02..99e2bbbcf7 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -52,7 +52,11 @@ use crate::platform::{SpinPlatformConfigStore, build_runtime_services}; // AppState // --------------------------------------------------------------------------- -/// Application state built once at startup and shared across all requests. +/// Application state shared by everything that serves one request. +/// +/// Not built once per process, because `edgezero_adapter_spin::run_app` +/// calls `build_app` inside the per-request entry point, so this is +/// rebuilt on every request. pub struct AppState { settings: Arc, orchestrator: Arc, diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index 45c92759bd..f26c5ca891 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -1105,10 +1105,15 @@ impl IntegrationRegistry { } if let Some(module) = registration.js_module { // The served `?v=` hash and its memo trust this value, so a - // stale literal is a startup error rather than a stale script. - // On Fastly the registry is built per request, so this costs - // one SHA-256 of each carried module per request there, and - // once per process on the other adapters. + // stale literal is a startup error rather than a stale + // script. The cost is one SHA-256 of each carried module + // per registry build, and only the Axum dev server builds + // the registry once, because its `main` builds the router + // before serving. Fastly starts a fresh Wasm instance per + // request, and `edgezero_adapter_cloudflare::run_app` and + // `edgezero_adapter_spin::run_app` both call `build_app` + // inside the per-request entry point, so those three pay + // it on every request. let actual = hex::encode(Sha256::digest(module.source.as_bytes())); if actual != module.sha256 { return Err(Report::new(TrustedServerError::Configuration { diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index ef74287852..6af90dde79 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -695,7 +695,7 @@ enabled = true #### When the selector names something that cannot supply a provider -The application refuses to build and the message names the module and the reason, in three cases. Every adapter treats this as a startup error, and because the Fastly adapter builds the application per request, it shows there on every request rather than once. +The application refuses to build and the message names the module and the reason, in three cases. Every adapter treats this as a startup error. Only the Axum dev server builds the application once, before it starts serving, so only there does the failure show once. Fastly, Cloudflare Workers and Spin all build the application inside the per-request entry point, so on those three the failure shows on every request. - The module is not registered at all. The message also lists the registered modules that do supply a geo provider, so a typo is easy to spot. From 02edd06fdd544676d6a32489d36a8b11aaf26c30 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 29 Aug 2026 12:38:10 +0100 Subject: [PATCH 025/124] Pin the renderer wire form without pinning JSON key order `the_open_renderer_serializes_to_the_same_bytes_as_the_aps_variant_did` compared `serde_json::to_string` against two literal strings whose keys happen to be in alphabetical order. That order holds only while `serde_json::Map` is a `BTreeMap`, which is only while the crate's `preserve_order` feature is off. The feature is reachable in this workspace. `trusted-server-cli` depends on `edgezero-cli`, which depends on `handlebars`, which declares `serde_json` with `features = ["preserve_order"]` unconditionally. Cargo unifies features across everything built for one target, so any command that builds the CLI and core together for the host target turns the map into an `IndexMap` inside core. `cargo tree -p trusted-server-core --target x86_64-pc-windows-msvc -e features -i serde_json` shows no `preserve_order`; adding `-p trusted-server-cli` to the same command shows three. A maintainer running `cargo test --workspace --target ` gets the second build, and the test would fail there for a reason that has nothing to do with the wire form. Both sides now go through a `with_sorted_keys` helper before comparison, so the key order is fixed by the test rather than by which map `serde_json` was compiled with. The literals are unchanged, byte for byte, and nothing is weakened: two JSON objects serialize to the same sorted bytes only when they carry exactly the same keys with exactly the same values, which is the promise the literals were there to make. --- .../trusted-server-core/src/auction/types.rs | 50 ++++++++++++++++--- 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/crates/trusted-server-core/src/auction/types.rs b/crates/trusted-server-core/src/auction/types.rs index d639e549ae..f3bb3a5a7f 100644 --- a/crates/trusted-server-core/src/auction/types.rs +++ b/crates/trusted-server-core/src/auction/types.rs @@ -742,14 +742,50 @@ mod tests { ); } + /// Rewrites every object in `value` with its keys in sorted order, so + /// serializing the result gives one fixed key order. + /// + /// `serde_json::Map` is a `BTreeMap`, which serializes keys in sorted + /// order, only while the crate's `preserve_order` feature is off. With the + /// feature on it is an `IndexMap` and the order follows insertion instead. + /// Nothing in this crate asks for the feature, but Cargo unifies features + /// across everything built for one target, and `trusted-server-cli` pulls + /// it in through `edgezero-cli` and then `handlebars`. A maintainer + /// running `cargo test --workspace --target ` therefore builds this + /// crate with `preserve_order` on, and a test that pinned insertion order + /// would fail there for no reason. Sorting both sides removes the + /// dependence on which map `serde_json` was built with. + fn with_sorted_keys(value: &serde_json::Value) -> serde_json::Value { + match value { + serde_json::Value::Object(map) => { + let mut keys = map.keys().collect::>(); + keys.sort_unstable(); + let mut sorted = serde_json::Map::with_capacity(keys.len()); + for key in keys { + let child = map.get(key).expect("should find a key the map just listed"); + sorted.insert(key.clone(), with_sorted_keys(child)); + } + serde_json::Value::Object(sorted) + } + serde_json::Value::Array(items) => { + serde_json::Value::Array(items.iter().map(with_sorted_keys).collect()) + } + scalar => scalar.clone(), + } + } + #[test] fn the_open_renderer_serializes_to_the_same_bytes_as_the_aps_variant_did() { // Literal strings captured from the closed-enum form before this // change, through the same `serde_json::to_value` path production // uses: `BidExt::to_ext` for the OpenRTB response extension, and - // `build_bid_map` for `window.tsjs.bids`. Both hand the page an object - // whose keys are ordered by `serde_json::Map`, so these are the bytes - // a browser actually receives. + // `build_bid_map` for `window.tsjs.bids`. + // + // Both sides go through `with_sorted_keys` first, because the key + // order `serde_json` emits is not ours to pin, and that function + // explains why. Sorting settles the order without weakening what is + // pinned, since two objects serialize to the same sorted bytes only + // when they carry exactly the same keys with exactly the same values. let full = BidRenderer::from_typed( APS_RENDERER_TYPE, &ApsRendererV1 { @@ -781,13 +817,13 @@ mod tests { ) .expect("should build APS renderer descriptor"); - let full_bytes = serde_json::to_string( + let full_bytes = serde_json::to_string(&with_sorted_keys( &serde_json::to_value(&full).expect("should convert renderer to a JSON value"), - ) + )) .expect("should serialize renderer"); - let absent_bytes = serde_json::to_string( + let absent_bytes = serde_json::to_string(&with_sorted_keys( &serde_json::to_value(&absent).expect("should convert renderer to a JSON value"), - ) + )) .expect("should serialize renderer"); assert_eq!( From 2e606400eba94498a1f17f5ab6c0eb488c2e10e1 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 30 Aug 2026 14:11:37 +0100 Subject: [PATCH 026/124] Repair the seam against the stack it now sits on Fallout from rebasing this branch onto the five-PR stack rather than onto main, where both sides had changed the same functions. The Axum adapter's test helper still called `build_orchestrator`, which this branch's import list replaced with `build_orchestrator_with_providers`. It now calls the latter with no external providers, which is the same thing. The Cloudflare adapter had a test calling `build_per_request_services` with the old `(ctx, settings)` argument pair. That function now takes `(state, ctx)` so it can apply a module's geo provider, and the test has no application state, so it calls `build_runtime_services` directly, which is what it actually wanted. Four doc comments still said an unset `[geo] provider` leaves the host's own lookup in place. Reconciling the two geo designs settled that unset resolves nothing and makes no host geo call, matching the permission model's default, and that `"platform"` is the explicit opt-in to the host lookup. The comments now say that on all four adapters. Addresses: rebase of this branch onto split/5, conflicts resolved in the composition roots and the geo selector --- crates/trusted-server-adapter-axum/src/app.rs | 8 +++++--- crates/trusted-server-adapter-cloudflare/src/app.rs | 7 +++++-- crates/trusted-server-adapter-fastly/src/app.rs | 5 +++-- crates/trusted-server-adapter-spin/src/app.rs | 7 +++++-- 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 5408923013..08b08f7b38 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -137,8 +137,9 @@ pub fn build_state_with_registrations( /// Build per-request [`RuntimeServices`], applying the module-supplied geo /// provider selected by `[geo] provider`. /// -/// When the selector is unset the registry resolves no provider and this -/// adapter's own host lookup stands, so the request path is unchanged. +/// Unset and `"none"` both resolve nothing, so no client IP reaches a host geo +/// service. `"platform"` opts in to this adapter's own lookup, and any other +/// key names an integration module that declares a geo provider. fn build_per_request_services(state: &AppState, ctx: &RequestContext) -> RuntimeServices { let services = build_runtime_services(ctx, &state.settings); match state.registry.geo_provider() { @@ -782,7 +783,8 @@ mod tests { fn state_with_uninjected_provider() -> AppState { let settings = Settings::from_toml(UNINJECTED_PROVIDER_TOML) .expect("should parse settings selecting an uninjected provider"); - let orchestrator = build_orchestrator(&settings).expect("should build orchestrator"); + let orchestrator = + build_orchestrator_with_providers(&settings, &[]).expect("should build orchestrator"); let registry = IntegrationRegistry::new(&settings).expect("should build registry"); AppState { settings: Arc::new(settings), diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index 2f58be39a0..8cc4963a34 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -183,8 +183,11 @@ pub fn build_state_with_registrations( /// provider selected by `[geo] provider`. /// /// No Edge Cookie provider is carried when the composition root found nothing -/// safe to keep, and no geo provider is applied when the selector is unset and -/// the registry resolves none, so in both cases the request path is unchanged. +/// safe to keep, so the request path resolves the selection for itself. +/// +/// For geo, unset and `"none"` both resolve nothing, so no client IP reaches a +/// host geo service. `"platform"` opts in to this adapter's own lookup, and any +/// other key names an integration module that declares a geo provider. fn build_per_request_services(state: &AppState, ctx: &RequestContext) -> RuntimeServices { let services = build_runtime_services(ctx, &state.settings) .with_resolved_ec_provider(state.ec_provider.clone()); diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 6aeb44c86f..307b894d2c 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -380,8 +380,9 @@ fn build_per_request_services(state: &AppState, ctx: &RequestContext) -> Runtime None => builder.build(), }; - // A module-supplied geo provider selected by `[geo] provider` replaces the - // Fastly lookup, and an unset selector leaves the host lookup in place. + // Unset and `"none"` both resolve nothing, so no client IP reaches a host + // geo service. `"platform"` opts in to the Fastly lookup below, and any + // other key names an integration module that declares a geo provider. match state.registry.geo_provider() { Some(provider) => services.with_geo(provider), None => services, diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index 99e2bbbcf7..cbe2e3ddc0 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -155,8 +155,11 @@ pub fn build_state_with_registrations( /// provider selected by `[geo] provider`. /// /// No Edge Cookie provider is carried when the composition root found nothing -/// safe to keep, and no geo provider is applied when the selector is unset and -/// the registry resolves none, so in both cases the request path is unchanged. +/// safe to keep, so the request path resolves the selection for itself. +/// +/// For geo, unset and `"none"` both resolve nothing, so no client IP reaches a +/// host geo service. `"platform"` opts in to this adapter's own lookup, and any +/// other key names an integration module that declares a geo provider. fn build_per_request_services(state: &AppState, ctx: &RequestContext) -> RuntimeServices { let services = build_runtime_services(ctx, &state.settings) .with_resolved_ec_provider(state.ec_provider.clone()); From 8dba426f67b119aa760ed45a03c1fc5c7635b53d Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 30 Aug 2026 14:34:22 +0100 Subject: [PATCH 027/124] Let a module id count as a selected geo provider Two checks were written when the host lookup was the only geo provider there could be, and both treat any other value as no geo at all. validate_jurisdiction_acknowledgment asked whether the selector was `"platform"`, so a deployment selecting a module's geo provider was told it had none and made to set assume_single_jurisdiction to acknowledge something untrue. It now asks the question the other way round: only an unset selector and the explicit `none` resolve nothing, and everything else resolves a location. build_geo_provider's doc said an unknown provider is rejected by GeoConfig::validate_provider_selection, which is no longer where that happens. The function returns DisabledGeo for a module id because it cannot see the registry, and the adapter then substitutes the module's provider. The doc now says that the value is the base the adapter starts from rather than what serves the request, and that the registry is the layer that rejects a selector naming a module with no geo provider. The seam probe's test settings also predate the permission model, so they now carry the required [geo] default_country. The two tests that select a geo provider write their own [geo] table, so the fixture adds one only when the caller has not. Addresses: reconciling the geo selector across the permission model and the seam --- .../tests/seam_probe.rs | 17 +++++++++++++++++ crates/trusted-server-core/src/platform/mod.rs | 12 +++++++++--- crates/trusted-server-core/src/settings.rs | 6 +++++- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/crates/trusted-server-adapter-axum/tests/seam_probe.rs b/crates/trusted-server-adapter-axum/tests/seam_probe.rs index ade7fc9beb..364aca54c3 100644 --- a/crates/trusted-server-adapter-axum/tests/seam_probe.rs +++ b/crates/trusted-server-adapter-axum/tests/seam_probe.rs @@ -31,6 +31,20 @@ const MAX_BODY_BYTES: usize = 4 * 1024 * 1024; /// The settings baked into the binary carry placeholder secrets that /// `get_settings()` rejects by design, so every test states its own. fn settings_with(extra: &str) -> Settings { + // The permission model requires a `[geo] default_country`, because there + // must always be a permission baseline for a request the geo provider + // leaves unmatched. Tests that select a geo provider write their own + // `[geo]` table, so only supply one when the caller has not. + let geo = if extra.contains("[geo]") { + String::new() + } else { + " +[geo] +default_country = \"US\" +assume_single_jurisdiction = true +" + .to_owned() + }; Settings::from_toml(&format!( r#" [[handlers]] @@ -48,6 +62,7 @@ fn settings_with(extra: &str) -> Settings { passphrase = "test-secret-key-32-bytes-minimum" {extra} + {geo} "# )) .expect("should parse seam probe test settings") @@ -197,6 +212,7 @@ async fn proxy_route_reports_the_modules_geo_and_that_the_preparer_ran() { r#" [geo] provider = "seam_probe" + default_country = "US" {PROBE_BLOCK} "# )); @@ -320,6 +336,7 @@ fn geo_selector_naming_a_module_without_a_provider_fails_at_startup() { r#" [geo] provider = "seam_probe" + default_country = "US" [integrations.seam_probe] country = "ZZ" diff --git a/crates/trusted-server-core/src/platform/mod.rs b/crates/trusted-server-core/src/platform/mod.rs index 730d97f580..a8185040a8 100644 --- a/crates/trusted-server-core/src/platform/mod.rs +++ b/crates/trusted-server-core/src/platform/mod.rs @@ -89,9 +89,15 @@ use crate::settings::Settings; /// the top of the `permissions.yaml` rules tree. `provider = "none"` spells the /// same choice explicitly. The host platform's own geo lookup is opt-in: /// `provider = "platform"` returns `host_default`, which the adapter passes -/// as its platform geo implementation. A selected-but-unknown provider is -/// rejected at startup by -/// [`GeoConfig::validate_provider_selection`](crate::settings::GeoConfig::validate_provider_selection). +/// as its platform geo implementation. +/// +/// Any other value names an integration module that declares a geo provider. +/// This function returns [`DisabledGeo`] for one, because it cannot see the +/// registry, and the adapter then replaces it with the module's provider from +/// `IntegrationRegistry::geo_provider`. So the value returned here is the base +/// the adapter starts from, not necessarily what serves the request. A selector +/// naming a module that supplies no geo provider is rejected when the registry +/// is built, which is the only layer that can tell. #[must_use] pub fn build_geo_provider( settings: &Settings, diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 90610b737d..ec8fc723a9 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -1005,7 +1005,11 @@ impl GeoConfig { &self, ec: &Ec, ) -> Result<(), Report> { - let geo_disabled = !matches!(self.provider.as_deref(), Some("platform")); + // Location is resolved by the host lookup (`platform`) or by any + // integration module that declares a geo provider. Only an unset + // selector and the explicit `none` resolve nothing, so only those + // two leave every request on the default country. + let geo_disabled = matches!(self.provider.as_deref(), None | Some("none")); // The selector is a typed enum on this branch, so statelessness is the // absent selector or the explicit `none`, matched rather than compared // as a string. From b950abf5223f0260a1a298b82a160971372116be Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 30 Aug 2026 15:10:51 +0100 Subject: [PATCH 028/124] Carry identity and device on the registration alongside geo Answers the architectural finding on #1043 rather than deferring it. Vendor identity reached core through RuntimeServices injection while everything else a module supplies was declared on its integration registration, so there were two extension mechanisms and identity was on the second one. A registration can now declare an Edge Cookie provider and a device provider the same way it declares a geo provider. The registry resolves each against its selector, warns when a module declares a capability the selector does not choose, and the adapters apply all three to RuntimeServices in one place. RuntimeServices gains a device slot, which it had no way to carry before, and with_ec_provider and with_device_provider to match with_geo. Identity and device differ from geo in one way that matters. Both have providers built into core, so a selector naming a built-in is not an error at the registry, and resolution returns None for those and lets core resolve them as before. Two closed allowlists had to open, being the same fault in two more places. The device selector accepted only `builtin` and `fastly`, so any module id was rejected before the registry saw it. The jurisdiction check asked whether the geo selector was `platform`, so a deployment selecting a module's geo provider was told it had none. Neither could stand once a module can supply these. Proven by the probe, which now declares all three capabilities from one registration. Two tests select the module for identity and for device and assert the resolved provider is the module's own. The probe's identity provider mints a value the built-in HMAC provider cannot produce, so a passing assertion means the module's provider ran rather than core's. Addresses: Aram, `ec/provider.rs`, vendor identity should lean on the integration system rather than a second extension mechanism --- crates/integrations/seam-probe/src/lib.rs | 74 +++++++++ crates/trusted-server-adapter-axum/src/app.rs | 14 +- .../tests/seam_probe.rs | 99 ++++++++++++ .../src/app.rs | 39 +++-- .../trusted-server-adapter-fastly/src/app.rs | 42 +++-- crates/trusted-server-adapter-spin/src/app.rs | 39 +++-- .../src/integrations/registry.rs | 144 +++++++++++++++++- .../trusted-server-core/src/platform/types.rs | 38 +++++ crates/trusted-server-core/src/settings.rs | 23 ++- 9 files changed, 448 insertions(+), 64 deletions(-) diff --git a/crates/integrations/seam-probe/src/lib.rs b/crates/integrations/seam-probe/src/lib.rs index eaba4815a0..bfac1c7406 100644 --- a/crates/integrations/seam-probe/src/lib.rs +++ b/crates/integrations/seam-probe/src/lib.rs @@ -28,7 +28,12 @@ use serde_json::json; use trusted_server_core::auction::AuctionProviderBuilder; use trusted_server_core::auction::provider::{AuctionProvider, ProviderRequestOutcome}; use trusted_server_core::auction::types::{AuctionContext, AuctionRequest, AuctionResponse}; +use trusted_server_core::ec::device::{DeviceProvider, DeviceSignals}; +use trusted_server_core::ec::provider::{ + EdgeCookieProvider, GeneratedEdgeCookie, IdentityInput, ProviderCode, +}; use trusted_server_core::error::TrustedServerError; +use trusted_server_core::evidence::RequestInfo; use trusted_server_core::integrations::{ CarriedJsModule, IntegrationBuilder, IntegrationEndpoint, IntegrationProxy, IntegrationRegistration, @@ -196,6 +201,68 @@ impl SeamProbeGeo { } } +/// The probe's Edge Cookie provider, declared on its registration so identity +/// reaches core the same way location does. +/// +/// It mints a value derived from the request evidence rather than a constant, +/// so a test can tell the difference between the module's provider running and +/// core's built-in one running. +#[derive(Debug)] +pub struct SeamProbeEc; + +/// The four-character code core stamps on identifiers this provider owns. +const SEAM_PROBE_EC_CODE: ProviderCode = ProviderCode::new("sprb"); + +impl EdgeCookieProvider for SeamProbeEc { + fn id(&self) -> &'static str { + SEAM_PROBE_ID + } + + fn code(&self) -> ProviderCode { + SEAM_PROBE_EC_CODE + } + + fn generate( + &self, + request_info: &dyn RequestInfo, + _input: &IdentityInput<'_>, + ) -> Result> { + // Derived from evidence so the test can prove this ran rather than the + // built-in provider, and prove the evidence actually arrived. + let ip = request_info.client_ip(); + let id = format!("seam-probe-{}", if ip.is_empty() { "no-ip" } else { ip }); + Ok(GeneratedEdgeCookie { + id: Some(id), + response_headers: Vec::new(), + }) + } + + fn accepts_id(&self, value: &str) -> bool { + value.starts_with("seam-probe-") + } +} + +/// The probe's device provider, declared on the same registration. +#[derive(Debug)] +pub struct SeamProbeDevice; + +impl DeviceProvider for SeamProbeDevice { + fn id(&self) -> &'static str { + SEAM_PROBE_ID + } + + fn detect(&self, _request_info: &dyn RequestInfo) -> DeviceSignals { + DeviceSignals { + is_mobile: 1, + platform_class: Some("seam-probe".to_owned()), + known_browser: Some(true), + looks_like_browser: true, + ja4_class: None, + h2_fp_hash: None, + } + } +} + impl PlatformGeo for SeamProbeGeo { fn lookup(&self, _client_ip: Option) -> Result, Report> { Ok(Some(GeoInfo { @@ -345,6 +412,13 @@ pub fn register( if config.declares_geo { registration = registration.with_geo_provider(Arc::new(SeamProbeGeo::new(config.country))); } + { + // Identity and device are declared the same way location is, so one + // module supplies all three and there is no second mechanism. + registration = registration + .with_ec_provider(Arc::new(SeamProbeEc)) + .with_device_provider(Arc::new(SeamProbeDevice)); + } Ok(Some(registration.build())) } diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 08b08f7b38..7a15993fc5 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -141,11 +141,17 @@ pub fn build_state_with_registrations( /// service. `"platform"` opts in to this adapter's own lookup, and any other /// key names an integration module that declares a geo provider. fn build_per_request_services(state: &AppState, ctx: &RequestContext) -> RuntimeServices { - let services = build_runtime_services(ctx, &state.settings); - match state.registry.geo_provider() { - Some(provider) => services.with_geo(provider), - None => services, + let mut services = build_runtime_services(ctx, &state.settings); + if let Some(provider) = state.registry.geo_provider() { + services = services.with_geo(provider); } + if let Some(provider) = state.registry.ec_provider() { + services = services.with_ec_provider(provider); + } + if let Some(provider) = state.registry.device_provider() { + services = services.with_device_provider(provider); + } + services } // --------------------------------------------------------------------------- diff --git a/crates/trusted-server-adapter-axum/tests/seam_probe.rs b/crates/trusted-server-adapter-axum/tests/seam_probe.rs index 364aca54c3..50d5740711 100644 --- a/crates/trusted-server-adapter-axum/tests/seam_probe.rs +++ b/crates/trusted-server-adapter-axum/tests/seam_probe.rs @@ -437,3 +437,102 @@ fn auction_provider_name_from_an_outside_builder_satisfies_the_configured_list() "should name the provider nothing provides: {error}" ); } + +/// Settings selecting a module-supplied provider. +/// +/// These cannot use [`settings_with`], because that fixture carries the +/// deprecated `[ec] passphrase`, and configuration validation rejects the old +/// key and a provider selection together. +fn settings_selecting_module(extra: &str) -> Settings { + Settings::from_toml(&format!( + r#" + [[handlers]] + path = "^/_ts/admin" + username = "admin" + password = "admin-pass" + + [publisher] + domain = "test-publisher.example.com" + cookie_domain = ".test-publisher.example.com" + origin_url = "https://origin.test-publisher.example.com" + proxy_secret = "seam-probe-test-proxy-secret" + + [geo] + default_country = "US" + assume_single_jurisdiction = true + + {extra} + + {PROBE_BLOCK} + "# + )) + .expect("should parse settings selecting a module-supplied provider") +} + +// --------------------------------------------------------------------------- +// Identity and device declared on the registration +// --------------------------------------------------------------------------- + +/// `[ec] provider` naming a module resolves to that module's Edge Cookie +/// provider, so identity reaches core through the integration registration +/// rather than through a second extension mechanism. +/// +/// This is the check that matters for the architectural finding on #1043. The +/// probe's provider mints an identifier that starts `seam-probe-`, which the +/// built-in HMAC provider can never produce, so a passing assertion means the +/// module's provider ran and not core's. +#[test] +fn ec_selector_naming_a_module_resolves_that_modules_provider() { + let settings = settings_selecting_module( + r#" + [ec] + provider = "seam_probe" + + [ec.providers.seam_probe] + "#, + ); + + let registry = trusted_server_core::integrations::IntegrationRegistry::with_registrations( + &settings, + &[seam_probe::builder()], + ) + .expect("should build a registry with the probe registered"); + + let provider = registry + .ec_provider() + .expect("`[ec] provider = \"seam_probe\"` should resolve the module's provider"); + + assert_eq!( + provider.id(), + "seam_probe", + "the resolved provider should be the one the module declared" + ); +} + +/// `[device] provider` naming a module resolves that module's device provider +/// the same way, so all three capabilities travel one route. +#[test] +fn device_selector_naming_a_module_resolves_that_modules_provider() { + let settings = settings_selecting_module( + r#" + [device] + provider = "seam_probe" + "#, + ); + + let registry = trusted_server_core::integrations::IntegrationRegistry::with_registrations( + &settings, + &[seam_probe::builder()], + ) + .expect("should build a registry with the probe registered"); + + let provider = registry + .device_provider() + .expect("`[device] provider = \"seam_probe\"` should resolve the module's provider"); + + assert_eq!( + provider.id(), + "seam_probe", + "the resolved provider should be the one the module declared" + ); +} diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index 8cc4963a34..095d7a16e0 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -73,8 +73,9 @@ pub struct AppState { /// Resolving reads no request data, so the result is kept and handed to /// every request through /// [`RuntimeServices::resolved_ec_provider`](trusted_server_core::platform::RuntimeServices::resolved_ec_provider). - /// `None` for a deployment that selects no provider. - ec_provider: Option>, + /// `None` for a deployment that selects no provider, and for one whose + /// provider must be resolved per request. + resolved_ec_provider: Option>, } /// Build the application state, loading settings and constructing all per-application components. @@ -155,22 +156,24 @@ pub fn build_state_with_registrations( integrations: &[IntegrationBuilder], auction_providers: &[AuctionProviderBuilder], ) -> Result, Report> { + let orchestrator = build_orchestrator_with_providers(&settings, auction_providers)?; + let registry = IntegrationRegistry::with_registrations(&settings, integrations)?; + // Composition root: resolve the provider selection once, before any request // is served, so a selection this adapter can never supply fails here rather // than on the first request. Keeping what the resolution produced is what - // stops the request path resolving the same settings again. This adapter - // supplies no host signals and injects no vendor Edge Cookie provider, so - // both arguments are `None`, and each is passed here once this adapter - // supplies it. - let ec_provider = build_reusable_provider(&settings.ec, None, None)?; - let orchestrator = build_orchestrator_with_providers(&settings, auction_providers)?; - let registry = IntegrationRegistry::with_registrations(&settings, integrations)?; + // stops the request path resolving the same settings again. The registry is + // built first because a module can supply the vendor Edge Cookie provider + // the selector names, and resolving without it would reject a selection + // this deployment can in fact satisfy. This adapter supplies no host + // signals, so that argument stays `None` until it does. + let resolved_ec_provider = build_reusable_provider(&settings.ec, None, registry.ec_provider())?; Ok(Arc::new(AppState { settings: Arc::new(settings), orchestrator: Arc::new(orchestrator), registry: Arc::new(registry), - ec_provider, + resolved_ec_provider, })) } @@ -189,12 +192,18 @@ pub fn build_state_with_registrations( /// host geo service. `"platform"` opts in to this adapter's own lookup, and any /// other key names an integration module that declares a geo provider. fn build_per_request_services(state: &AppState, ctx: &RequestContext) -> RuntimeServices { - let services = build_runtime_services(ctx, &state.settings) - .with_resolved_ec_provider(state.ec_provider.clone()); - match state.registry.geo_provider() { - Some(provider) => services.with_geo(provider), - None => services, + let mut services = build_runtime_services(ctx, &state.settings) + .with_resolved_ec_provider(state.resolved_ec_provider.clone()); + if let Some(provider) = state.registry.geo_provider() { + services = services.with_geo(provider); + } + if let Some(provider) = state.registry.ec_provider() { + services = services.with_ec_provider(provider); + } + if let Some(provider) = state.registry.device_provider() { + services = services.with_device_provider(provider); } + services } /// Builds the geo-aware [`EcContext`] for consent-gated endpoints (`/auction`, diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 307b894d2c..3bb0d20119 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -177,8 +177,9 @@ pub(crate) struct AppState { /// Resolving reads no request data, so the result is kept and handed to /// every request through /// [`RuntimeServices::resolved_ec_provider`](trusted_server_core::platform::RuntimeServices::resolved_ec_provider). - /// `None` for a deployment that selects no provider. - pub(crate) ec_provider: Option>, + /// `None` for a deployment that selects no provider, and for one whose + /// provider must be resolved per request. + pub(crate) resolved_ec_provider: Option>, } /// Build the application state, loading settings and constructing all per-application components. @@ -232,26 +233,28 @@ pub(crate) fn build_state_with_registrations( ) -> Result, Report> { warn_if_certificate_check_disabled(&settings); + let orchestrator = build_orchestrator_with_providers(&settings, auction_providers)?; + let registry = IntegrationRegistry::with_registrations(&settings, integrations)?; + // Composition root: resolve the provider selection once, before any request // is served, so a selection this adapter can never supply fails here rather // than on the first request, and keep what the resolution produced so the - // request path does not resolve the same settings again. This adapter - // injects no vendor Edge Cookie provider, so `None` is the injected - // argument, and one is passed here once this adapter supplies it. + // request path does not resolve the same settings again. The registry is + // built first because a module can supply the vendor Edge Cookie provider + // the selector names, and resolving without it would reject a selection + // this deployment can in fact satisfy. // // This adapter injects host signals on every request, so a startup instance // with no captured signals answers the only question the check asks, // which is whether the service exists at all. That same emptiness is why // `build_reusable_provider` hands back nothing for a provider built from - // those signals, leaving it to be resolved per request against the - // signals that request actually carried. - let ec_provider = build_reusable_provider( + // those fingerprints, leaving it to be resolved per request against the + // fingerprints that request actually carried. + let resolved_ec_provider = build_reusable_provider( &settings.ec, Some(Arc::new(FastlyHostSignals::default())), - None, + registry.ec_provider(), )?; - let orchestrator = build_orchestrator_with_providers(&settings, auction_providers)?; - let registry = IntegrationRegistry::with_registrations(&settings, integrations)?; let auction_telemetry_sink = crate::tinybird::auction_sink_from_settings(&settings); let default_kv_store = Arc::new(UnavailableKvStore) as Arc; @@ -262,7 +265,7 @@ pub(crate) fn build_state_with_registrations( registry: Arc::new(registry), default_kv_store, auction_telemetry_sink, - ec_provider, + resolved_ec_provider, })) } @@ -375,7 +378,7 @@ fn build_per_request_services(state: &AppState, ctx: &RequestContext) -> Runtime // again. Nothing is set for a deployment that selects no provider, or one // whose provider is built from this request's own host signals, and both // are resolved on the request path instead. - let services = match state.ec_provider.clone() { + let services = match state.resolved_ec_provider.clone() { Some(provider) => builder.resolved_ec_provider(provider).build(), None => builder.build(), }; @@ -383,10 +386,17 @@ fn build_per_request_services(state: &AppState, ctx: &RequestContext) -> Runtime // Unset and `"none"` both resolve nothing, so no client IP reaches a host // geo service. `"platform"` opts in to the Fastly lookup below, and any // other key names an integration module that declares a geo provider. - match state.registry.geo_provider() { - Some(provider) => services.with_geo(provider), - None => services, + let mut services = services; + if let Some(provider) = state.registry.geo_provider() { + services = services.with_geo(provider); + } + if let Some(provider) = state.registry.ec_provider() { + services = services.with_ec_provider(provider); + } + if let Some(provider) = state.registry.device_provider() { + services = services.with_device_provider(provider); } + services } fn publisher_fallback_methods() -> [Method; 7] { diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index cbe2e3ddc0..4f646dd7ce 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -69,8 +69,9 @@ pub struct AppState { /// Resolving reads no request data, so the result is kept and handed to /// every request through /// [`RuntimeServices::resolved_ec_provider`](trusted_server_core::platform::RuntimeServices::resolved_ec_provider). - /// `None` for a deployment that selects no provider. - ec_provider: Option>, + /// `None` for a deployment that selects no provider, and for one whose + /// provider must be resolved per request. + resolved_ec_provider: Option>, } /// Build the application state, loading settings and constructing all per-application components. @@ -127,22 +128,24 @@ pub fn build_state_with_registrations( integrations: &[IntegrationBuilder], auction_providers: &[AuctionProviderBuilder], ) -> Result, Report> { + let orchestrator = build_orchestrator_with_providers(&settings, auction_providers)?; + let registry = IntegrationRegistry::with_registrations(&settings, integrations)?; + // Composition root: resolve the provider selection once, before any request // is served, so a selection this adapter can never supply fails here rather // than on the first request. Keeping what the resolution produced is what - // stops the request path resolving the same settings again. This adapter - // supplies no host signals and injects no vendor Edge Cookie provider, so - // both arguments are `None`, and each is passed here once this adapter - // supplies it. - let ec_provider = build_reusable_provider(&settings.ec, None, None)?; - let orchestrator = build_orchestrator_with_providers(&settings, auction_providers)?; - let registry = IntegrationRegistry::with_registrations(&settings, integrations)?; + // stops the request path resolving the same settings again. The registry is + // built first because a module can supply the vendor Edge Cookie provider + // the selector names, and resolving without it would reject a selection + // this deployment can in fact satisfy. This adapter supplies no host + // signals, so that argument stays `None` until it does. + let resolved_ec_provider = build_reusable_provider(&settings.ec, None, registry.ec_provider())?; Ok(Arc::new(AppState { settings: Arc::new(settings), orchestrator: Arc::new(orchestrator), registry: Arc::new(registry), - ec_provider, + resolved_ec_provider, })) } @@ -161,12 +164,18 @@ pub fn build_state_with_registrations( /// host geo service. `"platform"` opts in to this adapter's own lookup, and any /// other key names an integration module that declares a geo provider. fn build_per_request_services(state: &AppState, ctx: &RequestContext) -> RuntimeServices { - let services = build_runtime_services(ctx, &state.settings) - .with_resolved_ec_provider(state.ec_provider.clone()); - match state.registry.geo_provider() { - Some(provider) => services.with_geo(provider), - None => services, + let mut services = build_runtime_services(ctx, &state.settings) + .with_resolved_ec_provider(state.resolved_ec_provider.clone()); + if let Some(provider) = state.registry.geo_provider() { + services = services.with_geo(provider); + } + if let Some(provider) = state.registry.ec_provider() { + services = services.with_ec_provider(provider); + } + if let Some(provider) = state.registry.device_provider() { + services = services.with_device_provider(provider); } + services } // --------------------------------------------------------------------------- diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index f26c5ca891..8c016aef4d 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -11,7 +11,9 @@ use sha2::{Digest as _, Sha256}; use crate::constants::HEADER_X_TS_EC; use crate::ec::EcContext; +use crate::ec::device::DeviceProvider; use crate::ec::kv::KvIdentityGraph; +use crate::ec::provider::{EcProviderSelection, EdgeCookieProvider}; use crate::error::TrustedServerError; use crate::geo::GeoInfo; use crate::http_util::is_navigation_request; @@ -624,9 +626,21 @@ pub struct IntegrationRegistration { pub request_filters: Vec>, /// Geo provider this module supplies, selectable by `[geo] provider`. /// - /// Declaring one does not make it active: the module is only asked to - /// resolve location when `[geo] provider` names this module's id. + /// Declaring one does not make it active, because the module is only asked + /// to resolve location when `[geo] provider` names this module's id. pub geo_provider: Option>, + /// Edge Cookie provider this module supplies, selectable by `[ec] provider`. + /// + /// Declaring one does not make it active, because the module is only asked + /// to mint an identifier when `[ec] provider` names this module's id. This + /// is the same route geo takes, so identity is not a second extension + /// mechanism sitting beside the integration system. + pub ec_provider: Option>, + /// Device provider this module supplies, selectable by `[device] provider`. + /// + /// Declaring one does not make it active, because the module is only asked + /// to classify a request when `[device] provider` names this module's id. + pub device_provider: Option>, } impl IntegrationRegistration { @@ -656,6 +670,8 @@ impl IntegrationRegistrationBuilder { head_injectors: Vec::new(), request_filters: Vec::new(), geo_provider: None, + ec_provider: None, + device_provider: None, }, } } @@ -713,6 +729,28 @@ impl IntegrationRegistrationBuilder { self } + /// Declare the Edge Cookie provider this module supplies. + /// + /// The provider only mints identifiers when `[ec] provider` names this + /// module's id, and a declared provider the selector does not choose is + /// reported as a startup warning, the same as geo. + #[must_use] + pub fn with_ec_provider(mut self, provider: Arc) -> Self { + self.registration.ec_provider = Some(provider); + self + } + + /// Declare the device provider this module supplies. + /// + /// The provider only classifies requests when `[device] provider` names + /// this module's id, and a declared provider the selector does not choose + /// is reported as a startup warning, the same as geo. + #[must_use] + pub fn with_device_provider(mut self, provider: Arc) -> Self { + self.registration.device_provider = Some(provider); + self + } + /// Mark this integration's JS module for deferred loading via /// `
still streaming
"#; let mut settings = create_test_settings(); settings - .integrations + .integration .insert_config( "nextjs", &serde_json::json!({ - "enabled": true, "rewrite_attributes": ["href", "link", "url"], }), ) @@ -20216,7 +20225,9 @@ mod tests { #[async_trait::async_trait(?Send)] impl AuctionProvider for GatedAuctionProvider { fn provider_name(&self) -> &'static str { - "seam-test" + // A demand name is snake_case, and it has to match the + // `[demand]` entry the test selects for the auction to route to it. + "seam_test" } async fn request_bids( @@ -20263,20 +20274,17 @@ mod tests { for encoding in ["", "gzip", "deflate", "br"] { let mut settings = create_test_settings(); settings.auction.enabled = true; - settings.auction.providers = - crate::auction::AuctionConfig::legacy_provider_map(&["seam-test"]); + settings.auction.provider_names = vec!["seam_test".to_owned()]; settings.auction.timeout_ms = 60_000; - settings.auction.mediator = None; settings - .integrations + .integration .insert_config( "nextjs", &serde_json::json!({ - "enabled": true, "rewrite_attributes": ["href", "link", "url"], }), ) - .expect("should enable Next.js"); + .expect("should select Next.js"); let client = Arc::new(GatedAuctionHttpClient { inner: StubHttpClient::new(), released: std::sync::atomic::AtomicBool::new(false), diff --git a/crates/trusted-server-core/src/test_support.rs b/crates/trusted-server-core/src/test_support.rs index c0c7678ba9..2d23d0c694 100644 --- a/crates/trusted-server-core/src/test_support.rs +++ b/crates/trusted-server-core/src/test_support.rs @@ -203,10 +203,12 @@ pub mod nextjs_auction { [ec] passphrase = "test-secret-key-32-bytes-minimum" - # The fixture serves one publisher in one place, so it says so - # rather than selecting a geo provider it has no use for. + # The fixture supplies its own geo provider through the runtime + # services, so it selects the host lookup to reach it. Without a + # selection the jurisdiction is unknown, and a server-side auction + # fails closed on an unknown jurisdiction. [geo] - assume_single_jurisdiction = true + provider = "platform" "#, ) .expect("should parse Next.js auction fixture settings"); From 52feb1c10626e8ecf156e202168f5f94ff83b290 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 25 Sep 2026 12:54:23 +0100 Subject: [PATCH 060/124] Fix the audit port and point the diagnostics at the demand table Three of these are mine. Renaming the settings table in the audit module also renamed a local binding, a struct field and, worst, dropped the id constant between a derive and the struct it applied to, so the module did not compile. The fourth is main's new ad template diagnostics reading auction.providers, which is gone. They report the [demand] provider list instead, and the test that configured two providers selects two demand sources so the output is still sorted rather than in selection order. --- .../src/commands/audit/generate/mod.rs | 8 +-- .../src/commands/config/ad_templates.rs | 49 ++++++++++--------- 2 files changed, 29 insertions(+), 28 deletions(-) diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs index 7d60671d3d..abb54a2b47 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -158,10 +158,10 @@ struct DraftConfig { js_asset_proxy_candidate_count: usize, } -#[derive(Debug, Clone)] /// Id of the integration the audit writes its discovered assets for. const JS_ASSET_PROXY_ID: &str = "js_asset_proxy"; +#[derive(Debug, Clone)] struct JsAssetProxySection { /// The block to write when the audit found at least one asset. toml: String, @@ -398,10 +398,10 @@ fn write_success_summary( outputs.artifact.js_asset_count, outputs.artifact.third_party_asset_count, outputs.ad_slot_count, - if integration.is_empty() { + if integrations.is_empty() { "none".to_string() } else { - integration.join(", ") + integrations.join(", ") }, asset_proxy_note, if written.is_empty() { @@ -1232,7 +1232,7 @@ const INTERSTITIAL_SCRIPT_CEILING: usize = 3; /// different in each case. fn looks_like_an_interstitial(artifact: &AuditArtifact) -> Option { if artifact.js_asset_count > INTERSTITIAL_SCRIPT_CEILING - || !artifact.detected_integration.is_empty() + || !artifact.detected_integrations.is_empty() { return None; } diff --git a/crates/trusted-server-cli/src/commands/config/ad_templates.rs b/crates/trusted-server-cli/src/commands/config/ad_templates.rs index cfb2d13972..c63be217f4 100644 --- a/crates/trusted-server-cli/src/commands/config/ad_templates.rs +++ b/crates/trusted-server-cli/src/commands/config/ad_templates.rs @@ -183,16 +183,16 @@ fn run_lint(args: &AdTemplatesLintArgs, out: &mut dyn Write) -> Result<(), Strin .map_err(output_error)?; writeln!( out, - "auction.providers: {}", - if loaded.settings.auction.providers.is_empty() { + "demand.provider: {}", + if loaded.settings.demand.selected().is_empty() { "(none)".to_string() } else { loaded .settings - .auction - .providers - .keys() - .map(|id| escape_terminal_text(id.as_str()).into_owned()) + .demand + .selected() + .into_iter() + .map(|name| escape_terminal_text(name).into_owned()) .collect::>() .join(", ") } @@ -213,10 +213,10 @@ fn run_lint(args: &AdTemplatesLintArgs, out: &mut dyn Write) -> Result<(), Strin "status: slots are configured, but [auction].enabled is false" ) .map_err(output_error)?; - } else if loaded.settings.auction.providers.is_empty() { + } else if loaded.settings.demand.selected().is_empty() { writeln!( out, - "status: slots are configured, but [auction].providers is empty" + "status: slots are configured, but [demand] provider is empty" ) .map_err(output_error)?; } else { @@ -351,7 +351,7 @@ fn run_explain(args: &AdTemplatesExplainArgs, out: &mut dyn Write) -> Result<(), .creative_opportunities .as_ref() .is_some_and(|config| config.enabled); - let providers_configured = !loaded.settings.auction.providers.is_empty(); + let providers_configured = !loaded.settings.demand.selected().is_empty(); let gate = evaluate_ad_stack_gate(AdStackGateInput { method_get: method_pass, @@ -702,15 +702,16 @@ mod tests { #[test] fn lint_reports_configured_slot_count_and_auction_state() { - // Insert the alphabetically earlier provider after pbs-main in source - // order, so the output must sort identifiers rather than preserve input. + // The example config selects `pbs_main`, and this names the + // alphabetically earlier `aps_main` after it, so the output must sort + // the names rather than preserve the order they were selected in. let config_text = format!( - "{}\n[auction.providers.aps-main]\n\ - protocol = \"openrtb-2.6\"\n\ - profile = \"aps\"\n\ + "{}\n[demand]\n\ + provider = [\"pbs_main\", \"aps_main\"]\n\ + [demand.aps_main]\n\ + implementation = \"aps\"\n\ endpoint = \"https://aps.example.com/e/pb/bid\"\n\ routing = \"all_eligible\"\n\ - [auction.providers.aps-main.profile_config]\n\ account_id = \"example-aps-account-id\"\n", config_with_slots() ); @@ -735,9 +736,9 @@ mod tests { assert_eq!( output .lines() - .find(|line| line.starts_with("auction.providers:")), - Some("auction.providers: aps-main, pbs-main"), - "should report provider identifiers in deterministic order: {output}" + .find(|line| line.starts_with("demand.provider:")), + Some("demand.provider: aps_main, pbs_main"), + "should report demand names in deterministic order: {output}" ); assert!(!output.contains("legacy fallback")); } @@ -767,14 +768,14 @@ mod tests { assert_eq!( output .lines() - .find(|line| line.starts_with("auction.providers:")), - Some("auction.providers: (none)"), - "should report no providers" + .find(|line| line.starts_with("demand.provider:")), + Some("demand.provider: (none)"), + "should report no demand sources" ); assert!( - output.lines().any( - |line| line == "status: slots are configured, but [auction].providers is empty" - ), + output + .lines() + .any(|line| line == "status: slots are configured, but [demand] provider is empty"), "should explain why configured slots are ineligible: {output}" ); } From 742ecfd107a6da7d8d1c0ed6171619aa5b514b0b Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 25 Sep 2026 12:57:53 +0100 Subject: [PATCH 061/124] Stop importing the plan compiler the adapters no longer call Both the Axum and Fastly adapters call the module-aware compiler now, and the only bare calls left are in test modules that import it themselves. --- crates/trusted-server-adapter-axum/src/app.rs | 3 +-- crates/trusted-server-adapter-fastly/src/app.rs | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 8bda84abc4..d63429f3d2 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -11,8 +11,7 @@ use edgezero_core::router::RouterService; use error_stack::Report; use trusted_server_core::auction::endpoints::handle_auction; use trusted_server_core::auction::{ - AuctionOrchestrator, build_orchestrator_with_plan, compile_auction_plan, - compile_auction_plan_with, + AuctionOrchestrator, build_orchestrator_with_plan, compile_auction_plan_with, }; use trusted_server_core::cache_policy::EdgeCacheHeader; use trusted_server_core::ec::EcContext; diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 0eb2885c95..b7b17088c1 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -104,8 +104,7 @@ use error_stack::Report; use trusted_server_core::auction::AuctionTelemetrySink; use trusted_server_core::auction::endpoints::handle_auction; use trusted_server_core::auction::{ - AuctionOrchestrator, build_orchestrator_with_plan, compile_auction_plan, - compile_auction_plan_with, + AuctionOrchestrator, build_orchestrator_with_plan, compile_auction_plan_with, }; use trusted_server_core::cache_policy::EdgeCacheHeader; use trusted_server_core::config_payload::DEFAULT_SECRET_STORE_ID; From 408fad37aefa0d71e899c9a2f8f83cea1a4f4a90 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 25 Sep 2026 13:01:18 +0100 Subject: [PATCH 062/124] Re-pad the API reference table and drop a test re-import Renaming the integration table shortened a cell, so two rows needed re-padding. The Fastly test module re-imported the plain plan compiler through the app module and calls it by its full path, so it stops. --- crates/trusted-server-adapter-fastly/src/app.rs | 2 +- docs/guide/api-reference.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index b7b17088c1..cca37e3e3f 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -1600,7 +1600,7 @@ mod tests { AppContext, AppState, AuctionDispatch, EcContext, EdgeCacheHeader, HandlerFuture, NAMED_ROUTES, NamedRouteHandler, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, RuntimeStoreConfig, TrustedServerApp, build_orchestrator_with_plan, build_per_request_services, - build_state_from_settings, compile_auction_plan, handle_publisher_request, + build_state_from_settings, handle_publisher_request, publisher_response_into_streaming_response, startup_error_router, }; use base64::Engine as _; diff --git a/docs/guide/api-reference.md b/docs/guide/api-reference.md index 0a0005ad8f..e4d5e02bff 100644 --- a/docs/guide/api-reference.md +++ b/docs/guide/api-reference.md @@ -1031,7 +1031,7 @@ rewriter, injector, post-processor, request filter, or auction mediator. | `permutive` | `enabled=true` | `POST /integrations/permutive/secure-signal/*` | | `permutive` | `enabled=true` | `POST /integrations/permutive/sync/*` | | `prebid` | `enabled=true;script_patterns=config-derived` | `GET /integrations/prebid/bundle.js` | -| `prebid` | `enabled=true;script_patterns=config-derived` | `GET ` | +| `prebid` | `enabled=true;script_patterns=config-derived` | `GET ` | | `sourcepoint` | `enabled=true` | `GET /integrations/sourcepoint/cdn/*` | | `sourcepoint` | `enabled=true` | `HEAD /integrations/sourcepoint/cdn/*` | | `sourcepoint` | `enabled=true` | `OPTIONS /integrations/sourcepoint/cdn/*` | @@ -1055,7 +1055,7 @@ available when a deployment needs either. | Didomi consent | `GET` or `POST` under the configured prefix (default `/integrations/didomi/consent/*`); path selects SDK or API origin; query and bounded POST body forwarded | Upstream status/body preserved; SDK responses receive the integration's CORS headers; API responses retain selected upstream headers; no local cache policy | `curl -i https://edge.example.com/integrations/didomi/consent/loader.js` | | GTM/gtag scripts | `GET` the generated `gtm.js`, `gtag.js`, or `gtag/js` paths; query forwarded or configured container ID supplied; successful script is rewritten | Non-success upstream status preserved; rewritten scripts use `cache_max_age`; oversized rewritten upstream bodies use shared integration errors | `curl -i 'https://edge.example.com/integrations/google_tag_manager/gtm.js?id=GTM-XXXX'` | | Google collect | `GET` or `POST` the generated `collect` or `g/collect` paths; query, selected headers, and bounded body proxy to the configured Google origin | Malformed `Content-Length` returns `400`; body over `max_beacon_body_size` returns `413`; stream-read failure returns `502`; upstream response otherwise preserved | Browser beacon; body schema belongs to Google Analytics | -| JS asset proxy | `GET` each configured `[[integration.js_asset_proxy.assets]]` path whose `proxy = "enabled"`; the exact `origin_url` is fetched and served first-party | Upstream failures use shared integration errors; successful responses honor the per-asset or integration `cache_ttl_seconds`; `blocked` assets register no route and strip matching tags | Path is operator-configured, for example `curl -i https://edge.example.com/js/vendor-tag.js` | +| JS asset proxy | `GET` each configured `[[integration.js_asset_proxy.assets]]` path whose `proxy = "enabled"`; the exact `origin_url` is fetched and served first-party | Upstream failures use shared integration errors; successful responses honor the per-asset or integration `cache_ttl_seconds`; `blocked` assets register no route and strip matching tags | Path is operator-configured, for example `curl -i https://edge.example.com/js/vendor-tag.js` | | GPT | `GET` `/script`, `/pagead/*`, or `/tag/*`; path/query proxy to the configured GPT origins and script content can be rewritten | Upstream status is preserved; successful scripts/assets apply integration cache rules; selected upstream CORS is preserved | `curl -i https://edge.example.com/integrations/gpt/script` | | Lockr SDK | `GET /integrations/lockr/sdk`; no body; fetches and returns the configured SDK as JavaScript | Successful SDK uses `cache_ttl_seconds`; upstream/transport failures follow integration mapping; no added CORS policy | `curl -i https://edge.example.com/integrations/lockr/sdk` | | Lockr API | `GET` or `POST /integrations/lockr/api/*`; path, query, selected headers, and bounded body proxy to `api_endpoint`; publisher credentials are stripped | Upstream status/body preserved; no local cache/CORS policy | Payload is Lockr-specific; use the SDK for normal calls | From 7757770137b57ff7a964c45abf64d9e9b2c6400d Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 25 Sep 2026 13:19:30 +0100 Subject: [PATCH 063/124] Let the adapter test fixtures import the plan compiler they use The Axum and Fastly test fixtures supply no integration builders, so they compile the plan without them. They were reaching the plain compiler through the adapter's own import, which the adapter no longer needs. --- crates/trusted-server-adapter-axum/src/app.rs | 4 ++++ crates/trusted-server-adapter-fastly/src/app.rs | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index d63429f3d2..dc56511710 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -858,6 +858,10 @@ fn build_router(state: &Arc) -> RouterService { mod tests { use edgezero_core::http::request_builder; use edgezero_core::params::PathParams; + // These fixtures supply no integration builders, so they compile the plan + // without them rather than through the module-aware compiler the adapter + // itself uses. + use trusted_server_core::auction::compile_auction_plan; use super::*; diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index cca37e3e3f..37c552e5da 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -1603,6 +1603,10 @@ mod tests { build_state_from_settings, handle_publisher_request, publisher_response_into_streaming_response, startup_error_router, }; + // These fixtures supply no integration builders, so they compile the plan + // without them rather than through the module-aware compiler the adapter + // itself uses. + use trusted_server_core::auction::compile_auction_plan; use base64::Engine as _; use bytes::Bytes; use edgezero_core::app::Hooks as _; From 8aa8ca1e25b132960fa5de38c5b8e40564febd33 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 25 Sep 2026 13:34:56 +0100 Subject: [PATCH 064/124] Bring main's CLI tests and documented fixture onto this stack Main wrote a disabled JS asset proxy block into the starter template and then replaced it. Here the template documents every integration as a commented example and the audit appends the block it fills in, so its tests count only entries at the start of a line and expect the commented example to still be a comment. One of them was also still reading the removed `[integrations]` table out of the parsed draft. The ad template diagnostics report `[demand] provider`. The two fixtures that isolated "no providers configured" remove the demand table rather than a key inside `[auction]`, and the one that wanted two sources adds the second to the existing table rather than writing a second `[demand]` header, which is a duplicate key. The report sorts the names it prints, which the old map gave for free and a selection list does not. The integration guide's documented geo fixture implements an asynchronous `lookup` that takes the request's services, and the crate its source is compiled in gains the async-trait dependency that needs, resolved from the workspace the same way error-stack already was. --- .../src/commands/audit/generate/mod.rs | 27 +++++--- .../src/commands/config/ad_templates.rs | 66 +++++++++++++------ .../tests/documentation_snippets.rs | 28 ++++---- docs/guide/integration-guide.md | 4 +- 4 files changed, 85 insertions(+), 40 deletions(-) diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs index abb54a2b47..eb340c1e21 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -2427,19 +2427,21 @@ mod tests { .contains("Native integration may be preferable: [integration.gpt]") ); assert!( - !draft.toml.contains("example-vendor-loader"), - "should remove starter-template placeholder asset" + draft + .toml + .contains("# path = \"/assets/example-vendor-loader.js\""), + "should leave the template's commented example as a comment" ); assert!( draft.toml.contains( "# Proxy behavior and first-party asset routing. Kept active with defaults.\n[proxy]" ), - "should preserve documentation for the section following the replaced block" + "should leave the documentation of the sections it appends below" ); let parsed = toml::from_str::(&draft.toml).expect("draft should parse as TOML"); assert!( - parsed["integrations"]["js_asset_proxy"] + parsed["integration"]["js_asset_proxy"] .get("cache_ttl_seconds") .is_none(), "generated config should inherit upstream cache headers by default" @@ -2531,9 +2533,14 @@ mod tests { assert_eq!(draft.js_asset_proxy_candidate_count, 1); assert_eq!( + // The template's own example is a comment, so only an entry at the + // start of a line is one the audit wrote. draft .toml - .matches("[[integration.js_asset_proxy.assets]]") + .matches( + " +[[integration.js_asset_proxy.assets]]" + ) .count(), 1, "should only emit one candidate entry" @@ -2623,12 +2630,16 @@ mod tests { .contains("No eligible third-party HTTPS script assets") ); assert!( - !draft.toml.contains("[[integration.js_asset_proxy.assets]]"), + !draft + .toml + .contains("\n[[integration.js_asset_proxy.assets]]"), "should not emit asset array entries without candidates" ); assert!( - !draft.toml.contains("example-vendor-loader"), - "should remove starter-template placeholder asset" + draft + .toml + .contains("# path = \"/assets/example-vendor-loader.js\""), + "should leave the template's commented example as a comment" ); toml::from_str::(&draft.toml).expect("draft should parse as TOML"); } diff --git a/crates/trusted-server-cli/src/commands/config/ad_templates.rs b/crates/trusted-server-cli/src/commands/config/ad_templates.rs index c63be217f4..79a4f06de7 100644 --- a/crates/trusted-server-cli/src/commands/config/ad_templates.rs +++ b/crates/trusted-server-cli/src/commands/config/ad_templates.rs @@ -187,14 +187,17 @@ fn run_lint(args: &AdTemplatesLintArgs, out: &mut dyn Write) -> Result<(), Strin if loaded.settings.demand.selected().is_empty() { "(none)".to_string() } else { - loaded + // Sorted rather than in selection order, so the report reads the + // same however the document happens to be written. + let mut names = loaded .settings .demand .selected() .into_iter() .map(|name| escape_terminal_text(name).into_owned()) - .collect::>() - .join(", ") + .collect::>(); + names.sort_unstable(); + names.join(", ") } ) .map_err(output_error)?; @@ -666,10 +669,14 @@ mod tests { .as_table_mut() .expect("should find the auction table"); auction.insert("enabled".to_string(), toml::Value::Boolean(true)); - // Remove the provider map and its bidder references structurally to - // isolate the runtime verdict from the provider advisory. - auction.remove("providers"); + // Remove the bidder references, and the demand sources the auction + // would route to, structurally, to isolate the runtime verdict from the + // provider advisory. auction.remove("bidders"); + config_value + .as_table_mut() + .expect("should find the document root") + .remove("demand"); let config_text = toml::to_string(&config_value).expect("should serialize the fixture config"); let (_temp, config) = project_with_config(&config_text); @@ -702,19 +709,35 @@ mod tests { #[test] fn lint_reports_configured_slot_count_and_auction_state() { - // The example config selects `pbs_main`, and this names the - // alphabetically earlier `aps_main` after it, so the output must sort - // the names rather than preserve the order they were selected in. - let config_text = format!( - "{}\n[demand]\n\ - provider = [\"pbs_main\", \"aps_main\"]\n\ - [demand.aps_main]\n\ - implementation = \"aps\"\n\ - endpoint = \"https://aps.example.com/e/pb/bid\"\n\ - routing = \"all_eligible\"\n\ - account_id = \"example-aps-account-id\"\n", - config_with_slots() - ); + // The example config selects `pbs_main`. Naming the alphabetically + // earlier `aps_main` after it means the output must sort the names + // rather than preserve the order they were selected in. + let mut config_value: toml::Value = + toml::from_str(&config_with_slots()).expect("should parse the fixture config"); + let root = config_value + .as_table_mut() + .expect("should find the document root"); + let demand = root + .get_mut("demand") + .and_then(toml::Value::as_table_mut) + .expect("should find the demand table"); + demand + .get_mut("provider") + .and_then(toml::Value::as_array_mut) + .expect("should find the demand selection") + .push(toml::Value::String("aps_main".to_string())); + let mut aps = toml::value::Table::new(); + for (key, value) in [ + ("implementation", "aps"), + ("endpoint", "https://aps.example.com/e/pb/bid"), + ("routing", "all_eligible"), + ("account_id", "example-aps-account-id"), + ] { + aps.insert(key.to_string(), toml::Value::String(value.to_string())); + } + demand.insert("aps_main".to_string(), toml::Value::Table(aps)); + let config_text = + toml::to_string(&config_value).expect("should serialize the fixture config"); let (_temp, config) = project_with_config(&config_text); let mut out = Vec::new(); @@ -751,8 +774,11 @@ mod tests { .as_table_mut() .expect("should find auction table"); auction.insert("enabled".to_string(), toml::Value::Boolean(true)); - auction.remove("providers"); auction.remove("bidders"); + config_value + .as_table_mut() + .expect("should find the document root") + .remove("demand"); let config_text = toml::to_string(&config_value).expect("should serialize the fixture config"); let (_temp, config) = project_with_config(&config_text); diff --git a/crates/trusted-server-integration-tests/tests/documentation_snippets.rs b/crates/trusted-server-integration-tests/tests/documentation_snippets.rs index d9c36d2338..0e56dbc119 100644 --- a/crates/trusted-server-integration-tests/tests/documentation_snippets.rs +++ b/crates/trusted-server-integration-tests/tests/documentation_snippets.rs @@ -38,18 +38,24 @@ fn integration_guide_runtime_services_fixture_compiles() { let core_path = repository_root.join("crates/trusted-server-core"); let workspace_manifest = fs::read_to_string(repository_root.join("Cargo.toml")) .expect("should read the workspace manifest"); - let error_stack_requirement = workspace_manifest - .lines() - .find_map(|line| { - let assignment = line.trim().strip_prefix("error-stack")?.trim_start(); - let value = assignment.strip_prefix('=')?; - let start = value.find('"')? + 1; - let end = start + value[start..].find('"')?; - Some(value[start..end].to_owned()) - }) - .expect("should find the workspace error-stack requirement"); + let requirement_of = |crate_name: &str| { + workspace_manifest + .lines() + .find_map(|line| { + let assignment = line.trim().strip_prefix(crate_name)?.trim_start(); + let value = assignment.strip_prefix('=')?; + let start = value.find('"')? + 1; + let end = start + value[start..].find('"')?; + Some(value[start..end].to_owned()) + }) + .unwrap_or_else(|| panic!("should find the workspace {crate_name} requirement")) + }; + let error_stack_requirement = requirement_of("error-stack"); + // The documented fixture implements `PlatformGeo`, whose `lookup` is + // asynchronous, so it carries the same attribute the trait does. + let async_trait_requirement = requirement_of("async-trait"); let manifest = format!( - "[package]\nname = \"documentation-snippet\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n[workspace]\n\n[dependencies]\nerror-stack = \"{error_stack_requirement}\"\ntrusted-server-core = {{ path = {:?} }}\n", + "[package]\nname = \"documentation-snippet\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n[workspace]\n\n[dependencies]\nasync-trait = \"{async_trait_requirement}\"\nerror-stack = \"{error_stack_requirement}\"\ntrusted-server-core = {{ path = {:?} }}\n", core_path ); fs::write(fixture_root.join("Cargo.toml"), manifest) diff --git a/docs/guide/integration-guide.md b/docs/guide/integration-guide.md index f127d4207c..626d521756 100644 --- a/docs/guide/integration-guide.md +++ b/docs/guide/integration-guide.md @@ -130,10 +130,12 @@ impl PlatformBackend for FixtureBackend { struct FixtureGeo; +#[async_trait::async_trait(?Send)] impl PlatformGeo for FixtureGeo { - fn lookup( + async fn lookup( &self, _client_ip: Option, + _services: &RuntimeServices, ) -> Result, Report> { Ok(None) } From 44b56d015aaa2d007b6b31c9d8b0474d54638573 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 25 Sep 2026 14:05:43 +0100 Subject: [PATCH 065/124] Put the test import where rustfmt wants it The Fastly test module's import of the plain plan compiler went in at the top of its group rather than in sorted order. --- crates/trusted-server-adapter-fastly/src/app.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 37c552e5da..0015adf891 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -1603,10 +1603,6 @@ mod tests { build_state_from_settings, handle_publisher_request, publisher_response_into_streaming_response, startup_error_router, }; - // These fixtures supply no integration builders, so they compile the plan - // without them rather than through the module-aware compiler the adapter - // itself uses. - use trusted_server_core::auction::compile_auction_plan; use base64::Engine as _; use bytes::Bytes; use edgezero_core::app::Hooks as _; @@ -1618,6 +1614,10 @@ mod tests { use edgezero_core::params::PathParams; use edgezero_core::router::RouterService; use std::net::{IpAddr, Ipv4Addr}; + // These fixtures supply no integration builders, so they compile the plan + // without them rather than through the module-aware compiler the adapter + // itself uses. + use trusted_server_core::auction::compile_auction_plan; use error_stack::Report; use futures::executor::block_on; From b8d451b8c16ea71955cbec6a8c9af78c6b87408f Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 6 Oct 2026 23:48:36 +0100 Subject: [PATCH 066/124] The seam says module where an operator selects a capability The lower branches settled the word: what an operator selects in `[ec]`, `[geo]`, `[device]`, `[permission_signal]` and `[integration]` is a module, and `provider` keeps its one meaning in the auction, where `[demand]` and `[adserver]` select the demand sources and the ad server. This applies that to the seam's own lines, which were written before the word was settled. The `[integration]` selector is now `module`, in the settings, the example template, the starter configuration the CLI writes, the audit draft and every fixture. The registry's declared capabilities follow: a module declares a `geo_module`, an `ec_module` or a `device_module`, the registry resolves `resolve_geo_module`, `resolve_ec_module` and `resolve_device_module`, and the reserved selector values are `GEO_MODULE_NONE`, `GEO_MODULE_PLATFORM`, `DEVICE_MODULE_BUILTIN` and `DEVICE_MODULE_FASTLY`. Startup refusals name the `module` key they are about, so an operator reading one is told the key to edit. The generic provider table, the auction plan, the demand and ad server implementations and the CLI's ad template lint keep `provider`, and the configuration rules, the configuration guide, the example template, the CHANGELOG and the agent notes state the convention with both spellings rather than one word for everything. --- AGENTS.md | 47 +- CHANGELOG.md | 10 +- README.md | 2 +- TESTING.md | 2 +- crates/integrations/seam-probe/src/lib.rs | 54 +-- crates/trusted-server-adapter-axum/src/app.rs | 18 +- .../src/platform.rs | 2 +- .../tests/seam_probe.rs | 124 +++--- .../src/app.rs | 16 +- .../src/platform.rs | 2 +- .../trusted-server-adapter-fastly/src/app.rs | 34 +- .../trusted-server-adapter-fastly/src/main.rs | 12 +- .../src/middleware.rs | 6 +- crates/trusted-server-adapter-spin/src/app.rs | 16 +- .../src/platform.rs | 2 +- .../src/commands/audit/generate/mod.rs | 20 +- .../src/commands/config/init.rs | 4 +- .../tests/config_env_overlay.rs | 10 +- crates/trusted-server-core/src/config.rs | 7 +- .../trusted-server-core/src/config_payload.rs | 15 +- crates/trusted-server-core/src/ec/device.rs | 4 +- crates/trusted-server-core/src/ec/mod.rs | 50 +-- .../trusted-server-core/src/html_processor.rs | 2 +- .../src/integrations/datadome.rs | 4 +- .../src/integrations/didomi.rs | 4 +- .../src/integrations/google_tag_manager.rs | 6 +- .../src/integrations/gpt.rs | 4 +- .../src/integrations/gpt_diagnostics.rs | 6 +- .../src/integrations/js_asset_proxy.rs | 4 +- .../src/integrations/lockr.rs | 4 +- .../src/integrations/mod.rs | 14 +- .../src/integrations/nextjs/mod.rs | 6 +- .../src/integrations/osano.rs | 6 +- .../src/integrations/permutive.rs | 2 +- .../src/integrations/prebid.rs | 6 +- .../src/integrations/registry.rs | 416 +++++++++--------- .../src/integrations/sourcepoint.rs | 4 +- .../src/integrations/testlight.rs | 4 +- .../trusted-server-core/src/platform/geo.rs | 4 +- .../trusted-server-core/src/platform/mod.rs | 2 +- .../src/platform/test_support.rs | 2 +- .../src/platform/traits.rs | 6 +- .../trusted-server-core/src/platform/types.rs | 42 +- .../trusted-server-core/src/provider_table.rs | 6 +- crates/trusted-server-core/src/publisher.rs | 6 +- crates/trusted-server-core/src/settings.rs | 86 ++-- .../trusted-server-core/src/test_support.rs | 10 +- .../configs/trusted-server.integration.toml | 4 +- docs/guide/api-reference.md | 2 +- docs/guide/auction-orchestration.md | 10 +- docs/guide/cli.md | 4 +- docs/guide/configuration-rules.md | 125 +++--- docs/guide/configuration.md | 150 ++++--- docs/guide/error-reference.md | 10 +- docs/guide/getting-started.md | 2 +- docs/guide/integrations/adserver_mock.md | 36 +- docs/guide/integrations/aps.md | 2 +- docs/guide/integrations/datadome.md | 20 +- docs/guide/integrations/didomi.md | 4 +- docs/guide/integrations/google_tag_manager.md | 2 +- docs/guide/integrations/gpt-diagnostics.md | 4 +- docs/guide/integrations/gpt.md | 4 +- docs/guide/integrations/lockr.md | 2 +- docs/guide/integrations/nextjs.md | 6 +- docs/guide/integrations/osano.md | 2 +- docs/guide/integrations/permutive.md | 2 +- docs/guide/integrations/prebid.md | 6 +- docs/guide/integrations/sourcepoint.md | 4 +- docs/guide/permission-model.md | 2 +- scripts/template-cache-local-test.sh | 2 +- trusted-server.example.toml | 33 +- 71 files changed, 777 insertions(+), 774 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 8d71bc8cda..381fb8207f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -369,24 +369,25 @@ Bad: `"fix: added feature flags"` Each vendor-differentiated capability is pluggable behind its own trait, so a deployment selects an implementation and the core stays neutral. Every pluggable thing follows one configuration convention, which is a top-level -table named for the job, a `provider` key that selects what runs, and a -`[.]` table for a provider that has settings. The rules and what +table named for the job, a `module` key that selects what runs (`provider` for +the auction's `[demand]` and `[adserver]` tables), and a `[.]` +table for a selected name that has settings. The rules and what is checked when are in `docs/guide/configuration-rules.md`, which is the page -to read before changing any provider configuration. - -| Capability | Trait | Selector | Built-in (core) | Vendor / host crates | -| --------------------- | ---------------------------------------- | ------------------------- | -------------------------------------------- | ---------------------------- | -| Edge Cookie identity | `EdgeCookieModule` (`ec/module.rs`) | `[ec] provider` | `hmac`, `host_signals`, `client_fixed` (opt-in, no default) | `crates/edgecookie/` | -| Device detection | `DeviceModule` (`ec/device.rs`) | `[device] provider` | `builtin`, User-Agent only (the default) | `crates/device/` | -| Geo / IP intelligence | `PlatformGeo` (`platform/traits.rs`) | `[geo] provider` | None, no location (the default), or `platform` | `crates/geo/` | -| Permission signals | `PermissionSignalModule` (`permission_signal/mod.rs`) | `[permission_signal] provider` (an ordered list) | `gpc`, `gpp_sale_opt_out`, `us_privacy`, `tcf`, all of them with no list | `crates/permission-signal/` | -| Auction demand | `DemandImplementation` (`auction/demand.rs`) | `[demand] provider` (a list) | `openrtb`, `prebid_server`, `aps` | an integration builder | -| Ad server | `AdServerImplementation` (`auction/demand.rs`) | `[adserver] provider` | `adserver_mock` | an integration builder | -| Page integrations | `IntegrationBuilder` (`integrations/mod.rs`) | `[integration] provider` (a list) | `datadome`, `didomi`, `google_tag_manager`, `gpt`, `gpt_diagnostics`, `js_asset_proxy`, `lockr`, `nextjs`, `osano`, `permutive`, `prebid`, `sourcepoint`, `testlight` | an adapter-supplied builder | +to read before changing any module configuration. + +| Capability | Trait | Selector | Built-in (core) | Vendor / host crates | +| --------------------- | ----------------------------------------------------- | ---------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------- | +| Edge Cookie identity | `EdgeCookieModule` (`ec/module.rs`) | `[ec] module` | `hmac`, `host_signals`, `client_fixed` (opt-in, no default) | `crates/edgecookie/` | +| Device detection | `DeviceModule` (`ec/device.rs`) | `[device] module` | `builtin`, User-Agent only (the default) | `crates/device/` | +| Geo / IP intelligence | `PlatformGeo` (`platform/traits.rs`) | `[geo] module` | None, no location (the default), or `platform` | `crates/geo/` | +| Permission signals | `PermissionSignalModule` (`permission_signal/mod.rs`) | `[permission_signal] module` (an ordered list) | `gpc`, `gpp_sale_opt_out`, `us_privacy`, `tcf`, all of them with no list | `crates/permission-signal/` | +| Auction demand | `DemandImplementation` (`auction/demand.rs`) | `[demand] provider` (a list) | `openrtb`, `prebid_server`, `aps` | an integration builder | +| Ad server | `AdServerImplementation` (`auction/demand.rs`) | `[adserver] provider` | `adserver_mock` | an integration builder | +| Page integrations | `IntegrationBuilder` (`integrations/mod.rs`) | `[integration] module` (a list) | `datadome`, `didomi`, `google_tag_manager`, `gpt`, `gpt_diagnostics`, `js_asset_proxy`, `lockr`, `nextjs`, `osano`, `permutive`, `prebid`, `sourcepoint`, `testlight` | an adapter-supplied builder | `openrtb`, `prebid_server`, `aps` and `adserver_mock` supply implementations only. They are not page integrations and cannot be named in -`[integration] provider`. +`[integration] module`. Principles for adding or changing a module: @@ -425,7 +426,7 @@ IntegrationRegistration::builder(ID) ``` - Integration IDs match JS directory names: `prebid` (deferred), `lockr`, `permutive`, `datadome`, `didomi`, `testlight`. -- An integration runs when `[integration] provider` names it. There is no `enabled` flag. +- An integration runs when `[integration] module` names it. There is no `enabled` flag. - `creative` is JS-only (no Rust registration), and `nextjs` is Rust-only. `openrtb`, `prebid_server`, `aps` and `adserver_mock` register demand or ad server implementations rather than page integrations. - Integrations opt into deferred loading via `.with_deferred_js()` on the registration builder. Deferred modules are served as separate `
still streaming
"#; let mut settings = create_test_settings(); settings - .integration - .insert_config( - "nextjs", + .insert_module_config( + "framework", + "framework.nextjs", &serde_json::json!({ "rewrite_attributes": ["href", "link", "url"], }), @@ -21753,9 +21765,9 @@ mod tests { settings.auction.provider_names = vec!["seam_test".to_owned()]; settings.auction.timeout_ms = 60_000; settings - .integration - .insert_config( - "nextjs", + .insert_module_config( + "framework", + "framework.nextjs", &serde_json::json!({ "rewrite_attributes": ["href", "link", "url"], }), @@ -22575,9 +22587,9 @@ mod tests { // Configure nextjs so a stream processor is registered. let mut settings = create_test_settings(); settings - .integration - .insert_config( - "nextjs", + .insert_module_config( + "framework", + "framework.nextjs", &serde_json::json!({ "rewrite_attributes": ["href", "link", "url"], }), @@ -22661,9 +22673,9 @@ mod tests { fn document_state_placeholders_substitute_through_streaming_path() { let mut settings = create_test_settings(); settings - .integration - .insert_config( - "nextjs", + .insert_module_config( + "framework", + "framework.nextjs", &serde_json::json!({ "rewrite_attributes": ["href", "link", "url"], }), @@ -23011,7 +23023,7 @@ mod tests { "no token should be minted without the diagnostics integration" ); - settings.integration.select("gpt_diagnostics"); + settings.select_module("ad-tag", "ad-tag.google.diagnostics"); let first = diagnostics_auction_id(&settings).expect("enabled diagnostics should mint a token"); let second = @@ -24505,8 +24517,8 @@ mod tests { fn settings_with_co() -> Settings { let toml = format!( - "{}\n[auction]\nenabled = true\n\n[creative_opportunities]\ngam_network_id = \"12345\"\n", - crate_test_settings_str() + "{}\n[creative_opportunities]\ngam_network_id = \"12345\"\n", + crate_test_settings_str().replace("[auction]\n", "[auction]\nenabled = true\n") ); Settings::from_toml(&toml).expect("should parse settings with creative_opportunities") } @@ -24514,23 +24526,26 @@ mod tests { /// Settings for a deployment that has no `[creative_opportunities]` /// section, so page-bids answers `404`. fn settings_without_co() -> Settings { - let toml = format!("{}\n[auction]\nenabled = true\n", crate_test_settings_str()); + let toml = format!( + "{}\n", + crate_test_settings_str().replace("[auction]\n", "[auction]\nenabled = true\n") + ); Settings::from_toml(&toml) .expect("should parse settings without creative_opportunities") } fn settings_with_co_auction_disabled() -> Settings { let toml = format!( - "{}\n[auction]\nenabled = false\n\n[creative_opportunities]\ngam_network_id = \"12345\"\n", - crate_test_settings_str() + "{}\n[creative_opportunities]\ngam_network_id = \"12345\"\n", + crate_test_settings_str().replace("[auction]\n", "[auction]\nenabled = false\n") ); Settings::from_toml(&toml).expect("should parse settings with creative_opportunities") } fn settings_with_co_templates_disabled() -> Settings { let toml = format!( - "{}\n[auction]\nenabled = true\n\n[creative_opportunities]\nenabled = false\ngam_network_id = \"12345\"\n", - crate_test_settings_str() + "{}\n[creative_opportunities]\nenabled = false\ngam_network_id = \"12345\"\n", + crate_test_settings_str().replace("[auction]\n", "[auction]\nenabled = true\n") ); Settings::from_toml(&toml).expect("should parse settings with disabled templates") } @@ -24731,7 +24746,7 @@ mod tests { let mut settings = settings_with_co(); settings.demand = crate::auction::test_support::demand_named(&[AUCTION_ID_TEST_PROVIDER]); - settings.integration.select("gpt_diagnostics"); + settings.select_module("ad-tag", "ad-tag.google.diagnostics"); let slots = article_slot(); let winning_stub = Arc::new(StubHttpClient::new()); winning_stub.push_response(200, b"winner".to_vec()); @@ -24885,7 +24900,7 @@ mod tests { let mut settings = settings_with_co(); settings.demand = crate::auction::test_support::demand_named(&[AUCTION_ID_TEST_PROVIDER]); - settings.integration.select("gpt_diagnostics"); + settings.select_module("ad-tag", "ad-tag.google.diagnostics"); let first = winning_auction_id(&settings) .await @@ -24898,10 +24913,7 @@ mod tests { "each auction for the same visitor should mint its own token" ); - settings - .integration - .module - .retain(|id| id != "gpt_diagnostics"); + settings.remove_module("ad-tag", "ad-tag.google.diagnostics"); assert_eq!( winning_auction_id(&settings).await, None, @@ -25573,9 +25585,9 @@ mod tests { fn settings_with_capturing_provider() -> Settings { let toml = format!( - "{}\n[auction]\nenabled = true\n\n[demand]\nmodules = [\"{CAPTURING_PROVIDER}\"]\n\n[demand.{CAPTURING_PROVIDER}]\nimplementation = \"openrtb\"\nendpoint = \"https://capture.example/openrtb2/auction\"\nrouting = \"all_eligible\"\n\n\ + "{}\n[demand]\nmodules = [\"{CAPTURING_PROVIDER}\"]\n\n[demand.{CAPTURING_PROVIDER}]\nimplementation = \"openrtb\"\nendpoint = \"https://capture.example/openrtb2/auction\"\nrouting = \"all_eligible\"\n\n\ [creative_opportunities]\ngam_network_id = \"12345\"\n", - crate_test_settings_str() + crate_test_settings_str().replace("[auction]\n", "[auction]\nenabled = true\n") ); settings_from_toml(&toml) } diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 76160ab844..38bd65c02c 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -27,7 +27,7 @@ use crate::ec::module::{ use crate::error::TrustedServerError; use crate::host_header::validate_host_header_override_value; use crate::platform::PlatformImageOptimizerRegion; -use crate::provider_table::{ProviderChoice, ProviderList}; +use crate::provider_table::{ProviderChoice, ProviderList, SectionModules}; use crate::redacted::Redacted; #[cfg(test)] @@ -222,252 +222,109 @@ impl Publisher { /// Which integrations run, and the settings each one is given. /// -/// Mapped from the `[integration]` TOML section, which follows the convention -/// every selectable type uses, where [`module`](Self::module) names what runs -/// and a named block holds one module's settings. Here that block is -/// `[integration.]`, and it is written only for an integration that has -/// settings to give. -#[derive(Default, Clone, Deserialize, Serialize)] -pub struct IntegrationSettings { - /// The integrations that run, named by id, for example - /// `module = ["gpt", "prebid"]`. - /// - /// An integration runs when, and only when, its id is on this list, so - /// there is no second switch inside its own block and leaving the list out - /// runs none of them. A repeated id, and a block for an integration that - /// is not named here, are both refused by - /// [`validate_selection`](Self::validate_selection). An id that no builder - /// supplies is refused where the registry is built, which is the only - /// place an adapter's and a vendor crate's builders are known. - /// - /// The order of the list carries no meaning, because integrations run in - /// the order their builders are registered. This is unlike - /// `[permission_signal] module`, where the order is the policy. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub module: Vec, - #[serde(flatten)] - entries: HashMap, -} +/// The sections of module types core does not read itself, such as `[cmp]` or +/// `[tag]`, each named by its type's folder under `crates`. +/// +/// Every top-level table that is not one of Trusted Server's own settings is +/// read as one of these. Whether its type is one a module on offer has is +/// only known where the registry is built, so it is checked there. +#[derive(Clone, Default, PartialEq, Serialize)] +#[serde(transparent)] +pub struct TypeSections(BTreeMap); -impl std::fmt::Debug for IntegrationSettings { +impl std::fmt::Debug for TypeSections { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let mut integration_ids = self.entries.keys().collect::>(); - integration_ids.sort_unstable(); - formatter - .debug_struct("IntegrationSettings") - .field("module", &self.module) - .field("integration_ids", &integration_ids) - .finish() + formatter.debug_map().entries(self.0.iter()).finish() } } -/// The settings type an integration reads from its `[integration.]` block. -/// -/// The type states which settings the integration takes and how they are -/// validated, and nothing else. Whether the integration runs is not its -/// business, because `[integration] module` names what runs. -pub trait IntegrationConfig: DeserializeOwned + Validate {} +impl<'de> Deserialize<'de> for TypeSections { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let entries = serde_json::Map::::deserialize(deserializer)?; + let mut sections = BTreeMap::new(); + for (name, value) in entries { + let JsonValue::Object(table) = value else { + return Err(serde::de::Error::custom(format!( + "unknown field `{name}`, which is neither a setting Trusted Server reads nor \ + the section of a module type, which is a table" + ))); + }; + if name.contains('.') || !crate::module_name::is_valid(&name) { + return Err(serde::de::Error::custom(format!( + "unknown field `{name}`. The section of a module type is named by the \ + type's folder, of lower case letters, digits, `_` or `-`" + ))); + } + let section = SectionModules::from_entries(table).map_err(|message| { + serde::de::Error::custom(format!( + "[{name}] is not a section Trusted Server reads itself, so it is read as \ + the section of a module type, and it {message}" + )) + })?; + sections.insert(name, section); + } + Ok(Self(sections)) + } +} -impl IntegrationSettings { - /// Whether `integration_id` is named in `[integration] module`. +impl TypeSections { + /// Whether no type section is present. #[must_use] - pub fn is_selected(&self, integration_id: &str) -> bool { - self.module + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + /// Each type section, by name. + pub fn iter(&self) -> impl Iterator { + self.0 .iter() - .any(|selected| selected == integration_id) + .map(|(name, section)| (name.as_str(), section)) } - /// Names `integration_id` in `[integration] module`, so it runs. - /// - /// Naming one that is already on the list changes nothing, so a caller - /// never has to check first. - #[cfg_attr(not(test), allow(dead_code))] - pub fn select(&mut self, integration_id: impl Into) { - let integration_id = integration_id.into(); - if !self.is_selected(&integration_id) { - self.module.push(integration_id); - } + /// The section of `type_name`, when present. + #[must_use] + pub fn section(&self, type_name: &str) -> Option<&SectionModules> { + self.0.get(type_name) } - /// Selects an integration and stores the settings it runs with, which is - /// what naming it in `[integration] module` and writing its - /// `[integration.]` block do together. - /// - /// # Errors - /// - /// Returns an error if the configuration cannot be serialized to JSON. - #[cfg_attr(not(test), allow(dead_code))] - pub fn insert_config( - &mut self, - integration_id: impl Into, - value: &T, - ) -> Result<(), Report> - where - T: Serialize, - { - let json = - serde_json::to_value(value).change_context(TrustedServerError::Configuration { - message: "Failed to serialize integration configuration".to_string(), - })?; - let integration_id = integration_id.into(); - self.select(integration_id.clone()); - self.entries.insert(integration_id, json); - Ok(()) + /// The section of `type_name`, created empty when absent. + pub fn section_mut(&mut self, type_name: &str) -> &mut SectionModules { + self.0.entry(type_name.to_owned()).or_default() } - /// Validates the selection against the blocks that are present. - /// - /// Three shapes are refused, because each reads as though it does - /// something it does not: an id named twice, an `enabled` key left in an - /// integration's block, and a block for an integration that is not - /// selected. Startup and `ts config validate` both run this, so none of - /// them is quietly ignored. - /// - /// Whether an id names an integration this build supplies is a separate - /// question, answered where the registry is built, because only there are - /// the adapter's and a vendor crate's builders known. + /// Refuses a section that selects nothing, and checks each selection. /// /// # Errors /// - /// Returns [`TrustedServerError::Configuration`] naming the ids at fault - /// and the fix for them. - pub fn validate_selection(&self) -> Result<(), Report> { - let mut named = HashSet::new(); - for integration_id in &self.module { - if !named.insert(integration_id.as_str()) { + /// Naming the first section at fault. + pub fn validate(&self) -> Result<(), Report> { + for (name, section) in &self.0 { + if section.selected().is_empty() { return Err(Report::new(TrustedServerError::Configuration { message: format!( - "[integration] module names `{integration_id}` more than once. \ - Name each integration that runs exactly once" + "[{name}] selects no module. A module type's section names what runs \ + with `module` or `modules`, so select one or remove the section" ), })); } + section + .validate(name) + .map_err(|message| Report::new(TrustedServerError::Configuration { message }))?; } - - // Blocks live in a map, so both lists are sorted before they are - // reported and an operator gets the same message every time. - let mut carries_enabled = Vec::new(); - let mut unselected = Vec::new(); - for (integration_id, block) in &self.entries { - if block.get("enabled").is_some() { - carries_enabled.push(integration_id.as_str()); - } - if !self.is_selected(integration_id) { - unselected.push(integration_id.as_str()); - } - } - carries_enabled.sort_unstable(); - unselected.sort_unstable(); - - if !carries_enabled.is_empty() { - return Err(Report::new(TrustedServerError::Configuration { - message: format!( - "[integration.{}] sets `enabled`, which is no longer read. An integration \ - runs when its id is named in [integration] module, so remove the key \ - and name the integration there instead", - carries_enabled.join("] and [integration."), - ), - })); - } - - if !unselected.is_empty() { - return Err(Report::new(TrustedServerError::Configuration { - message: format!( - "[integration.{}] is configured but not named in [integration] module. \ - Add the integration to that list to run it, or remove the block", - unselected.join("] and [integration."), - ), - })); - } - Ok(()) } - - fn remove_legacy_static_secret_store_selectors(&mut self) { - let Some(datadome) = self - .entries - .get_mut("datadome") - .and_then(JsonValue::as_object_mut) - else { - return; - }; - - let mut removed = datadome.remove("server_side_key_secret_store").is_some(); - if let Some(bypass) = datadome - .get_mut("protection_test_bypass") - .and_then(JsonValue::as_object_mut) - { - removed |= bypass.remove("credential_secret_store").is_some(); - } - if removed { - log::warn!( - "DataDome secret-store selectors are deprecated and ignored; static credentials resolve through the default app-config secret store" - ); - } - } - - /// Reads and validates a selected integration's typed configuration, or - /// returns `None` when `[integration] module` does not name it. - /// - /// A selected integration with no block of its own is read from an empty - /// one, so an integration that takes no settings runs on its id alone and - /// one that requires a setting reports the setting it is missing. The - /// parse and validation messages carry the underlying error, because a - /// report renders only its outermost message in `ts config validate`. - /// - /// # Errors - /// - /// Returns an error if the configuration cannot be parsed from JSON or fails validation. - pub fn get_typed( - &self, - integration_id: &str, - ) -> Result, Report> - where - T: IntegrationConfig, - { - if !self.is_selected(integration_id) { - return Ok(None); - } - - let raw = self - .entries - .get(integration_id) - .cloned() - .unwrap_or_else(|| JsonValue::Object(serde_json::Map::new())); - - let config: T = serde_json::from_value(raw).map_err(|error| { - Report::new(TrustedServerError::Configuration { - message: format!( - "Integration '{integration_id}' configuration could not be parsed: {error}" - ), - }) - })?; - - config.validate().map_err(|err| { - Report::new(TrustedServerError::Configuration { - message: format!( - "Integration '{integration_id}' configuration failed validation: {err}" - ), - }) - })?; - - Ok(Some(config)) - } } -impl Deref for IntegrationSettings { - type Target = HashMap; - - fn deref(&self) -> &Self::Target { - &self.entries - } -} - -impl DerefMut for IntegrationSettings { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.entries - } -} +/// The settings type a module reads from its table, beneath the section that +/// selects it. +/// +/// The type states which settings the module takes and how they are +/// validated, and nothing else. Whether the module runs is not its business, +/// because the section's selection names what runs. +pub trait IntegrationConfig: DeserializeOwned + Validate {} /// A partner (SSP, DSP, identity vendor) configured in `[[ec.partners]]`. /// @@ -2685,7 +2542,6 @@ impl ProxyAssetRoute { } #[derive(Debug, Clone, Deserialize, Serialize)] -#[serde(deny_unknown_fields)] pub struct Proxy { /// Enable TLS certificate verification when proxying to HTTPS origins. /// Defaults to true for secure production use. @@ -2708,6 +2564,10 @@ pub struct Proxy { /// Path-prefix-based asset proxy routes evaluated before publisher fallback. #[serde(default, deserialize_with = "vec_from_seq_or_map")] pub asset_routes: Vec, + /// The modules this section selects, with each one's settings in the + /// table at its name. + #[serde(flatten)] + pub modules: SectionModules, } fn default_certificate_check() -> bool { @@ -2728,6 +2588,7 @@ impl Default for Proxy { certificate_check: default_certificate_check(), allowed_domains: Vec::new(), asset_routes: Vec::new(), + modules: SectionModules::default(), } } } @@ -3613,19 +3474,12 @@ fn is_default_permission_signal_config(value: &PermissionSignalConfig) -> bool { *value == PermissionSignalConfig::default() } -// `[integration]` is a new section under this name, so a serialized blob that -// carries it is rejected by a base-revision binary that has never heard of it. -// An unconfigured section is omitted for the same reason the selector tables -// above are. -fn is_default_integration_config(value: &IntegrationSettings) -> bool { - value.module.is_empty() && value.entries.is_empty() -} - /// Message a configuration still carrying the removed `[integrations]` table /// is rejected with. -const REMOVED_INTEGRATIONS_TABLE_MESSAGE: &str = "Configuration table `[integrations]` was removed. Move each `[integrations.]` block to \ - `[integration.]`, name the integrations that run in `[integration] module`, and delete \ - every `enabled` key, as described in the CHANGELOG.md breaking migration"; +const REMOVED_INTEGRATIONS_TABLE_MESSAGE: &str = "Configuration table `[integrations]` was removed. Each module is selected in the \ + section of its type, as [] module = \"\" or modules = [\"\", ...], with its \ + settings in the table at its name, [.], as described in the CHANGELOG.md \ + breaking migration"; /// The removed `[integrations]` table. /// @@ -3679,6 +3533,14 @@ refused_table! { unchanged" } +refused_table! { + /// The `[integration]` table, which is no longer read. + RemovedIntegrationTable => "Configuration table `[integration]` is no longer read. Each \ + module is selected in the section of its type, as [] module = \"\" or \ + modules = [\"\", ...], with its settings in the table at its name, \ + [.]" +} + refused_table! { /// The `[permission_signal]` table, renamed `[permission-signal]`. RenamedPermissionSignalTable => "Configuration table `[permission_signal]` is now \ @@ -3934,7 +3796,6 @@ fn validate_trusted_client_ip_shared_secret( } #[derive(Debug, Default, Clone, Deserialize, Serialize, Validate)] -#[serde(deny_unknown_fields)] pub struct Settings { #[validate(nested)] pub publisher: Publisher, @@ -3966,8 +3827,18 @@ pub struct Settings { reason = "the field exists so that reading the removed table fails with directions" )] integrations: RemovedIntegrationsTable, - #[serde(default, skip_serializing_if = "is_default_integration_config")] - pub integration: IntegrationSettings, + /// The `[integration]` table, kept so a configuration carrying it is + /// told where modules are selected now. + #[serde(default, skip_serializing)] + #[allow( + dead_code, + reason = "the field exists so that reading the removed table fails with directions" + )] + integration: RemovedIntegrationTable, + /// The sections of module types core does not read itself, such as + /// `[cmp]` or `[tag]`, each named by its type's folder under `crates`. + #[serde(flatten)] + pub sections: TypeSections, #[serde(default, deserialize_with = "vec_from_seq_or_map")] #[validate(nested)] pub handlers: Vec, @@ -4110,8 +3981,7 @@ impl Settings { self.image_optimizer.normalize(); self.debug.auction_html_comment_options.normalize(); self.tinybird.normalize(); - self.integration - .remove_legacy_static_secret_store_selectors(); + self.remove_legacy_static_secret_store_selectors(); self.consent.validate(); } @@ -4134,7 +4004,7 @@ impl Settings { settings.ec.migrate_legacy_ec_layout()?; settings.ec.validate_module_selection()?; settings.ec.validate_resolve_allowed_origins()?; - settings.integration.validate_selection()?; + settings.validate_module_sections()?; settings.device.validate_module_selection()?; settings.geo.validate_module_selection()?; GeoConfig::validate_permission_policy()?; @@ -4517,23 +4387,194 @@ impl Settings { Ok(()) } - /// Retrieves a selected integration's configuration of a specific type. + /// Every section that selects modules, with its name: `[proxy]`, + /// `[auction]` and each module type's own section. + pub fn module_sections(&self) -> impl Iterator { + [ + ("proxy", &self.proxy.modules), + ("auction", &self.auction.modules), + ] + .into_iter() + .chain(self.sections.iter()) + } + + /// The section that selects the module `name`, and the name as written + /// there, whether in full or with the section's type folder left off. + #[must_use] + pub fn module_selection(&self, name: &str) -> Option<(&str, &str)> { + self.module_sections().find_map(|(section, modules)| { + modules + .selected() + .iter() + .find(|written| crate::module_name::resolve(section, written, &[name]).is_some()) + .map(|written| (section, written.as_str())) + }) + } + + /// Whether a section selects the module `name`. + #[must_use] + pub fn selects_module(&self, name: &str) -> bool { + self.module_selection(name).is_some() + } + + /// Reads and validates a selected module's settings, from the table at the + /// name it is written under beneath the section that selects it, or + /// returns `None` when no section selects it. /// - /// Hands back `None` when `[integration] module` does not name the - /// integration, so a caller that reads its own configuration is also - /// asking whether it runs. + /// A selected module with no table reads an empty one, so a module that + /// takes no settings runs on its selection alone and one that requires a + /// setting reports the setting it is missing. The parse and validation + /// messages carry the underlying error, because a report renders only its + /// outermost message in `ts config validate`. /// /// # Errors /// - /// Returns an error if the integration configuration exists but cannot be deserialized as the requested type. - pub fn integration_config( - &self, - integration_id: &str, - ) -> Result, Report> + /// When the table cannot be read as `T` or fails its validation, naming the + /// table. + pub fn module_config(&self, name: &str) -> Result, Report> where T: IntegrationConfig, { - self.integration.get_typed(integration_id) + let Some((section, written)) = self.module_selection(name) else { + return Ok(None); + }; + let table = self + .module_sections() + .find(|(candidate, _)| *candidate == section) + .and_then(|(_, modules)| modules.settings_of(written)) + .cloned() + .unwrap_or_default(); + let config: T = serde_json::from_value(JsonValue::Object(table)).map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: format!("[{section}.{written}] could not be read: {error}"), + }) + })?; + config.validate().map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: format!("[{section}.{written}] failed validation: {error}"), + }) + })?; + Ok(Some(config)) + } + + /// The modules `section` selects, to change, with the section created + /// empty when it is a module type's and absent. + fn section_modules_mut(&mut self, section: &str) -> &mut SectionModules { + match section { + "proxy" => &mut self.proxy.modules, + "auction" => &mut self.auction.modules, + _ => self.sections.section_mut(section), + } + } + + /// Selects the module `name` in `section`, written with the section's + /// type folder left off, with no table of its own. + pub fn select_module(&mut self, section: &str, name: &str) { + let written = crate::module_name::short_form(section, name).to_owned(); + self.section_modules_mut(section).select(&written); + } + + /// Stops selecting the module `name` in `section` and drops its table. + pub fn remove_module(&mut self, section: &str, name: &str) { + let written = crate::module_name::short_form(section, name).to_owned(); + self.section_modules_mut(section).remove(&written); + } + + /// Selects the module `name` in `section`, with `settings` as its table, + /// which is what writing both in a document does. + /// + /// The section is the name's type folder, except for the sections that + /// Trusted Server reads itself and that also select modules, where the + /// name is written in full. + /// + /// # Errors + /// + /// Returns an error if `settings` cannot be serialized to a table. + pub fn insert_module_config( + &mut self, + section: &str, + name: &str, + settings: &T, + ) -> Result<(), Report> + where + T: Serialize, + { + let JsonValue::Object(table) = + serde_json::to_value(settings).change_context(TrustedServerError::Configuration { + message: "Failed to serialize module configuration".to_string(), + })? + else { + return Err(Report::new(TrustedServerError::Configuration { + message: format!("the settings of `{name}` are not a table"), + })); + }; + let written = crate::module_name::short_form(section, name).to_owned(); + self.section_modules_mut(section).insert(&written, table); + Ok(()) + } + + /// Checks every section's selection and tables, and that no module is + /// selected in two sections. + /// + /// # Errors + /// + /// Naming the section at fault. + pub fn validate_module_sections(&self) -> Result<(), Report> { + self.sections.validate()?; + for (section, modules) in [ + ("proxy", &self.proxy.modules), + ("auction", &self.auction.modules), + ] { + modules + .validate(section) + .map_err(|message| Report::new(TrustedServerError::Configuration { message }))?; + } + let mut seen: Vec<(&str, String)> = Vec::new(); + for (section, modules) in self.module_sections() { + for written in modules.selected() { + let full = if written.contains('.') { + written.clone() + } else { + format!("{section}.{written}") + }; + if let Some((earlier, _)) = seen.iter().find(|(_, name)| *name == full) { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "`{written}` is selected in both [{earlier}] and [{section}]. A \ + module is selected in one section" + ), + })); + } + seen.push((section, full)); + } + } + Ok(()) + } + + /// Drops the `DataDome` secret-store selectors a previous release read, + /// from the module's table when it is configured. + fn remove_legacy_static_secret_store_selectors(&mut self) { + let Some(datadome) = self + .sections + .0 + .get_mut("bot-protection") + .and_then(|section| section.settings_of_mut("datadome")) + else { + return; + }; + + let mut removed = datadome.remove("server_side_key_secret_store").is_some(); + if let Some(bypass) = datadome + .get_mut("protection_test_bypass") + .and_then(JsonValue::as_object_mut) + { + removed |= bypass.remove("credential_secret_store").is_some(); + } + if removed { + log::warn!( + "DataDome secret-store selectors are deprecated and ignored; static credentials resolve through the default secret store" + ); + } } } @@ -5032,7 +5073,8 @@ mod tests { // country, so both selector tables are reset to unset here. settings.geo = GeoConfig::default(); settings.device = DeviceConfig::default(); - settings.integration = IntegrationSettings::default(); + settings.sections = TypeSections::default(); + settings.auction.modules.clear(); let value = serde_json::to_value(&settings).expect("should serialize settings"); @@ -5207,14 +5249,13 @@ module = \"none\"", max_body_bytes: 0, }; - // `IntegrationSettings` stores integration configs as opaque JSON and - // relies on a hand-written `Debug` impl to suppress their values. That - // impl is the only thing keeping resolved DataDome credentials out of - // this output, so pin it here. + // A module's table is opaque JSON, and the section's hand-written + // `Debug` impl is the only thing keeping resolved DataDome + // credentials out of this output, so pin it here. settings - .integration - .insert_config( - "datadome", + .insert_module_config( + "bot-protection", + crate::integrations::datadome::MODULE, &json!({ "server_side_key_secret_name": CANARY_DATADOME_SERVER_SIDE_KEY, }), @@ -5701,8 +5742,9 @@ module = \"none\"", #[test] fn toml_settings_reject_legacy_auction_provider_list_with_migration_guidance() { let toml = format!( - "{}\n[auction]\nproviders = [\"prebid\"]\n", + "{}\n", crate_test_settings_str() + .replace("[auction]\n", "[auction]\nproviders = [\"prebid\"]\n") ); let error = Settings::from_toml(&toml) @@ -5981,21 +6023,21 @@ module = \"none\"", let settings = settings.expect("should parse valid TOML"); let prebid_cfg = settings - .integration_config::("prebid") + .module_config::(crate::integrations::prebid::MODULE) .expect("Prebid config query should succeed") .expect("Prebid config should load from test settings"); assert_eq!(prebid_cfg.timeout_ms, 1000); assert!( settings - .integration_config::("nextjs") + .module_config::(crate::integrations::nextjs::MODULE) .expect("Next.js config query should succeed") .is_none(), - "an integration the module list does not name should not run" + "a module no section selects should not run" ); assert_eq!( - settings.integration.module, - vec!["prebid".to_owned()], - "the fixture should run exactly the integration it names" + settings.auction.modules.selected(), + ["prebid".to_owned()], + "the fixture should run exactly the module it selects" ); assert_eq!(settings.publisher.domain, "test-publisher.com"); assert_eq!(settings.publisher.cookie_domain, ".test-publisher.com"); @@ -8294,212 +8336,228 @@ source_domain = "partner.example.com" ); } - /// An integration `[integration] module` does not name has no - /// configuration, whatever else the settings hold. + /// A module no section selects has no configuration, whatever else the + /// settings hold. #[test] - fn an_integration_that_is_not_named_has_no_configuration() { - use crate::integrations::testlight::TestlightConfig; + fn a_module_that_is_not_selected_has_no_configuration() { + use crate::integrations::testlight::{self, TestlightConfig}; let settings = create_test_settings(); assert!( - !settings.integration.is_selected("testlight"), - "the shared fixture should not name testlight" + !settings.selects_module(testlight::MODULE), + "the shared fixture should not select testlight" ); assert!( settings - .integration_config::("testlight") - .expect("reading an unnamed integration should succeed") + .module_config::(testlight::MODULE) + .expect("reading an unselected module should succeed") .is_none(), - "an integration that is not named should have no configuration" + "a module that is not selected should have no configuration" ); } - /// A named integration with no block of its own is read from an empty one, - /// so one that takes no settings runs on its id alone. + /// A selected module with no table of its own is read from an empty one, + /// so one that takes no settings runs on its selection alone. #[test] - fn a_named_integration_with_no_block_is_read_from_an_empty_one() { - use crate::integrations::osano::OsanoConfig; + fn a_selected_module_with_no_table_is_read_from_an_empty_one() { + use crate::integrations::osano::{self, OsanoConfig}; let mut settings = create_test_settings(); - settings.integration.select("osano"); + settings.select_module("cmp", osano::MODULE); assert!( settings - .integration_config::("osano") - .expect("an integration that takes no settings should read from an empty block") + .module_config::(osano::MODULE) + .expect("a module that takes no settings should read from an empty table") .is_some(), - "naming the integration should be the whole configuration" + "selecting the module should be the whole configuration" ); } - /// The same empty block makes an integration that requires a setting report - /// the setting it is missing, rather than starting without it. + /// The same empty table makes a module that requires a setting report the + /// setting it is missing, rather than starting without it. #[test] - fn a_named_integration_without_a_required_setting_names_it() { - use crate::integrations::testlight::TestlightConfig; + fn a_selected_module_without_a_required_setting_names_it() { + use crate::integrations::testlight::{self, TestlightConfig}; let mut settings = create_test_settings(); - settings.integration.select("testlight"); + settings.select_module("auction", testlight::MODULE); let error = settings - .integration_config::("testlight") - .expect_err("should reject a named integration with no endpoint"); + .module_config::(testlight::MODULE) + .expect_err("should reject a selected module with no endpoint"); let rendered = error.to_string(); assert!( rendered.contains("testlight") && rendered.contains("endpoint"), - "should name the integration and the missing setting: {rendered}" + "should name the module and the missing setting: {rendered}" ); } #[test] fn removed_integration_fields_are_rejected() { - for (integration_id, removed_field) in - [("prebid", "server_url"), ("datadome", "account_id")] - { - let mut settings = create_test_settings(); - settings - .integration - .insert_config( - integration_id, - &json!({ - (removed_field): "removed-value", - }), - ) - .expect("should insert removed integration config field"); - - let error = match integration_id { - "prebid" => settings - .integration_config::(integration_id) - .expect_err("should reject the removed Prebid field"), - "datadome" => settings - .integration_config::( - integration_id, - ) - .expect_err("should reject the removed DataDome field"), - _ => unreachable!("test integration ID should be known"), - }; - assert!( - format!("{error:?}").contains(removed_field), - "should identify removed field `{removed_field}`: {error:?}" - ); - } + use crate::integrations::{datadome, prebid}; + + let mut settings = create_test_settings(); + settings + .insert_module_config( + "auction", + prebid::MODULE, + &json!({ "server_url": "removed-value" }), + ) + .expect("should insert the removed Prebid field"); + let error = settings + .module_config::(prebid::MODULE) + .expect_err("should reject the removed Prebid field"); + assert!( + format!("{error:?}").contains("server_url"), + "should identify the removed field: {error:?}" + ); + + let mut settings = create_test_settings(); + settings + .insert_module_config( + "bot-protection", + datadome::MODULE, + &json!({ "account_id": "removed-value" }), + ) + .expect("should insert the removed DataDome field"); + let error = settings + .module_config::(datadome::MODULE) + .expect_err("should reject the removed DataDome field"); + assert!( + format!("{error:?}").contains("account_id"), + "should identify the removed field: {error:?}" + ); } - /// A block written for an integration the module list does not name is - /// refused, rather than sitting in the configuration doing nothing. + /// A table written for a module its section does not select is refused, + /// rather than sitting in the configuration doing nothing. #[test] - fn a_block_for_an_integration_that_is_not_named_is_refused() { + fn a_table_for_a_module_that_is_not_selected_is_refused() { let toml = format!( - "{}\n[integration.osano]\n", - crate_test_settings_str().replace( - "module = [\"prebid\"]", - "module = [\"prebid\"]\n\n[integration.nextjs]\nrewrite_attributes = [\"href\"]", - ) + "{}\n[testing]\nmodule = \"example\"\n\n[testing.another]\n", + crate_test_settings_str() ); - let error = - Settings::from_toml(&toml).expect_err("should reject blocks nothing on the list names"); + let error = Settings::from_toml(&toml).expect_err("should reject a table nothing selects"); let rendered = format!("{error:?}"); assert!( - rendered.contains("[integration.nextjs]") && rendered.contains("[integration.osano]"), - "should name every block that is not on the list: {rendered}" - ); - assert!( - rendered.contains("[integration] module"), - "should say where to name the integration instead: {rendered}" + rendered.contains("[testing.another] is configured") + && rendered.contains("does not select"), + "should name the table and the selection it is missing from: {rendered}" ); } - /// The removed `enabled` key is refused where it is written, so a - /// configuration carried over from the previous release cannot read as - /// switched off while the integration runs. + /// A module type's section that selects nothing is refused, which is also + /// what a misspelt section of Trusted Server's own meets. #[test] - fn an_enabled_key_left_in_a_block_is_refused() { - let toml = crate_test_settings_str().replace( - "[integration.prebid]", - "[integration.prebid]\nenabled = false", + fn a_section_that_selects_no_module_is_refused() { + let toml = format!( + "{}\n[framework.nextjs]\nrewrite_attributes = [\"href\"]\n", + crate_test_settings_str() ); - let error = Settings::from_toml(&toml).expect_err("should reject a leftover enabled key"); - let rendered = format!("{error:?}"); + let error = + Settings::from_toml(&toml).expect_err("should reject a section selecting nothing"); assert!( - rendered.contains("[integration.prebid]") && rendered.contains("enabled"), - "should name the block and the key: {rendered}" - ); - assert!( - rendered.contains("[integration] module"), - "should say what switches an integration on instead: {rendered}" + format!("{error:?}").contains("[framework] selects no module"), + "should name the section: {error:?}" ); } - /// Naming one integration twice is a mistake rather than a way of running - /// it twice, so it is refused. + /// A key left in a module's table that its settings do not have, such as + /// the removed `enabled`, is refused when the module reads its table, so a + /// configuration cannot read as switched off while the module runs. #[test] - fn naming_an_integration_twice_is_refused() { + fn an_enabled_key_left_in_a_table_is_refused() { let toml = crate_test_settings_str() - .replace("module = [\"prebid\"]", "module = [\"prebid\", \"prebid\"]"); + .replace("[auction.prebid]", "[auction.prebid]\nenabled = false"); + let settings = Settings::from_toml(&toml).expect("core reads no module's table itself"); - let error = Settings::from_toml(&toml).expect_err("should reject a repeated id"); + let error = settings + .module_config::(crate::integrations::prebid::MODULE) + .expect_err("should reject a leftover enabled key"); + let rendered = format!("{error:?}"); assert!( - format!("{error:?}").contains("more than once"), - "should report the repeated id: {error:?}" + rendered.contains("[auction.prebid]") && rendered.contains("enabled"), + "should name the table and the key: {rendered}" ); } - /// The table this release removed is refused with the move spelled out, - /// rather than with a bare unknown-field error. + /// Naming one module twice is a mistake rather than a way of running it + /// twice, so it is refused. #[test] - fn the_removed_integrations_table_is_refused_with_directions() { - let toml = crate_test_settings_str().replace("[integration]", "[integrations]"); + fn naming_a_module_twice_is_refused() { + let toml = crate_test_settings_str().replace( + "modules = [\"prebid\"]", + "modules = [\"prebid\", \"prebid\"]", + ); - let error = Settings::from_toml(&toml).expect_err("should reject the removed table"); - let rendered = format!("{error:?}"); + let error = Settings::from_toml(&toml).expect_err("should reject a repeated name"); assert!( - rendered.contains("[integration.]") && rendered.contains("[integration] module"), - "should say where the blocks moved: {rendered}" - ); - assert!( - rendered.contains("CHANGELOG.md"), - "should point at the migration: {rendered}" + format!("{error:?}").contains("more than once"), + "should report the repeated name: {error:?}" ); } + /// The tables removed from the configuration are refused with the move + /// spelled out, rather than with a bare unknown-field error. + #[test] + fn the_removed_integration_tables_are_refused_with_directions() { + for (table, expected) in [ + ("[integrations.example]", "CHANGELOG.md"), + ( + "[integration]\nmodules = [\"example\"]", + "is no longer read", + ), + ] { + let toml = format!("{}\n{table}\n", crate_test_settings_str()); + + let error = Settings::from_toml(&toml).expect_err("should reject the removed table"); + let rendered = format!("{error:?}"); + + assert!( + rendered.contains("the section of its type") && rendered.contains(expected), + "should say where modules are selected now: {rendered}" + ); + } + } + /// The same refusal reaches a configuration blob, which is the shape the /// runtime loads rather than TOML. #[test] - fn json_settings_refuse_the_removed_integrations_table() { - let mut value = serde_json::to_value(create_test_settings()) - .expect("should serialize the test settings fixture to JSON"); - let settings = value - .as_object_mut() - .expect("settings should serialize as an object"); - let integration = settings - .remove("integration") - .expect("the fixture should serialize its integration table"); - settings.insert("integrations".to_owned(), integration); + fn json_settings_refuse_the_removed_integration_tables() { + for table in ["integrations", "integration"] { + let mut value = serde_json::to_value(create_test_settings()) + .expect("should serialize the test settings fixture to JSON"); + value + .as_object_mut() + .expect("settings should serialize as an object") + .insert(table.to_owned(), json!({ "example": {} })); - let error = - Settings::from_json_value(value).expect_err("should reject the removed table in JSON"); + let error = Settings::from_json_value(value) + .expect_err("should reject the removed table in JSON"); - assert!( - format!("{error:?}").contains("[integration] module"), - "should say where the blocks moved: {error:?}" - ); + assert!( + format!("{error:?}").contains("the section of its type"), + "should say where modules are selected now: {error:?}" + ); + } } #[test] - fn invalid_settings_for_a_named_integration_fail_registry_startup() { + fn invalid_settings_for_a_selected_module_fail_registry_startup() { let mut settings = create_test_settings(); settings - .integration - .insert_config( - "gpt", + .insert_module_config( + "ad-tag", + crate::integrations::gpt::MODULE, &json!({ "script_url": "not a url", }), @@ -8513,12 +8571,12 @@ source_domain = "partner.example.com" .expect("should compile auction plan"), ), ) { - Ok(_) => panic!("a named integration with invalid settings should fail startup"), + Ok(_) => panic!("a selected module with invalid settings should fail startup"), Err(err) => err, }; assert!( - err.to_string().contains("Integration 'gpt'"), - "should identify the invalid integration config" + format!("{err:?}").contains("[ad-tag.google]"), + "should identify the invalid module table: {err:?}" ); } @@ -8574,11 +8632,8 @@ source_domain = "partner.example.com" #[test] fn test_auction_creative_processing_defaults_when_omitted() { - let toml_str = crate_test_settings_str() - + r#" - [auction] - enabled = true - "#; + let toml_str = + crate_test_settings_str().replace("[auction]\n", "[auction]\nenabled = true\n"); let settings = Settings::from_toml(&toml_str).expect("should parse valid TOML"); @@ -8594,12 +8649,10 @@ source_domain = "partner.example.com" #[test] fn test_auction_rewrite_creatives_accepts_explicit_false() { - let toml_str = crate_test_settings_str() - + r#" - [auction] - enabled = true - rewrite_creatives = false - "#; + let toml_str = crate_test_settings_str().replace( + "[auction]\n", + "[auction]\nenabled = true\nrewrite_creatives = false\n", + ); let settings = Settings::from_toml(&toml_str).expect("should parse valid TOML"); @@ -8620,12 +8673,7 @@ source_domain = "partner.example.com" #[test] fn test_auction_allowed_context_keys_from_toml() { - let toml_str = crate_test_settings_str() - + r#" - [auction] - enabled = true - allowed_context_keys = ["permutive_segments", "lockr_ids"] - "#; + let toml_str = crate_test_settings_str().replace("[auction]\n", "[auction]\nenabled = true\nallowed_context_keys = [\"permutive_segments\", \"lockr_ids\"]\n"); let settings = Settings::from_toml(&toml_str).expect("should parse valid TOML"); assert_eq!( settings.auction.allowed_context_keys, @@ -8635,12 +8683,10 @@ source_domain = "partner.example.com" #[test] fn test_auction_empty_allowed_context_keys_blocks_all() { - let toml_str = crate_test_settings_str() - + r#" - [auction] - enabled = true - allowed_context_keys = [] - "#; + let toml_str = crate_test_settings_str().replace( + "[auction]\n", + "[auction]\nenabled = true\nallowed_context_keys = []\n", + ); let settings = Settings::from_toml(&toml_str).expect("should parse valid TOML"); assert!( settings.auction.allowed_context_keys.is_empty(), @@ -8659,6 +8705,7 @@ source_domain = "partner.example.com" "*.Example.Org".to_string(), ], asset_routes: vec![], + modules: SectionModules::default(), }; proxy.normalize(); assert_eq!( @@ -8679,6 +8726,7 @@ source_domain = "partner.example.com" "cdn.example.com".to_string(), ], asset_routes: vec![], + modules: SectionModules::default(), }; proxy.normalize(); assert_eq!( @@ -8694,6 +8742,7 @@ source_domain = "partner.example.com" certificate_check: true, allowed_domains: vec!["*".to_string(), "tracker.com".to_string()], asset_routes: vec![], + modules: SectionModules::default(), }; proxy.normalize(); assert_eq!( @@ -8709,6 +8758,7 @@ source_domain = "partner.example.com" certificate_check: true, allowed_domains: vec!["*".to_string()], asset_routes: vec![], + modules: SectionModules::default(), }; proxy.normalize(); assert!( @@ -8723,6 +8773,7 @@ source_domain = "partner.example.com" certificate_check: true, allowed_domains: vec![" ".to_string(), "\t".to_string()], asset_routes: vec![], + modules: SectionModules::default(), }; proxy.normalize(); assert!( @@ -8741,6 +8792,7 @@ source_domain = "partner.example.com" origin_url: " https://assets.example.com ".to_string(), ..Default::default() }], + modules: SectionModules::default(), }; proxy.normalize(); assert_eq!( @@ -8765,6 +8817,7 @@ source_domain = "partner.example.com" target_path: Some(" /rewritten/$1 ".to_string()), ..Default::default() }], + modules: SectionModules::default(), }; proxy.normalize(); @@ -9244,6 +9297,7 @@ source_domain = "partner.example.com" ..Default::default() }, ], + modules: SectionModules::default(), }; let route = proxy @@ -9272,6 +9326,7 @@ source_domain = "partner.example.com" ..Default::default() }, ], + modules: SectionModules::default(), }; let route = proxy diff --git a/crates/trusted-server-core/src/test_support.rs b/crates/trusted-server-core/src/test_support.rs index a9cccf3452..6af6d66001 100644 --- a/crates/trusted-server-core/src/test_support.rs +++ b/crates/trusted-server-core/src/test_support.rs @@ -31,13 +31,13 @@ pub mod tests { # is acknowledged the same way a deployment would. assume_single_jurisdiction = true - [integration] - module = ["prebid"] + [auction] + modules = ["prebid"] - [integration.prebid] + [auction.prebid] external_bundle_url = "https://assets.example/prebid/trusted-prebid.js" - [integration.prebid.bundle.modules] + [auction.prebid.bundle.modules] bidder = ["exampleBidderBidAdapter"] [ec] @@ -53,14 +53,21 @@ pub mod tests { .to_owned() } - /// The shared fixture TOML with `integration_id` named in - /// `[integration] module` as well, for a test that appends that - /// integration's own block. + /// The shared fixture TOML with the module `name` selected in `section` + /// as well, written with the section's type folder left off, for a test + /// that appends that module's own table. #[must_use] - pub fn crate_test_settings_str_running(integration_id: &str) -> String { - crate_test_settings_str().replace( - "module = [\"prebid\"]", - &format!("module = [\"prebid\", \"{integration_id}\"]"), + pub fn crate_test_settings_str_running(section: &str, name: &str) -> String { + let written = crate::module_name::short_form(section, name); + if section == "auction" { + return crate_test_settings_str().replace( + "modules = [\"prebid\"]", + &format!("modules = [\"prebid\", \"{written}\"]"), + ); + } + format!( + "{}\n[{section}]\nmodules = [\"{written}\"]\n", + crate_test_settings_str() ) } @@ -211,9 +218,9 @@ pub mod nextjs_auction { ) .expect("should parse Next.js auction fixture settings"); settings - .integration - .insert_config( - "nextjs", + .insert_module_config( + "framework", + crate::integrations::nextjs::MODULE, &serde_json::json!({ "rewrite_attributes": ["href", "link", "url"], }), diff --git a/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml b/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml index dcf97eb353..b33f461c83 100644 --- a/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml +++ b/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml @@ -46,10 +46,10 @@ secret_store_id = "secrets" # Only the GPT diagnostics overlay runs, which is what the browser tests need. # Every other integration used to sit here as an `enabled = false` stub. A -# settings table its type's `module` does not name now refuses startup, so -# the stubs are gone instead of sitting unselected. -[integration] -module = ["gpt_diagnostics"] +# settings table its section does not select now refuses startup, so the +# stubs are gone instead of sitting unselected. +[ad-tag] +modules = ["google.diagnostics"] [proxy] certificate_check = false diff --git a/docs/guide/api-reference.md b/docs/guide/api-reference.md index ccdec3de53..dda2026577 100644 --- a/docs/guide/api-reference.md +++ b/docs/guide/api-reference.md @@ -1058,7 +1058,7 @@ compiled integration registry, not an arbitrary filename lookup. ``` **Module Selection:** -All integration modules are built at compile time. At runtime, the server concatenates only the modules of the integrations `[integration] module` names in `trusted-server.toml`. No rebuild is required to change the module set. +All integration modules are built at compile time. At runtime, the server concatenates only the modules of the integrations the module sections select in `trusted-server.toml`. No rebuild is required to change the module set. --- @@ -1067,8 +1067,7 @@ All integration modules are built at compile time. At runtime, the server concat Every row below records a compiled integration registration predicate. An integration with no HTTP route can still contribute a browser module, rewriter, injector, post-processor, request filter, or ad server. In the -predicates below, `named` means the integration's id is in -`[integration] module`. +predicates below, `named` means a section selects the integration's module. | Integration | Registration predicate | HTTP routes | | -------------------- | ----------------------------------------------------------- | ------------------------------------------------- | @@ -1126,7 +1125,7 @@ predicates below, `named` means the integration's id is in ### Integration proxy contracts All integration routes are registered only when the documented predicate is -true. An integration `[integration] module` does not name registers no route, +true. An integration no section selects registers no route, so its path continues through normal routing and can reach the publisher fallback. Duplicate registrations are startup errors. None of these routes has built-in caller authentication or @@ -1141,7 +1140,7 @@ available when a deployment needs either. | Didomi consent | `GET` or `POST` under the configured prefix (default `/integrations/didomi/consent/*`); path selects SDK or API origin; query and bounded POST body forwarded | Upstream status/body preserved; SDK responses receive the integration's CORS headers; API responses retain selected upstream headers; no local cache policy | `curl -i https://edge.example.com/integrations/didomi/consent/loader.js` | | GTM/gtag scripts | `GET` the generated `gtm.js`, `gtag.js`, or `gtag/js` paths; query forwarded or configured container ID supplied; successful script is rewritten | Non-success upstream status preserved; rewritten scripts use `cache_max_age`; oversized rewritten upstream bodies use shared integration errors | `curl -i 'https://edge.example.com/integrations/google_tag_manager/gtm.js?id=GTM-XXXX'` | | Google collect | `GET` or `POST` the generated `collect` or `g/collect` paths; query, selected headers, and bounded body proxy to the configured Google origin | Malformed `Content-Length` returns `400`; body over `max_beacon_body_size` returns `413`; stream-read failure returns `502`; upstream response otherwise preserved | Browser beacon; body schema belongs to Google Analytics | -| JS asset proxy | `GET` each configured `[[integration.js_asset_proxy.assets]]` path whose `proxy = "enabled"`; the exact `origin_url` is fetched and served first-party | Upstream failures use shared integration errors; successful responses honor the per-asset or integration `cache_ttl_seconds`; `blocked` assets register no route and strip matching tags | Path is operator-configured, for example `curl -i https://edge.example.com/js/vendor-tag.js` | +| JS asset proxy | `GET` each configured `[[proxy.js_asset_proxy.assets]]` path whose `proxy = "enabled"`; the exact `origin_url` is fetched and served first-party | Upstream failures use shared integration errors; successful responses honor the per-asset or integration `cache_ttl_seconds`; `blocked` assets register no route and strip matching tags | Path is operator-configured, for example `curl -i https://edge.example.com/js/vendor-tag.js` | | GPT | `GET` `/script`, `/pagead/*`, or `/tag/*`; path/query proxy to the configured GPT origins and script content can be rewritten | Upstream status is preserved; successful scripts/assets apply integration cache rules; selected upstream CORS is preserved | `curl -i https://edge.example.com/integrations/gpt/script` | | Lockr SDK | `GET /integrations/lockr/sdk`; no body; fetches and returns the configured SDK as JavaScript | Successful SDK uses `cache_ttl_seconds`; upstream/transport failures follow integration mapping; no added CORS policy | `curl -i https://edge.example.com/integrations/lockr/sdk` | | Lockr API | `GET` or `POST /integrations/lockr/api/*`; path, query, selected headers, and bounded body proxy to `api_endpoint`; publisher credentials are stripped | Upstream status/body preserved; no local cache/CORS policy | Payload is Lockr-specific; use the SDK for normal calls | diff --git a/docs/guide/auction-orchestration.md b/docs/guide/auction-orchestration.md index 33cf153c39..4fe2464127 100644 --- a/docs/guide/auction-orchestration.md +++ b/docs/guide/auction-orchestration.md @@ -707,6 +707,7 @@ Each proxied URL includes a `tstoken` HMAC signature for tamper protection. See ```toml [auction] +modules = ["prebid"] enabled = true sanitize_creatives = false # Opt-in, blanks script-based creatives when enabled rewrite_creatives = true @@ -746,10 +747,7 @@ module = "adserver_mock" endpoint = "https://adserver.example.com/decide" timeout_ms = 500 -[integration] -module = ["prebid"] - -[integration.prebid] +[auction.prebid] timeout_ms = 1000 debug = false client_side_bidders = ["example-browser"] @@ -836,7 +834,7 @@ late response can remain eligible. Local decision and delivery also finish after network launch closes, so `timeout_ms` is not a hard wall-clock ceiling and an auction can exceed it. -Browser Prebid `timeout_ms` and `debug` stay under `[integration.prebid]` and +Browser Prebid `timeout_ms` and `debug` stay under `[auction.prebid]` and are independent of every demand source value. The server endpoint, timeout, routes, debug, test mode, overrides, consent forwarding and notification suppression belong to the `[demand.]` table, not to the browser diff --git a/docs/guide/cli.md b/docs/guide/cli.md index 5c8af917ed..8ab18a4127 100644 --- a/docs/guide/cli.md +++ b/docs/guide/cli.md @@ -304,9 +304,9 @@ publisher-specific settings, then run: ts config validate ``` -The draft also names in `[integration] module` the integrations it can -configure from what it found, and writes their blocks. Where it found -third-party scripts it writes `[integration.js_asset_proxy]` with each one +The draft also selects, in the section of each module's type, the modules it +can configure from what it found, and writes their tables. Where it found +third-party scripts it writes `[proxy.js_asset_proxy]` with each one `proxy = "disabled"`, so they are inventory only, and nothing is served or rewritten until you review a candidate and change its `proxy` value to `"enabled"` or `"blocked"`. Some candidates may be runtime-injected scripts, @@ -676,10 +676,10 @@ APIs. > is not accepted as an alias. ```toml -[integration] -module = ["prebid"] +[auction] +modules = ["prebid"] -[integration.prebid.bundle.modules] +[auction.prebid.bundle.modules] bidder = ["rubiconBidAdapter", "kargoBidAdapter"] user_id = ["sharedIdSystem"] analytics = ["atsAnalyticsAdapter"] diff --git a/docs/guide/configuration-rules.md b/docs/guide/configuration-rules.md index 7b35a65833..dc8ae0fec4 100644 --- a/docs/guide/configuration-rules.md +++ b/docs/guide/configuration-rules.md @@ -21,8 +21,9 @@ setting = "value" ``` 1. **The type is the job.** Each type is one top-level table, named for what - its modules do: `ec`, `geo`, `device`, `permission-signal`, `demand`, - `ad-server` and `integration`. + its modules do: `ec`, `geo`, `device`, `permission-signal`, `demand` and + `ad-server`, and the section of each module type, named by the type's + folder under `crates/`, such as `cmp` or `tag`. 2. **The selector chooses what runs.** A type that runs one module takes `module`, a string. A type that runs several takes `modules`, a list. 3. **`[.]` holds the settings.** A name with nothing to set @@ -39,22 +40,29 @@ setting = "value" 6. **Secrets are key names.** A secret setting holds the name of a key in `trusted_server_secrets`, never the secret itself. -| Type | Runs | Selector | Implementations in this repository | -| ------------------- | ----------------- | ------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `ec` | one | `module`, a string | `hmac`, `host_signals`, `client_fixed` (demonstration builds), or an integration that supplies identity | -| `geo` | one | `module`, a string | `platform`, `none`, or an integration that supplies location | -| `device` | one | `module`, a string | `builtin` (the default), `fastly`, or an integration that supplies device signals | -| `permission-signal` | several, in order | `modules`, a list | `gpc`, `gpp`, `us-privacy`, `tcf` | -| `demand` | several | `modules`, a list | `openrtb`, `prebid_server`, `aps` | -| `ad-server` | one | `module`, a string | `adserver_mock` | -| `integration` | several | `modules`, a list | `datadome`, `didomi`, `google_tag_manager`, `gpt`, `gpt_diagnostics`, `js_asset_proxy`, `lockr`, `nextjs`, `osano`, `permutive`, `prebid`, `sourcepoint`, `testlight` | +| Type | Runs | Selector | Implementations in this repository | +| ------------------- | ----------------- | ------------------ | ------------------------------------------------------------------------------------------------------- | +| `ec` | one | `module`, a string | `hmac`, `host_signals`, `client_fixed` (demonstration builds), or an integration that supplies identity | +| `geo` | one | `module`, a string | `platform`, `none`, or an integration that supplies location | +| `device` | one | `module`, a string | `builtin` (the default), `fastly`, or an integration that supplies device signals | +| `permission-signal` | several, in order | `modules`, a list | `gpc`, `gpp`, `us-privacy`, `tcf` | +| `demand` | several | `modules`, a list | `openrtb`, `prebid_server`, `aps` | +| `ad-server` | one | `module`, a string | `adserver_mock` | +| `cmp` | one | `module`, a string | `didomi`, `sourcepoint`, `osano` | +| `tag` | several | `modules`, a list | `google-tag-manager` | +| `ad-tag` | several | `modules`, a list | `google`, `google.diagnostics` | +| `bot-protection` | one | `module`, a string | `datadome` | +| `identity` | one | `module`, a string | `lockr` | +| `audience` | one | `module`, a string | `permutive` | +| `framework` | one | `module`, a string | `nextjs` | +| `auction` | several | `modules`, a list | `prebid`, `testing.testlight` | +| `proxy` | several | `modules`, a list | `js_asset_proxy` | `openrtb`, `prebid_server`, `aps` and `adserver_mock` supply implementations -only. They are not page integrations and cannot be named in -`[integration] module`. +only. No module section selects them. -A module that an integration supplies needs that integration named in -`[integration] module` too, because the module has to be registered before +A module that an integration supplies needs that integration selected in the +section of its type too, because the module has to be registered before another type can select what it offers. A few types keep settings of their own beside the selector, where the setting @@ -64,15 +72,15 @@ partner registry and the cluster thresholds, and `[geo]` holds ### Leaving the selector out -| Type | With no `module` or `modules` line | -| ------------------- | --------------------------------------------------------- | -| `ec` | no Edge Cookie is created | -| `geo` | no location is resolved and no host geo service is called | -| `device` | `builtin` runs, which reads the User-Agent only | -| `permission-signal` | every linked module runs, in the order shown above | -| `demand` | no demand source is called | -| `ad-server` | the highest bid wins, with no ad server | -| `integration` | no integration runs | +| Type | With no `module` or `modules` line | +| ----------------------------------------------- | --------------------------------------------------------------------------------- | +| `ec` | no Edge Cookie is created | +| `geo` | no location is resolved and no host geo service is called | +| `device` | `builtin` runs, which reads the User-Agent only | +| `permission-signal` | every linked module runs, in the order shown above | +| `demand` | no demand source is called | +| `ad-server` | the highest bid wins, with no ad server | +| a module type's section, such as `cmp` or `tag` | the section is refused, so remove the whole section when none of its modules runs | ## What is checked before a request is served @@ -111,9 +119,9 @@ implementations compiled into the CLI. host, timeouts, routing modes, notification bounds, a bidder route naming a demand source `[demand] modules` does not select, and any setting the chosen implementation rejects. -- Every selected integration's own settings, and the refusal of a block for an - integration `[integration] module` does not name, of an `enabled` key left - behind in a block, and of the removed `[integrations]` table. +- Every selected module's own settings, and the refusal of a table for a + module its section does not select, of a section that selects nothing, and + of the removed `[integrations]` and `[integration]` tables. - Every secret setting holding a non-empty key name rather than a value, with a secret store declared to hold it. - Basic-auth coverage of the admin namespace, and the placeholder values the @@ -132,7 +140,7 @@ loaded, and these join it. four will start.** Start an instance on the new configuration to find out. - Assembling the integration registry, which is where a module that supplies an identity, location or device module is matched to the type that selected - it, and where an `[integration] module` entry no builder in this build + it, and where a name a section selects that no builder in this build supplies is refused. A module declaring an identity or device module that no type selects is logged as a warning here. - The resolved secret values, which a key name alone cannot show. A weak or @@ -198,14 +206,14 @@ proxy_secret = "publisher_proxy_secret" # key name [proxy] allowed_domains = ["assets.example.com"] -[integration] -module = ["prebid", "gpt"] +[ad-tag] +modules = ["google"] -[integration.prebid] +[auction.prebid] external_bundle_url = "https://assets.example.com/prebid/trusted-prebid.js" timeout_ms = 1500 # the browser's Prebid timeout -[integration.gpt] +[ad-tag.google] gam_attribution_enabled = true [ec] @@ -241,6 +249,7 @@ endpoint = "https://adserver.example.com/decide" timeout_ms = 500 [auction] +modules = ["prebid"] enabled = true timeout_ms = 2000 # the whole auction @@ -280,7 +289,7 @@ endpoint = "https://house.example.com/openrtb2/auction" | Previous | Now | | ------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------- | -| `[integrations.]` with `enabled = true` | `` in `[integration] module`, and `[integration.]` only for settings | +| `[integrations.]` with `enabled = true` | the module in the section of its type, and `[.]` only for settings | | `[ec.providers.]` | `[ec.]` | | `[permission_signal] sources` | `[permission-signal] modules` | | `host-signals`, `client-fixed`, `gpp-sale-opt-out`, `gpp_sale_opt_out`, `us_privacy` | `host_signals`, `client_fixed`, `gpp`, `us-privacy` | diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 9843c349e9..46ab0913f4 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -217,30 +217,30 @@ fail and the service will return its startup-error response. `[.]` settings table, as [Configuration Rules](/guide/configuration-rules) describes. -| Section | Selects | Purpose | -| -------------------------- | --------------- | ----------------------------------------------------------- | -| `[ad-server]` | one module | The ad server that picks the winner | -| `[auction]` | nothing | Auction orchestration, bidder routes, and mediation | -| `[cache]` | nothing | Static and rehosted asset cache policy | -| `[consent]` | nothing | Consent interpretation, forwarding, and conflict resolution | -| `[creative_opportunities]` | nothing | Server-side page ad opportunities and templates | -| `[debug]` | nothing | Explicit non-production diagnostics | -| `[demand]` | several modules | The auction's demand sources | -| `[device]` | one module | Device classification | -| `[ec]` | one module | Edge Cookie identity, persistence, and partner sync | -| `[geo]` | one module | Which module resolves location, if any | -| `[[handlers]]` | nothing | Ordered HTTP Basic-auth rules | -| `[image_optimizer]` | nothing | Reusable Fastly Image Optimizer profiles | -| `[integration]` | several modules | Partner and browser integrations | -| `[permission-signal]` | several modules | Which permission signals are acted on, in order | -| `[proxy]` | nothing | Proxy allowlist, TLS policy, and asset routes | -| `[publisher]` | nothing | Publisher domain, origin, and proxy signing key | -| `[request_signing]` | nothing | Outbound Ed25519 request signing and management-store IDs | -| `[response_headers]` | nothing | Headers added to Trusted Server responses | -| `[rewrite]` | nothing | First-party URL rewrite exclusions | -| `[tester_cookie]` | nothing | Optional tester-cookie endpoints | -| `[tinybird]` | nothing | Direct Tinybird auction telemetry | -| `[trusted_client_ip]` | nothing | Authenticated front-door client-IP forwarding | +| Section | Selects | Purpose | +| ----------------------------------------------------------------------------------------------------------------- | --------------------- | --------------------------------------------------------------------------------------------------------------------- | +| `[ad-server]` | one module | The ad server that picks the winner | +| `[auction]` | several modules | Auction orchestration, bidder routes, and the modules the auction runs, `prebid` among them | +| `[cache]` | nothing | Static and rehosted asset cache policy | +| `[consent]` | nothing | Consent interpretation, forwarding, and conflict resolution | +| `[creative_opportunities]` | nothing | Server-side page ad opportunities and templates | +| `[debug]` | nothing | Explicit non-production diagnostics | +| `[demand]` | several modules | The auction's demand sources | +| `[device]` | one module | Device classification | +| `[ec]` | one module | Edge Cookie identity, persistence, and partner sync | +| `[geo]` | one module | Which module resolves location, if any | +| `[[handlers]]` | nothing | Ordered HTTP Basic-auth rules | +| `[image_optimizer]` | nothing | Reusable Fastly Image Optimizer profiles | +| `[permission-signal]` | several modules | Which permission signals are acted on, in order | +| `[proxy]` | several modules | Proxy allowlist, TLS policy, asset routes, and the first-party script proxy module | +| `[publisher]` | nothing | Publisher domain, origin, and proxy signing key | +| `[request_signing]` | nothing | Outbound Ed25519 request signing and management-store IDs | +| `[response_headers]` | nothing | Headers added to Trusted Server responses | +| `[rewrite]` | nothing | First-party URL rewrite exclusions | +| `[tester_cookie]` | nothing | Optional tester-cookie endpoints | +| `[tinybird]` | nothing | Direct Tinybird auction telemetry | +| `[trusted_client_ip]` | nothing | Authenticated front-door client-IP forwarding | +| `[]`, such as `[cmp]`, `[tag]`, `[ad-tag]`, `[bot-protection]`, `[identity]`, `[audience]` or `[framework]` | one module or several | The section of a module type, named by the type's folder under `crates/`, selecting the modules of that type that run | ## Example: Production Setup @@ -265,10 +265,7 @@ enabled = true config_store_id = "01GXXX" secret_store_id = "01GYYY" -[integration] -module = ["prebid"] - -[integration.prebid] +[auction.prebid] client_side_bidders = ["example-browser-bidder"] external_bundle_url = "https://assets.example.com/prebid/trusted-prebid.js" @@ -276,6 +273,7 @@ external_bundle_url = "https://assets.example.com/prebid/trusted-prebid.js" allowed_domains = ["assets.example.com"] [auction] +modules = ["prebid"] enabled = true timeout_ms = 2000 @@ -689,13 +687,13 @@ A module has a table only when it has settings of its own. Both modules that der rather than of one module, so they sit directly in `[ec]` whichever module is selected. -A module an integration supplies also needs that integration named in -`[integration] module`. +A module an integration supplies also needs that integration selected in the +section of its type. ### `[ec.hmac]` -A module an integration supplies also needs that integration named in -`[integration] module`. +A module an integration supplies also needs that integration selected in the +section of its type. ### `[ec.hmac]` @@ -1624,42 +1622,51 @@ tracked in [#908](https://github.com/IABTechLab/trusted-server/issues/908). ## Integration Configurations -`[integration] module` lists the integrations that run, and each one that has -settings gets its own `[integration.]` table. There is no `enabled` flag, -because an integration that is not on the list does not run. The full rule set -is in [Configuration Rules](/guide/configuration-rules). Every integration that -deploy validation knows is listed below. - -| Section | Reference | -| ---------------------------------- | ------------------------------------------------------------------ | -| `[integration.datadome]` | [DataDome](/guide/integrations/datadome) | -| `[integration.didomi]` | [Didomi](/guide/integrations/didomi) | -| `[integration.google_tag_manager]` | [Google Tag Manager](/guide/integrations/google_tag_manager) | -| `[integration.gpt]` | [GPT](/guide/integrations/gpt) | -| `[integration.gpt_diagnostics]` | [GPT diagnostics](/guide/integrations/gpt-diagnostics) | -| `[integration.js_asset_proxy]` | [JS Asset Proxy](#js-asset-proxy-integration) (no dedicated guide) | -| `[integration.lockr]` | [lockr](/guide/integrations/lockr) | -| `[integration.nextjs]` | [Next.js](/guide/integrations/nextjs) | -| `[integration.osano]` | [Osano](/guide/integrations/osano) | -| `[integration.permutive]` | [Permutive](/guide/integrations/permutive) | -| `[integration.prebid]` | [Prebid](/guide/integrations/prebid) | -| `[integration.sourcepoint]` | [Sourcepoint](/guide/integrations/sourcepoint) | -| `[integration.testlight]` | [Testlight](/guide/integrations/testlight) | - -### Naming the integrations that run +A page integration is a module, selected in the section of its type, which +is named by the type's folder under `crates/`, and each one that has +settings gets its own `[
.]` table. There is no `enabled` flag +and no `[integration]` table, because a module no section selects does not +run. The full rule set is in [Configuration Rules](/guide/configuration-rules). +Every module that deploy validation knows is listed below. + +| Section | Reference | +| ----------------------------- | ------------------------------------------------------------------ | +| `[bot-protection.datadome]` | [DataDome](/guide/integrations/datadome) | +| `[cmp.didomi]` | [Didomi](/guide/integrations/didomi) | +| `[tag.google-tag-manager]` | [Google Tag Manager](/guide/integrations/google_tag_manager) | +| `[ad-tag.google]` | [GPT](/guide/integrations/gpt) | +| `[ad-tag.google.diagnostics]` | [GPT diagnostics](/guide/integrations/gpt-diagnostics) | +| `[proxy.js_asset_proxy]` | [JS Asset Proxy](#js-asset-proxy-integration) (no dedicated guide) | +| `[identity.lockr]` | [lockr](/guide/integrations/lockr) | +| `[framework.nextjs]` | [Next.js](/guide/integrations/nextjs) | +| `[cmp.osano]` | [Osano](/guide/integrations/osano) | +| `[audience.permutive]` | [Permutive](/guide/integrations/permutive) | +| `[auction.prebid]` | [Prebid](/guide/integrations/prebid) | +| `[cmp.sourcepoint]` | [Sourcepoint](/guide/integrations/sourcepoint) | +| `[auction.testing.testlight]` | [Testlight](/guide/integrations/testlight) | + +### Selecting the modules that run ```toml -[integration] -module = ["prebid", "gpt", "nextjs"] +[auction] +modules = ["prebid"] + +[ad-tag] +modules = ["google"] + +[framework] +module = "nextjs" ``` -The integrations this repository ships are `datadome`, `didomi`, -`google_tag_manager`, `gpt`, `gpt_diagnostics`, `js_asset_proxy`, `lockr`, -`nextjs`, `osano`, `permutive`, `prebid`, `sourcepoint` and `testlight`. A -name this build does not have refuses startup, and the message lists the ones -it does. A `[integration.]` table for an integration the list does not -name refuses startup too, so a block left behind after an integration is -switched off is caught rather than sitting unread. +The modules this repository ships, by section, are `[cmp]` `didomi`, +`sourcepoint` and `osano`; `[tag]` `google-tag-manager`; `[ad-tag]` `google` +and `google.diagnostics`; `[bot-protection]` `datadome`; `[identity]` +`lockr`; `[audience]` `permutive`; `[framework]` `nextjs`; `[auction]` +`prebid` and `testing.testlight`; and `[proxy]` `js_asset_proxy`. A name +this build does not have refuses startup, and the message lists the ones it +does. A `[
.]` table for a module its section does not select +refuses startup too, as does a section that selects nothing, so a table left +behind after a module is switched off is caught rather than sitting unread. `openrtb`, `prebid_server`, `aps` and `adserver_mock` supply demand and ad server implementations only. They are not page integrations and cannot be @@ -1670,7 +1677,7 @@ the others, see the relevant integration guides. ### DataDome Integration -**Section**: `[integration.datadome]` +**Section**: `[bot-protection.datadome]` The [DataDome guide](/guide/integrations/datadome) explains request behavior and exclusion-rule syntax. This table covers every canonical top-level field: @@ -1717,7 +1724,7 @@ to new configurations. ### Didomi Integration -**Section**: `[integration.didomi]` +**Section**: `[cmp.didomi]` | Field | Type | Default | Contract | | ------------ | -------------- | -------------------------------- | ------------------------------------------------------------------- | @@ -1730,7 +1737,7 @@ See [Didomi](/guide/integrations/didomi) for the routed endpoint shapes. ### Google Tag Manager Integration -**Section**: `[integration.google_tag_manager]` +**Section**: `[tag.google-tag-manager]` | Field | Type | Default | Contract | | ---------------------- | ------- | ---------------------------------- | --------------------------------------------------- | @@ -1744,7 +1751,7 @@ See [Google Tag Manager](/guide/integrations/google_tag_manager). ### GPT Integration -**Section**: `[integration.gpt]` +**Section**: `[ad-tag.google]` | Field | Type | Default | Contract | | ------------------------- | -------------- | ----------------------- | ------------------------------------------------------ | @@ -1759,7 +1766,7 @@ See [GPT](/guide/integrations/gpt). ### GPT Diagnostics Integration -**Section**: `[integration.gpt_diagnostics]` +**Section**: `[ad-tag.google.diagnostics]` The only field is `enabled`, a Boolean that defaults to `false`. When enabled, the standalone diagnostics tag is available, but individual browser sessions @@ -1767,7 +1774,7 @@ still require the activation flow in [GPT diagnostics](/guide/integrations/gpt-d ### JS Asset Proxy Integration -**Section**: `[integration.js_asset_proxy]` +**Section**: `[proxy.js_asset_proxy]` Serves explicitly configured third-party JavaScript assets from first-party paths. Each asset maps one exact publisher-facing path to one exact HTTPS @@ -1782,7 +1789,7 @@ guide; the registered routes appear in the | `cache_ttl_seconds` | Integer | None | Optional downstream cache TTL for every asset | | `assets` | Array | `[]` | Asset mappings; required when enabled | -Each `[[integration.js_asset_proxy.assets]]` entry: +Each `[[proxy.js_asset_proxy.assets]]` entry: | Field | Type | Default | Contract | | ------------------- | ------- | --------- | --------------------------------------------------------- | @@ -1793,7 +1800,7 @@ Each `[[integration.js_asset_proxy.assets]]` entry: ### lockr Integration -**Section**: `[integration.lockr]` +**Section**: `[identity.lockr]` | Field | Type | Default | Contract | | ------------------- | --------------- | --------------------------------------------------- | ------------------------------------ | @@ -1810,7 +1817,7 @@ See [lockr](/guide/integrations/lockr). ### Prebid Integration -`[integration.prebid]` owns browser behavior only. The server endpoint, the +`[auction.prebid]` owns browser behavior only. The server endpoint, the demand source timeout, routing, debug and test controls, consent forwarding, bidder-param overrides, and notification suppression belong to a `[demand]` source with `implementation = "prebid_server"`. @@ -1832,26 +1839,26 @@ source with `implementation = "prebid_server"`. Server-side bidder codes are derived from validated `[auction.bidders.*]` routes and injected into the browser. There is no second server bidder list in -`[integration.prebid]`. A browser bidder stays client-side only when named in +`[auction.prebid]`. A browser bidder stays client-side only when named in `client_side_bidders` and its adapter is present in the generated bundle. **Example**: ```toml -[integration] -module = ["prebid"] +[auction] +modules = ["prebid"] -[integration.prebid] +[auction.prebid] timeout_ms = 1000 debug = false client_side_bidders = ["rubicon"] external_bundle_url = "https://assets.example.com/prebid/trusted-prebid.js" script_patterns = ["/prebid.js", "/prebid.min.js"] -[[integration.prebid.managed_user_ids]] +[[auction.prebid.managed_user_ids]] name = "sharedId" -[integration.prebid.managed_user_ids.storage] +[auction.prebid.managed_user_ids.storage] type = "cookie" name = "_sharedid" expires = 15 @@ -1860,7 +1867,7 @@ refresh_in_seconds = 1800 [proxy] allowed_domains = ["assets.example.com"] -[integration.prebid.bundle.modules] +[auction.prebid.bundle.modules] bidder = ["rubiconBidAdapter"] user_id = ["sharedIdSystem"] analytics = ["atsAnalyticsAdapter"] @@ -1892,7 +1899,7 @@ module = "pbs_main" **Environment override**: ```bash -env 'TRUSTED_SERVER__INTEGRATION__PREBID__TIMEOUT_MS=1000' \ +env 'TRUSTED_SERVER__AUCTION__PREBID__TIMEOUT_MS=1000' \ 'TRUSTED_SERVER__DEMAND__PBS_MAIN__DEBUG=true' \ ts config validate ``` @@ -1903,7 +1910,7 @@ then validate and push the edited file. **Managed User ID modules**: -Each `[[integration.prebid.managed_user_ids]]` entry names a Prebid +Each `[[auction.prebid.managed_user_ids]]` entry names a Prebid `userSync.userIds` module that Trusted Server installs on the page and reinstates whenever publisher JavaScript replaces the User ID configuration. Trusted Server does not interpret module-specific fields; every registered @@ -1920,7 +1927,7 @@ module uses the same vendor-neutral surface. The managed `name` must match a | `storage.refresh_in_seconds` | Integer | Prebid's own default | Seconds before the module may refresh the stored value; must be at least 1 | The module must be present in the built bundle. Name it under -`[integration.prebid.bundle.modules].user_id`, or omit that list to take the +`[auction.prebid.bundle.modules].user_id`, or omit that list to take the generator's default preset. `ts prebid client` resolves each managed `name` through the checked-in `user_id_modules.json` registry, rejects unknown names, ambiguous names, and two names that resolve to the same module, and confirms the @@ -1967,7 +1974,7 @@ conflicts. ### Next.js Integration -**Section**: `[integration.nextjs]` +**Section**: `[framework.nextjs]` | Field | Type | Default | Contract | | ---------------------------- | ------------- | ------------------------- | -------------------------------------------------- | @@ -1977,10 +1984,10 @@ conflicts. **Example**: ```toml -[integration] -module = ["nextjs"] +[framework] +module = "nextjs" -[integration.nextjs] +[framework.nextjs] rewrite_attributes = ["href", "link", "url", "src"] max_combined_payload_bytes = 10485760 ``` @@ -1988,30 +1995,30 @@ max_combined_payload_bytes = 10485760 **Environment Override**: ```bash -TRUSTED_SERVER__INTEGRATION__NEXTJS__MAX_COMBINED_PAYLOAD_BYTES=10485760 +TRUSTED_SERVER__FRAMEWORK__NEXTJS__MAX_COMBINED_PAYLOAD_BYTES=10485760 ``` Edit `rewrite_attributes` in TOML because the overlay cannot replace arrays. ### Osano Integration -**Section**: `[integration.osano]` +**Section**: `[cmp.osano]` -Osano has nothing to set, so naming it in `[integration] module` is the whole +Osano has nothing to set, so `[cmp] module = "osano"` is the whole configuration and it needs no table. **Example**: ```toml -[integration] -module = ["osano"] +[cmp] +module = "osano" ``` The Osano mirror runs in the browser, so consent cookies it writes are available to Trusted Server on requests after the page where Osano consent APIs become ready. See [Osano Integration](/guide/integrations/osano) for details. ### Permutive Integration -**Section**: `[integration.permutive]` +**Section**: `[audience.permutive]` | Field | Type | Default | Contract | | ------------------------- | ------- | -------------------------------------- | -------------------------------------------- | @@ -2026,10 +2033,10 @@ The Osano mirror runs in the browser, so consent cookies it writes are available **Example**: ```toml -[integration] -module = ["permutive"] +[audience] +module = "permutive" -[integration.permutive] +[audience.permutive] organization_id = "org-12345" workspace_id = "ws-67890" project_id = "proj-abcde" @@ -2041,7 +2048,7 @@ rewrite_sdk = true ### Sourcepoint Integration -**Section**: `[integration.sourcepoint]` +**Section**: `[cmp.sourcepoint]` | Field | Type | Default | Contract | | ------------------- | -------------- | ------------------------------ | ----------------------------------------------------------------- | @@ -2055,7 +2062,7 @@ See [Sourcepoint](/guide/integrations/sourcepoint). ### Testlight Integration -**Section**: `[integration.testlight]` +**Section**: `[auction.testing.testlight]` | Field | Type | Default | Contract | | ----------------- | ------- | ---------------------------------- | ----------------------------------- | @@ -2067,10 +2074,10 @@ See [Sourcepoint](/guide/integrations/sourcepoint). **Example**: ```toml -[integration] -module = ["testlight"] +[auction] +modules = ["testing.testlight"] -[integration.testlight] +[auction.testing.testlight] endpoint = "https://testlight.example/openrtb2/auction" timeout_ms = 1500 rewrite_scripts = true @@ -2164,7 +2171,7 @@ For Prebid Server, move `server_url` to `endpoint` and the server timeout to `/openrtb2/auction`, query parameters survive, and configured non-root custom endpoint paths remain exact. Browser timeout, debug, bundle, script interception, refresh exclusions and `client_side_bidders` stay under -`[integration.prebid]`. Configure timeout or debug under both owners when both +`[auction.prebid]`. Configure timeout or debug under both owners when both browser and server behavior should retain the old value. For APS, move the endpoint and timeout to the table, then move account, @@ -2379,8 +2386,8 @@ env 'TRUSTED_SERVER__AUCTION__ENABLED=true' \ ts config validate ``` -A selector list is an array, so `[demand] modules` and -`[integration] module` cannot be changed by an overlay. Edit the TOML, then +A selector list is an array, so `[demand] modules` and a section's +`modules` cannot be changed by an overlay. Edit the TOML, then validate and push. ## Creative Opportunities Configuration diff --git a/docs/guide/error-reference.md b/docs/guide/error-reference.md index 9d13fa592f..a38d7195e2 100644 --- a/docs/guide/error-reference.md +++ b/docs/guide/error-reference.md @@ -235,7 +235,7 @@ Upstream request timeout after 1000ms timeout_ms = 2000 ``` -Browser `[integration.prebid].timeout_ms` is independent and does not control +Browser `[auction.prebid].timeout_ms` is independent and does not control Prebid Server transport. 2. Verify upstream service is responsive: @@ -297,7 +297,7 @@ Prebid Server returned 400: Invalid OpenRTB request debug = true ``` -`[integration.prebid].debug` controls browser Prebid.js only. +`[auction.prebid].debug` controls browser Prebid.js only. 2. Check logs for request/response details 3. Verify bidders are supported by your Prebid Server @@ -332,10 +332,10 @@ Next.js links still pointing to origin domain 2. Update `rewrite_attributes` to match actual keys: ```toml -[integration] -module = ["nextjs"] +[framework] +module = "nextjs" -[integration.nextjs] +[framework.nextjs] rewrite_attributes = ["href", "link", "url", "src"] # Add keys you find ``` @@ -363,10 +363,10 @@ Failed to fetch Permutive SDK: 404 Not Found 2. Update configuration: ```toml -[integration] -module = ["permutive"] +[audience] +module = "permutive" -[integration.permutive] +[audience.permutive] organization_id = "myorg" workspace_id = "workspace-123" ``` @@ -387,15 +387,15 @@ curl https://myorg.edge.permutive.app/workspace-123-web.js No route matched for /integrations/custom/endpoint ``` -**Cause:** `[integration] module` does not name the integration, or the route is not registered +**Cause:** No section selects the integration's module, or the route is not registered **Solution:** -1. Name the integration so it runs: +1. Select the module in the section of its type so it runs: ```toml -[integration] -module = ["custom"] +[testing] +modules = ["custom"] ``` 2. Verify integration is compiled in (check build logs) @@ -648,13 +648,13 @@ cargo install viceroy --version 0.17.0 --locked --force ### Enable Debug Logging -Browser Prebid.js debug remains under `[integration.prebid]`: +Browser Prebid.js debug remains under `[auction.prebid]`: ```toml -[integration] -module = ["prebid"] +[auction] +modules = ["prebid"] -[integration.prebid] +[auction.prebid] debug = true ``` diff --git a/docs/guide/getting-started.md b/docs/guide/getting-started.md index 48575af1b5..7e267b19ca 100644 --- a/docs/guide/getting-started.md +++ b/docs/guide/getting-started.md @@ -174,7 +174,7 @@ only the asset proxy entries you want to serve or block. Then validate it. Edit `trusted-server.toml` to configure: -- the integrations that run, in `[integration] module`, with their settings under `[integration.]` +- the page modules that run, each selected in the section of its type (`[cmp]`, `[tag]`, `[ad-tag]`, `[bot-protection]`, `[identity]`, `[audience]`, `[framework]`, and `[auction]` for Prebid), with their settings under `[
.]` - the demand sources, in `[demand] modules`, each with its settings under `[demand.]` - the ad server, if one runs, in `[ad-server] module`, with its settings under `[ad-server.]` - server bidder routes under `[auction.bidders.]` @@ -183,7 +183,7 @@ Edit `trusted-server.toml` to configure: - stable key names for `trusted_server_secrets` Do not put a Prebid Server URL or server bidder list under -`[integration.prebid]`. Those server values belong to a `[demand.]` table +`[auction.prebid]`. Those server values belong to a `[demand.]` table whose `implementation` is `prebid_server`, and APS has no integration table at all, because it is selected in `[demand]` too. The rules every one of these tables follows are in [Configuration Rules](/guide/configuration-rules). diff --git a/docs/guide/integrations/aps.md b/docs/guide/integrations/aps.md index 9602fdff2b..b077b02793 100644 --- a/docs/guide/integrations/aps.md +++ b/docs/guide/integrations/aps.md @@ -25,8 +25,8 @@ The integration does not implement: ## Configuration -APS is a demand implementation, not a page integration, so it is never named in -`[integration] module`. Everything APS owns, including rendering ownership, +APS is a demand implementation, not a page integration, so no module section +selects it. Everything APS owns, including rendering ownership, lives in the `[demand.]` table that names `implementation = "aps"`. APS renderer support is registered whenever the compiled auction plan contains an APS demand source. See [Configuration Rules](/guide/configuration-rules) for diff --git a/docs/guide/integrations/datadome.md b/docs/guide/integrations/datadome.md index 684dd4523d..179bd59bcc 100644 --- a/docs/guide/integrations/datadome.md +++ b/docs/guide/integrations/datadome.md @@ -32,10 +32,10 @@ The DataDome integration can: Add the following to your `trusted-server.toml`: ```toml -[integration] -module = ["datadome"] +[bot-protection] +module = "datadome" -[integration.datadome] +[bot-protection.datadome] # First-party JavaScript/proxy layer sdk_origin = "https://js.datadome.co" @@ -62,7 +62,7 @@ inject_client_side_tag = true client_side_tag_url = "/integrations/datadome/tags.js" client_side_configuration = { ajaxListenerPath = true } -[[integration.datadome.protection_exclusion_rules]] +[[bot-protection.datadome.protection_exclusion_rules]] id = "default-static-assets" type = "path_regex" patterns = ["(?i)\\.(avi|flv|mka|mkv|mov|mp4|mpeg|mpg|mp3|flac|ogg|ogm|opus|wav|webm|webp|bmp|gif|ico|jpeg|jpg|png|svg|svgz|swf|eot|otf|ttf|woff|woff2|css|less|js|map)$"] @@ -100,10 +100,10 @@ patterns = ["(?i)\\.(avi|flv|mka|mkv|mov|mp4|mpeg|mpg|mp3|flac|ogg|ogm|opus|wav| Set `client_side_key` to have Trusted Server inject the DataDome browser tag into processed HTML responses: ```toml -[integration] -module = ["datadome"] +[bot-protection] +module = "datadome" -[integration.datadome] +[bot-protection.datadome] client_side_key = "YOUR_DATADOME_JS_KEY" inject_client_side_tag = true ``` @@ -121,10 +121,10 @@ Trusted Server emits the DataDome configuration before the Trusted Server JavaSc If your site already manages the DataDome tag, disable auto-injection: ```toml -[integration] -module = ["datadome"] +[bot-protection] +module = "datadome" -[integration.datadome] +[bot-protection.datadome] inject_client_side_tag = false ``` @@ -167,7 +167,7 @@ When `enable_protection = true`, Trusted Server calls DataDome before normal rou A request is protected when all of the following are true: -1. `[integration] module` names the DataDome integration. +1. `[bot-protection] module` selects DataDome. 2. `enable_protection = true`. 3. The method is not listed in `protection_excluded_methods`. 4. The path is not one of Trusted Server's internal routes. @@ -188,10 +188,10 @@ Protection API: ```toml # Runtime activation also requires FASTLY_IS_STAGING=1. -[integration] -module = ["datadome"] +[bot-protection] +module = "datadome" -[integration.datadome.protection_test_bypass] +[bot-protection.datadome.protection_test_bypass] enabled = true credential_secret_name = "datadome_test_bypass" ``` @@ -271,10 +271,10 @@ fields. For example: Use structured rules for all DataDome protection exclusions. Each rule has an `id`, optional `methods`, and a typed matcher. The default configuration includes a `path_regex` rule for common static assets. ```toml -[integration] -module = ["datadome"] +[bot-protection] +module = "datadome" -[[integration.datadome.protection_exclusion_rules]] +[[bot-protection.datadome.protection_exclusion_rules]] id = "legacy-static-get-head" methods = ["GET", "HEAD"] type = "path_regex" @@ -284,7 +284,7 @@ patterns = [ "^/robots\\.txt$", ] -[[integration.datadome.protection_exclusion_rules]] +[[bot-protection.datadome.protection_exclusion_rules]] id = "next-rsc" methods = ["GET", "HEAD"] type = "query_param_non_empty" @@ -304,10 +304,10 @@ Supported rule types are: Config Store-backed CIDR sources accept newline-, comma-, whitespace-, or JSON-array encoded CIDR lists. They are useful for large or frequently updated vendor crawler lists. ```toml -[integration] -module = ["datadome"] +[bot-protection] +module = "datadome" -[[integration.datadome.protection_excluded_ip_cidr_sources]] +[[bot-protection.datadome.protection_excluded_ip_cidr_sources]] config_store = "datadome-ip-bypass" key = "googlebot_ips" ``` @@ -382,13 +382,13 @@ sequenceDiagram Override configuration via environment variables: ```bash -TRUSTED_SERVER__INTEGRATION__DATADOME__SDK_ORIGIN=https://js.datadome.co -TRUSTED_SERVER__INTEGRATION__DATADOME__API_ORIGIN=https://api-js.datadome.co -TRUSTED_SERVER__INTEGRATION__DATADOME__CACHE_TTL_SECONDS=3600 -TRUSTED_SERVER__INTEGRATION__DATADOME__REWRITE_SDK=true -TRUSTED_SERVER__INTEGRATION__DATADOME__ENABLE_PROTECTION=true -TRUSTED_SERVER__INTEGRATION__DATADOME__SERVER_SIDE_KEY_SECRET_NAME=datadome_server_side_key -TRUSTED_SERVER__INTEGRATION__DATADOME__CLIENT_SIDE_KEY=your-client-side-key +TRUSTED_SERVER__BOT-PROTECTION__DATADOME__SDK_ORIGIN=https://js.datadome.co +TRUSTED_SERVER__BOT-PROTECTION__DATADOME__API_ORIGIN=https://api-js.datadome.co +TRUSTED_SERVER__BOT-PROTECTION__DATADOME__CACHE_TTL_SECONDS=3600 +TRUSTED_SERVER__BOT-PROTECTION__DATADOME__REWRITE_SDK=true +TRUSTED_SERVER__BOT-PROTECTION__DATADOME__ENABLE_PROTECTION=true +TRUSTED_SERVER__BOT-PROTECTION__DATADOME__SERVER_SIDE_KEY_SECRET_NAME=datadome_server_side_key +TRUSTED_SERVER__BOT-PROTECTION__DATADOME__CLIENT_SIDE_KEY=your-client-side-key ``` ## Client-side script guard @@ -407,11 +407,11 @@ This keeps DataDome scripts routed through first-party context, even when insert ### Script not loading -Check that `[integration] module` names the integration: +Check that `[bot-protection]` selects the module: ```toml -[integration] -module = ["datadome"] +[bot-protection] +module = "datadome" ``` If you rely on auto-injection, verify `client_side_key` is non-empty and `inject_client_side_tag = true`. @@ -429,10 +429,10 @@ curl -X POST https://www.example.com/integrations/datadome/js/check Check that both fields are configured: ```toml -[integration] -module = ["datadome"] +[bot-protection] +module = "datadome" -[integration.datadome] +[bot-protection.datadome] enable_protection = true server_side_key_secret_name = "datadome_server_side_key" ``` diff --git a/docs/guide/integrations/didomi.md b/docs/guide/integrations/didomi.md index dd76741bbb..c07b3b846d 100644 --- a/docs/guide/integrations/didomi.md +++ b/docs/guide/integrations/didomi.md @@ -26,10 +26,10 @@ for the upstream contract. Add the integration to the operator-owned `trusted-server.toml`: ```toml -[integration] -module = ["didomi"] +[cmp] +module = "didomi" -[integration.didomi] +[cmp.didomi] geo_query_parameters = true # proxy_path = "my-custom-consent" # sdk_origin = "https://sdk.privacy-center.org" @@ -63,10 +63,10 @@ already exists in the TOML input. `proxy_path` helps avoid a predictable integration path: ```toml -[integration] -module = ["didomi"] +[cmp] +module = "didomi" -[integration.didomi] +[cmp.didomi] geo_query_parameters = true proxy_path = "my-custom-consent" ``` diff --git a/docs/guide/integrations/google_tag_manager.md b/docs/guide/integrations/google_tag_manager.md index 6b035265e6..89ae85083d 100644 --- a/docs/guide/integrations/google_tag_manager.md +++ b/docs/guide/integrations/google_tag_manager.md @@ -34,10 +34,10 @@ This is a deliberate design choice to limit data forwarded to Google. If your us Add the GTM configuration to `trusted-server.toml`: ```toml -[integration] -module = ["google_tag_manager"] +[tag] +modules = ["google-tag-manager"] -[integration.google_tag_manager] +[tag.google-tag-manager] container_id = "GTM-XXXXXX" # upstream_url = "https://www.googletagmanager.com" # Optional override (must be https) # allowed_tag_ids = ["G-XXXXXXXX"] # Tag ids besides container_id that gtag/js may serve first-party diff --git a/docs/guide/integrations/gpt-diagnostics.md b/docs/guide/integrations/gpt-diagnostics.md index 2d6514fe67..d33abe0a7a 100644 --- a/docs/guide/integrations/gpt-diagnostics.md +++ b/docs/guide/integrations/gpt-diagnostics.md @@ -32,8 +32,8 @@ only for slots served through the existing GPT integration. The module is unavailable unless the deployment names it: ```toml -[integration] -module = ["gpt_diagnostics"] +[ad-tag] +modules = ["google.diagnostics"] ``` Deployment configuration makes the module available; it does not activate any browser @@ -477,7 +477,7 @@ trigger GPT or Prebid work, gate an auction, or delay delivery. ### The API or panel is absent -1. Confirm `[integration] module` names `gpt_diagnostics` in the deployed configuration. +1. Confirm `[ad-tag] modules` selects `google.diagnostics` in the deployed configuration. 2. Activate the browser session with an exact recognized `ts_console` value. 3. Confirm the Trusted Server script bundle loaded successfully. 4. Use `ts_console=false` and then `ts_console=true` on a new document to reset diff --git a/docs/guide/integrations/gpt.md b/docs/guide/integrations/gpt.md index 796ddba5cb..39f45af4fe 100644 --- a/docs/guide/integrations/gpt.md +++ b/docs/guide/integrations/gpt.md @@ -51,10 +51,10 @@ There are three layers: Add GPT configuration to `trusted-server.toml`: ```toml -[integration] -module = ["gpt"] +[ad-tag] +modules = ["google"] -[integration.gpt] +[ad-tag.google] gam_attribution_enabled = false script_url = "https://securepubads.g.doubleclick.net/tag/js/gpt.js" cache_ttl_seconds = 3600 @@ -71,8 +71,8 @@ rewrite_script = true | `rewrite_script` | boolean | No | `true` | Whether to rewrite GPT script URLs in HTML | The environment override -`TRUSTED_SERVER__INTEGRATION__GPT__GAM_ATTRIBUTION_ENABLED` works only when -`gam_attribution_enabled` is already present under `[integration.gpt]` in the +`TRUSTED_SERVER__AD-TAG__GOOGLE__GAM_ATTRIBUTION_ENABLED` works only when +`gam_attribution_enabled` is already present under `[ad-tag.google]` in the TOML file. The environment overlay cannot create a missing configuration leaf. ## Endpoints @@ -156,7 +156,7 @@ value `ts=true`. It is applied before publisher GPT initialization and remains for the browser document's lifetime, so initial, lazy, refresh, publisher-owned, and SPA-route requests inherit it unless another targeting consumer clears or overrides the key. The attribution switch is independently controlled and -defaults to `false`, but `[integration] module` must also name `gpt`, which +defaults to `false`, but `[ad-tag] modules` must also select `google`, which be `true`. This key is distinct from the existing slot-level `ts_initial=1` value. diff --git a/docs/guide/integrations/lockr.md b/docs/guide/integrations/lockr.md index 3eaf153f95..510c991f14 100644 --- a/docs/guide/integrations/lockr.md +++ b/docs/guide/integrations/lockr.md @@ -41,10 +41,10 @@ Lockr is an identity resolution and privacy platform that helps publishers manag Add Lockr configuration to `trusted-server.toml`: ```toml -[integration] -module = ["lockr"] +[identity] +module = "lockr" -[integration.lockr] +[identity.lockr] api_endpoint = "https://api.lockr.io" organization_id = "your-org-id" project_id = "your-project-id" @@ -61,9 +61,9 @@ project_id = "your-project-id" ### Environment Variables ```bash -TRUSTED_SERVER__INTEGRATION__LOCKR__API_ENDPOINT=https://api.lockr.io -TRUSTED_SERVER__INTEGRATION__LOCKR__ORGANIZATION_ID=your-org-id -TRUSTED_SERVER__INTEGRATION__LOCKR__PROJECT_ID=your-project-id +TRUSTED_SERVER__IDENTITY__LOCKR__API_ENDPOINT=https://api.lockr.io +TRUSTED_SERVER__IDENTITY__LOCKR__ORGANIZATION_ID=your-org-id +TRUSTED_SERVER__IDENTITY__LOCKR__PROJECT_ID=your-project-id ``` ## Features diff --git a/docs/guide/integrations/nextjs.md b/docs/guide/integrations/nextjs.md index 043ae3d189..026b977e03 100644 --- a/docs/guide/integrations/nextjs.md +++ b/docs/guide/integrations/nextjs.md @@ -26,10 +26,10 @@ Next.js applications generate framework-specific JSON data (`__NEXT_DATA__`) and ## Configuration ```toml -[integration] -module = ["nextjs"] +[framework] +module = "nextjs" -[integration.nextjs] +[framework.nextjs] rewrite_attributes = ["href", "link", "url"] max_combined_payload_bytes = 10485760 ``` @@ -145,8 +145,8 @@ Combine Next.js SSR/SSG with Trusted Server edge logic. Name it only if you're using Next.js: ```toml -[integration] -module = ["nextjs"] +[framework] +module = "nextjs" ``` ### 2. Configure Rewrite Attributes @@ -154,10 +154,10 @@ module = ["nextjs"] Add custom attributes if your Next.js app uses non-standard fields: ```toml -[integration] -module = ["nextjs"] +[framework] +module = "nextjs" -[integration.nextjs] +[framework.nextjs] rewrite_attributes = ["href", "link", "url", "customImageUrl"] ``` diff --git a/docs/guide/integrations/osano.md b/docs/guide/integrations/osano.md index 682572b2ae..373f7a1ed0 100644 --- a/docs/guide/integrations/osano.md +++ b/docs/guide/integrations/osano.md @@ -29,8 +29,8 @@ It writes the corresponding first-party cookies when Osano reports ready consent Add the following to `trusted-server.toml`: ```toml -[integration] -module = ["osano"] +[cmp] +module = "osano" ``` No additional server-side settings are required for the initial Osano integration. diff --git a/docs/guide/integrations/permutive.md b/docs/guide/integrations/permutive.md index 86bf03c5c7..0813d2eaec 100644 --- a/docs/guide/integrations/permutive.md +++ b/docs/guide/integrations/permutive.md @@ -15,10 +15,10 @@ Permutive is a real-time data platform that helps publishers build and activate ## Configuration ```toml -[integration] -module = ["permutive"] +[audience] +module = "permutive" -[integration.permutive] +[audience.permutive] organization_id = "your-org-id" workspace_id = "your-workspace-id" project_id = "your-project-id" diff --git a/docs/guide/integrations/prebid.md b/docs/guide/integrations/prebid.md index cba390f3bf..aec31bc6b9 100644 --- a/docs/guide/integrations/prebid.md +++ b/docs/guide/integrations/prebid.md @@ -16,21 +16,19 @@ Prebid is the leading open-source header bidding solution that allows publishers Prebid configuration has two independent owners: -- `[integration.prebid]` owns browser Prebid.js behavior, being bundle +- `[auction.prebid]` owns browser Prebid.js behavior, being bundle selection and injection, browser timeout and debug, account injection, script interception, client-side bidders, and refresh exclusions. It runs - when `prebid` is named in `[integration] module`. + when `[auction] modules` selects `prebid`. - A `[demand.]` table that sets `implementation = "prebid_server"`, its `notifications`, and `[auction.bidders]` own every Prebid Server request. - Prebid Server is a demand implementation, not a page integration, so it is - never named in `[integration] module`. See + Prebid Server is a demand implementation, not a page integration, so no + module section selects it. See [Configuration Rules](/guide/configuration-rules). ```toml -[integration] -module = ["prebid"] -[integration.prebid] +[auction.prebid] timeout_ms = 1000 debug = false client_side_bidders = ["example-browser"] @@ -41,11 +39,11 @@ external_bundle_url = "https://assets.example.com/prebid/trusted-prebid.js" # external_bundle_sri = "sha384-" # Optional operator-owned Prebid User ID modules, forwarded to Prebid verbatim. -[[integration.prebid.managed_user_ids]] +[[auction.prebid.managed_user_ids]] name = "identityLink" params = { pid = "999", notUse3P = false } -[integration.prebid.managed_user_ids.storage] +[auction.prebid.managed_user_ids.storage] type = "cookie" name = "idl_env" expires = 15 @@ -53,7 +51,7 @@ refresh_in_seconds = 1800 # External bundle generation inputs used by `ts prebid client`. # Values are exact Prebid module stems without `.js`. -[integration.prebid.bundle.modules] +[auction.prebid.bundle.modules] bidder = ["rubiconBidAdapter"] user_id = ["sharedIdSystem", "identityLinkIdSystem"] analytics = ["atsAnalyticsAdapter"] @@ -62,6 +60,7 @@ analytics = ["atsAnalyticsAdapter"] allowed_domains = ["assets.example.com"] [auction] +modules = ["prebid"] enabled = true timeout_ms = 2000 @@ -206,7 +205,7 @@ versioned contract change. There is no time-based expiry in this fix. ## External Bundle Generation Use `ts prebid client` to build the publisher-specific browser bundle from -`[integration.prebid.bundle.modules]` selections: +`[auction.prebid.bundle.modules]` selections: ```bash ts prebid client @@ -250,7 +249,7 @@ that bundle with the server and push its new hash and SRI. The sentinel ### Upgrading from `bundle.adapters` and `bundle.user_id_modules` Before deploying this server version, move the old bundle fields under -`[integration.prebid.bundle.modules]` and expand short bidder names to exact +`[auction.prebid.bundle.modules]` and expand short bidder names to exact upstream stems. For example, `adapters = ["rubicon"]` becomes `bidder = ["rubiconBidAdapter"]`; `client_side_bidders` continues to use the runtime code `rubicon`. @@ -577,7 +576,7 @@ impression or measurement purpose but must not participate in Trusted Server's Prebid refresh auction: ```toml -[integration.prebid] +[auction.prebid] excluded_gam_ad_unit_path_suffixes = ["/trackingonly"] ``` @@ -627,7 +626,7 @@ owned by Trusted Server. ### Configuration ```toml -[integration.prebid] +[auction.prebid] client_side_bidders = ["example-browser"] [auction.bidders.example-server] @@ -644,10 +643,10 @@ Client-side bidders need their exact Prebid.js module stems in the generated bundle: ```toml -[integration.prebid] +[auction.prebid] client_side_bidders = ["rubicon", "appnexus", "openx"] -[integration.prebid.bundle.modules] +[auction.prebid.bundle.modules] bidder = ["rubiconBidAdapter", "appnexusBidAdapter", "openxBidAdapter"] user_id = ["sharedIdSystem", "uid2IdSystem"] ``` @@ -712,7 +711,7 @@ Add analytics modules by exact stem. When the pinned Prebid.js package includes ATS, use this build selection: ```toml -[integration.prebid.bundle.modules] +[auction.prebid.bundle.modules] bidder = ["rubiconBidAdapter"] analytics = ["atsAnalyticsAdapter"] ``` @@ -743,7 +742,7 @@ Trusted Server can own one or more Prebid `userSync.userIds` entries so operators configure identity centrally instead of asking publishers to edit their Prebid JavaScript. -Each `[[integration.prebid.managed_user_ids]]` entry is forwarded to Prebid.js +Each `[[auction.prebid.managed_user_ids]]` entry is forwarded to Prebid.js verbatim. Trusted Server validates only what Prebid needs to address the module — a usable entry name and storage key, positive expiry and refresh values — and never interprets `params`. Supported names come from the checked-in @@ -772,15 +771,15 @@ is stale, or was modified after generation. Core remains vendor-neutral: it forwards each managed entry's `params` to Prebid.js without interpreting them. ```toml -[integration.prebid.bundle] +[auction.prebid.bundle] adapters = ["rubicon"] user_id_modules = ["identityLinkIdSystem"] -[[integration.prebid.managed_user_ids]] +[[auction.prebid.managed_user_ids]] name = "identityLink" params = { pid = "999", notUse3P = false } -[integration.prebid.managed_user_ids.storage] +[auction.prebid.managed_user_ids.storage] type = "cookie" name = "idl_env" expires = 15 @@ -788,7 +787,7 @@ refresh_in_seconds = 1800 ``` Run `ts prebid client`, upload the generated content-addressed bundle, copy its -hash metadata into `[integration.prebid]`, and validate the configuration +hash metadata into `[auction.prebid]`, and validate the configuration before rollout. ### Worked example: LiveRamp RampID diff --git a/docs/guide/integrations/sourcepoint.md b/docs/guide/integrations/sourcepoint.md index f4f33b100a..8d5bce98cc 100644 --- a/docs/guide/integrations/sourcepoint.md +++ b/docs/guide/integrations/sourcepoint.md @@ -17,10 +17,10 @@ The Sourcepoint integration: Add the following to `trusted-server.toml`: ```toml -[integration] -module = ["sourcepoint"] +[cmp] +module = "sourcepoint" -[integration.sourcepoint] +[cmp.sourcepoint] rewrite_sdk = true cdn_origin = "https://cdn.privacy-mgmt.com" # Optional: forward a custom Sourcepoint authCookie name upstream. @@ -29,7 +29,7 @@ cache_ttl_seconds = 3600 ``` ::: warning Migration note -The Sourcepoint browser module is now opt-in through `[integration] module`. Existing deployments that relied on unconditional Sourcepoint JavaScript inclusion should name `sourcepoint` there before upgrading. +The Sourcepoint browser module is now opt-in through `[cmp] module`. Existing deployments that relied on unconditional Sourcepoint JavaScript inclusion should name `sourcepoint` there before upgrading. ::: ### Configuration Options diff --git a/scripts/template-cache-local-test.sh b/scripts/template-cache-local-test.sh index d888b31dac..6744a9bcfd 100755 --- a/scripts/template-cache-local-test.sh +++ b/scripts/template-cache-local-test.sh @@ -290,14 +290,14 @@ s = replace_once( # A real auction points at the slow HTTPS stub so the timings mean something. s = replace_once( s, - '[integration]\nmodule = []', - '[integration]\nmodule = ["prebid"]', - "integration selector", + '# modules = ["prebid"]', + 'modules = ["prebid"]', + "auction selects prebid", ) s = replace_once( s, - '# [integration.prebid]\n', - '[integration.prebid]\n' + '# [auction.prebid]\n', + '[auction.prebid]\n' 'external_bundle_url = "https://assets.example.com/prebid/trusted-prebid-stub.js"\n', "Prebid integration settings", ) diff --git a/trusted-server.example.toml b/trusted-server.example.toml index fb78a424dd..037f494e08 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -310,20 +310,19 @@ pull_sync_concurrency = 3 # documented in crates/trusted-server-core/src/permission_signal/README.md # and in the Permission Signals guide. -# Page integration. [integration] module lists the ones that run, and each -# one that has settings gets its own [integration.] table. An -# integration that is not on the list does not run, and a settings table the -# list does not name refuses startup. The built-in names are datadome, -# didomi, google_tag_manager, gpt, gpt_diagnostics, js_asset_proxy, lockr, -# nextjs, osano, permutive, prebid, sourcepoint and testlight. Add the ones -# this deployment needs to the list below and uncomment their tables further -# down. -[integration] -module = [] - -# First-party proxying of named third-party scripts. Add "js_asset_proxy" to -# the list above to switch it on. -# [integration.js_asset_proxy] +# Page integrations are modules, each selected in the section of its type, +# which is named by the type's folder under crates/: [cmp] for a consent +# prompt, [tag] and [ad-tag] for tags, [bot-protection], [identity], +# [audience], [framework], [auction] for the modules the auction runs and +# [proxy] for the script proxy. A section names what runs with `module` +# where one runs or `modules` where several run, a module with settings gets +# its own [
.] table, and a section that selects nothing or a +# table its section does not select refuses startup. Nothing is selected by +# default. The sections and tables are further down, commented out. + +# First-party proxying of named third-party scripts. Uncomment the +# `modules` line under [proxy] below to switch it on. +# [proxy.js_asset_proxy] # Uncomment to override upstream cache headers for every asset below. # This replaces upstream directives, including private and no-store. # Use only when each asset's bytes are identical for every visitor. @@ -331,13 +330,16 @@ module = [] # Asset fetches use a fixed TrustedServer/1.0 User-Agent. Do not proxy assets # that vary by browser User-Agent or use integrity hashes for UA-specific bytes. # -# [[integration.js_asset_proxy.assets]] +# [[proxy.js_asset_proxy.assets]] # path = "/assets/example-vendor-loader.js" # origin_url = "https://cdn.example.com/vendor-loader.js" # proxy = "enabled" # Proxy behavior and first-party asset routing. Kept active with defaults. [proxy] +# The modules [proxy] selects. Uncomment to run the first-party script proxy +# configured in [proxy.js_asset_proxy] above. +# modules = ["js_asset_proxy"] # Verify TLS certs when proxying to HTTPS origins (default true; false only for # local self-signed dev). # certificate_check = true @@ -441,6 +443,9 @@ timeout_ms = 2000 # Context keys the JS client may forward into auction requests (allowlist; # empty blocks all). allowed_context_keys = [] +# The modules [auction] selects, with their tables under OPTIONAL below. Add +# "testing.testlight" for the Testlight test module. +# modules = ["prebid"] # Example demand sources. Remove or customize before enabling auctions. The # table name is this source's identity, and it is the implementation unless the @@ -649,14 +654,14 @@ auction_timeout_ms = 500 # ============================================================================= # OPTIONAL - Integration settings # ============================================================================= -# Every integration is off by default, because [integration] module above is -# empty. To switch one on, add its name to that list and uncomment its settings -# table below. An integration with nothing to set needs no table at all. A -# table the list does not name refuses startup, so a block left behind after an -# integration is switched off is caught rather than sitting unread. Required -# fields are noted per table. +# Every module is off by default, because no section below is uncommented. To +# switch one on, uncomment its section's selection and its settings table. A +# module with nothing to set needs no table at all. A table its section does +# not select refuses startup, so a table left behind after a module is +# switched off is caught rather than sitting unread. Required fields are +# noted per table. # ============================================================================= -# [integration.prebid] +# [auction.prebid] # timeout_ms = 1000 # debug = false # client_side_bidders = [] # bidders running via native Prebid.js adapters @@ -669,7 +674,7 @@ auction_timeout_ms = 500 # external_bundle_sri = "" # Bundle build inputs consumed by `ts prebid client`, not by the edge runtime. # Values are exact upstream module stems without `.js`. -# [integration.prebid.bundle.modules] +# [auction.prebid.bundle.modules] # bidder = ["rubiconBidAdapter"] # Omit user_id for the curated default preset; use [] to select none. # user_id = ["sharedIdSystem"] @@ -684,7 +689,7 @@ auction_timeout_ms = 500 # the checked-in User ID registry. Each `name` must appear only once. # # The module must be present in the built bundle: name it under -# [integration.prebid.bundle.modules].user_id, or omit that list to take the +# [auction.prebid.bundle.modules].user_id, or omit that list to take the # generator's default preset. `ts prebid client` resolves every managed name # through the checked-in User ID registry and fails if the generated manifest # omits its required module without updating the configured hash or SRI. @@ -694,30 +699,37 @@ auction_timeout_ms = 500 # OpenRTB EID source; without one the ID still reaches the auction but is never # written to KV. # -# [[integration.prebid.managed_user_ids]] +# [[auction.prebid.managed_user_ids]] # name = "sharedId" # -# [integration.prebid.managed_user_ids.storage] +# [auction.prebid.managed_user_ids.storage] # type = "cookie" # or "html5" # name = "_sharedid" # expires = 15 # days; omit to keep Prebid's default # refresh_in_seconds = 1800 # omit to keep Prebid's default # Next.js first-party rewriting for App Router / RSC payloads. -# [integration.nextjs] +# [framework] +# module = "nextjs" + +# [framework.nextjs] # rewrite_attributes = ["href", "link", "siteBaseUrl", "siteProductionDomain", "url"] # max_combined_payload_bytes = 10485760 # 10 MiB -# Testlight OpenRTB test integration. `endpoint` is required. -# [integration.testlight] +# Testlight OpenRTB test integration, selected as "testing.testlight" in +# [auction] modules. `endpoint` is required. +# [auction.testing.testlight] # endpoint = "https://testlight.example.com/openrtb2/auction" # timeout_ms = 1200 # rewrite_scripts = true -# Didomi CMP SDK/API first-party proxy. `ts audit` adds "didomi" to the -# [integration] module list and uncomments this table when it detects Didomi -# on the audited page. -# [integration.didomi] +# Didomi CMP SDK/API first-party proxy. `ts audit` selects "didomi" in [cmp] +# when it detects Didomi on the audited page; the table below stays yours to +# uncomment and fill. +# [cmp] +# module = "didomi" + +# [cmp.didomi] # geo_query_parameters = false # sdk_origin = "https://sdk.example.com" # api_origin = "https://api.example.com" @@ -726,18 +738,20 @@ auction_timeout_ms = 500 # `cdn_origin` is intentionally omitted: the upstream CDN origin is pinned to # Sourcepoint's own host and is not operator-selectable. Leave it unset so the # validated default applies, because overriding it with any other host fails -# config validation. -# [integration.sourcepoint] +# config validation. Selected with [cmp] module = "sourcepoint". +# [cmp.sourcepoint] # rewrite_sdk = true # cache_ttl_seconds = 3600 # auth_cookie_name = "sp_auth" # optional: forward a custom authCookie upstream -# Osano consent management. It has nothing to set, so naming "osano" in the -# [integration] module list above is the whole configuration and it needs no -# table of its own. +# Osano consent management. It has nothing to set, so [cmp] module = "osano" +# is the whole configuration and it needs no table of its own. # Permutive DMP. `organization_id` and `workspace_id` are required. -# [integration.permutive] +# [audience] +# module = "permutive" + +# [audience.permutive] # organization_id = "your-permutive-organization-id" # required (non-empty) # workspace_id = "your-permutive-workspace-id" # required (non-empty) # project_id = "your-permutive-project-id" @@ -745,7 +759,10 @@ auction_timeout_ms = 500 # secure_signals_endpoint = "https://secure-signals.example.com" # lockr identity SDK. `app_id` is required. -# [integration.lockr] +# [identity] +# module = "lockr" + +# [identity.lockr] # app_id = "your-lockr-app-id" # required (non-empty) # api_endpoint = "https://identity.example.com" # sdk_url = "https://identity.example.com/trusted-server.js" @@ -753,9 +770,12 @@ auction_timeout_ms = 500 # rewrite_sdk = true # DataDome bot protection. Proxies tags.js + signal-collection API first-party. -# `ts audit` adds "datadome" to the [integration] module list and uncomments -# this table when it detects DataDome. -# [integration.datadome] +# `ts audit` selects "datadome" in [bot-protection] when it detects DataDome; +# the table below stays yours to uncomment and fill. +# [bot-protection] +# module = "datadome" + +# [bot-protection.datadome] # sdk_origin = "https://sdk.example.com" # api_origin = "https://api.example.com" # cache_ttl_seconds = 3600 @@ -777,17 +797,22 @@ auction_timeout_ms = 500 # API validation and is stripped before the origin sees it. The credential is # loaded from the Secret Store at runtime (>= 32 bytes of high-entropy material). # Never enable in production. -# [integration.datadome.protection_test_bypass] +# [bot-protection.datadome.protection_test_bypass] # enabled = true # credential_secret_store = "ts_secrets" # credential_secret_name = "datadome_protection_test_bypass" -# Google Publisher Tag (GPT) first-party proxy. `ts audit` adds "gpt" to the -# [integration] module list and uncomments this table when it detects GPT. -# GAM attribution stays off until `gam_attribution_enabled` is set true. The -# environment overlay cannot create a missing leaf, so uncomment the line -# before relying on TRUSTED_SERVER__INTEGRATION__GPT__GAM_ATTRIBUTION_ENABLED. -# [integration.gpt] +# Google Publisher Tag (GPT) first-party proxy. `ts audit` selects "google" in +# [ad-tag] when it detects GPT; the table below stays yours. GAM attribution +# stays off until `gam_attribution_enabled` is set true. The environment +# overlay can only replace a leaf the document already has, and it names a +# section as written, so the variable for this leaf is +# TRUSTED_SERVER__AD-TAG__GOOGLE__GAM_ATTRIBUTION_ENABLED, which a shell's +# `export` cannot set. +# [ad-tag] +# modules = ["google"] + +# [ad-tag.google] # gam_attribution_enabled = false # script_url = "https://ads.example.com/gpt.js" # cache_ttl_seconds = 3600 @@ -795,14 +820,17 @@ auction_timeout_ms = 500 # GPT runtime diagnostics browser overlay. Selected manually rather than by # `ts audit`, it serves a diagnostics module gated behind an activation query -# param and a session cookie. It has nothing to set, so naming -# "gpt_diagnostics" in the [integration] module list above is the whole -# configuration. - -# Google Tag Manager first-party proxy. `ts audit` adds "google_tag_manager" to -# the [integration] module list, uncomments this table and fills container_id -# when it detects GTM. `container_id` is required. -# [integration.google_tag_manager] +# param and a session cookie. It has nothing to set, so adding +# "google.diagnostics" to [ad-tag] modules is the whole configuration. + +# Google Tag Manager first-party proxy. `ts audit` selects "google-tag-manager" +# in [tag] and writes this table with the container_id it finds when it +# detects GTM. +# `container_id` is required. +# [tag] +# modules = ["google-tag-manager"] + +# [tag.google-tag-manager] # Invalid placeholder on purpose, because selecting GTM without a real # GTM-XXXXXX id fails validation. # container_id = "GTM-REPLACE-ME" From ad734fd4a9acfa5b8ef9103d8fa186ba08f40d71 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 7 Oct 2026 09:57:33 +0100 Subject: [PATCH 072/124] The audit generator borrows the detected integration's name The native-integration note formats the detected integration's section and name from a String, so the lookup takes a borrow and the fallback clones. --- crates/trusted-server-cli/src/commands/audit/generate/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs index 842bd4e9f4..ea5477a779 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -600,8 +600,8 @@ fn build_js_asset_proxy_section( toml.push_str(&format!("# Detected integration: {integration}\n")); toml.push_str(&format!( "# Native integration may be preferable: [{}]\n", - module_section(integration).map_or_else( - || integration.to_owned(), + module_section(&integration).map_or_else( + || integration.clone(), |(section, written)| format!("{section}.{written}") ) )); From d3e2366cf56e9fed8a68bc79f5ea540d77dcfd44 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 7 Oct 2026 10:58:49 +0100 Subject: [PATCH 073/124] An auction implementation is named by its module path The four implementations this repository ships are named `auction-protocol.openrtb`, `auction.prebid-server`, `auction.aps` and `ad-server.mock`, the module paths the design gives the crates they move to, in place of the ids `openrtb`, `prebid_server`, `aps` and `adserver_mock`. A `[demand.]` table names its implementation on an `implementation` line by that path, because `demand` is not the type an implementation is named under, and `[ad-server] module = "mock"` resolves within its own type as a name does in every other section. The plan resolves a written name with the module name resolver, and when the build has no such implementation it lists the ones it has as the section would write them. APS keeps `aps` as its bidder and response name on the wire. The table's validation still named the `provider` key in its messages after the key became `modules` and `module`. The messages name the key the table reads, and an `implementation` line is checked as a module name. `ts prebid server inspect` recognises a Prebid Server table by its `implementation` line. The tests, the fixtures, the example configuration, the guides, the changelog and the agent notes use the names, and the migration tables gain the row for the `prebid-server` and `aps` profiles. --- AGENTS.md | 8 +- CHANGELOG.md | 11 +- TESTING.md | 8 +- .../tests/routes.rs | 5 +- .../src/app.rs | 10 +- .../trusted-server-adapter-fastly/src/app.rs | 7 +- crates/trusted-server-adapter-spin/src/app.rs | 10 +- crates/trusted-server-cli/README.md | 2 +- .../src/commands/pbs/inspect.rs | 17 +- crates/trusted-server-cli/tests/pbs_cli.rs | 2 +- .../trusted-server-core/src/auction/README.md | 19 +- .../trusted-server-core/src/auction/demand.rs | 4 +- .../src/auction/endpoints.rs | 4 +- .../src/auction/formats.rs | 4 +- crates/trusted-server-core/src/auction/mod.rs | 10 +- .../src/auction/openrtb.rs | 8 +- .../src/auction/openrtb/tests.rs | 92 ++++--- .../src/auction/orchestrator.rs | 14 +- .../trusted-server-core/src/auction/plan.rs | 242 ++++++++++-------- .../src/auction/routing.rs | 17 +- .../src/auction/test_support.rs | 4 +- crates/trusted-server-core/src/config.rs | 14 +- .../src/creative_opportunities.rs | 2 +- .../src/integrations/adserver_mock.rs | 16 +- .../src/integrations/aps.rs | 13 +- .../src/integrations/mod.rs | 7 +- .../src/integrations/openrtb.rs | 27 +- .../src/integrations/prebid.rs | 6 +- .../src/integrations/prebid_server.rs | 11 +- .../trusted-server-core/src/provider_table.rs | 120 ++++++--- crates/trusted-server-core/src/publisher.rs | 4 +- .../trusted-server-core/src/test_support.rs | 5 +- .../configs/trusted-server.integration.toml | 4 +- docs/guide/ad-serving.md | 14 +- docs/guide/api-reference.md | 106 ++++---- docs/guide/auction-orchestration.md | 54 ++-- docs/guide/auction-testing.md | 2 +- docs/guide/configuration-rules.md | 29 ++- docs/guide/configuration.md | 47 ++-- docs/guide/error-reference.md | 4 +- docs/guide/getting-started.md | 2 +- docs/guide/integrations-overview.md | 2 +- docs/guide/integrations/adserver_mock.md | 6 +- docs/guide/integrations/aps.md | 16 +- docs/guide/integrations/prebid.md | 10 +- trusted-server.example.toml | 25 +- 46 files changed, 595 insertions(+), 449 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 04f1af74b6..c1dcd865e7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -381,11 +381,11 @@ to read before changing any module configuration. | Device detection | `DeviceModule` (`ec/device.rs`) | `[device] module` | `builtin`, User-Agent only (the default) | `crates/device/` | | Geo / IP intelligence | `PlatformGeo` (`platform/traits.rs`) | `[geo] module` | None, no location (the default), or `platform` | `crates/geo/` | | Permission signals | `PermissionSignalModule` (`permission_signal/mod.rs`) | `[permission-signal] modules` (an ordered list) | `gpc`, `gpp`, `us-privacy`, `tcf`, all of them with no list | `crates/permission-signal/` | -| Auction demand | `DemandImplementation` (`auction/demand.rs`) | `[demand] modules` (a list) | `openrtb`, `prebid_server`, `aps` | an integration builder | -| Ad server | `AdServerImplementation` (`auction/demand.rs`) | `[ad-server] module` | `adserver_mock` | an integration builder | +| Auction demand | `DemandImplementation` (`auction/demand.rs`) | `[demand] modules` (a list) | `auction-protocol.openrtb`, `auction.prebid-server`, `auction.aps` | an integration builder | +| Ad server | `AdServerImplementation` (`auction/demand.rs`) | `[ad-server] module` | `mock` (`ad-server.mock`) | an integration builder | | Page integrations | `IntegrationBuilder` (`integrations/mod.rs`) | the section of the module's type: `[cmp] module`, `[tag] modules`, `[ad-tag] modules`, `[bot-protection] module`, `[identity] module`, `[audience] module`, `[framework] module`, `[auction] modules`, `[proxy] modules` | `cmp.didomi`, `cmp.sourcepoint`, `cmp.osano`, `tag.google-tag-manager`, `ad-tag.google`, `ad-tag.google.diagnostics`, `bot-protection.datadome`, `identity.lockr`, `audience.permutive`, `framework.nextjs`, `auction.prebid`, `testing.testlight`, `js_asset_proxy` | an adapter-supplied builder | -`openrtb`, `prebid_server`, `aps` and `adserver_mock` supply implementations +`auction-protocol.openrtb`, `auction.prebid-server`, `auction.aps` and `ad-server.mock` supply implementations only. No module section selects them. Principles for adding or changing a module: @@ -426,7 +426,7 @@ IntegrationRegistration::builder(ID) - Integration IDs match JS directory names: `prebid` (deferred), `lockr`, `permutive`, `datadome`, `didomi`, `testlight`. - An integration runs when the section of its type selects its module, which is named by its crate folder (`cmp.didomi`, `tag.google-tag-manager`, `ad-tag.google`, `bot-protection.datadome`, `identity.lockr`, `audience.permutive`, `framework.nextjs`, `auction.prebid`, `testing.testlight`) and written in its section with the type folder left off. There is no `enabled` flag and no `[integration]` table. -- `creative` is JS-only (no Rust registration), and `nextjs` is Rust-only. `openrtb`, `prebid_server`, `aps` and `adserver_mock` register demand or ad server implementations rather than page integrations. +- `creative` is JS-only (no Rust registration), and `nextjs` is Rust-only. `auction-protocol.openrtb`, `auction.prebid-server`, `auction.aps` and `ad-server.mock` register demand or ad server implementations rather than page integrations. - Integrations opt into deferred loading via `.with_deferred_js()` on the registration builder. Deferred modules are served as separate `"# ); } + + #[test] + fn module_constant_is_the_crate_folder() { + assert_eq!( + super::MODULE, + trusted_server_core::module_name!(), + "should be named by the folder this crate lives in" + ); + } } diff --git a/crates/trusted-server-core/src/config_payload.rs b/crates/trusted-server-core/src/config_payload.rs index 156afc077f..f821a207d9 100644 --- a/crates/trusted-server-core/src/config_payload.rs +++ b/crates/trusted-server-core/src/config_payload.rs @@ -179,7 +179,6 @@ fn json_bool_or_string_is_true(value: Option<&serde_json::Value>) -> bool { mod tests { use super::*; use crate::ec::module::{HMAC_MODULE_KEY, HOST_SIGNALS_MODULE_KEY}; - use crate::integrations::didomi::DidomiIntegrationConfig; use crate::platform::{PlatformError, StoreId}; use crate::redacted::Redacted; use crate::settings::{ @@ -408,32 +407,45 @@ mod tests { ); } + /// The settings of an example module, with a flag that defaults to off. + #[derive(Debug, serde::Deserialize, serde::Serialize, validator::Validate)] + #[serde(deny_unknown_fields)] + struct ExampleModuleSettings { + #[serde(default)] + opted_in: bool, + endpoint: String, + } + + impl crate::settings::IntegrationConfig for ExampleModuleSettings {} + #[test] - fn didomi_geo_query_parameters_survive_blob_round_trip() { + fn a_module_s_table_survives_the_blob_round_trip() { let mut original = test_settings(); original .insert_module_config( - "cmp", - "cmp.didomi", - &DidomiIntegrationConfig { - geo_query_parameters: true, - proxy_path: None, - sdk_origin: "https://sdk.example.com".to_string(), - api_origin: "https://api.example.com".to_string(), + "example", + "example.notice", + &ExampleModuleSettings { + opted_in: true, + endpoint: "https://api.example.com".to_string(), }, ) - .expect("should insert Didomi configuration"); + .expect("should insert the example module's table"); let reconstructed = load_settings(&envelope_json(&original)).expect("should reconstruct settings"); let config = reconstructed - .module_config::(crate::integrations::didomi::MODULE) - .expect("should read Didomi configuration") - .expect("should enable Didomi"); + .module_config::("example.notice") + .expect("should read the example module's table") + .expect("should still select the example module"); assert!( - config.geo_query_parameters, - "should preserve Didomi geo opt-in" + config.opted_in, + "should preserve a flag the table set away from its default" + ); + assert_eq!( + config.endpoint, "https://api.example.com", + "should preserve the table's other settings" ); } diff --git a/crates/trusted-server-core/src/integrations/mod.rs b/crates/trusted-server-core/src/integrations/mod.rs index f4e67f8f97..09fdbbfde7 100644 --- a/crates/trusted-server-core/src/integrations/mod.rs +++ b/crates/trusted-server-core/src/integrations/mod.rs @@ -16,7 +16,6 @@ use crate::settings::Settings; pub mod adserver_mock; pub mod aps; pub mod datadome; -pub mod didomi; pub mod google_tag_manager; pub mod gpt; pub mod gpt_diagnostics; @@ -577,7 +576,6 @@ const BUILT_IN_BUILDERS: &[IntegrationBuilder] = &[ js_asset_proxy::BUILDER, testlight::BUILDER, nextjs::BUILDER, - didomi::BUILDER, google_tag_manager::BUILDER, datadome::BUILDER, gpt::BUILDER, diff --git a/crates/trusted-server-modules/Cargo.toml b/crates/trusted-server-modules/Cargo.toml index feed62403e..abd637ee31 100644 --- a/crates/trusted-server-modules/Cargo.toml +++ b/crates/trusted-server-modules/Cargo.toml @@ -15,6 +15,7 @@ workspace = true [dependencies] trusted-server-audience-permutive = { workspace = true } +trusted-server-cmp-didomi = { workspace = true } trusted-server-cmp-osano = { workspace = true } trusted-server-cmp-sourcepoint = { workspace = true } trusted-server-core = { workspace = true } diff --git a/crates/trusted-server-modules/src/lib.rs b/crates/trusted-server-modules/src/lib.rs index 08e8c123f4..f9916b283e 100644 --- a/crates/trusted-server-modules/src/lib.rs +++ b/crates/trusted-server-modules/src/lib.rs @@ -22,6 +22,7 @@ pub fn builders() -> Vec { vec![ trusted_server_audience_permutive::builder(), trusted_server_identity_lockr::builder(), + trusted_server_cmp_didomi::builder(), trusted_server_cmp_sourcepoint::builder(), trusted_server_cmp_osano::builder(), ] @@ -54,6 +55,7 @@ mod tests { [ "audience.permutive", "identity.lockr", + "cmp.didomi", "cmp.sourcepoint", "cmp.osano", ], From 91af0a11d6528cc31aad440bc40aaadc4cd0cd50 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 8 Oct 2026 00:37:15 +0100 Subject: [PATCH 091/124] Google Tag Manager moves out of core The Google Tag Manager module is `tag.google-tag-manager`, and it now lives in `crates/tag/google-tag-manager`. The stock list offers it in the place its hooks ran before, and the settings that select and configure it do not change. One of its tests runs the HTML pipeline over the publisher page fixture core's own HTML tests use. Core's test support offers that page, so the half megabyte file stays in one place. --- .cargo/config.toml | 12 +- CHANGELOG.md | 2 +- Cargo.lock | 19 ++ Cargo.toml | 2 + crates/tag/google-tag-manager/Cargo.toml | 38 ++++ .../google-tag-manager/src/lib.rs} | 163 +++++++++--------- .../src/integrations/mod.rs | 2 - .../trusted-server-core/src/test_support.rs | 8 + .../google_tag_manager/script_guard.ts | 2 +- crates/trusted-server-modules/Cargo.toml | 1 + crates/trusted-server-modules/src/lib.rs | 2 + docs/guide/integrations/google_tag_manager.md | 2 +- 12 files changed, 161 insertions(+), 92 deletions(-) create mode 100644 crates/tag/google-tag-manager/Cargo.toml rename crates/{trusted-server-core/src/integrations/google_tag_manager.rs => tag/google-tag-manager/src/lib.rs} (97%) diff --git a/.cargo/config.toml b/.cargo/config.toml index 2aa3248e71..95dde5e19f 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -26,10 +26,10 @@ test_details = "test --target aarch64-apple-darwin" # native crate needs no change here. Axum (native), Cloudflare # (wasm32-unknown-unknown), Spin, the CLI (native), and integration-tests # (native) are simply not listed. -build-fastly = "build -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" -check-fastly = "check -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" -clippy-fastly = "clippy -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-js -p trusted-server-openrtb --all-targets --all-features --target wasm32-wasip1 -- -D warnings" -test-fastly = "test -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" +build-fastly = "build -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" +check-fastly = "check -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" +clippy-fastly = "clippy -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-js -p trusted-server-openrtb --all-targets --all-features --target wasm32-wasip1 -- -D warnings" +test-fastly = "test -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" # Feature-on counterpart of `test-fastly`. `test-fastly` does NOT pass # --all-features, so without this the reusable-sandbox code is linted by # `clippy-fastly` but its tests never execute. @@ -40,8 +40,8 @@ test-fastly-reuse = "test -p trusted-server-adapter-fastly --target wasm32-wasip # run with the Axum adapter in CI. build-axum = "build -p trusted-server-adapter-axum" check-axum = "check -p trusted-server-adapter-axum" -clippy-axum = "clippy -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi --all-targets --all-features -- -D warnings" -test-axum = "test -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi" +clippy-axum = "clippy -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager --all-targets --all-features -- -D warnings" +test-axum = "test -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager" # --- Cloudflare adapter (native host + wasm32-unknown-unknown) --- # Build/check target the WASM runtime (requires the `cloudflare` feature); diff --git a/CHANGELOG.md b/CHANGELOG.md index 027cf000f4..86cfed0b8e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- Integration modules are moving out of `trusted-server-core` into crates of their own under `crates///`, each named by that folder, so `crates/cmp/osano` is the module `cmp.osano`. The settings that select and configure a module do not change. `crates/trusted-server-modules` lists the modules a stock build ships from crates of their own, in the order their hooks run. Every adapter builds its state and loads its settings with that list, and `ts config validate` and `ts config push` run each stock module's own rules. Code that named a moved module under `trusted_server_core::integrations` names its crate instead, such as `trusted_server_cmp_osano`. Moved so far: Osano, Lockr, Permutive, Sourcepoint and Didomi. +- Integration modules are moving out of `trusted-server-core` into crates of their own under `crates///`, each named by that folder, so `crates/cmp/osano` is the module `cmp.osano`. The settings that select and configure a module do not change. `crates/trusted-server-modules` lists the modules a stock build ships from crates of their own, in the order their hooks run. Every adapter builds its state and loads its settings with that list, and `ts config validate` and `ts config push` run each stock module's own rules. Code that named a moved module under `trusted_server_core::integrations` names its crate instead, such as `trusted_server_cmp_osano`. Moved so far: Osano, Lockr, Permutive, Sourcepoint, Didomi and Google Tag Manager. - The Fastly adapter is a library with a thin binary over it. A deployment that ships a vendor crate writes a binary of its own that calls `trusted_server_adapter_fastly::run_with(vec![...])` with that crate's integration builder, in place of editing the adapter. The builders are composed with the built-in ones, and a builder runs only when the settings select its module. The Wasm artifact keeps its name, `trusted-server-adapter-fastly.wasm`. - **Breaking:** `[integrations]` is gone. Each page integration is a module selected in the section of its type, with its settings in the table at its name: `[cmp] module = "didomi"` (or `"sourcepoint"` or `"osano"`), `[tag] modules = ["google-tag-manager"]`, `[ad-tag] modules = ["google", "google.diagnostics"]`, `[bot-protection] module = "datadome"`, `[identity] module = "lockr"`, `[audience] module = "permutive"`, `[framework] module = "nextjs"`, `[auction] modules = ["prebid"]` (with `"testing.testlight"` for the test module) and `[proxy] modules = ["js_asset_proxy"]`, so `[integrations.gpt]` is `[ad-tag.google]` and `[integrations.prebid]` is `[auction.prebid]`. A module's name is `.`, which is its crate's path under `crates/` once it has a crate of its own and a constant in core until then, and it is written in its section with the type left off. A section that selects nothing, a table its section does not select, and a name no module in the deployment supplies are each refused at startup. An `[integrations]` or `[integration]` table is refused with directions. Environment overlays name a section as written, so `TRUSTED_SERVER__INTEGRATIONS__GPT__GAM_ATTRIBUTION_ENABLED` is `TRUSTED_SERVER__AD-TAG__GOOGLE__GAM_ATTRIBUTION_ENABLED`. - **Breaking:** Every pluggable component now shares one configuration syntax. Each selectable type is a top-level table named for the job, a `module` key inside it selects what runs, or `modules` for a type that runs several, and a `[.]` table holds that name's settings when it has any. The types are `ec`, `geo`, `device`, `permission-signal`, `demand` and `ad-server`, and the section of each page integration type, such as `cmp` or `tag`. A module from a crate is named by its folder below `crates/`, such as `permission-signal.gpc`, and within its own table the type folder may be left off. A `demand` or `ad-server` name is snake_case, because it may be a label of your own. A settings table its type's selector does not name refuses startup, so a block left behind after a module is switched off is caught rather than sitting unread, and every module rejects settings it does not know, so a misspelled key fails rather than being ignored. A table's name is the implementation unless the table carries an `implementation` line, and a demand table always carries one, naming the implementation by its module path (`auction-protocol.openrtb`, `auction.prebid-server` or `auction.aps`), which is how two Prebid Servers run side by side under names of their own. The ad server implementation is `ad-server.mock`, written `mock` in `[ad-server]`. A `demand` or `ad-server` endpoint must be HTTPS, or HTTP to `127.0.0.1`, `::1` or `localhost`. The word mediator is gone. It is "ad server" in prose and `ad-server` in configuration, `[debug.auction_html_comment_options] include_mediator_response` is now `include_adserver_response`, and the auction response metadata value `parallel_mediation` is now `parallel_adserver`. `auction-protocol.openrtb`, `auction.prebid-server`, `auction.aps` and `ad-server.mock` supply implementations only, so a page integration's section that names one is refused. The full rules and what is checked at each gate are in [Configuration Rules](docs/guide/configuration-rules.md). Migrate as follows: diff --git a/Cargo.lock b/Cargo.lock index 5565c7a363..136a099bbd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5762,6 +5762,7 @@ dependencies = [ "trusted-server-cmp-sourcepoint", "trusted-server-core", "trusted-server-identity-lockr", + "trusted-server-tag-google-tag-manager", ] [[package]] @@ -5816,6 +5817,24 @@ dependencies = [ "trusted-server-core", ] +[[package]] +name = "trusted-server-tag-google-tag-manager" +version = "0.1.0" +dependencies = [ + "async-trait", + "edgezero-core", + "error-stack", + "futures", + "http", + "log", + "regex", + "serde", + "serde_json", + "trusted-server-core", + "url", + "validator", +] + [[package]] name = "try-lock" version = "0.2.5" diff --git a/Cargo.toml b/Cargo.toml index 3a0ab3b965..f8594d7336 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,7 @@ members = [ "crates/audience/permutive", "crates/cmp/sourcepoint", "crates/cmp/didomi", + "crates/tag/google-tag-manager", "crates/trusted-server-adapter-axum", "crates/trusted-server-adapter-cloudflare", "crates/trusted-server-adapter-fastly", @@ -144,6 +145,7 @@ trusted-server-identity-lockr = { path = "crates/identity/lockr" } trusted-server-audience-permutive = { path = "crates/audience/permutive" } trusted-server-cmp-sourcepoint = { path = "crates/cmp/sourcepoint" } trusted-server-cmp-didomi = { path = "crates/cmp/didomi" } +trusted-server-tag-google-tag-manager = { path = "crates/tag/google-tag-manager" } trusted-server-js = { path = "crates/trusted-server-js" } trusted-server-modules = { path = "crates/trusted-server-modules" } trusted-server-openrtb = { path = "crates/trusted-server-openrtb" } diff --git a/crates/tag/google-tag-manager/Cargo.toml b/crates/tag/google-tag-manager/Cargo.toml new file mode 100644 index 0000000000..2ef9759fea --- /dev/null +++ b/crates/tag/google-tag-manager/Cargo.toml @@ -0,0 +1,38 @@ +[package] +name = "trusted-server-tag-google-tag-manager" +description = "Google Tag Manager first-party tag delivery as a tag module." +authors = { workspace = true } +edition = { workspace = true } +license = { workspace = true } +publish = { workspace = true } +version = { workspace = true } + +[lib] +doctest = false + +[lints] +workspace = true + +[dependencies] +async-trait = { workspace = true } +edgezero-core = { workspace = true } +error-stack = { workspace = true } +futures = { workspace = true } +http = { workspace = true } +log = { workspace = true } +regex = { workspace = true } +serde = { workspace = true } +trusted-server-core = { workspace = true } +url = { workspace = true } +validator = { workspace = true } + +[dev-dependencies] +serde_json = { workspace = true } +trusted-server-core = { workspace = true, features = ["test-utils"] } + +# The visible owner of this vendor crate, the way Prebid.js requires a named +# maintainer of every adapter. Google has not yet adopted this crate, so the +# Trusted Server maintainers own it until a vendor owner steps up. +[package.metadata.maintainers] +owner = "Trusted Server maintainers" +status = "seeking vendor owner" diff --git a/crates/trusted-server-core/src/integrations/google_tag_manager.rs b/crates/tag/google-tag-manager/src/lib.rs similarity index 97% rename from crates/trusted-server-core/src/integrations/google_tag_manager.rs rename to crates/tag/google-tag-manager/src/lib.rs index b1f2ab2a7b..535405abc1 100644 --- a/crates/trusted-server-core/src/integrations/google_tag_manager.rs +++ b/crates/tag/google-tag-manager/src/lib.rs @@ -12,6 +12,18 @@ //! | `GET/POST` | `.../collect` | Proxies GA analytics beacons | //! | `GET/POST` | `.../g/collect` | Proxies GA4 analytics beacons | +#![cfg_attr( + test, + allow( + clippy::print_stdout, + clippy::print_stderr, + clippy::panic, + clippy::dbg_macro, + clippy::unwrap_used, + reason = "tests use direct diagnostics and panic-on-failure helpers" + ) +)] + use std::sync::{Arc, LazyLock}; use async_trait::async_trait; @@ -23,32 +35,37 @@ use regex::Regex; use serde::{Deserialize, Serialize}; use validator::{Validate, ValidationError}; -use crate::error::TrustedServerError; -use crate::integrations::ScriptTextAccumulator; -use crate::integrations::{ +use trusted_server_core::error::TrustedServerError; +use trusted_server_core::integrations::ScriptTextAccumulator; +use trusted_server_core::integrations::{ AttributeRewriteAction, IntegrationAttributeContext, IntegrationAttributeRewriter, IntegrationEndpoint, IntegrationProxy, IntegrationRegistration, IntegrationScriptContext, IntegrationScriptRewriter, ScriptRewriteAction, UPSTREAM_SDK_MAX_RESPONSE_BYTES, collect_response_bounded, }; -use crate::platform::RuntimeServices; -use crate::proxy::{FIRST_PARTY_PASSTHROUGH_STRIP_HEADERS, ProxyRequestConfig, proxy_request}; -use crate::settings::{IntegrationConfig, Settings}; +use trusted_server_core::platform::RuntimeServices; +use trusted_server_core::proxy::{ + FIRST_PARTY_PASSTHROUGH_STRIP_HEADERS, ProxyRequestConfig, proxy_request, +}; +use trusted_server_core::settings::{IntegrationConfig, Settings}; const GTM_INTEGRATION_ID: &str = "google_tag_manager"; /// The name this module is selected by, in `[tag]`. pub const MODULE: &str = "tag.google-tag-manager"; -/// The builder the registry runs when a section selects [`MODULE`]. -pub(crate) const BUILDER: crate::integrations::IntegrationBuilder = - crate::integrations::IntegrationBuilder::new( +/// The builder a deployment hands to an adapter, which the registry runs when +/// a section selects [`MODULE`]. +#[must_use] +pub fn builder() -> trusted_server_core::integrations::IntegrationBuilder { + trusted_server_core::integrations::IntegrationBuilder::new( GTM_INTEGRATION_ID, - crate::integrations::CORE_SOURCE, + env!("CARGO_PKG_NAME"), register, validate, ) - .with_module_name(MODULE); + .with_module_name(MODULE) +} const DEFAULT_UPSTREAM: &str = "https://www.googletagmanager.com"; /// Host serving the GA beacon endpoints the `/collect` paths are pinned to. const GA_COLLECT_HOST: &str = "www.google-analytics.com"; @@ -822,7 +839,7 @@ impl GoogleTagManagerIntegration { // Explicitly strip X-Forwarded-For to prevent client IP leakage to Google. // The empty value will override any existing header during proxy forwarding. proxy_config = proxy_config.with_header( - crate::constants::HEADER_X_FORWARDED_FOR, + trusted_server_core::constants::HEADER_X_FORWARDED_FOR, http::HeaderValue::from_static(""), ); @@ -1101,15 +1118,19 @@ impl IntegrationScriptRewriter for GoogleTagManagerIntegration { #[cfg(test)] mod tests { use super::*; - use crate::html_processor::{HtmlProcessorConfig, create_html_processor}; - use crate::integrations::{ + use trusted_server_core::html_processor::{HtmlProcessorConfig, create_html_processor}; + use trusted_server_core::integrations::{ AttributeRewriteAction, IntegrationAttributeContext, IntegrationAttributeRewriter, IntegrationDocumentState, IntegrationRegistry, IntegrationScriptContext, IntegrationScriptRewriter, ScriptRewriteAction, }; - use crate::platform::test_support::{StubHttpClient, build_services_with_http_client}; - use crate::settings::Settings; - use crate::streaming_processor::{Compression, PipelineConfig, StreamingPipeline}; + use trusted_server_core::platform::test_support::{ + StubHttpClient, build_services_with_http_client, + }; + use trusted_server_core::settings::Settings; + use trusted_server_core::streaming_processor::{ + Compression, PipelineConfig, StreamingPipeline, + }; #[test] fn container_id_validation_matches_gtm_pattern() { @@ -1144,10 +1165,10 @@ mod tests { } } - use crate::platform::test_support::noop_services; - use crate::test_support::tests::create_test_settings; use http::Method; use std::io::Cursor; + use trusted_server_core::platform::test_support::noop_services; + use trusted_server_core::test_support::tests::create_test_settings; fn build_http_request(method: Method, uri: &str, body: EdgeBody) -> http::Request { http::Request::builder() @@ -1739,7 +1760,7 @@ mod tests { let stub = Arc::new(StubHttpClient::new()); stub.push_response_with_headers(status, b"upstream body".to_vec(), headers); let services = build_services_with_http_client( - Arc::clone(&stub) as Arc + Arc::clone(&stub) as Arc ); let settings = create_test_settings(); let integration = GoogleTagManagerIntegration::new(tag_config("GTM-CONFIGURED", &[])); @@ -1996,7 +2017,7 @@ mod tests { assert!( domains .iter() - .any(|pattern| crate::proxy::is_host_allowed(host, pattern)), + .any(|pattern| trusted_server_core::proxy::is_host_allowed(host, pattern)), "should permit {host}: {domains:?}" ); } @@ -2010,7 +2031,7 @@ mod tests { assert!( !domains .iter() - .any(|pattern| crate::proxy::is_host_allowed(host, pattern)), + .any(|pattern| trusted_server_core::proxy::is_host_allowed(host, pattern)), "should refuse {host}: {domains:?}" ); } @@ -2035,7 +2056,10 @@ mod tests { assert!( !domains .iter() - .any(|pattern| crate::proxy::is_host_allowed("www.googletagmanager.com", pattern)), + .any(|pattern| trusted_server_core::proxy::is_host_allowed( + "www.googletagmanager.com", + pattern + )), "a custom upstream replaces the Google script host: {domains:?}" ); } @@ -2076,7 +2100,7 @@ mod tests { let stub = Arc::new(StubHttpClient::new()); stub.push_response(status, body.to_vec()); let services = build_services_with_http_client( - Arc::clone(&stub) as Arc + Arc::clone(&stub) as Arc ); let settings = create_test_settings(); let integration = GoogleTagManagerIntegration::new(config); @@ -2957,7 +2981,7 @@ j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= EdgeBody::empty(), ); req.headers_mut().insert( - crate::constants::HEADER_X_FORWARDED_FOR, + trusted_server_core::constants::HEADER_X_FORWARDED_FOR, http::HeaderValue::from_static("198.51.100.42"), ); @@ -2975,9 +2999,9 @@ j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= // We check if X-Forwarded-For is explicitly overridden with an empty string, // which effectively strips it during proxy forwarding due to header override logic. let has_header_override = proxy_config.headers.iter().any(|(name, value)| { - name.as_str() - .eq_ignore_ascii_case(crate::constants::HEADER_X_FORWARDED_FOR.as_str()) - && value.is_empty() + name.as_str().eq_ignore_ascii_case( + trusted_server_core::constants::HEADER_X_FORWARDED_FOR.as_str(), + ) && value.is_empty() }); assert!( @@ -3153,14 +3177,8 @@ assume_single_jurisdiction = true ) .expect("should update gtm config"); - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) - .expect("should create registry"); + let registry = IntegrationRegistry::with_registrations(&settings, &[builder()]) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); let pipeline_config = PipelineConfig { @@ -3199,14 +3217,8 @@ assume_single_jurisdiction = true .expect("should update gtm config"); // 2. Setup Pipeline - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) - .expect("should create registry"); + let registry = IntegrationRegistry::with_registrations(&settings, &[builder()]) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); let pipeline_config = PipelineConfig { @@ -3217,8 +3229,7 @@ assume_single_jurisdiction = true let mut pipeline = StreamingPipeline::new(pipeline_config, processor); // 3. Load Fixture - // Path is relative to this file: ../html_processor.test.html - let html_content = include_str!("../html_processor.test.html"); + let html_content = trusted_server_core::test_support::fixtures::PUBLISHER_PAGE_HTML; // 4. Run Pipeline let mut output = Vec::new(); @@ -3271,14 +3282,8 @@ assume_single_jurisdiction = true .expect("should update config"); // Inlined Pipeline Creation - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) - .expect("should create registry"); + let registry = IntegrationRegistry::with_registrations(&settings, &[builder()]) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); let pipeline_config = PipelineConfig { @@ -3658,14 +3663,8 @@ assume_single_jurisdiction = true ) .expect("should update config"); - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) - .expect("should create registry"); + let registry = IntegrationRegistry::with_registrations(&settings, &[builder()]) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); @@ -3714,7 +3713,8 @@ assume_single_jurisdiction = true }), ) .expect("should update config"); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_registrations(&settings, &[builder()]) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); let mut pipeline = StreamingPipeline::new( @@ -3754,8 +3754,10 @@ assume_single_jurisdiction = true /// that can't plausibly contain a GTM domain. #[test] fn fragmented_next_data_survives_with_gtm_enabled() { - use crate::streaming_processor::{Compression, PipelineConfig, StreamingPipeline}; use std::io::Cursor; + use trusted_server_core::streaming_processor::{ + Compression, PipelineConfig, StreamingPipeline, + }; let mut settings = make_settings(); settings @@ -3777,14 +3779,8 @@ assume_single_jurisdiction = true ) .expect("should update nextjs config"); - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) - .expect("should create registry"); + let registry = IntegrationRegistry::with_registrations(&settings, &[builder()]) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); @@ -3825,8 +3821,10 @@ assume_single_jurisdiction = true /// rewritable origin URL appears later in the payload. #[test] fn fragmented_next_data_with_trailing_g_survives_gtm() { - use crate::streaming_processor::{Compression, PipelineConfig, StreamingPipeline}; use std::io::Cursor; + use trusted_server_core::streaming_processor::{ + Compression, PipelineConfig, StreamingPipeline, + }; let mut settings = make_settings(); settings @@ -3848,14 +3846,8 @@ assume_single_jurisdiction = true ) .expect("should update nextjs config"); - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) - .expect("should create registry"); + let registry = IntegrationRegistry::with_registrations(&settings, &[builder()]) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); @@ -3928,4 +3920,13 @@ assume_single_jurisdiction = true ); } } + + #[test] + fn module_constant_is_the_crate_folder() { + assert_eq!( + super::MODULE, + trusted_server_core::module_name!(), + "should be named by the folder this crate lives in" + ); + } } diff --git a/crates/trusted-server-core/src/integrations/mod.rs b/crates/trusted-server-core/src/integrations/mod.rs index 09fdbbfde7..4d71f16f7a 100644 --- a/crates/trusted-server-core/src/integrations/mod.rs +++ b/crates/trusted-server-core/src/integrations/mod.rs @@ -16,7 +16,6 @@ use crate::settings::Settings; pub mod adserver_mock; pub mod aps; pub mod datadome; -pub mod google_tag_manager; pub mod gpt; pub mod gpt_diagnostics; pub mod js_asset_proxy; @@ -576,7 +575,6 @@ const BUILT_IN_BUILDERS: &[IntegrationBuilder] = &[ js_asset_proxy::BUILDER, testlight::BUILDER, nextjs::BUILDER, - google_tag_manager::BUILDER, datadome::BUILDER, gpt::BUILDER, gpt_diagnostics::BUILDER, diff --git a/crates/trusted-server-core/src/test_support.rs b/crates/trusted-server-core/src/test_support.rs index 114296c632..de594be3ae 100644 --- a/crates/trusted-server-core/src/test_support.rs +++ b/crates/trusted-server-core/src/test_support.rs @@ -160,6 +160,14 @@ pub mod tests { "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.Ab1234"; } +/// Page fixtures shared by core's tests and the tests of a module crate. +#[cfg(any(test, feature = "test-utils"))] +pub mod fixtures { + /// A publisher page carrying the tags of several vendors, as an origin + /// would serve it. + pub const PUBLISHER_PAGE_HTML: &str = include_str!("html_processor.test.html"); +} + /// The operator-facing settings template, for tests that check what it /// documents. #[cfg(any(test, feature = "test-utils"))] diff --git a/crates/trusted-server-js/lib/src/integrations/google_tag_manager/script_guard.ts b/crates/trusted-server-js/lib/src/integrations/google_tag_manager/script_guard.ts index 2072c04032..874748bec8 100644 --- a/crates/trusted-server-js/lib/src/integrations/google_tag_manager/script_guard.ts +++ b/crates/trusted-server-js/lib/src/integrations/google_tag_manager/script_guard.ts @@ -21,7 +21,7 @@ const GTM_URL_PATTERN = /** * Supported paths that the server can proxy. * Must match the route patterns defined in the GoogleTagManagerIntegration handler - * in crates/trusted-server-core/src/integrations/google_tag_manager.rs + * in crates/tag/google-tag-manager/src/lib.rs */ const SUPPORTED_PATHS = ['/gtm.js', '/gtag/js', '/gtag.js', '/collect', '/g/collect']; diff --git a/crates/trusted-server-modules/Cargo.toml b/crates/trusted-server-modules/Cargo.toml index abd637ee31..f2e9209a8b 100644 --- a/crates/trusted-server-modules/Cargo.toml +++ b/crates/trusted-server-modules/Cargo.toml @@ -20,6 +20,7 @@ trusted-server-cmp-osano = { workspace = true } trusted-server-cmp-sourcepoint = { workspace = true } trusted-server-core = { workspace = true } trusted-server-identity-lockr = { workspace = true } +trusted-server-tag-google-tag-manager = { workspace = true } [dev-dependencies] serde_json = { workspace = true } diff --git a/crates/trusted-server-modules/src/lib.rs b/crates/trusted-server-modules/src/lib.rs index f9916b283e..e1b1550611 100644 --- a/crates/trusted-server-modules/src/lib.rs +++ b/crates/trusted-server-modules/src/lib.rs @@ -25,6 +25,7 @@ pub fn builders() -> Vec { trusted_server_cmp_didomi::builder(), trusted_server_cmp_sourcepoint::builder(), trusted_server_cmp_osano::builder(), + trusted_server_tag_google_tag_manager::builder(), ] } @@ -58,6 +59,7 @@ mod tests { "cmp.didomi", "cmp.sourcepoint", "cmp.osano", + "tag.google-tag-manager", ], "should offer the stock modules in the order their hooks run" ); diff --git a/docs/guide/integrations/google_tag_manager.md b/docs/guide/integrations/google_tag_manager.md index 89ae85083d..d5e8587d35 100644 --- a/docs/guide/integrations/google_tag_manager.md +++ b/docs/guide/integrations/google_tag_manager.md @@ -296,7 +296,7 @@ _Expected_: `200 OK` (or 204). ## Implementation Details -See [crates/trusted-server-core/src/integrations/google_tag_manager.rs](https://github.com/IABTechLab/trusted-server/blob/main/crates/trusted-server-core/src/integrations/google_tag_manager.rs). +See [crates/tag/google-tag-manager/src/lib.rs](https://github.com/IABTechLab/trusted-server/blob/main/crates/tag/google-tag-manager/src/lib.rs). ## Next Steps From 83929608bbd925dc65a6df8bbb0797f7831b4d32 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 8 Oct 2026 00:47:45 +0100 Subject: [PATCH 092/124] Testlight moves out of core The Testlight example integration is `testing.testlight`, and it now lives in `crates/testing/testlight`. The stock list offers it in the place its hooks ran before, and the settings that select and configure it do not change. It reads the Edge Cookie a request carries through `edge_cookie::recognized_ec_id`, so that module is public. The test that serves a page through the HTML pipeline and checks Testlight's script is replaced moves with it. The registry test that lists the modules a deployment does supply names a probe builder of its own. --- .cargo/config.toml | 12 +- CHANGELOG.md | 2 +- Cargo.lock | 16 +++ Cargo.toml | 2 + crates/testing/testlight/Cargo.toml | 33 +++++ .../testlight/src/lib.rs} | 127 +++++++++++++++--- .../trusted-server-core/src/html_processor.rs | 54 -------- .../src/integrations/mod.rs | 2 - .../src/integrations/registry.rs | 14 +- crates/trusted-server-core/src/lib.rs | 2 +- crates/trusted-server-modules/Cargo.toml | 1 + crates/trusted-server-modules/src/lib.rs | 2 + 12 files changed, 179 insertions(+), 88 deletions(-) create mode 100644 crates/testing/testlight/Cargo.toml rename crates/{trusted-server-core/src/integrations/testlight.rs => testing/testlight/src/lib.rs} (82%) diff --git a/.cargo/config.toml b/.cargo/config.toml index 95dde5e19f..a383a0b71e 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -26,10 +26,10 @@ test_details = "test --target aarch64-apple-darwin" # native crate needs no change here. Axum (native), Cloudflare # (wasm32-unknown-unknown), Spin, the CLI (native), and integration-tests # (native) are simply not listed. -build-fastly = "build -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" -check-fastly = "check -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" -clippy-fastly = "clippy -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-js -p trusted-server-openrtb --all-targets --all-features --target wasm32-wasip1 -- -D warnings" -test-fastly = "test -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" +build-fastly = "build -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" +check-fastly = "check -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" +clippy-fastly = "clippy -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-js -p trusted-server-openrtb --all-targets --all-features --target wasm32-wasip1 -- -D warnings" +test-fastly = "test -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" # Feature-on counterpart of `test-fastly`. `test-fastly` does NOT pass # --all-features, so without this the reusable-sandbox code is linted by # `clippy-fastly` but its tests never execute. @@ -40,8 +40,8 @@ test-fastly-reuse = "test -p trusted-server-adapter-fastly --target wasm32-wasip # run with the Axum adapter in CI. build-axum = "build -p trusted-server-adapter-axum" check-axum = "check -p trusted-server-adapter-axum" -clippy-axum = "clippy -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager --all-targets --all-features -- -D warnings" -test-axum = "test -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager" +clippy-axum = "clippy -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight --all-targets --all-features -- -D warnings" +test-axum = "test -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight" # --- Cloudflare adapter (native host + wasm32-unknown-unknown) --- # Build/check target the WASM runtime (requires the `cloudflare` feature); diff --git a/CHANGELOG.md b/CHANGELOG.md index 86cfed0b8e..183c79f94c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- Integration modules are moving out of `trusted-server-core` into crates of their own under `crates///`, each named by that folder, so `crates/cmp/osano` is the module `cmp.osano`. The settings that select and configure a module do not change. `crates/trusted-server-modules` lists the modules a stock build ships from crates of their own, in the order their hooks run. Every adapter builds its state and loads its settings with that list, and `ts config validate` and `ts config push` run each stock module's own rules. Code that named a moved module under `trusted_server_core::integrations` names its crate instead, such as `trusted_server_cmp_osano`. Moved so far: Osano, Lockr, Permutive, Sourcepoint, Didomi and Google Tag Manager. +- Integration modules are moving out of `trusted-server-core` into crates of their own under `crates///`, each named by that folder, so `crates/cmp/osano` is the module `cmp.osano`. The settings that select and configure a module do not change. `crates/trusted-server-modules` lists the modules a stock build ships from crates of their own, in the order their hooks run. Every adapter builds its state and loads its settings with that list, and `ts config validate` and `ts config push` run each stock module's own rules. Code that named a moved module under `trusted_server_core::integrations` names its crate instead, such as `trusted_server_cmp_osano`. Moved so far: Osano, Lockr, Permutive, Sourcepoint, Didomi, Google Tag Manager and Testlight. - The Fastly adapter is a library with a thin binary over it. A deployment that ships a vendor crate writes a binary of its own that calls `trusted_server_adapter_fastly::run_with(vec![...])` with that crate's integration builder, in place of editing the adapter. The builders are composed with the built-in ones, and a builder runs only when the settings select its module. The Wasm artifact keeps its name, `trusted-server-adapter-fastly.wasm`. - **Breaking:** `[integrations]` is gone. Each page integration is a module selected in the section of its type, with its settings in the table at its name: `[cmp] module = "didomi"` (or `"sourcepoint"` or `"osano"`), `[tag] modules = ["google-tag-manager"]`, `[ad-tag] modules = ["google", "google.diagnostics"]`, `[bot-protection] module = "datadome"`, `[identity] module = "lockr"`, `[audience] module = "permutive"`, `[framework] module = "nextjs"`, `[auction] modules = ["prebid"]` (with `"testing.testlight"` for the test module) and `[proxy] modules = ["js_asset_proxy"]`, so `[integrations.gpt]` is `[ad-tag.google]` and `[integrations.prebid]` is `[auction.prebid]`. A module's name is `.`, which is its crate's path under `crates/` once it has a crate of its own and a constant in core until then, and it is written in its section with the type left off. A section that selects nothing, a table its section does not select, and a name no module in the deployment supplies are each refused at startup. An `[integrations]` or `[integration]` table is refused with directions. Environment overlays name a section as written, so `TRUSTED_SERVER__INTEGRATIONS__GPT__GAM_ATTRIBUTION_ENABLED` is `TRUSTED_SERVER__AD-TAG__GOOGLE__GAM_ATTRIBUTION_ENABLED`. - **Breaking:** Every pluggable component now shares one configuration syntax. Each selectable type is a top-level table named for the job, a `module` key inside it selects what runs, or `modules` for a type that runs several, and a `[.]` table holds that name's settings when it has any. The types are `ec`, `geo`, `device`, `permission-signal`, `demand` and `ad-server`, and the section of each page integration type, such as `cmp` or `tag`. A module from a crate is named by its folder below `crates/`, such as `permission-signal.gpc`, and within its own table the type folder may be left off. A `demand` or `ad-server` name is snake_case, because it may be a label of your own. A settings table its type's selector does not name refuses startup, so a block left behind after a module is switched off is caught rather than sitting unread, and every module rejects settings it does not know, so a misspelled key fails rather than being ignored. A table's name is the implementation unless the table carries an `implementation` line, and a demand table always carries one, naming the implementation by its module path (`auction-protocol.openrtb`, `auction.prebid-server` or `auction.aps`), which is how two Prebid Servers run side by side under names of their own. The ad server implementation is `ad-server.mock`, written `mock` in `[ad-server]`. A `demand` or `ad-server` endpoint must be HTTPS, or HTTP to `127.0.0.1`, `::1` or `localhost`. The word mediator is gone. It is "ad server" in prose and `ad-server` in configuration, `[debug.auction_html_comment_options] include_mediator_response` is now `include_adserver_response`, and the auction response metadata value `parallel_mediation` is now `parallel_adserver`. `auction-protocol.openrtb`, `auction.prebid-server`, `auction.aps` and `ad-server.mock` supply implementations only, so a page integration's section that names one is refused. The full rules and what is checked at each gate are in [Configuration Rules](docs/guide/configuration-rules.md). Migrate as follows: diff --git a/Cargo.lock b/Cargo.lock index 136a099bbd..5794dfecb0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5763,6 +5763,7 @@ dependencies = [ "trusted-server-core", "trusted-server-identity-lockr", "trusted-server-tag-google-tag-manager", + "trusted-server-testing-testlight", ] [[package]] @@ -5835,6 +5836,21 @@ dependencies = [ "validator", ] +[[package]] +name = "trusted-server-testing-testlight" +version = "0.1.0" +dependencies = [ + "async-trait", + "edgezero-core", + "error-stack", + "futures", + "http", + "serde", + "serde_json", + "trusted-server-core", + "validator", +] + [[package]] name = "try-lock" version = "0.2.5" diff --git a/Cargo.toml b/Cargo.toml index f8594d7336..2a2599f721 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ members = [ "crates/cmp/sourcepoint", "crates/cmp/didomi", "crates/tag/google-tag-manager", + "crates/testing/testlight", "crates/trusted-server-adapter-axum", "crates/trusted-server-adapter-cloudflare", "crates/trusted-server-adapter-fastly", @@ -146,6 +147,7 @@ trusted-server-audience-permutive = { path = "crates/audience/permutive" } trusted-server-cmp-sourcepoint = { path = "crates/cmp/sourcepoint" } trusted-server-cmp-didomi = { path = "crates/cmp/didomi" } trusted-server-tag-google-tag-manager = { path = "crates/tag/google-tag-manager" } +trusted-server-testing-testlight = { path = "crates/testing/testlight" } trusted-server-js = { path = "crates/trusted-server-js" } trusted-server-modules = { path = "crates/trusted-server-modules" } trusted-server-openrtb = { path = "crates/trusted-server-openrtb" } diff --git a/crates/testing/testlight/Cargo.toml b/crates/testing/testlight/Cargo.toml new file mode 100644 index 0000000000..a624e64eae --- /dev/null +++ b/crates/testing/testlight/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "trusted-server-testing-testlight" +description = "The Testlight example integration, which rewrites an auction request body and proxies it to a test endpoint." +authors = { workspace = true } +edition = { workspace = true } +license = { workspace = true } +publish = { workspace = true } +version = { workspace = true } + +[lib] +doctest = false + +[lints] +workspace = true + +[dependencies] +async-trait = { workspace = true } +edgezero-core = { workspace = true } +error-stack = { workspace = true } +http = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +trusted-server-core = { workspace = true } +validator = { workspace = true } + +[dev-dependencies] +futures = { workspace = true } +trusted-server-core = { workspace = true, features = ["test-utils"] } + +# An example the project keeps for testing an auction integration end to end, +# so the Trusted Server maintainers own it. +[package.metadata.maintainers] +owner = "Trusted Server maintainers" diff --git a/crates/trusted-server-core/src/integrations/testlight.rs b/crates/testing/testlight/src/lib.rs similarity index 82% rename from crates/trusted-server-core/src/integrations/testlight.rs rename to crates/testing/testlight/src/lib.rs index 7893974e96..6cfbbf26db 100644 --- a/crates/trusted-server-core/src/integrations/testlight.rs +++ b/crates/testing/testlight/src/lib.rs @@ -1,3 +1,14 @@ +#![cfg_attr( + test, + allow( + clippy::print_stdout, + clippy::print_stderr, + clippy::panic, + clippy::dbg_macro, + clippy::unwrap_used, + reason = "tests use direct diagnostics and panic-on-failure helpers" + ) +)] use std::sync::Arc; use async_trait::async_trait; @@ -9,33 +20,36 @@ use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; use validator::Validate; -use crate::edge_cookie::recognized_ec_id; -use crate::error::TrustedServerError; -use crate::integrations::{ +use trusted_server_core::edge_cookie::recognized_ec_id; +use trusted_server_core::error::TrustedServerError; +use trusted_server_core::integrations::{ AttributeRewriteAction, INTEGRATION_MAX_BODY_BYTES, IntegrationAttributeContext, IntegrationAttributeRewriter, IntegrationEndpoint, IntegrationProxy, IntegrationRegistration, UPSTREAM_RTB_MAX_RESPONSE_BYTES, collect_body_bounded, collect_response_bounded, }; -use crate::platform::RuntimeServices; -use crate::proxy::{ProxyRequestConfig, proxy_request}; -use crate::settings::{IntegrationConfig, Settings}; -use crate::tsjs; +use trusted_server_core::platform::RuntimeServices; +use trusted_server_core::proxy::{ProxyRequestConfig, proxy_request}; +use trusted_server_core::settings::{IntegrationConfig, Settings}; +use trusted_server_core::tsjs; const TESTLIGHT_INTEGRATION_ID: &str = "testlight"; /// The name this module is selected by, in `[auction]`. pub const MODULE: &str = "testing.testlight"; -/// The builder the registry runs when a section selects [`MODULE`]. -pub(crate) const BUILDER: crate::integrations::IntegrationBuilder = - crate::integrations::IntegrationBuilder::new( +/// The builder a deployment hands to an adapter, which the registry runs when +/// a section selects [`MODULE`]. +#[must_use] +pub fn builder() -> trusted_server_core::integrations::IntegrationBuilder { + trusted_server_core::integrations::IntegrationBuilder::new( TESTLIGHT_INTEGRATION_ID, - crate::integrations::CORE_SOURCE, + env!("CARGO_PKG_NAME"), register, validate, ) .with_module_name(MODULE) - .selected_in("auction"); + .selected_in("auction") +} #[derive(Debug, Deserialize, Validate)] #[serde(deny_unknown_fields)] @@ -298,12 +312,14 @@ fn default_shim_src() -> String { #[cfg(test)] mod tests { use super::*; - use crate::platform::test_support::{StubHttpClient, build_services_with_http_client}; - use crate::test_support::tests::{VALID_SYNTHETIC_ID, create_test_settings}; - use crate::tsjs; use http::Method; use serde_json::json; use std::sync::Arc; + use trusted_server_core::platform::test_support::{ + StubHttpClient, build_services_with_http_client, + }; + use trusted_server_core::test_support::tests::{VALID_SYNTHETIC_ID, create_test_settings}; + use trusted_server_core::tsjs; #[test] fn build_requires_config() { @@ -443,7 +459,7 @@ mod tests { let stub = Arc::new(StubHttpClient::new()); stub.push_response(200, br#"{"ok":true}"#.to_vec()); let services = build_services_with_http_client( - Arc::clone(&stub) as Arc + Arc::clone(&stub) as Arc ); let settings = create_test_settings(); let integration = TestlightIntegration::new(TestlightConfig { @@ -458,7 +474,7 @@ mod tests { .body(EdgeBody::from(br#"{"imp":[{"id":"slot-1"}]}"#.to_vec())) .expect("should build request"); req.headers_mut().insert( - crate::constants::HEADER_X_TS_EC.clone(), + trusted_server_core::constants::HEADER_X_TS_EC.clone(), http::HeaderValue::from_static(VALID_SYNTHETIC_ID), ); @@ -498,7 +514,7 @@ mod tests { .body(EdgeBody::from(br#"{"imp":[{"id":"slot-1"}]}"#.to_vec())) .expect("should build request"); req.headers_mut().insert( - crate::constants::HEADER_X_TS_EC.clone(), + trusted_server_core::constants::HEADER_X_TS_EC.clone(), http::HeaderValue::from_str(ec_id).expect("should build EC header value"), ); req @@ -522,7 +538,7 @@ mod tests { let stub = Arc::new(StubHttpClient::new()); stub.push_response(200, br#"{"ok":true}"#.to_vec()); let services = build_services_with_http_client( - Arc::clone(&stub) as Arc + Arc::clone(&stub) as Arc ); let settings = create_test_settings(); @@ -549,7 +565,7 @@ mod tests { let stub = Arc::new(StubHttpClient::new()); stub.push_response(200, br#"{"ok":true}"#.to_vec()); let services = build_services_with_http_client( - Arc::clone(&stub) as Arc + Arc::clone(&stub) as Arc ); // The value is one the built-in module would recognize, so only // the absence of a selected module can withhold it. @@ -573,4 +589,75 @@ mod tests { ); }); } + + #[test] + fn module_constant_is_the_crate_folder() { + assert_eq!( + super::MODULE, + trusted_server_core::module_name!(), + "should be named by the folder this crate lives in" + ); + } + + #[test] + fn a_served_page_loads_the_shim_in_place_of_the_vendor_script() { + use std::io::Cursor; + + use trusted_server_core::html_processor::{HtmlProcessorConfig, create_html_processor}; + use trusted_server_core::integrations::IntegrationRegistry; + use trusted_server_core::streaming_processor::{ + Compression, PipelineConfig, StreamingPipeline, + }; + + let html = r#" + + "#; + + let mut settings = Settings::default(); + let shim_src = "https://edge.example.com/static/testlight.js".to_owned(); + settings + .insert_module_config( + "auction", + MODULE, + &json!({ + "endpoint": "https://example.com/openrtb2/auction", + "rewrite_scripts": true, + "shim_src": shim_src, + }), + ) + .expect("should insert testlight config"); + + let registry = IntegrationRegistry::with_registrations(&settings, &[builder()]) + .expect("should create registry"); + let config = HtmlProcessorConfig::from_settings( + &settings, + ®istry, + "origin.example.com", + "test.example.com", + "https", + ); + let mut pipeline = StreamingPipeline::new( + PipelineConfig { + input_compression: Compression::None, + output_compression: Compression::None, + chunk_size: 8192, + }, + create_html_processor(config), + ); + + let mut output = Vec::new(); + pipeline + .process(Cursor::new(html.as_bytes()), &mut output) + .expect("should process the page"); + + let processed = String::from_utf8_lossy(&output); + assert!( + processed.contains(&shim_src), + "the shim should replace the vendor's script reference" + ); + assert!( + !processed.contains("cdn.testlight.com"), + "the vendor's own URL should be gone from the page" + ); + } } diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index 9790bb9019..7612c4d7ee 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -1247,60 +1247,6 @@ mod tests { ); } - #[test] - fn test_integration_registry_rewrites_integration_scripts() { - let html = r#" - - "#; - - let mut settings = Settings::default(); - let shim_src = "https://edge.example.com/static/testlight.js".to_owned(); - settings - .insert_module_config( - "auction", - "testing.testlight", - &json!({ - "endpoint": "https://example.com/openrtb2/auction", - "rewrite_scripts": true, - "shim_src": shim_src, - }), - ) - .expect("should insert testlight config"); - - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) - .expect("should create registry"); - let mut config = create_test_config(); - config.integrations = registry; - - let processor = create_html_processor(config); - let pipeline_config = PipelineConfig { - input_compression: Compression::None, - output_compression: Compression::None, - chunk_size: 8192, - }; - let mut pipeline = StreamingPipeline::new(pipeline_config, processor); - - let mut output = Vec::new(); - let result = pipeline.process(Cursor::new(html.as_bytes()), &mut output); - result.unwrap(); - - let processed = String::from_utf8_lossy(&output); - assert!( - processed.contains(&shim_src), - "Integration shim should replace integration script reference" - ); - assert!( - !processed.contains("cdn.testlight.com"), - "Original integration URL should be removed" - ); - } - #[test] fn test_real_publisher_html_with_gzip() { use flate2::Compression as GzCompression; diff --git a/crates/trusted-server-core/src/integrations/mod.rs b/crates/trusted-server-core/src/integrations/mod.rs index 4d71f16f7a..d035496555 100644 --- a/crates/trusted-server-core/src/integrations/mod.rs +++ b/crates/trusted-server-core/src/integrations/mod.rs @@ -24,7 +24,6 @@ pub mod openrtb; pub mod prebid; pub mod prebid_server; mod registry; -pub mod testlight; #[cfg(test)] pub(crate) use registry::test_support as registry_test_support; @@ -573,7 +572,6 @@ const BUILT_IN_BUILDERS: &[IntegrationBuilder] = &[ // This must remain first: attribute rewriters chain replacements and // short-circuit removals. js_asset_proxy::BUILDER, - testlight::BUILDER, nextjs::BUILDER, datadome::BUILDER, gpt::BUILDER, diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index 91d56a7fa0..fadec1ea5f 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -4281,8 +4281,15 @@ mod tests { #[test] fn an_id_no_builder_supplies_is_refused_at_registry_build() { let settings = settings_naming("a_vendors_own_integration"); + let extra = [crate::integrations::IntegrationBuilder::new( + "probe", + "seam-probe", + probe_registration, + validate_nothing, + ) + .with_module_name("testing.probe")]; - let error = IntegrationRegistry::new(&settings) + let error = IntegrationRegistry::with_registrations(&settings, &extra) .err() .expect("should refuse an id no builder in this deployment supplies"); @@ -4292,9 +4299,8 @@ mod tests { "should name the id nothing supplies: {message}" ); assert!( - message.contains("The testing modules it supplies are") - && message.contains("testlight"), - "should list the module types this deployment does supply: {message}" + message.contains("The testing modules it supplies are") && message.contains("probe"), + "should list the modules of that type this deployment does supply: {message}" ); } diff --git a/crates/trusted-server-core/src/lib.rs b/crates/trusted-server-core/src/lib.rs index 372eb337ba..8570b8768b 100644 --- a/crates/trusted-server-core/src/lib.rs +++ b/crates/trusted-server-core/src/lib.rs @@ -46,7 +46,7 @@ pub mod cookies; pub mod creative; pub mod creative_opportunities; pub mod ec; -pub(crate) mod edge_cookie; +pub mod edge_cookie; pub mod error; pub mod evidence; pub mod geo; diff --git a/crates/trusted-server-modules/Cargo.toml b/crates/trusted-server-modules/Cargo.toml index f2e9209a8b..ba61a493f1 100644 --- a/crates/trusted-server-modules/Cargo.toml +++ b/crates/trusted-server-modules/Cargo.toml @@ -21,6 +21,7 @@ trusted-server-cmp-sourcepoint = { workspace = true } trusted-server-core = { workspace = true } trusted-server-identity-lockr = { workspace = true } trusted-server-tag-google-tag-manager = { workspace = true } +trusted-server-testing-testlight = { workspace = true } [dev-dependencies] serde_json = { workspace = true } diff --git a/crates/trusted-server-modules/src/lib.rs b/crates/trusted-server-modules/src/lib.rs index e1b1550611..db854342ee 100644 --- a/crates/trusted-server-modules/src/lib.rs +++ b/crates/trusted-server-modules/src/lib.rs @@ -20,6 +20,7 @@ use trusted_server_core::integrations::IntegrationBuilder; #[must_use] pub fn builders() -> Vec { vec![ + trusted_server_testing_testlight::builder(), trusted_server_audience_permutive::builder(), trusted_server_identity_lockr::builder(), trusted_server_cmp_didomi::builder(), @@ -54,6 +55,7 @@ mod tests { assert_eq!( names, [ + "testing.testlight", "audience.permutive", "identity.lockr", "cmp.didomi", From 215c5dca852229680e91cd6dc05ed2ec7553d2e8 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 8 Oct 2026 01:32:56 +0100 Subject: [PATCH 093/124] Next.js moves out of core The Next.js module is `framework.nextjs`, and it now lives in `crates/framework/nextjs`, with its six source files and two fixtures. The stock list offers it in the place its hooks ran before, and the settings that select and configure it do not change. Five of core's page pipeline tests used Next.js as their example of an integration that rewrites script payloads in two passes, a script rewriter that leaves placeholders and a stream processor that fills them. They test core's part of that (the document state, the hold on an unresolved group, the auction seam, the template cache), so they stay in core and select a stand-in that does the same two passes with no vendor's format. The stand-in is compiled for core's tests only. The test of Next.js's own overflow handling moves to its crate, and the registry test that used Next.js as an integration with no browser script uses a probe. Two of Google Tag Manager's tests check that it leaves another module's script rewrite alone, with Next.js as that module, so its crate takes the Next.js crate as a test dependency. The stock list crate gains the test that every stock module refuses a setting it does not know, which core's own list can no longer show for a module that has left it. --- .cargo/config.toml | 12 +- CHANGELOG.md | 2 +- Cargo.lock | 16 + Cargo.toml | 2 + crates/framework/nextjs/Cargo.toml | 34 ++ .../src}/fixtures/inlined-data-escaped.html | 0 .../src}/fixtures/inlined-data-nonce.html | 0 .../mod.rs => framework/nextjs/src/lib.rs} | 207 ++++++------ .../nextjs => framework/nextjs/src}/rsc.rs | 0 .../nextjs/src}/rsc_placeholders.rs | 6 +- .../nextjs/src}/rsc_stream.rs | 6 +- .../nextjs/src}/script_rewriter.rs | 8 +- .../nextjs => framework/nextjs/src}/shared.rs | 2 +- crates/tag/google-tag-manager/Cargo.toml | 3 + crates/tag/google-tag-manager/src/lib.rs | 16 +- .../trusted-server-core/src/html_processor.rs | 47 --- .../src/integrations/mod.rs | 6 +- .../src/integrations/registry.rs | 300 +++++++++++++++++- crates/trusted-server-core/src/publisher.rs | 129 +++----- crates/trusted-server-core/src/rsc_flight.rs | 2 +- crates/trusted-server-core/src/settings.rs | 8 +- .../trusted-server-core/src/test_support.rs | 2 +- crates/trusted-server-modules/Cargo.toml | 1 + crates/trusted-server-modules/src/lib.rs | 41 +++ docs/guide/integrations/nextjs.md | 6 +- docs/guide/rsc-hydration.md | 2 +- 26 files changed, 580 insertions(+), 278 deletions(-) create mode 100644 crates/framework/nextjs/Cargo.toml rename crates/{trusted-server-core/src/integrations/nextjs => framework/nextjs/src}/fixtures/inlined-data-escaped.html (100%) rename crates/{trusted-server-core/src/integrations/nextjs => framework/nextjs/src}/fixtures/inlined-data-nonce.html (100%) rename crates/{trusted-server-core/src/integrations/nextjs/mod.rs => framework/nextjs/src/lib.rs} (87%) rename crates/{trusted-server-core/src/integrations/nextjs => framework/nextjs/src}/rsc.rs (100%) rename crates/{trusted-server-core/src/integrations/nextjs => framework/nextjs/src}/rsc_placeholders.rs (99%) rename crates/{trusted-server-core/src/integrations/nextjs => framework/nextjs/src}/rsc_stream.rs (99%) rename crates/{trusted-server-core/src/integrations/nextjs => framework/nextjs/src}/script_rewriter.rs (99%) rename crates/{trusted-server-core/src/integrations/nextjs => framework/nextjs/src}/shared.rs (99%) diff --git a/.cargo/config.toml b/.cargo/config.toml index a383a0b71e..6cc71414f2 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -26,10 +26,10 @@ test_details = "test --target aarch64-apple-darwin" # native crate needs no change here. Axum (native), Cloudflare # (wasm32-unknown-unknown), Spin, the CLI (native), and integration-tests # (native) are simply not listed. -build-fastly = "build -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" -check-fastly = "check -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" -clippy-fastly = "clippy -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-js -p trusted-server-openrtb --all-targets --all-features --target wasm32-wasip1 -- -D warnings" -test-fastly = "test -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" +build-fastly = "build -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" +check-fastly = "check -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" +clippy-fastly = "clippy -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-js -p trusted-server-openrtb --all-targets --all-features --target wasm32-wasip1 -- -D warnings" +test-fastly = "test -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" # Feature-on counterpart of `test-fastly`. `test-fastly` does NOT pass # --all-features, so without this the reusable-sandbox code is linted by # `clippy-fastly` but its tests never execute. @@ -40,8 +40,8 @@ test-fastly-reuse = "test -p trusted-server-adapter-fastly --target wasm32-wasip # run with the Axum adapter in CI. build-axum = "build -p trusted-server-adapter-axum" check-axum = "check -p trusted-server-adapter-axum" -clippy-axum = "clippy -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight --all-targets --all-features -- -D warnings" -test-axum = "test -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight" +clippy-axum = "clippy -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs --all-targets --all-features -- -D warnings" +test-axum = "test -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs" # --- Cloudflare adapter (native host + wasm32-unknown-unknown) --- # Build/check target the WASM runtime (requires the `cloudflare` feature); diff --git a/CHANGELOG.md b/CHANGELOG.md index 183c79f94c..9ec838f67a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- Integration modules are moving out of `trusted-server-core` into crates of their own under `crates///`, each named by that folder, so `crates/cmp/osano` is the module `cmp.osano`. The settings that select and configure a module do not change. `crates/trusted-server-modules` lists the modules a stock build ships from crates of their own, in the order their hooks run. Every adapter builds its state and loads its settings with that list, and `ts config validate` and `ts config push` run each stock module's own rules. Code that named a moved module under `trusted_server_core::integrations` names its crate instead, such as `trusted_server_cmp_osano`. Moved so far: Osano, Lockr, Permutive, Sourcepoint, Didomi, Google Tag Manager and Testlight. +- Integration modules are moving out of `trusted-server-core` into crates of their own under `crates///`, each named by that folder, so `crates/cmp/osano` is the module `cmp.osano`. The settings that select and configure a module do not change. `crates/trusted-server-modules` lists the modules a stock build ships from crates of their own, in the order their hooks run. Every adapter builds its state and loads its settings with that list, and `ts config validate` and `ts config push` run each stock module's own rules. Code that named a moved module under `trusted_server_core::integrations` names its crate instead, such as `trusted_server_cmp_osano`. Moved so far: Osano, Lockr, Permutive, Sourcepoint, Didomi, Google Tag Manager, Testlight and Next.js.js. - The Fastly adapter is a library with a thin binary over it. A deployment that ships a vendor crate writes a binary of its own that calls `trusted_server_adapter_fastly::run_with(vec![...])` with that crate's integration builder, in place of editing the adapter. The builders are composed with the built-in ones, and a builder runs only when the settings select its module. The Wasm artifact keeps its name, `trusted-server-adapter-fastly.wasm`. - **Breaking:** `[integrations]` is gone. Each page integration is a module selected in the section of its type, with its settings in the table at its name: `[cmp] module = "didomi"` (or `"sourcepoint"` or `"osano"`), `[tag] modules = ["google-tag-manager"]`, `[ad-tag] modules = ["google", "google.diagnostics"]`, `[bot-protection] module = "datadome"`, `[identity] module = "lockr"`, `[audience] module = "permutive"`, `[framework] module = "nextjs"`, `[auction] modules = ["prebid"]` (with `"testing.testlight"` for the test module) and `[proxy] modules = ["js_asset_proxy"]`, so `[integrations.gpt]` is `[ad-tag.google]` and `[integrations.prebid]` is `[auction.prebid]`. A module's name is `.`, which is its crate's path under `crates/` once it has a crate of its own and a constant in core until then, and it is written in its section with the type left off. A section that selects nothing, a table its section does not select, and a name no module in the deployment supplies are each refused at startup. An `[integrations]` or `[integration]` table is refused with directions. Environment overlays name a section as written, so `TRUSTED_SERVER__INTEGRATIONS__GPT__GAM_ATTRIBUTION_ENABLED` is `TRUSTED_SERVER__AD-TAG__GOOGLE__GAM_ATTRIBUTION_ENABLED`. - **Breaking:** Every pluggable component now shares one configuration syntax. Each selectable type is a top-level table named for the job, a `module` key inside it selects what runs, or `modules` for a type that runs several, and a `[.]` table holds that name's settings when it has any. The types are `ec`, `geo`, `device`, `permission-signal`, `demand` and `ad-server`, and the section of each page integration type, such as `cmp` or `tag`. A module from a crate is named by its folder below `crates/`, such as `permission-signal.gpc`, and within its own table the type folder may be left off. A `demand` or `ad-server` name is snake_case, because it may be a label of your own. A settings table its type's selector does not name refuses startup, so a block left behind after a module is switched off is caught rather than sitting unread, and every module rejects settings it does not know, so a misspelled key fails rather than being ignored. A table's name is the implementation unless the table carries an `implementation` line, and a demand table always carries one, naming the implementation by its module path (`auction-protocol.openrtb`, `auction.prebid-server` or `auction.aps`), which is how two Prebid Servers run side by side under names of their own. The ad server implementation is `ad-server.mock`, written `mock` in `[ad-server]`. A `demand` or `ad-server` endpoint must be HTTPS, or HTTP to `127.0.0.1`, `::1` or `localhost`. The word mediator is gone. It is "ad server" in prose and `ad-server` in configuration, `[debug.auction_html_comment_options] include_mediator_response` is now `include_adserver_response`, and the auction response metadata value `parallel_mediation` is now `parallel_adserver`. `auction-protocol.openrtb`, `auction.prebid-server`, `auction.aps` and `ad-server.mock` supply implementations only, so a page integration's section that names one is refused. The full rules and what is checked at each gate are in [Configuration Rules](docs/guide/configuration-rules.md). Migrate as follows: diff --git a/Cargo.lock b/Cargo.lock index 5794dfecb0..2ba9527dc6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5668,6 +5668,20 @@ dependencies = [ "trusted-server-core", ] +[[package]] +name = "trusted-server-framework-nextjs" +version = "0.1.0" +dependencies = [ + "error-stack", + "log", + "regex", + "serde", + "serde_json", + "trusted-server-core", + "uuid", + "validator", +] + [[package]] name = "trusted-server-geo-fastly" version = "0.1.0" @@ -5761,6 +5775,7 @@ dependencies = [ "trusted-server-cmp-osano", "trusted-server-cmp-sourcepoint", "trusted-server-core", + "trusted-server-framework-nextjs", "trusted-server-identity-lockr", "trusted-server-tag-google-tag-manager", "trusted-server-testing-testlight", @@ -5832,6 +5847,7 @@ dependencies = [ "serde", "serde_json", "trusted-server-core", + "trusted-server-framework-nextjs", "url", "validator", ] diff --git a/Cargo.toml b/Cargo.toml index 2a2599f721..82b3042350 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ members = [ "crates/cmp/didomi", "crates/tag/google-tag-manager", "crates/testing/testlight", + "crates/framework/nextjs", "crates/trusted-server-adapter-axum", "crates/trusted-server-adapter-cloudflare", "crates/trusted-server-adapter-fastly", @@ -148,6 +149,7 @@ trusted-server-cmp-sourcepoint = { path = "crates/cmp/sourcepoint" } trusted-server-cmp-didomi = { path = "crates/cmp/didomi" } trusted-server-tag-google-tag-manager = { path = "crates/tag/google-tag-manager" } trusted-server-testing-testlight = { path = "crates/testing/testlight" } +trusted-server-framework-nextjs = { path = "crates/framework/nextjs" } trusted-server-js = { path = "crates/trusted-server-js" } trusted-server-modules = { path = "crates/trusted-server-modules" } trusted-server-openrtb = { path = "crates/trusted-server-openrtb" } diff --git a/crates/framework/nextjs/Cargo.toml b/crates/framework/nextjs/Cargo.toml new file mode 100644 index 0000000000..6d5edaf7d3 --- /dev/null +++ b/crates/framework/nextjs/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "trusted-server-framework-nextjs" +description = "Next.js support, which rewrites origin URLs in a page's data and in React Server Components payloads, as a framework module." +authors = { workspace = true } +edition = { workspace = true } +license = { workspace = true } +publish = { workspace = true } +version = { workspace = true } + +[lib] +doctest = false + +[lints] +workspace = true + +[dependencies] +error-stack = { workspace = true } +log = { workspace = true } +regex = { workspace = true } +serde = { workspace = true } +trusted-server-core = { workspace = true } +uuid = { workspace = true } +validator = { workspace = true } + +[dev-dependencies] +serde_json = { workspace = true } +trusted-server-core = { workspace = true, features = ["test-utils"] } + +# The visible owner of this vendor crate, the way Prebid.js requires a named +# maintainer of every adapter. Vercel has not yet adopted this crate, so the +# Trusted Server maintainers own it until a vendor owner steps up. +[package.metadata.maintainers] +owner = "Trusted Server maintainers" +status = "seeking vendor owner" diff --git a/crates/trusted-server-core/src/integrations/nextjs/fixtures/inlined-data-escaped.html b/crates/framework/nextjs/src/fixtures/inlined-data-escaped.html similarity index 100% rename from crates/trusted-server-core/src/integrations/nextjs/fixtures/inlined-data-escaped.html rename to crates/framework/nextjs/src/fixtures/inlined-data-escaped.html diff --git a/crates/trusted-server-core/src/integrations/nextjs/fixtures/inlined-data-nonce.html b/crates/framework/nextjs/src/fixtures/inlined-data-nonce.html similarity index 100% rename from crates/trusted-server-core/src/integrations/nextjs/fixtures/inlined-data-nonce.html rename to crates/framework/nextjs/src/fixtures/inlined-data-nonce.html diff --git a/crates/trusted-server-core/src/integrations/nextjs/mod.rs b/crates/framework/nextjs/src/lib.rs similarity index 87% rename from crates/trusted-server-core/src/integrations/nextjs/mod.rs rename to crates/framework/nextjs/src/lib.rs index e80c82d2f0..4c06663909 100644 --- a/crates/trusted-server-core/src/integrations/nextjs/mod.rs +++ b/crates/framework/nextjs/src/lib.rs @@ -1,27 +1,41 @@ +#![cfg_attr( + test, + allow( + clippy::print_stdout, + clippy::print_stderr, + clippy::panic, + clippy::dbg_macro, + clippy::unwrap_used, + reason = "tests use direct diagnostics and panic-on-failure helpers" + ) +)] use std::sync::Arc; use error_stack::Report; use serde::{Deserialize, Serialize}; use validator::Validate; -use crate::error::TrustedServerError; -use crate::integrations::IntegrationRegistration; -use crate::settings::{IntegrationConfig, Settings}; +use trusted_server_core::error::TrustedServerError; +use trusted_server_core::integrations::IntegrationRegistration; +use trusted_server_core::settings::{IntegrationConfig, Settings}; const NEXTJS_INTEGRATION_ID: &str = "nextjs"; /// The name this module is selected by, in `[framework]`. pub const MODULE: &str = "framework.nextjs"; -/// The builder the registry runs when a section selects [`MODULE`]. -pub(crate) const BUILDER: crate::integrations::IntegrationBuilder = - crate::integrations::IntegrationBuilder::new( +/// The builder a deployment hands to an adapter, which the registry runs when +/// a section selects [`MODULE`]. +#[must_use] +pub fn builder() -> trusted_server_core::integrations::IntegrationBuilder { + trusted_server_core::integrations::IntegrationBuilder::new( NEXTJS_INTEGRATION_ID, - crate::integrations::CORE_SOURCE, + env!("CARGO_PKG_NAME"), register, validate, ) - .with_module_name(MODULE); + .with_module_name(MODULE) +} mod rsc; mod rsc_placeholders; @@ -40,7 +54,7 @@ use script_rewriter::NextJsNextDataRewriter; pub struct NextJsIntegrationConfig { #[serde( default = "default_rewrite_attributes", - deserialize_with = "crate::settings::vec_from_seq_or_map" + deserialize_with = "trusted_server_core::settings::vec_from_seq_or_map" )] #[validate(length(min = 1))] pub rewrite_attributes: Vec, @@ -58,7 +72,7 @@ fn default_max_combined_payload_bytes() -> usize { 10 * 1024 * 1024 } -pub(super) fn configuration_error(message: impl Into) -> Report { +pub(crate) fn configuration_error(message: impl Into) -> Report { Report::new(TrustedServerError::Configuration { message: format!( "Integration '{NEXTJS_INTEGRATION_ID}' configuration error: {}", @@ -129,12 +143,14 @@ fn build( mod tests { use super::rsc_placeholders::RSC_PAYLOAD_PLACEHOLDER_PREFIX; use super::*; - use crate::html_processor::{HtmlProcessorConfig, create_html_processor}; - use crate::integrations::IntegrationRegistry; - use crate::streaming_processor::{Compression, PipelineConfig, StreamingPipeline}; - use crate::test_support::tests::create_test_settings; use serde_json::json; use std::io::Cursor; + use trusted_server_core::html_processor::{HtmlProcessorConfig, create_html_processor}; + use trusted_server_core::integrations::IntegrationRegistry; + use trusted_server_core::streaming_processor::{ + Compression, PipelineConfig, StreamingPipeline, + }; + use trusted_server_core::test_support::tests::create_test_settings; fn config_from_settings( settings: &Settings, @@ -167,14 +183,8 @@ mod tests { }), ) .expect("should update nextjs config"); - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) - .expect("should create registry"); + let registry = IntegrationRegistry::with_registrations(&settings, &[builder()]) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); let pipeline_config = PipelineConfig { @@ -259,14 +269,8 @@ mod tests { }), ) .expect("should update nextjs config"); - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) - .expect("should create registry"); + let registry = IntegrationRegistry::with_registrations(&settings, &[builder()]) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); let pipeline_config = PipelineConfig { @@ -335,14 +339,8 @@ mod tests { }), ) .expect("should update nextjs config"); - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) - .expect("should create registry"); + let registry = IntegrationRegistry::with_registrations(&settings, &[builder()]) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); let pipeline_config = PipelineConfig { @@ -387,14 +385,8 @@ mod tests { }), ) .expect("should update nextjs config"); - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) - .expect("should create registry"); + let registry = IntegrationRegistry::with_registrations(&settings, &[builder()]) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); let pipeline_config = PipelineConfig { @@ -442,14 +434,8 @@ mod tests { }), ) .expect("should update nextjs config"); - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) - .expect("should create registry"); + let registry = IntegrationRegistry::with_registrations(&settings, &[builder()]) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); let pipeline_config = PipelineConfig { @@ -511,14 +497,8 @@ mod tests { ) .expect("should update nextjs config"); - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) - .expect("should create registry"); + let registry = IntegrationRegistry::with_registrations(&settings, &[builder()]) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); let pipeline_config = PipelineConfig { @@ -586,14 +566,8 @@ mod tests { }), ) .expect("should update nextjs config"); - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) - .expect("should create registry"); + let registry = IntegrationRegistry::with_registrations(&settings, &[builder()]) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); // Use small chunk size to force fragmentation @@ -653,14 +627,8 @@ mod tests { }), ) .expect("should update nextjs config"); - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) - .expect("should create registry"); + let registry = IntegrationRegistry::with_registrations(&settings, &[builder()]) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); @@ -714,14 +682,8 @@ mod tests { }), ) .expect("should update nextjs config"); - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) - .expect("should create registry"); + let registry = IntegrationRegistry::with_registrations(&settings, &[builder()]) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); @@ -777,14 +739,8 @@ mod tests { }), ) .expect("should update nextjs config"); - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) - .expect("should create registry"); + let registry = IntegrationRegistry::with_registrations(&settings, &[builder()]) + .expect("should create registry"); let processor = create_html_processor(config_from_settings(&settings, ®istry)); let mut pipeline = StreamingPipeline::new( PipelineConfig { @@ -890,14 +846,8 @@ mod tests { }), ) .expect("should update nextjs config"); - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) - .expect("should create registry"); + let registry = IntegrationRegistry::with_registrations(&settings, &[builder()]) + .expect("should create registry"); let processor = create_html_processor(config_from_settings(&settings, ®istry)); let mut pipeline = StreamingPipeline::new( PipelineConfig { @@ -942,4 +892,57 @@ mod tests { ); } } + + #[test] + fn module_constant_is_the_crate_folder() { + assert_eq!( + super::MODULE, + trusted_server_core::module_name!(), + "should be named by the folder this crate lives in" + ); + } + + #[test] + fn output_overflow_restores_an_in_progress_script_at_every_split() { + use trusted_server_core::streaming_processor::StreamProcessor as _; + + let mut settings = create_test_settings(); + settings + .insert_module_config( + "framework", + MODULE, + &json!({"max_combined_payload_bytes": 128}), + ) + .expect("should select the nextjs integration"); + let registry = IntegrationRegistry::with_registrations(&settings, &[builder()]) + .expect("should create registry"); + let first = r#""#; + let script = r#"self.__next_f.push([1,"c"])"#; + let padding = "x".repeat(129); + let expected = format!("{first}{padding}"); + + for split in 1..script.len() { + let mut processor = create_html_processor(config_from_settings(&settings, ®istry)); + let mut output = processor + .process_chunk(first.as_bytes(), false) + .expect("should process unresolved RSC group"); + let second = format!("{padding}", &script[split..]); + output.extend( + processor + .process_chunk(third.as_bytes(), true) + .expect("should finish bypassed script"), + ); + assert_eq!( + String::from_utf8(output).expect("should retain UTF-8"), + expected, + "should restore all original bytes when overflow occurs at script split {split}" + ); + } + } } diff --git a/crates/trusted-server-core/src/integrations/nextjs/rsc.rs b/crates/framework/nextjs/src/rsc.rs similarity index 100% rename from crates/trusted-server-core/src/integrations/nextjs/rsc.rs rename to crates/framework/nextjs/src/rsc.rs diff --git a/crates/trusted-server-core/src/integrations/nextjs/rsc_placeholders.rs b/crates/framework/nextjs/src/rsc_placeholders.rs similarity index 99% rename from crates/trusted-server-core/src/integrations/nextjs/rsc_placeholders.rs rename to crates/framework/nextjs/src/rsc_placeholders.rs index 2e4268b860..08d81b3932 100644 --- a/crates/trusted-server-core/src/integrations/nextjs/rsc_placeholders.rs +++ b/crates/framework/nextjs/src/rsc_placeholders.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use crate::integrations::{ +use trusted_server_core::integrations::{ IntegrationScriptContext, IntegrationScriptRewriter, ScriptRewriteAction, }; @@ -344,8 +344,8 @@ fn remember_released(context: &mut String, released: &str) { #[cfg(test)] mod tests { use super::*; - use crate::integrations::IntegrationDocumentState; - use crate::integrations::nextjs::rsc_stream::NextJsDocumentState; + use crate::rsc_stream::NextJsDocumentState; + use trusted_server_core::integrations::IntegrationDocumentState; fn ctx( is_last_in_text_node: bool, diff --git a/crates/trusted-server-core/src/integrations/nextjs/rsc_stream.rs b/crates/framework/nextjs/src/rsc_stream.rs similarity index 99% rename from crates/trusted-server-core/src/integrations/nextjs/rsc_stream.rs rename to crates/framework/nextjs/src/rsc_stream.rs index ab12e13f1d..c927e70585 100644 --- a/crates/trusted-server-core/src/integrations/nextjs/rsc_stream.rs +++ b/crates/framework/nextjs/src/rsc_stream.rs @@ -2,10 +2,10 @@ use std::collections::VecDeque; use std::io; use std::sync::{Arc, Mutex}; -use crate::integrations::{ +use trusted_server_core::integrations::{ IntegrationDocumentState, IntegrationHtmlStreamContext, IntegrationHtmlStreamProcessorFactory, }; -use crate::streaming_processor::StreamProcessor; +use trusted_server_core::streaming_processor::StreamProcessor; use super::rsc::{ DEFAULT_MAX_COMBINED_PAYLOAD_BYTES, PendingTChunk, TChunkStep, next_tchunk, @@ -869,7 +869,7 @@ impl SegmentInspector { #[cfg(test)] mod tests { use super::*; - use crate::integrations::nextjs::rsc::{TChunkScan, scan_tchunks}; + use crate::rsc::{TChunkScan, scan_tchunks}; // Keep the reference independent of the incremental classifier: discover all // chunks first, then inspect each non-chunk segment from scratch. diff --git a/crates/trusted-server-core/src/integrations/nextjs/script_rewriter.rs b/crates/framework/nextjs/src/script_rewriter.rs similarity index 99% rename from crates/trusted-server-core/src/integrations/nextjs/script_rewriter.rs rename to crates/framework/nextjs/src/script_rewriter.rs index 273aa384fd..37de438f84 100644 --- a/crates/trusted-server-core/src/integrations/nextjs/script_rewriter.rs +++ b/crates/framework/nextjs/src/script_rewriter.rs @@ -3,8 +3,8 @@ use std::sync::Arc; use error_stack::Report; use regex::{Regex, escape}; -use crate::error::TrustedServerError; -use crate::integrations::{ +use trusted_server_core::error::TrustedServerError; +use trusted_server_core::integrations::{ IntegrationScriptContext, IntegrationScriptRewriter, ScriptRewriteAction, }; @@ -208,8 +208,8 @@ impl UrlRewriter { #[cfg(test)] mod tests { use super::*; - use crate::integrations::IntegrationDocumentState; - use crate::integrations::ScriptRewriteAction; + use trusted_server_core::integrations::IntegrationDocumentState; + use trusted_server_core::integrations::ScriptRewriteAction; fn test_config() -> Arc { Arc::new(NextJsIntegrationConfig { diff --git a/crates/trusted-server-core/src/integrations/nextjs/shared.rs b/crates/framework/nextjs/src/shared.rs similarity index 99% rename from crates/trusted-server-core/src/integrations/nextjs/shared.rs rename to crates/framework/nextjs/src/shared.rs index e4349fa50a..1ae78a7cab 100644 --- a/crates/trusted-server-core/src/integrations/nextjs/shared.rs +++ b/crates/framework/nextjs/src/shared.rs @@ -7,7 +7,7 @@ use std::sync::LazyLock; use regex::Regex; -use crate::host_rewrite::rewrite_bare_host_at_boundaries; +use trusted_server_core::host_rewrite::rewrite_bare_host_at_boundaries; // These are static code-defined literals, not config-derived patterns, so they // intentionally remain lazy statics instead of participating in diff --git a/crates/tag/google-tag-manager/Cargo.toml b/crates/tag/google-tag-manager/Cargo.toml index 2ef9759fea..b9bb1952e0 100644 --- a/crates/tag/google-tag-manager/Cargo.toml +++ b/crates/tag/google-tag-manager/Cargo.toml @@ -29,6 +29,9 @@ validator = { workspace = true } [dev-dependencies] serde_json = { workspace = true } trusted-server-core = { workspace = true, features = ["test-utils"] } +# Two tests check that this module leaves another module's script rewrite +# alone, and Next.js is the module they run beside. +trusted-server-framework-nextjs = { workspace = true } # The visible owner of this vendor crate, the way Prebid.js requires a named # maintainer of every adapter. Google has not yet adopted this crate, so the diff --git a/crates/tag/google-tag-manager/src/lib.rs b/crates/tag/google-tag-manager/src/lib.rs index 535405abc1..b739d9b67f 100644 --- a/crates/tag/google-tag-manager/src/lib.rs +++ b/crates/tag/google-tag-manager/src/lib.rs @@ -3779,8 +3779,12 @@ assume_single_jurisdiction = true ) .expect("should update nextjs config"); - let registry = IntegrationRegistry::with_registrations(&settings, &[builder()]) - .expect("should create registry"); + // Next.js first, as a stock build orders the two. + let registry = IntegrationRegistry::with_registrations( + &settings, + &[trusted_server_framework_nextjs::builder(), builder()], + ) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); @@ -3846,8 +3850,12 @@ assume_single_jurisdiction = true ) .expect("should update nextjs config"); - let registry = IntegrationRegistry::with_registrations(&settings, &[builder()]) - .expect("should create registry"); + // Next.js first, as a stock build orders the two. + let registry = IntegrationRegistry::with_registrations( + &settings, + &[trusted_server_framework_nextjs::builder(), builder()], + ) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index 7612c4d7ee..dd9becf1f2 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -1939,53 +1939,6 @@ mod tests { } } - #[test] - fn nextjs_output_overflow_restores_in_progress_script_at_every_split() { - let mut settings = create_test_settings(); - settings - .insert_module_config( - "framework", - "framework.nextjs", - &json!({"max_combined_payload_bytes": 128}), - ) - .expect("should select the nextjs integration"); - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new(crate::auction::compile_auction_plan(&settings).expect("should compile plan")), - ) - .expect("should create registry"); - let first = r#""#; - let script = r#"self.__next_f.push([1,"c"])"#; - let padding = "x".repeat(129); - let expected = format!("{first}{padding}"); - - for split in 1..script.len() { - let mut config = create_test_config(); - config.integrations = registry.clone(); - let mut processor = create_html_processor(config); - let mut output = processor - .process_chunk(first.as_bytes(), false) - .expect("should process unresolved RSC group"); - let second = format!("{padding}", &script[split..]); - output.extend( - processor - .process_chunk(third.as_bytes(), true) - .expect("should finish bypassed script"), - ); - assert_eq!( - String::from_utf8(output).expect("should retain UTF-8"), - expected, - "should restore all original bytes when overflow occurs at script split {split}" - ); - } - } - #[test] fn a_nonce_bearing_meta_policy_is_observed() { let observed = Arc::new(AtomicBool::new(false)); diff --git a/crates/trusted-server-core/src/integrations/mod.rs b/crates/trusted-server-core/src/integrations/mod.rs index d035496555..458c2e3c96 100644 --- a/crates/trusted-server-core/src/integrations/mod.rs +++ b/crates/trusted-server-core/src/integrations/mod.rs @@ -19,7 +19,6 @@ pub mod datadome; pub mod gpt; pub mod gpt_diagnostics; pub mod js_asset_proxy; -pub mod nextjs; pub mod openrtb; pub mod prebid; pub mod prebid_server; @@ -572,7 +571,10 @@ const BUILT_IN_BUILDERS: &[IntegrationBuilder] = &[ // This must remain first: attribute rewriters chain replacements and // short-circuit removals. js_asset_proxy::BUILDER, - nextjs::BUILDER, + // A stand-in for an integration that streams, which core's own tests + // select where they need one. + #[cfg(test)] + registry_test_support::payload_fixture::BUILDER, datadome::BUILDER, gpt::BUILDER, gpt_diagnostics::BUILDER, diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index fadec1ea5f..a95df6599e 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -2345,6 +2345,280 @@ pub(crate) mod test_support { ) -> Result> { Ok(true) } + + /// A stand-in for an integration that rewrites script payloads in two + /// passes, for core's own tests of the page pipeline. + /// + /// The script rewriter swaps the payload of each `fixture_payload("...")` + /// call for a placeholder that carries a namespace made per document, and + /// keeps the payload in the document state. The stream processor swaps + /// each placeholder back for its payload with the origin host rewritten. + /// A payload pushed with `fixture_payload_open` leaves its group + /// unresolved, and the processor holds its output from that placeholder + /// on until a `fixture_payload_close` arrives. + pub(crate) mod payload_fixture { + use std::io; + use std::sync::{Arc, Mutex, PoisonError}; + + use error_stack::Report; + + use crate::error::TrustedServerError; + use crate::integrations::registry::{ + IntegrationHtmlStreamContext, IntegrationHtmlStreamProcessorFactory, + IntegrationRegistration, IntegrationScriptContext, IntegrationScriptRewriter, + ScriptRewriteAction, ScriptTextAccumulator, + }; + use crate::integrations::{CORE_SOURCE, IntegrationBuilder}; + use crate::settings::Settings; + use crate::streaming_processor::StreamProcessor; + + /// The integration id the stand-in registers under. + pub(crate) const ID: &str = "payload_fixture"; + /// The name a test's settings select the stand-in by, in `[testing]`. + pub(crate) const MODULE: &str = "testing.payload-fixture"; + /// What every placeholder starts with, so a test can check that none + /// reaches a reader. + pub(crate) const PLACEHOLDER_PREFIX: &str = "__ts_fixture_"; + const PLACEHOLDER_END: &str = "__"; + + /// The builder core's test build lists beside its own. + pub(crate) const BUILDER: IntegrationBuilder = + IntegrationBuilder::new(ID, CORE_SOURCE, register, validate).with_module_name(MODULE); + + struct Payload { + placeholder: String, + original: String, + /// Whether the group this payload opened is still waiting for its + /// close. + unresolved: bool, + } + + /// What one document's script rewriter has captured so far. + struct Captured { + namespace: String, + payloads: Vec, + } + + impl Default for Captured { + fn default() -> Self { + Self { + namespace: uuid::Uuid::new_v4().simple().to_string(), + payloads: Vec::new(), + } + } + } + + fn captured( + state: &crate::integrations::registry::IntegrationDocumentState, + ) -> Arc> { + state.get_or_insert_with(ID, || Mutex::new(Captured::default())) + } + + struct ScriptRewriter; + + impl ScriptRewriter { + /// Swaps the payload of a whole script for a placeholder, or + /// leaves a script that pushes no payload as it is. + fn rewrite_whole(script: &str, ctx: &IntegrationScriptContext<'_>) -> Option { + let (call, opens, closes) = [ + ("fixture_payload_open(\"", true, false), + ("fixture_payload_close(\"", false, true), + ("fixture_payload(\"", false, false), + ] + .into_iter() + .find(|(call, _, _)| script.contains(*call))?; + let start = script.find(call)? + call.len(); + let end = script.rfind("\")")?; + if end < start { + return None; + } + + let shared = captured(ctx.document_state); + let mut captured = shared.lock().unwrap_or_else(PoisonError::into_inner); + let placeholder = format!( + "{PLACEHOLDER_PREFIX}{}_{}{PLACEHOLDER_END}", + captured.namespace, + captured.payloads.len() + ); + if closes { + for payload in &mut captured.payloads { + payload.unresolved = false; + } + } + captured.payloads.push(Payload { + placeholder: placeholder.clone(), + original: script[start..end].to_owned(), + unresolved: opens, + }); + + let mut rewritten = script.to_owned(); + rewritten.replace_range(start..end, &placeholder); + Some(rewritten) + } + } + + impl IntegrationScriptRewriter for ScriptRewriter { + fn integration_id(&self) -> &'static str { + ID + } + + fn selector(&self) -> &'static str { + "script" + } + + fn rewrite( + &self, + content: &str, + ctx: &IntegrationScriptContext<'_>, + ) -> ScriptRewriteAction { + let accumulator = ctx + .document_state + .get_or_insert_with(ID, ScriptTextAccumulator::default); + let mut buffer = accumulator.buffer(); + let claimed = !buffer.is_empty() || content.contains("fixture_payload"); + if !claimed { + return ScriptRewriteAction::Keep; + } + buffer.push_str(content); + if !ctx.is_last_in_text_node { + return ScriptRewriteAction::RemoveNode; + } + let script = std::mem::take(&mut *buffer); + let rewritten = Self::rewrite_whole(&script, ctx).unwrap_or(script); + ScriptRewriteAction::replace(rewritten) + } + } + + struct StreamFactory; + + impl IntegrationHtmlStreamProcessorFactory for StreamFactory { + fn integration_id(&self) -> &'static str { + ID + } + + fn create(&self, context: IntegrationHtmlStreamContext) -> Box { + Box::new(Processor { + context, + held: Vec::new(), + }) + } + } + + struct Processor { + context: IntegrationHtmlStreamContext, + /// Output not yet released, because it ends inside a placeholder + /// or starts at one whose group is unresolved. + held: Vec, + } + + fn find(haystack: &[u8], needle: &[u8], from: usize) -> Option { + haystack + .get(from..)? + .windows(needle.len()) + .position(|window| window == needle) + .map(|at| at + from) + } + + impl StreamProcessor for Processor { + fn process_chunk(&mut self, chunk: &[u8], is_last: bool) -> Result, io::Error> { + self.held.extend_from_slice(chunk); + let shared = captured(&self.context.document_state); + let captured = shared.lock().unwrap_or_else(PoisonError::into_inner); + let prefix = PLACEHOLDER_PREFIX.as_bytes(); + let mut out = Vec::with_capacity(self.held.len()); + let mut at = 0; + + while let Some(start) = find(&self.held, prefix, at) { + let Some(end) = + find(&self.held, PLACEHOLDER_END.as_bytes(), start + prefix.len()) + else { + // The placeholder runs past this chunk. + out.extend_from_slice(&self.held[at..start]); + at = start; + break; + }; + let end = end + PLACEHOLDER_END.len(); + let found = std::str::from_utf8(&self.held[start..end]) + .ok() + .and_then(|text| { + captured + .payloads + .iter() + .find(|payload| payload.placeholder == text) + }); + match found { + Some(payload) if payload.unresolved && !is_last => { + out.extend_from_slice(&self.held[at..start]); + at = start; + break; + } + Some(payload) => { + out.extend_from_slice(&self.held[at..start]); + out.extend_from_slice( + payload + .original + .replace(&self.context.origin_host, &self.context.request_host) + .as_bytes(), + ); + at = end; + } + None => { + out.extend_from_slice(&self.held[at..end]); + at = end; + } + } + } + + let stopped_at_placeholder = find(&self.held, prefix, at) == Some(at); + if is_last { + out.extend_from_slice(&self.held[at..]); + self.held.clear(); + } else if stopped_at_placeholder { + self.held.drain(..at); + } else { + // Keep back a tail that could be the start of a + // placeholder split across chunks. + let rest = &self.held[at..]; + let keep = (1..prefix.len().min(rest.len() + 1)) + .rev() + .find(|len| rest.ends_with(&prefix[..*len])) + .unwrap_or(0); + out.extend_from_slice(&rest[..rest.len() - keep]); + let tail = rest[rest.len() - keep..].to_vec(); + self.held = tail; + } + Ok(out) + } + } + + fn register( + settings: &Settings, + ) -> Result, Report> { + if settings.module_config::(MODULE)?.is_none() { + return Ok(None); + } + Ok(Some( + IntegrationRegistration::builder(ID) + .with_script_rewriter(Arc::new(ScriptRewriter)) + .with_html_stream_processor(Arc::new(StreamFactory)) + .build(), + )) + } + + /// The stand-in takes no settings, and refuses one it does not know + /// as any module does. + #[derive(Debug, serde::Deserialize, validator::Validate)] + #[serde(deny_unknown_fields)] + struct FixtureSettings {} + + impl crate::settings::IntegrationConfig for FixtureSettings {} + + fn validate(settings: &Settings) -> Result> { + settings + .module_config::(MODULE) + .map(|config| config.is_some()) + } + } } #[cfg(test)] @@ -3285,29 +3559,27 @@ mod tests { #[test] fn js_module_ids_skip_named_integrations_without_generated_js_module() { - let mut settings = crate::test_support::tests::create_test_settings(); - settings.select_module("framework", "framework.nextjs"); - - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), + let settings = settings_naming("testing.probe"); + let extra = [crate::integrations::IntegrationBuilder::new( + "probe", + "seam-probe", + probe_registration, + validate_nothing, ) - .expect("should create registry"); + .with_module_name("testing.probe")]; + + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should create registry"); let all = registry.js_module_ids(); assert!( - !all.contains(&"nextjs"), + !all.contains(&"probe"), "should not include named integrations without generated JS modules" ); let metadata = registry.registered_integrations(); assert!( - metadata - .iter() - .any(|integration| integration.id == "nextjs"), + metadata.iter().any(|integration| integration.id == "probe"), "should still register named Rust-only integrations" ); } diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 4f64eefd6a..ad3e460324 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -13027,30 +13027,25 @@ mod tests { } #[tokio::test] - async fn nextjs_stored_templates_are_identical_across_request_shapes() { + async fn two_pass_integration_templates_are_identical_across_request_shapes() { let mut settings = settings_with_mode("esi"); settings.publisher.origin_url = "https://origin.example.com".to_owned(); - settings - .insert_module_config( - "framework", - "framework.nextjs", - &serde_json::json!({ - "rewrite_attributes": ["href", "link", "url"], - }), - ) - .expect("should enable Next.js processing"); + settings.select_module( + "testing", + crate::integrations::registry_test_support::payload_fixture::MODULE, + ); let registry = IntegrationRegistry::new(&settings).expect("should create registry"); assert!( !registry.html_stream_processor_factories().is_empty(), - "should exercise the Next.js stream processor" + "should exercise the stand-in's stream processor" ); let settings = Arc::new(settings); - let html = br#"
"#; + let html = br#"
"#; for finalizer in [Finalizer::Streaming, Finalizer::Buffered] { let mut templates = Vec::new(); for request in [navigation_request(), prefetch_navigation_request()] { - // Independent fills force distinct per-request RSC namespaces. + // Independent fills force distinct per-document namespaces. let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); let services = services(Arc::clone(&stub), Arc::clone(&cache)); @@ -13067,11 +13062,7 @@ mod tests { let _ = body_of(response).await; let entries = cache.entries.lock().expect("should lock stored templates"); - assert_eq!( - entries.len(), - 1, - "should store a Next.js template on every fill" - ); + assert_eq!(entries.len(), 1, "should store a template on every fill"); let template = entries .values() .next() @@ -13081,14 +13072,14 @@ mod tests { let text = core::str::from_utf8(&template).expect("should store UTF-8 HTML"); assert!( text.contains("ts.example.com/page"), - "should rewrite the RSC URL" + "should rewrite the URL in the payload" ); assert!( text.contains(AD_ASSEMBLY_SEAM), "should store before per-reader assembly" ); for marker in [ - "__ts_rsc_", + crate::integrations::registry_test_support::payload_fixture::PLACEHOLDER_PREFIX, ".adSlots", ".bids=", "gpt-diagnostics", @@ -13103,7 +13094,7 @@ mod tests { } assert_eq!( templates[0], templates[1], - "should store identical bytes across request shapes and RSC namespaces" + "should store identical bytes across request shapes and per-document namespaces" ); } } @@ -21573,17 +21564,12 @@ mod tests { fn streaming_finalize_auction_hold_emits_prefix_before_origin_eof() { // A body-close literal in script data must not stop streaming. Only the // request token emitted by lol_html at the structural end is a seam. - let page = br#"
still streaming
"#; + let page = br#"
still streaming
"#; let mut settings = create_test_settings(); - settings - .insert_module_config( - "framework", - "framework.nextjs", - &serde_json::json!({ - "rewrite_attributes": ["href", "link", "url"], - }), - ) - .expect("should enable Next.js"); + settings.select_module( + "testing", + crate::integrations::registry_test_support::payload_fixture::MODULE, + ); let params = html_stream_params( "", Some(DispatchedAuction::empty_for_test( @@ -21601,11 +21587,11 @@ mod tests { let html = String::from_utf8(first.to_vec()).expect("should be valid UTF-8"); assert!( html.contains("") && html.contains("still streaming"), - "RSC script data and later article bytes must stream before EOF. Got: {html}" + "script data and later article bytes must stream before EOF. Got: {html}" ); assert!( html.contains("proxy.example.com/app") && !html.contains("origin.example.com/app"), - "Next.js rewriting must complete before the parser seam: {html}" + "the stand-in's rewriting must complete before the parser seam: {html}" ); assert!( html.contains(".adSlots=JSON.parse"), @@ -21751,21 +21737,16 @@ mod tests { } #[test] - fn parser_confirmed_auction_seam_streams_nextjs_for_every_encoding() { + fn parser_confirmed_auction_seam_streams_a_two_pass_integration_for_every_encoding() { for encoding in ["", "gzip", "deflate", "br"] { let mut settings = create_test_settings(); settings.auction.enabled = true; settings.auction.provider_names = vec!["seam_test".to_owned()]; settings.auction.timeout_ms = 60_000; - settings - .insert_module_config( - "framework", - "framework.nextjs", - &serde_json::json!({ - "rewrite_attributes": ["href", "link", "url"], - }), - ) - .expect("should select Next.js"); + settings.select_module( + "testing", + crate::integrations::registry_test_support::payload_fixture::MODULE, + ); let client = Arc::new(GatedAuctionHttpClient { inner: StubHttpClient::new(), released: std::sync::atomic::AtomicBool::new(false), @@ -21792,15 +21773,10 @@ mod tests { else { panic!("should dispatch a pending auction"); }; - let payload = r#"{"url":"https://origin.example.com/path","text":""}"#; - let split = payload.find("/path").expect("should find payload split"); - let first_payload = format!("1:T{:x},{}", payload.len(), &payload[..split]); - let first_script = - serde_json::to_string(&first_payload).expect("should encode first payload"); - let second_script = - serde_json::to_string(&payload[split..]).expect("should encode second payload"); - let prefix = format!( - "

before RSC

between scripts
still streaming
" + // One group split across two scripts, so the stand-in holds its + // output from the first script until the second arrives. + let prefix = String::from( + "

before the group

between scripts
still streaming
", ); let page = format!("{prefix}"); let encoded = match encoding { @@ -21863,7 +21839,7 @@ mod tests { ); assert!( prefix.contains("proxy.example.com") && !prefix.contains("origin.example.com"), - "should rewrite the split RSC group for {encoding}: {prefix}" + "should rewrite the split group for {encoding}: {prefix}" ); assert!( !prefix.contains("var b=JSON.parse("), @@ -21911,7 +21887,8 @@ mod tests { "should inject once" ); assert!( - !html.contains("ts-inline-body-close-") && !html.contains("__ts_rsc_"), + !html.contains("ts-inline-body-close-") + && !html.contains(crate::integrations::registry_test_support::payload_fixture::PLACEHOLDER_PREFIX), "should remove internal markers for {encoding}: {html}" ); assert_eq!( @@ -22577,17 +22554,12 @@ mod tests { /// routes through `Stream`, and the shared processor pipeline applies it. #[test] fn streaming_html_with_stream_processors_rewrites_body() { - // Configure nextjs so a stream processor is registered. + // Select the stand-in so a stream processor is registered. let mut settings = create_test_settings(); - settings - .insert_module_config( - "framework", - "framework.nextjs", - &serde_json::json!({ - "rewrite_attributes": ["href", "link", "url"], - }), - ) - .expect("should update nextjs config"); + settings.select_module( + "testing", + crate::integrations::registry_test_support::payload_fixture::MODULE, + ); let registry = IntegrationRegistry::with_plan( &settings, @@ -22600,7 +22572,7 @@ mod tests { assert!( !registry.html_stream_processor_factories().is_empty(), - "nextjs integration must register an HTML stream processor" + "the stand-in must register an HTML stream processor" ); assert_eq!( classify_response_route( @@ -22659,21 +22631,16 @@ mod tests { } /// Document-state survives from the parser pass into the stream processor. - /// `NextJsRscPlaceholderRewriter` writes into `IntegrationDocumentState` - /// during parsing; the request-local stream processor reads it and substitutes. + /// A script rewriter writes into `IntegrationDocumentState` during + /// parsing, and the request-local stream processor reads it and substitutes. /// Regression test: placeholders must be inserted and removed from final output. #[test] fn document_state_placeholders_substitute_through_streaming_path() { let mut settings = create_test_settings(); - settings - .insert_module_config( - "framework", - "framework.nextjs", - &serde_json::json!({ - "rewrite_attributes": ["href", "link", "url"], - }), - ) - .expect("should update nextjs config"); + settings.select_module( + "testing", + crate::integrations::registry_test_support::payload_fixture::MODULE, + ); let registry = IntegrationRegistry::with_plan( &settings, Arc::new( @@ -22683,8 +22650,8 @@ mod tests { ) .expect("should create integration registry"); - // Small, single-fragment RSC script — placeholder path (not fallback). - let html = br#""#; + // One small script whose payload is swapped for a placeholder. + let html = br#""#; let params = OwnedProcessResponseParams { csp_nonce_observed: None, template_cache_key: None, @@ -22715,11 +22682,13 @@ mod tests { &settings, ®istry, ) - .expect("should process RSC push"); + .expect("should process the payload script"); let processed = String::from_utf8(output).expect("valid UTF-8"); assert!( - !processed.contains("__ts_rsc_payload_"), + !processed.contains( + crate::integrations::registry_test_support::payload_fixture::PLACEHOLDER_PREFIX + ), "placeholder must be substituted before reaching output. Got: {processed}" ); assert!( diff --git a/crates/trusted-server-core/src/rsc_flight.rs b/crates/trusted-server-core/src/rsc_flight.rs index a9e3a40cec..dac4a4b6ae 100644 --- a/crates/trusted-server-core/src/rsc_flight.rs +++ b/crates/trusted-server-core/src/rsc_flight.rs @@ -29,7 +29,7 @@ enum RowState { /// URLs like `\/\/origin.example.com` (JSON-escaped slashes) will not be rewritten. This is /// acceptable because Flight responses from client-side navigation typically contain plain URLs, /// not doubly-escaped JSON-in-JS content. For inlined `__next_f` data in HTML (which can have -/// escape sequences), the HTML post-processor in `integrations/nextjs/` handles those cases. +/// escape sequences), the HTML post-processor in `crates/framework/nextjs` handles those cases. pub struct RscFlightUrlRewriter { origin_url: String, origin_http_url: Option, diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index e214e93f7e..c44167a790 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -4985,9 +4985,7 @@ mod tests { ); } - use crate::integrations::{ - IntegrationRegistry, nextjs::NextJsIntegrationConfig, prebid::PrebidIntegrationConfig, - }; + use crate::integrations::{IntegrationRegistry, prebid::PrebidIntegrationConfig}; use crate::redacted::Redacted; use crate::test_support::tests::{ crate_test_settings_str, crate_test_settings_str_with_ec_section, create_test_settings, @@ -6040,8 +6038,8 @@ module = \"none\"", assert_eq!(prebid_cfg.timeout_ms, 1000); assert!( settings - .module_config::(crate::integrations::nextjs::MODULE) - .expect("Next.js config query should succeed") + .module_config::(ENDPOINT_MODULE) + .expect("a query for a module no section selects should succeed") .is_none(), "a module no section selects should not run" ); diff --git a/crates/trusted-server-core/src/test_support.rs b/crates/trusted-server-core/src/test_support.rs index de594be3ae..cf58cb065d 100644 --- a/crates/trusted-server-core/src/test_support.rs +++ b/crates/trusted-server-core/src/test_support.rs @@ -293,7 +293,7 @@ pub mod nextjs_auction { settings .insert_module_config( "framework", - crate::integrations::nextjs::MODULE, + "framework.nextjs", &serde_json::json!({ "rewrite_attributes": ["href", "link", "url"], }), diff --git a/crates/trusted-server-modules/Cargo.toml b/crates/trusted-server-modules/Cargo.toml index ba61a493f1..d47991d00b 100644 --- a/crates/trusted-server-modules/Cargo.toml +++ b/crates/trusted-server-modules/Cargo.toml @@ -19,6 +19,7 @@ trusted-server-cmp-didomi = { workspace = true } trusted-server-cmp-osano = { workspace = true } trusted-server-cmp-sourcepoint = { workspace = true } trusted-server-core = { workspace = true } +trusted-server-framework-nextjs = { workspace = true } trusted-server-identity-lockr = { workspace = true } trusted-server-tag-google-tag-manager = { workspace = true } trusted-server-testing-testlight = { workspace = true } diff --git a/crates/trusted-server-modules/src/lib.rs b/crates/trusted-server-modules/src/lib.rs index db854342ee..2635ddd4a6 100644 --- a/crates/trusted-server-modules/src/lib.rs +++ b/crates/trusted-server-modules/src/lib.rs @@ -21,6 +21,7 @@ use trusted_server_core::integrations::IntegrationBuilder; pub fn builders() -> Vec { vec![ trusted_server_testing_testlight::builder(), + trusted_server_framework_nextjs::builder(), trusted_server_audience_permutive::builder(), trusted_server_identity_lockr::builder(), trusted_server_cmp_didomi::builder(), @@ -56,6 +57,7 @@ mod tests { names, [ "testing.testlight", + "framework.nextjs", "audience.permutive", "identity.lockr", "cmp.didomi", @@ -179,4 +181,43 @@ mod tests { assert!(valid, "{header} should resolve to a valid config"); } } + + /// Every stock page module refuses a setting it does not know, so a + /// misspelt key in its table fails deploy validation naming the table and + /// the key, where it would otherwise be ignored. + #[test] + fn every_stock_module_rejects_a_setting_it_does_not_know() { + use serde_json::json; + use trusted_server_core::config::validate_settings_for_deploy_with; + use trusted_server_core::module_name::short_form; + use trusted_server_core::test_support::tests::create_test_settings; + + let stock = builders(); + for builder in stock + .iter() + .filter(|builder| builder.supplies_integration()) + { + let name = builder + .module_name() + .expect("every stock page module names itself"); + let section = builder + .section() + .expect("every stock page module has a section"); + let mut settings = create_test_settings(); + settings + .insert_module_config(section, name, &json!({ "no_such_setting": true })) + .expect("should insert the planted table"); + + let error = match validate_settings_for_deploy_with(&settings, &stock) { + Ok(()) => panic!("`{name}` should refuse a setting it does not know"), + Err(error) => format!("{error:?}"), + }; + let written = short_form(section, name); + assert!( + error.contains(&format!("[{section}.{written}]")) + && error.contains("no_such_setting"), + "`{name}` should name its table and the unknown setting: {error}" + ); + } + } } diff --git a/docs/guide/integrations/nextjs.md b/docs/guide/integrations/nextjs.md index 026b977e03..7536de2d40 100644 --- a/docs/guide/integrations/nextjs.md +++ b/docs/guide/integrations/nextjs.md @@ -94,9 +94,9 @@ Next.js Pages Router embeds data in `__NEXT_DATA__` script: The Next.js integration is implemented across multiple files: -- [crates/trusted-server-core/src/integrations/nextjs/mod.rs](https://github.com/IABTechLab/trusted-server/blob/main/crates/trusted-server-core/src/integrations/nextjs/mod.rs) - Main integration -- [crates/trusted-server-core/src/integrations/nextjs/rsc.rs](https://github.com/IABTechLab/trusted-server/blob/main/crates/trusted-server-core/src/integrations/nextjs/rsc.rs) - RSC parsing -- [crates/trusted-server-core/src/integrations/nextjs/script_rewriter.rs](https://github.com/IABTechLab/trusted-server/blob/main/crates/trusted-server-core/src/integrations/nextjs/script_rewriter.rs) - Script rewriting +- [crates/framework/nextjs/src/lib.rs](https://github.com/IABTechLab/trusted-server/blob/main/crates/framework/nextjs/src/lib.rs) - Main integration +- [crates/framework/nextjs/src/rsc.rs](https://github.com/IABTechLab/trusted-server/blob/main/crates/framework/nextjs/src/rsc.rs) - RSC parsing +- [crates/framework/nextjs/src/script_rewriter.rs](https://github.com/IABTechLab/trusted-server/blob/main/crates/framework/nextjs/src/script_rewriter.rs) - Script rewriting ### Key Components diff --git a/docs/guide/rsc-hydration.md b/docs/guide/rsc-hydration.md index d6c9708d34..07a8dd444e 100644 --- a/docs/guide/rsc-hydration.md +++ b/docs/guide/rsc-hydration.md @@ -177,7 +177,7 @@ The rewrite logic handles multiple URL formats in RSC content: The current implementation lives in: -- `crates/trusted-server-core/src/integrations/nextjs/rsc.rs` +- `crates/framework/nextjs/src/rsc.rs` - `crates/trusted-server-core/src/rsc_flight.rs` ## Related Docs From 5f0e9c00f5cb23ed8de87e1208c65aed03bd9dc4 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 8 Oct 2026 01:46:40 +0100 Subject: [PATCH 094/124] Google Publisher Tags moves out of core The Google Publisher Tags module is `ad-tag.google`, and it now lives in `crates/ad-tag/google` with the bootstrap script it injects. The stock list offers it, and the settings that select and configure it do not change. Its diagnostics module, `ad-tag.google.diagnostics`, belongs to the same crate and stays in core until the page path can carry a module's own decision for a request. Five of core's tests used this module as their example of an integration that tags a page: the HTML processor's test of the attribute an injector asks for on the publisher bundle tag, the streaming test of the head arriving before the origin finishes, and the JavaScript asset proxy's three tests of its place ahead of an integration's own rewriter. They select a second stand-in compiled for core's tests only, which inserts a head script, asks for a bundle attribute and rewrites the address of its own script. The core test that a selected module with invalid settings fails startup uses an example module of its own. The browser tests and the initial render harness read the bootstrap script from its new place. --- .cargo/config.toml | 12 +- CHANGELOG.md | 2 +- Cargo.lock | 17 +++ Cargo.toml | 2 + crates/ad-tag/google/Cargo.toml | 36 ++++++ .../google/src}/gpt_bootstrap.js | 0 .../gpt.rs => ad-tag/google/src/lib.rs} | 54 +++++--- .../trusted-server-core/src/html_processor.rs | 47 +++---- .../src/integrations/js_asset_proxy.rs | 75 +++++++---- .../src/integrations/mod.rs | 5 +- .../src/integrations/registry.rs | 116 ++++++++++++++++++ crates/trusted-server-core/src/publisher.rs | 20 +-- crates/trusted-server-core/src/settings.rs | 43 ++++--- .../browser/initial-render/run.cjs | 6 +- .../lib/test/fixtures/paths.ts | 2 +- .../integrations/gpt/gpt_bootstrap.test.ts | 2 +- crates/trusted-server-modules/Cargo.toml | 1 + crates/trusted-server-modules/src/lib.rs | 2 + docs/guide/integrations/gpt.md | 2 +- 19 files changed, 345 insertions(+), 99 deletions(-) create mode 100644 crates/ad-tag/google/Cargo.toml rename crates/{trusted-server-core/src/integrations => ad-tag/google/src}/gpt_bootstrap.js (100%) rename crates/{trusted-server-core/src/integrations/gpt.rs => ad-tag/google/src/lib.rs} (97%) diff --git a/.cargo/config.toml b/.cargo/config.toml index 6cc71414f2..e61117d146 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -26,10 +26,10 @@ test_details = "test --target aarch64-apple-darwin" # native crate needs no change here. Axum (native), Cloudflare # (wasm32-unknown-unknown), Spin, the CLI (native), and integration-tests # (native) are simply not listed. -build-fastly = "build -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" -check-fastly = "check -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" -clippy-fastly = "clippy -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-js -p trusted-server-openrtb --all-targets --all-features --target wasm32-wasip1 -- -D warnings" -test-fastly = "test -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" +build-fastly = "build -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" +check-fastly = "check -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" +clippy-fastly = "clippy -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-js -p trusted-server-openrtb --all-targets --all-features --target wasm32-wasip1 -- -D warnings" +test-fastly = "test -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" # Feature-on counterpart of `test-fastly`. `test-fastly` does NOT pass # --all-features, so without this the reusable-sandbox code is linted by # `clippy-fastly` but its tests never execute. @@ -40,8 +40,8 @@ test-fastly-reuse = "test -p trusted-server-adapter-fastly --target wasm32-wasip # run with the Axum adapter in CI. build-axum = "build -p trusted-server-adapter-axum" check-axum = "check -p trusted-server-adapter-axum" -clippy-axum = "clippy -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs --all-targets --all-features -- -D warnings" -test-axum = "test -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs" +clippy-axum = "clippy -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google --all-targets --all-features -- -D warnings" +test-axum = "test -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google" # --- Cloudflare adapter (native host + wasm32-unknown-unknown) --- # Build/check target the WASM runtime (requires the `cloudflare` feature); diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ec838f67a..e4cddfad9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- Integration modules are moving out of `trusted-server-core` into crates of their own under `crates///`, each named by that folder, so `crates/cmp/osano` is the module `cmp.osano`. The settings that select and configure a module do not change. `crates/trusted-server-modules` lists the modules a stock build ships from crates of their own, in the order their hooks run. Every adapter builds its state and loads its settings with that list, and `ts config validate` and `ts config push` run each stock module's own rules. Code that named a moved module under `trusted_server_core::integrations` names its crate instead, such as `trusted_server_cmp_osano`. Moved so far: Osano, Lockr, Permutive, Sourcepoint, Didomi, Google Tag Manager, Testlight and Next.js.js. +- Integration modules are moving out of `trusted-server-core` into crates of their own under `crates///`, each named by that folder, so `crates/cmp/osano` is the module `cmp.osano`. The settings that select and configure a module do not change. `crates/trusted-server-modules` lists the modules a stock build ships from crates of their own, in the order their hooks run. Every adapter builds its state and loads its settings with that list, and `ts config validate` and `ts config push` run each stock module's own rules. Code that named a moved module under `trusted_server_core::integrations` names its crate instead, such as `trusted_server_cmp_osano`. Moved so far: Osano, Lockr, Permutive, Sourcepoint, Didomi, Google Tag Manager, Testlight, Next.js and Google Publisher Tags.js and Google Publisher Tags.js.js. - The Fastly adapter is a library with a thin binary over it. A deployment that ships a vendor crate writes a binary of its own that calls `trusted_server_adapter_fastly::run_with(vec![...])` with that crate's integration builder, in place of editing the adapter. The builders are composed with the built-in ones, and a builder runs only when the settings select its module. The Wasm artifact keeps its name, `trusted-server-adapter-fastly.wasm`. - **Breaking:** `[integrations]` is gone. Each page integration is a module selected in the section of its type, with its settings in the table at its name: `[cmp] module = "didomi"` (or `"sourcepoint"` or `"osano"`), `[tag] modules = ["google-tag-manager"]`, `[ad-tag] modules = ["google", "google.diagnostics"]`, `[bot-protection] module = "datadome"`, `[identity] module = "lockr"`, `[audience] module = "permutive"`, `[framework] module = "nextjs"`, `[auction] modules = ["prebid"]` (with `"testing.testlight"` for the test module) and `[proxy] modules = ["js_asset_proxy"]`, so `[integrations.gpt]` is `[ad-tag.google]` and `[integrations.prebid]` is `[auction.prebid]`. A module's name is `.`, which is its crate's path under `crates/` once it has a crate of its own and a constant in core until then, and it is written in its section with the type left off. A section that selects nothing, a table its section does not select, and a name no module in the deployment supplies are each refused at startup. An `[integrations]` or `[integration]` table is refused with directions. Environment overlays name a section as written, so `TRUSTED_SERVER__INTEGRATIONS__GPT__GAM_ATTRIBUTION_ENABLED` is `TRUSTED_SERVER__AD-TAG__GOOGLE__GAM_ATTRIBUTION_ENABLED`. - **Breaking:** Every pluggable component now shares one configuration syntax. Each selectable type is a top-level table named for the job, a `module` key inside it selects what runs, or `modules` for a type that runs several, and a `[.]` table holds that name's settings when it has any. The types are `ec`, `geo`, `device`, `permission-signal`, `demand` and `ad-server`, and the section of each page integration type, such as `cmp` or `tag`. A module from a crate is named by its folder below `crates/`, such as `permission-signal.gpc`, and within its own table the type folder may be left off. A `demand` or `ad-server` name is snake_case, because it may be a label of your own. A settings table its type's selector does not name refuses startup, so a block left behind after a module is switched off is caught rather than sitting unread, and every module rejects settings it does not know, so a misspelled key fails rather than being ignored. A table's name is the implementation unless the table carries an `implementation` line, and a demand table always carries one, naming the implementation by its module path (`auction-protocol.openrtb`, `auction.prebid-server` or `auction.aps`), which is how two Prebid Servers run side by side under names of their own. The ad server implementation is `ad-server.mock`, written `mock` in `[ad-server]`. A `demand` or `ad-server` endpoint must be HTTPS, or HTTP to `127.0.0.1`, `::1` or `localhost`. The word mediator is gone. It is "ad server" in prose and `ad-server` in configuration, `[debug.auction_html_comment_options] include_mediator_response` is now `include_adserver_response`, and the auction response metadata value `parallel_mediation` is now `parallel_adserver`. `auction-protocol.openrtb`, `auction.prebid-server`, `auction.aps` and `ad-server.mock` supply implementations only, so a page integration's section that names one is refused. The full rules and what is checked at each gate are in [Configuration Rules](docs/guide/configuration-rules.md). Migrate as follows: diff --git a/Cargo.lock b/Cargo.lock index 2ba9527dc6..136a64f377 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5378,6 +5378,22 @@ dependencies = [ "once_cell", ] +[[package]] +name = "trusted-server-ad-tag-google" +version = "0.1.0" +dependencies = [ + "async-trait", + "edgezero-core", + "error-stack", + "http", + "log", + "serde", + "serde_json", + "trusted-server-core", + "url", + "validator", +] + [[package]] name = "trusted-server-adapter-axum" version = "0.1.0" @@ -5770,6 +5786,7 @@ name = "trusted-server-modules" version = "0.1.0" dependencies = [ "serde_json", + "trusted-server-ad-tag-google", "trusted-server-audience-permutive", "trusted-server-cmp-didomi", "trusted-server-cmp-osano", diff --git a/Cargo.toml b/Cargo.toml index 82b3042350..2941d3d7f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ members = [ "crates/tag/google-tag-manager", "crates/testing/testlight", "crates/framework/nextjs", + "crates/ad-tag/google", "crates/trusted-server-adapter-axum", "crates/trusted-server-adapter-cloudflare", "crates/trusted-server-adapter-fastly", @@ -150,6 +151,7 @@ trusted-server-cmp-didomi = { path = "crates/cmp/didomi" } trusted-server-tag-google-tag-manager = { path = "crates/tag/google-tag-manager" } trusted-server-testing-testlight = { path = "crates/testing/testlight" } trusted-server-framework-nextjs = { path = "crates/framework/nextjs" } +trusted-server-ad-tag-google = { path = "crates/ad-tag/google" } trusted-server-js = { path = "crates/trusted-server-js" } trusted-server-modules = { path = "crates/trusted-server-modules" } trusted-server-openrtb = { path = "crates/trusted-server-openrtb" } diff --git a/crates/ad-tag/google/Cargo.toml b/crates/ad-tag/google/Cargo.toml new file mode 100644 index 0000000000..8fe3cc1947 --- /dev/null +++ b/crates/ad-tag/google/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "trusted-server-ad-tag-google" +description = "Google Publisher Tags first-party delivery and its guarded slot handoff as an ad tag module." +authors = { workspace = true } +edition = { workspace = true } +license = { workspace = true } +publish = { workspace = true } +version = { workspace = true } + +[lib] +doctest = false + +[lints] +workspace = true + +[dependencies] +async-trait = { workspace = true } +edgezero-core = { workspace = true } +error-stack = { workspace = true } +http = { workspace = true } +log = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +trusted-server-core = { workspace = true } +url = { workspace = true } +validator = { workspace = true } + +[dev-dependencies] +trusted-server-core = { workspace = true, features = ["test-utils"] } + +# The visible owner of this vendor crate, the way Prebid.js requires a named +# maintainer of every adapter. Google has not yet adopted this crate, so the +# Trusted Server maintainers own it until a vendor owner steps up. +[package.metadata.maintainers] +owner = "Trusted Server maintainers" +status = "seeking vendor owner" diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/ad-tag/google/src/gpt_bootstrap.js similarity index 100% rename from crates/trusted-server-core/src/integrations/gpt_bootstrap.js rename to crates/ad-tag/google/src/gpt_bootstrap.js diff --git a/crates/trusted-server-core/src/integrations/gpt.rs b/crates/ad-tag/google/src/lib.rs similarity index 97% rename from crates/trusted-server-core/src/integrations/gpt.rs rename to crates/ad-tag/google/src/lib.rs index 824796a550..b328539b44 100644 --- a/crates/trusted-server-core/src/integrations/gpt.rs +++ b/crates/ad-tag/google/src/lib.rs @@ -32,6 +32,18 @@ //! This is the sole mechanism that routes the GPT cascade through the proxy. //! The shim also hooks into the `googletag` API for targeting injection. +#![cfg_attr( + test, + allow( + clippy::print_stdout, + clippy::print_stderr, + clippy::panic, + clippy::dbg_macro, + clippy::unwrap_used, + reason = "tests use direct diagnostics and panic-on-failure helpers" + ) +)] + use std::sync::Arc; use async_trait::async_trait; @@ -42,31 +54,36 @@ use serde::{Deserialize, Serialize}; use url::Url; use validator::Validate; -use crate::constants::{HEADER_ACCEPT, HEADER_ACCEPT_ENCODING, HEADER_ACCEPT_LANGUAGE}; -use crate::error::TrustedServerError; -use crate::integrations::{ +use trusted_server_core::constants::{ + HEADER_ACCEPT, HEADER_ACCEPT_ENCODING, HEADER_ACCEPT_LANGUAGE, +}; +use trusted_server_core::error::TrustedServerError; +use trusted_server_core::integrations::{ AttributeRewriteAction, IntegrationAttributeContext, IntegrationAttributeRewriter, IntegrationEndpoint, IntegrationHeadInjector, IntegrationHtmlContext, IntegrationProxy, IntegrationRegistration, }; -use crate::platform::RuntimeServices; -use crate::proxy::{ProxyRequestConfig, proxy_request}; -use crate::settings::{IntegrationConfig, Settings}; +use trusted_server_core::platform::RuntimeServices; +use trusted_server_core::proxy::{ProxyRequestConfig, proxy_request}; +use trusted_server_core::settings::{IntegrationConfig, Settings}; const GPT_INTEGRATION_ID: &str = "gpt"; /// The name this module is selected by, in `[ad-tag]`. pub const MODULE: &str = "ad-tag.google"; -/// The builder the registry runs when a section selects [`MODULE`]. -pub(crate) const BUILDER: crate::integrations::IntegrationBuilder = - crate::integrations::IntegrationBuilder::new( +/// The builder a deployment hands to an adapter, which the registry runs when +/// a section selects [`MODULE`]. +#[must_use] +pub fn builder() -> trusted_server_core::integrations::IntegrationBuilder { + trusted_server_core::integrations::IntegrationBuilder::new( GPT_INTEGRATION_ID, - crate::integrations::CORE_SOURCE, + env!("CARGO_PKG_NAME"), register, validate, ) - .with_module_name(MODULE); + .with_module_name(MODULE) +} /// Primary Google domain that serves GPT scripts. const SECUREPUBADS_HOST: &str = "securepubads.g.doubleclick.net"; @@ -575,10 +592,10 @@ fn default_rewrite_script() -> bool { #[cfg(test)] mod tests { use super::*; - use crate::constants::HEADER_X_FORWARDED_FOR; - use crate::integrations::IntegrationDocumentState; - use crate::test_support::tests::create_test_settings; use http::Method; + use trusted_server_core::constants::HEADER_X_FORWARDED_FOR; + use trusted_server_core::integrations::IntegrationDocumentState; + use trusted_server_core::test_support::tests::create_test_settings; fn test_config() -> GptConfig { GptConfig { @@ -1566,4 +1583,13 @@ mod tests { "should not emit slim-Prebid URL tag when not configured" ); } + + #[test] + fn module_constant_is_the_crate_folder() { + assert_eq!( + super::MODULE, + trusted_server_core::module_name!(), + "should be named by the folder this crate lives in" + ); + } } diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index dd9becf1f2..7977698c71 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -912,20 +912,22 @@ mod tests { } #[test] - fn integration_head_injector_marks_only_attribution_enabled_gpt_bundle() { - fn process(gam_attribution_enabled: Option) -> String { - let integrations = if let Some(gam_attribution_enabled) = gam_attribution_enabled { + fn integration_head_injector_marks_only_the_bundle_it_asks_to_mark() { + use crate::integrations::registry_test_support::tag_fixture as tag; + + fn process(mark_bundle: Option) -> String { + let integrations = if let Some(mark_bundle) = mark_bundle { let mut settings = create_test_settings(); settings .insert_module_config( - "ad-tag", - "ad-tag.google", + "testing", + tag::MODULE, &json!({ - "gam_attribution_enabled": gam_attribution_enabled + "mark_bundle": mark_bundle }), ) - .expect("should insert GPT config"); - IntegrationRegistry::new(&settings).expect("should build GPT registry") + .expect("should insert the stand-in's settings"); + IntegrationRegistry::new(&settings).expect("should build the registry") } else { IntegrationRegistry::empty_for_tests() }; @@ -939,34 +941,35 @@ mod tests { String::from_utf8(output).expect("should produce valid UTF-8") } - let attributed = process(Some(true)); - let unattributed = process(Some(false)); - let without_gpt = process(None); + let marked = process(Some(true)); + let unmarked = process(Some(false)); + let without_the_integration = process(None); - for html in [&attributed, &unattributed, &without_gpt] { + for html in [&marked, &unmarked, &without_the_integration] { assert_eq!( html.matches("id=\"trustedserver-js\"").count(), 1, "should emit exactly one publisher bundle tag: {html}" ); } + let attribute = format!("{}=\"true\"", tag::BUNDLE_ATTRIBUTE); assert!( - attributed.contains("data-ts-gam-attribution=\"true\""), - "should mark only an attribution-enabled GPT publisher bundle" + marked.contains(&attribute), + "should mark the bundle an injector asks to mark" ); assert!( - !unattributed.contains("data-ts-gam-attribution"), - "should leave an attribution-disabled GPT publisher bundle unmarked" + !unmarked.contains(tag::BUNDLE_ATTRIBUTE), + "should leave the bundle unmarked when the injector asks for nothing" ); assert!( - !without_gpt.contains("data-ts-gam-attribution"), - "should leave a bundle unmarked when no section selects google" + !without_the_integration.contains(tag::BUNDLE_ATTRIBUTE), + "should leave the bundle unmarked when no section selects the integration" ); - let head_insert_index = attributed - .find("window.__tsjs_installGptShim") - .expect("should include the GPT head insert"); - let publisher_bundle_index = attributed + let head_insert_index = marked + .find(tag::HEAD_FLAG) + .expect("should include the integration's head insert"); + let publisher_bundle_index = marked .find("id=\"trustedserver-js\"") .expect("should include the publisher bundle"); assert!( diff --git a/crates/trusted-server-core/src/integrations/js_asset_proxy.rs b/crates/trusted-server-core/src/integrations/js_asset_proxy.rs index 999a9c88a7..b51ad1006e 100644 --- a/crates/trusted-server-core/src/integrations/js_asset_proxy.rs +++ b/crates/trusted-server-core/src/integrations/js_asset_proxy.rs @@ -852,92 +852,117 @@ mod tests { #[test] fn js_asset_proxy_rewriter_takes_precedence_over_native_rewriters() { let mut settings = create_test_settings(); - settings.select_module("ad-tag", "ad-tag.google"); + settings.select_module( + "testing", + crate::integrations::registry_test_support::tag_fixture::MODULE, + ); settings .insert_module_config( "proxy", "js_asset_proxy", &json!({ "assets": [{ - "path": "/assets/gpt.js", - "origin_url": "https://securepubads.g.doubleclick.net/tag/js/gpt.js", + "path": "/assets/sdk.js", + "origin_url": crate::integrations::registry_test_support::tag_fixture::SCRIPT_URL, "proxy": "enabled" }] }), ) .expect("should insert JS asset proxy config"); let registry = IntegrationRegistry::new(&settings).expect("should build registry"); - let html = r#""#; + let html = format!( + r#""#, + crate::integrations::registry_test_support::tag_fixture::SCRIPT_URL + ); - let processed = process_html_with_registry(html, registry); + let processed = process_html_with_registry(&html, registry); assert!( - processed.contains(r#""#), - "JS asset proxy should rewrite before GPT native rewriter: {processed}" + processed.contains(r#""#), + "JS asset proxy should rewrite before the integration's own rewriter: {processed}" ); assert!( - !processed.contains("/integrations/gpt/script"), - "GPT native rewrite should not override JS asset proxy" + !processed.contains( + crate::integrations::registry_test_support::tag_fixture::FIRST_PARTY_SCRIPT + ), + "the integration's own rewrite should not override JS asset proxy" ); } #[test] fn js_asset_proxy_blocking_takes_precedence_over_native_rewriters() { let mut settings = create_test_settings(); - settings.select_module("ad-tag", "ad-tag.google"); + settings.select_module( + "testing", + crate::integrations::registry_test_support::tag_fixture::MODULE, + ); settings .insert_module_config( "proxy", "js_asset_proxy", &json!({ "assets": [{ - "path": "/assets/gpt.js", - "origin_url": "https://securepubads.g.doubleclick.net/tag/js/gpt.js", + "path": "/assets/sdk.js", + "origin_url": crate::integrations::registry_test_support::tag_fixture::SCRIPT_URL, "proxy": "blocked" }] }), ) .expect("should insert JS asset proxy config"); let registry = IntegrationRegistry::new(&settings).expect("should build registry"); - let html = r#""#; + let html = format!( + r#""#, + crate::integrations::registry_test_support::tag_fixture::SCRIPT_URL + ); - let processed = process_html_with_registry(html, registry); + let processed = process_html_with_registry(&html, registry); assert!( - !processed.contains("googletag.cmd"), - "blocked JS asset should remove the script element before GPT can rewrite it" + !processed.contains("vendor.cmd"), + "blocked JS asset should remove the script element before the integration can rewrite it" ); assert!( - !processed.contains("/integrations/gpt/script"), - "GPT native rewrite should not keep a blocked script" + !processed.contains( + crate::integrations::registry_test_support::tag_fixture::FIRST_PARTY_SCRIPT + ), + "the integration's own rewrite should not keep a blocked script" ); } #[test] fn disabled_js_asset_proxy_candidate_allows_native_rewriters() { let mut settings = create_test_settings(); - settings.select_module("ad-tag", "ad-tag.google"); + settings.select_module( + "testing", + crate::integrations::registry_test_support::tag_fixture::MODULE, + ); settings .insert_module_config( "proxy", "js_asset_proxy", &json!({ "assets": [{ - "path": "/assets/gpt.js", - "origin_url": "https://securepubads.g.doubleclick.net/tag/js/gpt.js", + "path": "/assets/sdk.js", + "origin_url": crate::integrations::registry_test_support::tag_fixture::SCRIPT_URL, "proxy": "disabled" }] }), ) .expect("should insert JS asset proxy config"); let registry = IntegrationRegistry::new(&settings).expect("should build registry"); - let html = r#""#; + let html = format!( + r#""#, + crate::integrations::registry_test_support::tag_fixture::SCRIPT_URL + ); - let processed = process_html_with_registry(html, registry); + let processed = process_html_with_registry(&html, registry); assert!( - processed.contains(r#""#), - "disabled JS asset proxy entries should not suppress native integration rewrites" + processed.contains(&format!( + r#""#, + crate::integrations::registry_test_support::tag_fixture::FIRST_PARTY_SCRIPT + )), + "disabled JS asset proxy entries should not suppress an integration's own rewrite" ); } diff --git a/crates/trusted-server-core/src/integrations/mod.rs b/crates/trusted-server-core/src/integrations/mod.rs index 458c2e3c96..d9621c84f5 100644 --- a/crates/trusted-server-core/src/integrations/mod.rs +++ b/crates/trusted-server-core/src/integrations/mod.rs @@ -16,7 +16,6 @@ use crate::settings::Settings; pub mod adserver_mock; pub mod aps; pub mod datadome; -pub mod gpt; pub mod gpt_diagnostics; pub mod js_asset_proxy; pub mod openrtb; @@ -575,8 +574,10 @@ const BUILT_IN_BUILDERS: &[IntegrationBuilder] = &[ // select where they need one. #[cfg(test)] registry_test_support::payload_fixture::BUILDER, + // A stand-in for an integration that tags a page, for the same tests. + #[cfg(test)] + registry_test_support::tag_fixture::BUILDER, datadome::BUILDER, - gpt::BUILDER, gpt_diagnostics::BUILDER, // Implementations `[demand]` and `[ad-server]` can name. None of them is // a module a section selects. diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index a95df6599e..482440917a 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -2346,6 +2346,122 @@ pub(crate) mod test_support { Ok(true) } + /// A stand-in for an integration that tags a page, for core's own tests + /// of the HTML processor and the JavaScript asset proxy. + /// + /// Selected, it inserts one script at the start of `` and rewrites + /// the address of its own script to a first-party path. With + /// `mark_bundle` set it also asks for an attribute on the publisher + /// bundle tag. + pub(crate) mod tag_fixture { + use std::sync::Arc; + + use error_stack::Report; + + use crate::error::TrustedServerError; + use crate::integrations::registry::{ + AttributeRewriteAction, IntegrationAttributeContext, IntegrationAttributeRewriter, + IntegrationHeadInjector, IntegrationHtmlContext, IntegrationRegistration, + }; + use crate::integrations::{CORE_SOURCE, IntegrationBuilder}; + use crate::settings::Settings; + + /// The integration id the stand-in registers under. + pub(crate) const ID: &str = "tag_fixture"; + /// The name a test's settings select the stand-in by, in `[testing]`. + pub(crate) const MODULE: &str = "testing.tag-fixture"; + /// The script the stand-in's vendor would serve. + pub(crate) const SCRIPT_URL: &str = "https://cdn.tag-fixture.example/sdk.js"; + /// The first-party path the stand-in rewrites that script to. + pub(crate) const FIRST_PARTY_SCRIPT: &str = "/integrations/tag_fixture/script"; + /// What the stand-in's head insert sets, so a test can find it. + pub(crate) const HEAD_FLAG: &str = "window.__ts_tag_fixture=true;"; + /// The attribute the stand-in asks for on the publisher bundle tag. + pub(crate) const BUNDLE_ATTRIBUTE: &str = "data-ts-tag-fixture"; + + /// The builder core's test build lists beside its own. + pub(crate) const BUILDER: IntegrationBuilder = + IntegrationBuilder::new(ID, CORE_SOURCE, register, validate).with_module_name(MODULE); + + #[derive(Debug, serde::Deserialize, validator::Validate)] + #[serde(deny_unknown_fields)] + struct FixtureSettings { + #[serde(default)] + mark_bundle: bool, + } + + impl crate::settings::IntegrationConfig for FixtureSettings {} + + struct Tag { + mark_bundle: bool, + } + + impl IntegrationHeadInjector for Tag { + fn integration_id(&self) -> &'static str { + ID + } + + fn head_inserts(&self, _ctx: &IntegrationHtmlContext<'_>) -> Vec { + vec![format!("")] + } + + fn tsjs_script_tag_attributes(&self) -> Vec<(&'static str, &'static str)> { + if self.mark_bundle { + vec![(BUNDLE_ATTRIBUTE, "true")] + } else { + Vec::new() + } + } + } + + impl IntegrationAttributeRewriter for Tag { + fn integration_id(&self) -> &'static str { + ID + } + + fn handles_attribute(&self, attribute: &str) -> bool { + attribute == "src" + } + + fn rewrite( + &self, + _attr_name: &str, + attr_value: &str, + _ctx: &IntegrationAttributeContext<'_>, + ) -> AttributeRewriteAction { + if attr_value == SCRIPT_URL { + AttributeRewriteAction::Replace(FIRST_PARTY_SCRIPT.to_owned()) + } else { + AttributeRewriteAction::Keep + } + } + } + + fn register( + settings: &Settings, + ) -> Result, Report> { + let Some(config) = settings.module_config::(MODULE)? else { + return Ok(None); + }; + let tag = Arc::new(Tag { + mark_bundle: config.mark_bundle, + }); + Ok(Some( + IntegrationRegistration::builder(ID) + .without_js() + .with_attribute_rewriter(tag.clone()) + .with_head_injector(tag) + .build(), + )) + } + + fn validate(settings: &Settings) -> Result> { + settings + .module_config::(MODULE) + .map(|config| config.is_some()) + } + } + /// A stand-in for an integration that rewrites script payloads in two /// passes, for core's own tests of the page pipeline. /// diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index ad3e460324..920853e2b8 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -21488,17 +21488,19 @@ mod tests { } #[test] - fn streaming_finalize_emits_gam_attribution_head_before_origin_eof() { + fn streaming_finalize_emits_an_integration_s_head_before_origin_eof() { + use crate::integrations::registry_test_support::tag_fixture as tag; + let mut settings = create_test_settings(); settings .insert_module_config( - "ad-tag", - "ad-tag.google", + "testing", + tag::MODULE, &serde_json::json!({ - "gam_attribution_enabled": true + "mark_bundle": true }), ) - .expect("should insert GPT config"); + .expect("should insert the stand-in's settings"); let body = streaming_finalize_response_with_settings( html_stream_params("", None), @@ -21511,12 +21513,12 @@ mod tests { .expect("should emit UTF-8 HTML"); assert!( - html.contains("__tsjs_gam_attribution_enabled=true"), - "first rewritten head chunk should carry the primary activation flag: {html}" + html.contains(tag::HEAD_FLAG), + "first rewritten head chunk should carry the integration's head insert: {html}" ); assert!( - html.contains("data-ts-gam-attribution=\"true\""), - "first rewritten head chunk should authorize the bundle fallback: {html}" + html.contains(&format!("{}=\"true\"", tag::BUNDLE_ATTRIBUTE)), + "first rewritten head chunk should carry the bundle tag's attribute: {html}" ); } diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index c44167a790..43543bd2e0 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -4930,7 +4930,6 @@ mod tests { use regex::Regex; use serde_json::json; use std::collections::BTreeSet; - use std::sync::Arc; use crate::ec::resolve::NotAnOrigin; @@ -8357,7 +8356,7 @@ source_domain = "partner.example.com" #[derive(Debug, Deserialize, Validate)] #[serde(deny_unknown_fields)] struct OneRequiredSetting { - #[expect(dead_code, reason = "read only by deserialization")] + #[validate(url)] endpoint: String, } @@ -8579,29 +8578,45 @@ source_domain = "partner.example.com" #[test] fn invalid_settings_for_a_selected_module_fail_registry_startup() { + fn build( + settings: &Settings, + ) -> Result, Report> + { + Ok(settings + .module_config::(ENDPOINT_MODULE)? + .map(|_| crate::integrations::IntegrationRegistration::builder("endpoint").build())) + } + + fn validate(settings: &Settings) -> Result> { + settings + .module_config::(ENDPOINT_MODULE) + .map(|config| config.is_some()) + } + let mut settings = create_test_settings(); settings .insert_module_config( - "ad-tag", - crate::integrations::gpt::MODULE, + EXAMPLE_SECTION, + ENDPOINT_MODULE, &json!({ - "script_url": "not a url", + "endpoint": "not a url", }), ) - .expect("should insert GPT config"); + .expect("should insert the example module's table"); + let extra = [crate::integrations::IntegrationBuilder::new( + "endpoint", + "settings-tests", + build, + validate, + ) + .with_module_name(ENDPOINT_MODULE)]; - let err = match IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) { + let err = match IntegrationRegistry::with_registrations(&settings, &extra) { Ok(_) => panic!("a selected module with invalid settings should fail startup"), Err(err) => err, }; assert!( - format!("{err:?}").contains("[ad-tag.google]"), + format!("{err:?}").contains("[example.endpoint]"), "should identify the invalid module table: {err:?}" ); } diff --git a/crates/trusted-server-integration-tests/browser/initial-render/run.cjs b/crates/trusted-server-integration-tests/browser/initial-render/run.cjs index 681ba584a1..d69652bda6 100644 --- a/crates/trusted-server-integration-tests/browser/initial-render/run.cjs +++ b/crates/trusted-server-integration-tests/browser/initial-render/run.cjs @@ -67,7 +67,7 @@ const scriptContents = new Map( [ path.join( repo, - "crates/trusted-server-core/src/integrations/gpt_bootstrap.js", + "crates/ad-tag/google/src/gpt_bootstrap.js", ), path.join(dist, "tsjs-core.js"), path.join(dist, "tsjs-gpt.js"), @@ -163,7 +163,7 @@ const scenarios = [ scripts.push( path.join( repo, - "crates/trusted-server-core/src/integrations/gpt_bootstrap.js", + "crates/ad-tag/google/src/gpt_bootstrap.js", ), ); scripts.push( @@ -474,7 +474,7 @@ const scenarios = [ files: [ path.join( repo, - "crates/trusted-server-core/src/integrations/gpt_bootstrap.js", + "crates/ad-tag/google/src/gpt_bootstrap.js", ), path.join(dist, "tsjs-core.js"), path.join(dist, "tsjs-gpt.js"), diff --git a/crates/trusted-server-js/lib/test/fixtures/paths.ts b/crates/trusted-server-js/lib/test/fixtures/paths.ts index 199b105c11..90c34dc575 100644 --- a/crates/trusted-server-js/lib/test/fixtures/paths.ts +++ b/crates/trusted-server-js/lib/test/fixtures/paths.ts @@ -13,5 +13,5 @@ import { resolve } from 'node:path'; */ export const GPT_BOOTSTRAP_PATH = resolve( import.meta.dirname, - '../../../../trusted-server-core/src/integrations/gpt_bootstrap.js' + '../../../../ad-tag/google/src/gpt_bootstrap.js' ); diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts index 46ba778dc9..1f8df3a3a6 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts @@ -10,7 +10,7 @@ import { GPT_BOOTSTRAP_PATH } from '../../fixtures/paths'; * Executable coverage for the edge-injected `gpt_bootstrap.js` — the * head-inline fallback that keeps initial server-side ads working when the * main TSJS bundle fails to load. The file ships from - * `crates/trusted-server-core/src/integrations/gpt_bootstrap.js` and is + * `crates/ad-tag/google/src/gpt_bootstrap.js` and is * evaluated here verbatim, so the degradation path (fallback `adInit` and * fallback `scheduleInitialAdInit`) is executed, not string-matched. */ diff --git a/crates/trusted-server-modules/Cargo.toml b/crates/trusted-server-modules/Cargo.toml index d47991d00b..f41b434a1f 100644 --- a/crates/trusted-server-modules/Cargo.toml +++ b/crates/trusted-server-modules/Cargo.toml @@ -14,6 +14,7 @@ doctest = false workspace = true [dependencies] +trusted-server-ad-tag-google = { workspace = true } trusted-server-audience-permutive = { workspace = true } trusted-server-cmp-didomi = { workspace = true } trusted-server-cmp-osano = { workspace = true } diff --git a/crates/trusted-server-modules/src/lib.rs b/crates/trusted-server-modules/src/lib.rs index 2635ddd4a6..5779011c68 100644 --- a/crates/trusted-server-modules/src/lib.rs +++ b/crates/trusted-server-modules/src/lib.rs @@ -28,6 +28,7 @@ pub fn builders() -> Vec { trusted_server_cmp_sourcepoint::builder(), trusted_server_cmp_osano::builder(), trusted_server_tag_google_tag_manager::builder(), + trusted_server_ad_tag_google::builder(), ] } @@ -64,6 +65,7 @@ mod tests { "cmp.sourcepoint", "cmp.osano", "tag.google-tag-manager", + "ad-tag.google", ], "should offer the stock modules in the order their hooks run" ); diff --git a/docs/guide/integrations/gpt.md b/docs/guide/integrations/gpt.md index 39f45af4fe..e707dedd2c 100644 --- a/docs/guide/integrations/gpt.md +++ b/docs/guide/integrations/gpt.md @@ -247,7 +247,7 @@ treated as invalid for experiment reporting. ## Implementation -- **Rust**: [crates/trusted-server-core/src/integrations/gpt.rs](https://github.com/IABTechLab/trusted-server/blob/main/crates/trusted-server-core/src/integrations/gpt.rs) +- **Rust**: [crates/ad-tag/google/src/lib.rs](https://github.com/IABTechLab/trusted-server/blob/main/crates/ad-tag/google/src/lib.rs) - **TypeScript**: [crates/trusted-server-js/lib/src/integrations/gpt/](https://github.com/IABTechLab/trusted-server/blob/main/crates/trusted-server-js/lib/src/integrations/gpt/) ## Next Steps From ccdae3822bb1af798978983c0e9ffb595795ef36 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 8 Oct 2026 02:19:23 +0100 Subject: [PATCH 095/124] A module acts on one request without core naming it Two modules decide something about one request and act on it later, and the page path named both. It called the Google Publisher Tags diagnostics module to prepare the request and to finish the response, carried that module's decision to the HTML processor, which wrote its two scripts, and carried a flag for DataDome's tag beside it. Neither module can leave core while core does that for it. A request preparer or a request filter now leaves a value in the request's `IntegrationRequestState` under its integration id. The page path copies those values into the document's state before parsing starts, so the module's own head injector reads them, and hands them to the response finalizer a builder declares with `with_response_finalizer`. A head injector gets a second place to write, straight after the script bundle, which is where the diagnostics module loads its standalone script. The diagnostics module and DataDome both use these and core calls neither. A request that carries any value keeps to the origin path, so its document is never read from a shared template and never stored as one, and its HTML response is sent `private, no-store`. That was true of both modules before, each by its own route. It is now one rule that holds for any module, checked where the request is routed and again where the document is built. The registry runs the request preparers once for a request, so the page path asks it to prepare a request that reached it unprepared and names no module to do it. The token an auction publishes with its winning bids was made when the diagnostics module ran. A builder now declares with `with_auction_token` that its browser script reads the token, the auction plan records whether a selected module does, and the page path asks the orchestrator. Core's tests of all this use a third stand-in compiled for its tests only. It marks a document navigation that carries its query, writes a head script and its standalone module's tag for a marked request, and sets a cookie on the response. The template cache tests that set a diagnostics decision by hand, the template rendering tests and the auction token tests use it. The tests that drive either module itself still do, and hand the page path the request state where they handed it a decision or a flag. --- CHANGELOG.md | 1 + crates/trusted-server-adapter-axum/src/app.rs | 4 +- .../benches/html_processor_bench.rs | 3 +- crates/trusted-server-core/src/auction/mod.rs | 68 ++- .../src/auction/orchestrator.rs | 15 + .../trusted-server-core/src/auction/plan.rs | 14 + .../trusted-server-core/src/html_processor.rs | 98 ++-- .../src/integrations/datadome.rs | 12 +- .../src/integrations/datadome/protection.rs | 25 +- .../src/integrations/gpt_diagnostics.rs | 101 ++-- .../src/integrations/js_asset_proxy.rs | 3 +- .../src/integrations/mod.rs | 61 +- .../src/integrations/registry.rs | 554 +++++++++++++++++- crates/trusted-server-core/src/publisher.rs | 444 +++++++------- docs/guide/integration-guide.md | 34 +- 15 files changed, 1108 insertions(+), 329 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e4cddfad9b..43775ba7ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,6 +63,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- A module can act on one request from start to finish without `trusted-server-core` naming it. A request preparer or a request filter leaves a value in the request's `IntegrationRequestState`, which is copied into the document's state for the module's page hooks and handed to the response finalizer the builder declares with `with_response_finalizer`. A request that carries any value keeps to the origin path, so its document is never read from or stored as a shared template, and its HTML response is sent `private, no-store`. A head injector can also write straight after the script bundle with `after_bundle_inserts`, and a builder declares with `with_auction_token` that its browser script reads the token an auction publishes with its winning bids. The registry runs the request preparers once for a request. - Added the `[auction].rewrite_creatives` (default `true`) and `[auction].sanitize_creatives` (default `false`) options. `rewrite_creatives` rewrites winning-bid adm to first-party endpoints across `POST /auction` and publisher SSAT/page-bids delivery (proxy/click URL conversion, bidder `` removal; creative TSJS injection on `POST /auction` only). Enabling `sanitize_creatives` strips executable markup from winning-bid adm before delivery. - `creative_opportunities.slot.gam_unit_path` is now a template supporting `{network_id}`, `{slot_id}`, and `{section}`, so a publisher whose ad unit varies by site section expresses it in one slot rule instead of one per (slot × section). `{section}` derives from the request path: `[creative_opportunities].section_segment` selects which path segment names the section (0-based, default `0`; set `1` for locale-prefixed URLs), and `section_root` supplies the value for paths with no such segment. `section_root` is required when a template uses `{section}`. Existing static and absent `gam_unit_path` configs are unchanged. Startup rejects a blank `gam_network_id` only when an absent/default path or `{network_id}` template consumes it. Trusted Server conservatively caps whole rendered dynamic paths at 100 UTF-8 bytes, informed by Google's 100-character per-ad-unit-code limit; an over-limit request-specific path omits that slot without failing the response. During typed/startup finalization, every placeholder-bearing template that omits `section_segment` materializes `section_segment = 0`, so an older binary rejects the blob loudly. Static and absent paths remain legacy-schema compatible only when both `section_root` and `section_segment` are omitted. Before rolling back below this feature, replace or remove dynamic paths, remove both keys, re-push and finalize the config, then roll back the binary. - Added opt-in APS HTTP debug metadata for controlled test sites, exposing the direct request and response under `/auction` provider metadata using the Prebid Server `debug.httpcalls` shape. diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 7e5fe88140..3baf8e687d 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -351,8 +351,8 @@ async fn build_ec_context( /// The request arrives already prepared. Every route that reaches here is /// registered to [`fallback_handler`], which runs the request through /// [`execute_handler`], and [`execute_handler`] calls -/// `registry.prepare_request` before it calls this function, so preparing -/// again here would run each module's preparer twice for one request. +/// `registry.prepare_request` before it calls this function, so there is +/// nothing left to prepare here. async fn dispatch_fallback( state: &AppState, services: &RuntimeServices, diff --git a/crates/trusted-server-core/benches/html_processor_bench.rs b/crates/trusted-server-core/benches/html_processor_bench.rs index e0d5caa6f8..bbb5412f21 100644 --- a/crates/trusted-server-core/benches/html_processor_bench.rs +++ b/crates/trusted-server-core/benches/html_processor_bench.rs @@ -16,12 +16,11 @@ fn make_config() -> HtmlProcessorConfig { permissions_script: None, ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, - gpt_diagnostics: None, + request_state: trusted_server_core::integrations::IntegrationRequestState::default(), // The benchmark measures URL rewriting, not ad injection, and // `ad_slots_script` is `None` here — matching the previous behaviour, // which inferred no body-close work from that. body_close: BodyCloseInjection::None, - suppress_datadome_client_side_tag: false, } } diff --git a/crates/trusted-server-core/src/auction/mod.rs b/crates/trusted-server-core/src/auction/mod.rs index 4e89cad109..5e0f73c002 100644 --- a/crates/trusted-server-core/src/auction/mod.rs +++ b/crates/trusted-server-core/src/auction/mod.rs @@ -90,7 +90,15 @@ pub fn compile_auction_plan_with( .filter_map(IntegrationBuilder::adserver) .collect(), }) - .map(|plan| plan.with_enabled(settings.auction.enabled)) + .map(|plan| { + plan.with_enabled(settings.auction.enabled) + .with_auction_token(builders.iter().any(|builder| { + builder.reads_auction_token() + && builder + .module_name() + .is_some_and(|name| settings.selects_module(name)) + })) + }) } /// Build a new auction orchestrator from one shared compiled plan. @@ -135,6 +143,64 @@ pub fn build_orchestrator( build_orchestrator_with_plan(plan) } +#[cfg(test)] +mod auction_token_tests { + use super::*; + use crate::integrations::registry_test_support::{ + probe_registration, request_fixture, validate_nothing, + }; + use crate::test_support::tests::create_test_settings; + + #[test] + fn the_plan_publishes_a_token_only_where_a_selected_module_reads_one() { + let mut settings = create_test_settings(); + assert!( + !compile_auction_plan(&settings) + .expect("should compile the plan") + .publishes_auction_token(), + "should publish no token where no module is selected" + ); + + settings.select_module("testing", request_fixture::MODULE); + assert!( + compile_auction_plan(&settings) + .expect("should compile the plan") + .publishes_auction_token(), + "should publish a token once a module that reads one is selected" + ); + } + + #[test] + fn a_selected_module_that_reads_no_token_does_not_turn_one_on() { + let mut settings = create_test_settings(); + settings.select_module("testing", "testing.probe"); + let reads_nothing = + [ + IntegrationBuilder::new( + "probe", + "seam-probe", + probe_registration, + validate_nothing, + ) + .with_module_name("testing.probe"), + ]; + assert!( + !compile_auction_plan_with(&settings, &reads_nothing) + .expect("should compile the plan") + .publishes_auction_token(), + "should publish no token for a selected module that reads none" + ); + + let reads_one = [reads_nothing[0].with_auction_token()]; + assert!( + compile_auction_plan_with(&settings, &reads_one) + .expect("should compile the plan") + .publishes_auction_token(), + "should publish a token for a module a vendor crate supplies that reads one" + ); + } +} + #[cfg(test)] mod plan_sharing_tests { use super::*; diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 3fc04a2e4f..8639525342 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -830,6 +830,21 @@ impl AuctionOrchestrator { Arc::ptr_eq(&self.plan, plan) } + /// Return whether each auction publishes a token with its winning bids. + #[must_use] + pub fn publishes_auction_token(&self) -> bool { + self.plan.publishes_auction_token() + } + + /// The same orchestrator in a deployment where a module reads the + /// auction token. + #[cfg(test)] + #[must_use] + pub(crate) fn publishing_auction_token(mut self) -> Self { + self.plan = Arc::new((*self.plan).clone().with_auction_token(true)); + self + } + /// Register an auction provider in the legacy parity harness. #[cfg(test)] pub(crate) fn register_provider(&mut self, provider: Arc) { diff --git a/crates/trusted-server-core/src/auction/plan.rs b/crates/trusted-server-core/src/auction/plan.rs index 4f2f2ad897..bf78b40da4 100644 --- a/crates/trusted-server-core/src/auction/plan.rs +++ b/crates/trusted-server-core/src/auction/plan.rs @@ -283,6 +283,7 @@ pub struct AuctionPlan { bidder_routes: BTreeMap, signing_enabled: bool, adserver: Option, + publishes_auction_token: bool, } impl AuctionPlan { @@ -297,6 +298,18 @@ impl AuctionPlan { self } + /// Return whether each auction publishes a token with its winning bids, + /// which it does when a module that runs reads one. + #[must_use] + pub fn publishes_auction_token(&self) -> bool { + self.publishes_auction_token + } + + pub(crate) fn with_auction_token(mut self, publishes: bool) -> Self { + self.publishes_auction_token = publishes; + self + } + /// Compile a deterministic plan without adapter-specific validation. /// /// # Errors @@ -447,6 +460,7 @@ impl AuctionPlan { bidder_routes, signing_enabled, adserver, + publishes_auction_token: false, }) } } diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index 7977698c71..40fe7f3cdd 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -13,13 +13,10 @@ use lol_html::{ text, }; -use crate::integrations::datadome::{DATADOME_INTEGRATION_ID, DataDomeClientTagSuppressed}; -use crate::integrations::gpt_diagnostics::{ - GPT_DIAGNOSTICS_INTEGRATION_ID, GptDiagnosticsRequestDecision, -}; use crate::integrations::{ AttributeRewriteOutcome, IntegrationAttributeContext, IntegrationDocumentState, - IntegrationHtmlContext, IntegrationRegistry, IntegrationScriptContext, ScriptRewriteAction, + IntegrationHtmlContext, IntegrationRegistry, IntegrationRequestState, IntegrationScriptContext, + ScriptRewriteAction, }; use crate::publisher::build_empty_bids_script; use crate::settings::Settings; @@ -99,13 +96,14 @@ pub struct HtmlProcessorConfig { /// Maximum bytes an integration may retain while processing one script or /// unresolved streaming group. pub max_buffered_body_bytes: usize, - /// Request-scoped conditional diagnostics delivery decision. - pub gpt_diagnostics: Option, + /// What modules left on the request for their page hooks, copied into the + /// document's state before parsing starts. Empty for a document that may + /// be stored and served to other readers, where nothing made for one + /// request may appear. + pub request_state: IntegrationRequestState, /// What the `` seam injects. Decided by the caller rather than inferred /// from [`Self::ad_slots_script`]. pub body_close: BodyCloseInjection, - /// Whether to omit Trusted Server's automatic `DataDome` client-side tag. - pub suppress_datadome_client_side_tag: bool, /// Set when the document delivers a response-bound CSP nonce in its own markup. /// /// `None` on every path that cannot store a shared template, so an ordinary inline @@ -132,9 +130,8 @@ impl HtmlProcessorConfig { ad_slots_script: None, ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: settings.publisher.max_buffered_body_bytes, - gpt_diagnostics: None, + request_state: IntegrationRequestState::default(), body_close: BodyCloseInjection::None, - suppress_datadome_client_side_tag: false, csp_nonce_observed: None, } } @@ -179,10 +176,10 @@ impl HtmlProcessorConfig { self } - /// Attach the request-scoped conditional diagnostics decision. + /// Attach what modules left on the request for their page hooks. #[must_use] - pub fn with_gpt_diagnostics(mut self, decision: Option) -> Self { - self.gpt_diagnostics = decision; + pub fn with_request_state(mut self, request_state: IntegrationRequestState) -> Self { + self.request_state = request_state; self } @@ -195,13 +192,6 @@ impl HtmlProcessorConfig { self.csp_nonce_observed = observed; self } - - /// Attach the request-scoped `DataDome` client-tag suppression decision. - #[must_use] - pub fn with_datadome_client_tag_suppression(mut self, suppress: bool) -> Self { - self.suppress_datadome_client_side_tag = suppress; - self - } } /// Create an HTML processor with URL replacement and integration hooks. @@ -218,9 +208,7 @@ impl HtmlProcessorConfig { pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcessor { let stream_processor_factories = config.integrations.html_stream_processor_factories(); let document_state = IntegrationDocumentState::default(); - if config.suppress_datadome_client_side_tag { - document_state.get_or_insert_with(DATADOME_INTEGRATION_ID, || DataDomeClientTagSuppressed); - } + config.request_state.seed(&document_state); // Simplified URL patterns structure - stores only core data and generates variants on-demand struct UrlPatterns { @@ -293,7 +281,6 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso let permissions_script = config.permissions_script.clone(); let body_close = config.body_close.clone(); let ad_bids_state = config.ad_bids_state.clone(); - let gpt_diagnostics = config.gpt_diagnostics.clone(); // No source-comment neutralization here: rewriting a publisher comment that happens // to match the reserved marker would change publisher content bytes. Collisions are @@ -324,7 +311,6 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso let document_state = document_state.clone(); let ad_slots_script = ad_slots_script.clone(); let permissions_script = permissions_script.clone(); - let gpt_diagnostics = gpt_diagnostics.clone(); move |el| { if !injected_tsjs.get() { let mut snippet = String::new(); @@ -349,12 +335,6 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso for insert in integrations.head_inserts(&ctx) { snippet.push_str(&insert); } - if let Some(bootstrap) = gpt_diagnostics - .as_ref() - .and_then(GptDiagnosticsRequestDecision::bootstrap_script) - { - snippet.push_str(&bootstrap); - } // Main bundle: core + non-deferred integrations (synchronous). let immediate_parts = integrations.js_parts_immediate(); let script_attributes = integrations.tsjs_script_tag_attributes(); @@ -362,16 +342,10 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso &immediate_parts, &script_attributes, )); - // Active diagnostics loads synchronously after core so its - // GPT listeners precede publisher scripts in the origin head. - // The decision says whether to inject; the registry's part - // says what to inject. Nothing is injected without a part. - if let Some(module_tag) = gpt_diagnostics.as_ref().and_then(|decision| { - integrations - .js_part(GPT_DIAGNOSTICS_INTEGRATION_ID) - .and_then(|part| decision.module_script_tag(&part)) - }) { - snippet.push_str(&module_tag); + // What an integration loads after the bundle and ahead of + // the page's own scripts in the origin head. + for insert in integrations.after_bundle_inserts(&ctx) { + snippet.push_str(&insert); } // Deferred bundles: large modules like prebid loaded after // HTML parsing completes. Empty when none are enabled. @@ -781,8 +755,7 @@ mod tests { permissions_script: None, ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), } } @@ -1002,7 +975,11 @@ mod tests { ), ) .expect("should build integration registry"); - config.gpt_diagnostics = Some(decision); + assert!( + decision.active(), + "should activate diagnostics for the query" + ); + config.request_state = IntegrationRequestState::of(&request); let processor = create_html_processor(config); let pipeline_config = PipelineConfig { @@ -1132,6 +1109,11 @@ mod tests { .expect("should configure DataDome integration"); let registry = IntegrationRegistry::new(&settings) .expect("should create integration registry with DataDome"); + let mut suppressed = IntegrationRequestState::default(); + suppressed.set( + crate::integrations::datadome::DATADOME_INTEGRATION_ID, + crate::integrations::datadome::DataDomeClientTagSuppressed, + ); let config = HtmlProcessorConfig::from_settings( &settings, ®istry, @@ -1139,7 +1121,7 @@ mod tests { "test.example.com", "https", ) - .with_datadome_client_tag_suppression(true); + .with_request_state(suppressed); let mut processor = create_html_processor(config); let output = processor @@ -1525,8 +1507,7 @@ mod tests { permissions_script: None, ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let mut processor = create_html_processor(config); let output = processor @@ -1603,8 +1584,7 @@ mod tests { permissions_script: None, ad_bids_state: state, max_buffered_body_bytes: 16 * 1024 * 1024, - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let mut processor = create_html_processor(config); let output = processor @@ -1643,8 +1623,7 @@ mod tests { permissions_script: None, ad_bids_state: state, max_buffered_body_bytes: 16 * 1024 * 1024, - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let mut processor = create_html_processor(config); // Malformed HTML with two elements (common in CMS template pages) @@ -1682,8 +1661,7 @@ mod tests { permissions_script: None, ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let mut processor = create_html_processor(config); let output = processor @@ -1739,8 +1717,7 @@ mod tests { permissions_script: None, ad_bids_state: state, max_buffered_body_bytes: 16 * 1024 * 1024, - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let mut processor = create_html_processor(config); let output = processor @@ -1770,8 +1747,7 @@ mod tests { permissions_script: None, ad_bids_state: state, max_buffered_body_bytes: 16 * 1024 * 1024, - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let mut processor = create_html_processor(config); let output = processor @@ -1796,8 +1772,7 @@ mod tests { permissions_script: None, ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), } } @@ -2014,8 +1989,7 @@ mod tests { permissions_script: None, ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let source = format!(r#""#); diff --git a/crates/trusted-server-core/src/integrations/datadome.rs b/crates/trusted-server-core/src/integrations/datadome.rs index 4866480315..64be039592 100644 --- a/crates/trusted-server-core/src/integrations/datadome.rs +++ b/crates/trusted-server-core/src/integrations/datadome.rs @@ -110,11 +110,21 @@ pub(super) const MIN_TEST_BYPASS_CREDENTIAL_BYTES: usize = 32; /// Fixed request header used by the staging-only protection test bypass. pub(crate) const HEADER_DATADOME_TEST_BYPASS: &str = "x-ts-datadome-bypass"; -/// Request marker indicating that Trusted Server should omit its automatic +/// Marker indicating that Trusted Server should omit its automatic /// `DataDome` client-side tag for the current response. #[derive(Debug, Clone, Copy)] pub(crate) struct DataDomeClientTagSuppressed; +/// Leaves the marker on `request` for the head injector of the document the +/// request produces. +pub(crate) fn suppress_client_tag(request: &mut http::Request) { + crate::integrations::IntegrationRequestState::insert( + request, + DATADOME_INTEGRATION_ID, + DataDomeClientTagSuppressed, + ); +} + /// Regex pattern for matching and rewriting `DataDome` URLs in script content. /// /// Pattern breakdown: diff --git a/crates/trusted-server-core/src/integrations/datadome/protection.rs b/crates/trusted-server-core/src/integrations/datadome/protection.rs index 4ba1078a7d..3239848729 100644 --- a/crates/trusted-server-core/src/integrations/datadome/protection.rs +++ b/crates/trusted-server-core/src/integrations/datadome/protection.rs @@ -44,12 +44,14 @@ impl DataDomeIntegration { let test_bypass_matched = self.take_protection_test_bypass_header(input.request); if test_bypass_matched { // Both markers travel together. The first is DataDome's own - // tag-suppression signal, read by its head injector. The second tells + // tag-suppression signal, left for its head injector. The second tells // core the response is personalized to this request and cannot be // shared through a cache or a template. - let extensions = input.request.extensions_mut(); - extensions.insert(super::DataDomeClientTagSuppressed); - extensions.insert(crate::response_privacy::PersonalizedResponse); + super::suppress_client_tag(input.request); + input + .request + .extensions_mut() + .insert(crate::response_privacy::PersonalizedResponse); log_protection_test_bypass(&input); return RequestFilterDecision::Continue(RequestFilterEffects::default()); } @@ -163,12 +165,14 @@ impl DataDomeIntegration { } => { if suppress_client_tag { // Both markers travel together. The first is DataDome's own - // tag-suppression signal, read by its head injector. The second + // tag-suppression signal, left for its head injector. The second // tells core the response is personalized to this request and // cannot be shared through a cache or a template. - let extensions = input.request.extensions_mut(); - extensions.insert(super::DataDomeClientTagSuppressed); - extensions.insert(crate::response_privacy::PersonalizedResponse); + super::suppress_client_tag(input.request); + input + .request + .extensions_mut() + .insert(crate::response_privacy::PersonalizedResponse); } log_protection_skip(input, &rule_id, reason, suppress_client_tag); return false; @@ -924,9 +928,8 @@ mod tests { } fn has_client_tag_suppression_marker(request: &Request) -> bool { - request - .extensions() - .get::() + crate::integrations::IntegrationRequestState::of(request) + .get::(super::super::DATADOME_INTEGRATION_ID) .is_some() } diff --git a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs b/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs index e5fd01ad2c..3748252382 100644 --- a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs +++ b/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs @@ -12,14 +12,19 @@ use validator::Validate; use edgezero_core::body::Body as EdgeBody; +use std::sync::Arc; + use crate::error::TrustedServerError; use crate::http_util::is_navigation_request; -use crate::response_privacy::enforce_synthesized_html_cache_privacy; +use crate::response_privacy::enforce_terminal_private_cache_privacy; use crate::settings::{IntegrationConfig, Settings}; use crate::tsjs; use crate::tsjs_bundle::JsModulePart; -use super::IntegrationRegistration; +use super::{ + IntegrationHeadInjector, IntegrationHtmlContext, IntegrationRegistration, + IntegrationRequestState, +}; /// Stable integration identifier. pub const GPT_DIAGNOSTICS_INTEGRATION_ID: &str = "gpt_diagnostics"; @@ -36,7 +41,9 @@ pub(crate) const BUILDER: crate::integrations::IntegrationBuilder = validate, ) .with_module_name(MODULE) - .with_request_preparer(prepare_request_hook); + .with_request_preparer(prepare_request_hook) + .with_response_finalizer(finalize_response_hook) + .with_auction_token(); /// Reserved activation query parameter. pub const GPT_DIAGNOSTICS_QUERY: &str = "ts_console"; /// Host-only browser-session activation cookie. @@ -126,19 +133,6 @@ impl GptDiagnosticsRequestDecision { ) }) } - /// An active decision, for tests in other modules that need one. - /// - /// The fields are private and built by `prepare_request` from a cookie or query - /// parameter; there is no other way to obtain an active decision across a module - /// boundary. - #[cfg(test)] - pub(crate) fn active_for_tests() -> Self { - Self { - active: true, - clean_browser_path_and_query: None, - cookie_action: GptDiagnosticsCookieAction::None, - } - } } #[cfg(test)] @@ -259,24 +253,45 @@ pub fn register( Ok(Some( IntegrationRegistration::builder(GPT_DIAGNOSTICS_INTEGRATION_ID) .with_standalone_js() + .with_head_injector(Arc::new(DiagnosticsHead)) .build(), )) } -/// Whether `[ad-tag] modules` names the diagnostics module. -/// -/// This says whether the deployment runs diagnostics at all, rather than the -/// per-document activation state, so a caller that only needs to know whether -/// diagnostics could consume a value uses this, while document behavior uses -/// [`GptDiagnosticsRequestDecision::active`]. -/// A configuration that cannot be parsed reads as not running. -#[must_use] -pub fn runs(settings: &Settings) -> bool { - settings - .module_config::(MODULE) - .ok() - .flatten() - .is_some() +/// Writes what an active decision adds to a document's head. +struct DiagnosticsHead; + +impl DiagnosticsHead { + /// The decision the preparer left for this document, when it left one. + fn decision(ctx: &IntegrationHtmlContext<'_>) -> Option> { + ctx.document_state + .get::(GPT_DIAGNOSTICS_INTEGRATION_ID) + } +} + +impl IntegrationHeadInjector for DiagnosticsHead { + fn integration_id(&self) -> &'static str { + GPT_DIAGNOSTICS_INTEGRATION_ID + } + + fn head_inserts(&self, ctx: &IntegrationHtmlContext<'_>) -> Vec { + Self::decision(ctx) + .and_then(|decision| decision.bootstrap_script()) + .into_iter() + .collect() + } + + /// The module loads synchronously after the bundle, so its listeners are + /// in place before the publisher's scripts in the origin head run. + fn after_bundle_inserts(&self, ctx: &IntegrationHtmlContext<'_>) -> Vec { + Self::decision(ctx) + .and_then(|decision| { + JsModulePart::compile_time(GPT_DIAGNOSTICS_INTEGRATION_ID) + .and_then(|module| decision.module_script_tag(&module)) + }) + .into_iter() + .collect() + } } /// Evaluate activation and sanitize the request before generic cookie handling. @@ -335,6 +350,11 @@ pub fn prepare_request( } request.extensions_mut().insert(decision.clone()); + // Only a decision that changes the document or the response is left for + // the page path, so an ordinary request stays on the shared template path. + if decision.requires_private_no_store() { + IntegrationRequestState::insert(request, GPT_DIAGNOSTICS_INTEGRATION_ID, decision.clone()); + } Ok(decision) } @@ -351,14 +371,17 @@ pub(crate) fn prepare_request_hook( prepare_request(settings, request).map(|_| ()) } -/// Read the request decision, defaulting to inactive when not prepared. -#[must_use] -pub fn request_decision(request: &Request) -> GptDiagnosticsRequestDecision { - request - .extensions() - .get::() - .cloned() - .unwrap_or_default() +/// Builder hook: finalizes the response from the decision the preparer left +/// for the page path, and leaves the response alone when it left none. +pub(crate) fn finalize_response_hook( + request_state: &IntegrationRequestState, + response: &mut Response, +) { + if let Some(decision) = + request_state.get::(GPT_DIAGNOSTICS_INTEGRATION_ID) + { + finalize_response(&decision, response); + } } /// Apply activation cookie and strict cache privacy to an origin response. @@ -385,7 +408,7 @@ pub fn finalize_response( // `RequestFilterEffects` mutation such as `Cache-Control: public` replaced it, // and the adapter's terminal guard had no marker to re-enforce from, so // request-scoped diagnostics HTML became shared-cacheable. - enforce_synthesized_html_cache_privacy(response); + enforce_terminal_private_cache_privacy(response); } } diff --git a/crates/trusted-server-core/src/integrations/js_asset_proxy.rs b/crates/trusted-server-core/src/integrations/js_asset_proxy.rs index b51ad1006e..9b7990ae38 100644 --- a/crates/trusted-server-core/src/integrations/js_asset_proxy.rs +++ b/crates/trusted-server-core/src/integrations/js_asset_proxy.rs @@ -658,8 +658,7 @@ mod tests { ad_slots_script: None, ad_bids_state: Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: crate::integrations::IntegrationRequestState::default(), permissions_script: None, }); let pipeline_config = PipelineConfig { diff --git a/crates/trusted-server-core/src/integrations/mod.rs b/crates/trusted-server-core/src/integrations/mod.rs index d9621c84f5..a001243a58 100644 --- a/crates/trusted-server-core/src/integrations/mod.rs +++ b/crates/trusted-server-core/src/integrations/mod.rs @@ -5,7 +5,7 @@ use std::time::Duration; use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt}; use futures::StreamExt as _; -use http::Request; +use http::{Request, Response}; use url::Url; use crate::auction::demand::{AdServerImplementation, DemandImplementation}; @@ -31,10 +31,10 @@ pub use registry::{ IntegrationDocumentState, IntegrationEndpoint, IntegrationHeadInjector, IntegrationHtmlContext, IntegrationHtmlStreamContext, IntegrationHtmlStreamProcessorFactory, IntegrationMetadata, IntegrationProxy, IntegrationRegistration, IntegrationRegistrationBuilder, IntegrationRegistry, - IntegrationRequestFilter, IntegrationScriptContext, IntegrationScriptRewriter, - ProxyDispatchInput, RequestFilterDecision, RequestFilterEffects, RequestFilterInput, - RequestFilterRegistryInput, RequestFilterRegistryOutcome, ScriptRewriteAction, - ScriptTextAccumulator, + IntegrationRequestFilter, IntegrationRequestState, IntegrationScriptContext, + IntegrationScriptRewriter, ProxyDispatchInput, RequestFilterDecision, RequestFilterEffects, + RequestFilterInput, RequestFilterRegistryInput, RequestFilterRegistryOutcome, + ScriptRewriteAction, ScriptTextAccumulator, }; /// Registers or retrieves a platform backend for the given URL. @@ -329,6 +329,14 @@ pub type IntegrationValidateFn = fn(&Settings) -> Result) -> Result<(), Report>; +/// Finishes the response the page path returns for one request, from what +/// the module's request hooks left for it in the request's +/// [`IntegrationRequestState`]. +/// +/// Runs whether or not a section selects the integration, as the preparer +/// does, and has nothing to do for a request its module left nothing on. +pub type IntegrationFinalizeResponseFn = fn(&IntegrationRequestState, &mut Response); + /// Source label for the built-in integrations. pub const CORE_SOURCE: &str = "trusted-server-core"; @@ -378,6 +386,8 @@ pub struct IntegrationBuilder { build: IntegrationBuilderFn, validate: IntegrationValidateFn, prepare_request: Option, + finalize_response: Option, + reads_auction_token: bool, supplies_integration: bool, demand: Option<&'static DemandImplementation>, adserver: Option<&'static AdServerImplementation>, @@ -413,6 +423,8 @@ impl IntegrationBuilder { build, validate, prepare_request: None, + finalize_response: None, + reads_auction_token: false, supplies_integration: true, demand: None, adserver: None, @@ -435,6 +447,8 @@ impl IntegrationBuilder { build: no_registration, validate: nothing_to_validate, prepare_request: None, + finalize_response: None, + reads_auction_token: false, supplies_integration: false, demand: None, adserver: None, @@ -488,6 +502,34 @@ impl IntegrationBuilder { self } + /// Attaches a function that finishes the response the page path returns, + /// from what the module's request hooks left for the request. + #[must_use] + pub const fn with_response_finalizer( + mut self, + finalize: IntegrationFinalizeResponseFn, + ) -> Self { + self.finalize_response = Some(finalize); + self + } + + /// Declares that the module's browser script reads the token an auction + /// publishes with its winning bids. + /// + /// A token is made for each auction when a section selects such a + /// module, and none is made in a deployment where nothing reads it. + #[must_use] + pub const fn with_auction_token(mut self) -> Self { + self.reads_auction_token = true; + self + } + + /// Whether the module's browser script reads the auction token. + #[must_use] + pub const fn reads_auction_token(&self) -> bool { + self.reads_auction_token + } + /// Runs this builder when a section selects the module `name`, which is a /// crate's path under `crates` with `.` between the parts, such as /// `cmp.example`. An integration still in core holds the name its crate @@ -563,6 +605,11 @@ impl IntegrationBuilder { pub(crate) fn prepare_request(&self) -> Option { self.prepare_request } + + /// The response finalizer, when one is attached. + pub(crate) fn finalize_response(&self) -> Option { + self.finalize_response + } } /// The built-in integrations, in hook order. @@ -577,6 +624,10 @@ const BUILT_IN_BUILDERS: &[IntegrationBuilder] = &[ // A stand-in for an integration that tags a page, for the same tests. #[cfg(test)] registry_test_support::tag_fixture::BUILDER, + // A stand-in for an integration that acts on one request, for the same + // tests. + #[cfg(test)] + registry_test_support::request_fixture::BUILDER, datadome::BUILDER, gpt_diagnostics::BUILDER, // Implementations `[demand]` and `[ad-server]` can name. None of them is diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index 482440917a..2593e7cc41 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -194,6 +194,104 @@ impl IntegrationDocumentState { } } +/// Values a module's request hooks leave for its own page hooks, for one +/// request. +/// +/// A request preparer or a request filter leaves a value under its +/// integration id with [`IntegrationRequestState::insert`]. When the request +/// produces an HTML document for that one reader, every value is copied into +/// the document's [`IntegrationDocumentState`] before parsing starts, so the +/// module's head injector, rewriters and stream processors read it there, and +/// the same values are handed to the module's response finalizer. +/// +/// A request that carries any value keeps to the origin path. Its document is +/// never read from a shared template and never stored as one, and an HTML +/// response with a body is sent `private, no-store`. A document built to be +/// shared starts with none of these values. +#[derive(Clone, Default)] +pub struct IntegrationRequestState { + values: IntegrationDocumentStateMap, +} + +impl std::fmt::Debug for IntegrationRequestState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("IntegrationRequestState") + .field("keys", &self.values.keys().collect::>()) + .finish() + } +} + +impl IntegrationRequestState { + /// Leaves `value` on `request` for the page hooks of `integration_id`, + /// in place of a value of the same type left earlier. + pub fn insert(request: &mut Request, integration_id: &'static str, value: T) + where + T: Any + Send + Sync + 'static, + { + let mut state = request + .extensions_mut() + .remove::() + .unwrap_or_default(); + state.set(integration_id, value); + request.extensions_mut().insert(state); + } + + /// Holds `value` for the page hooks of `integration_id`, in place of a + /// value of the same type held earlier. + pub fn set(&mut self, integration_id: &'static str, value: T) + where + T: Any + Send + Sync + 'static, + { + self.values + .insert((integration_id, TypeId::of::()), Arc::new(value)); + } + + /// The values left on `request`, which is none for a request no module + /// left one on. + #[must_use] + pub fn of(request: &Request) -> Self { + request + .extensions() + .get::() + .cloned() + .unwrap_or_default() + } + + /// The value of type `T` left for `integration_id`. + #[must_use] + pub fn get(&self, integration_id: &'static str) -> Option> + where + T: Any + Send + Sync + 'static, + { + let value = self.values.get(&(integration_id, TypeId::of::()))?; + Arc::clone(value).downcast::().ok() + } + + /// Whether no module left a value. + #[must_use] + pub fn is_empty(&self) -> bool { + self.values.is_empty() + } + + /// Copies every value into the state of a document that is starting. + /// + /// # Panics + /// + /// Panics if the document state's lock is poisoned. + pub(crate) fn seed(&self, document: &IntegrationDocumentState) { + if self.values.is_empty() { + return; + } + let mut guard = document + .inner + .lock() + .expect("should lock integration document state"); + for (key, value) in &self.values { + guard.insert(*key, Arc::clone(value)); + } + } +} + /// Per-document buffer for script text fragments split across chunks. /// /// `lol_html` can deliver one text node as several chunks, so a rewriter that @@ -624,6 +722,13 @@ pub trait IntegrationHeadInjector: Send + Sync { /// Return HTML snippets to insert at the start of ``. fn head_inserts(&self, ctx: &IntegrationHtmlContext<'_>) -> Vec; + /// Return HTML snippets to insert straight after the main script bundle + /// and before any deferred one, for a script that needs the bundle to + /// have run and has to run before the page's own scripts. + fn after_bundle_inserts(&self, _ctx: &IntegrationHtmlContext<'_>) -> Vec { + Vec::new() + } + /// Return attributes to add to the publisher TSJS bundle tag. fn tsjs_script_tag_attributes(&self) -> Vec<(&'static str, &'static str)> { Vec::new() @@ -845,6 +950,10 @@ impl IntegrationRegistrationBuilder { type RouteValue = (Arc, &'static str); +/// Marks a request the preparers have run for. +#[derive(Clone, Copy)] +struct RequestPrepared; + struct IntegrationRegistryInner { // Method-specific routers for O(log n) lookups get_router: Router, @@ -878,6 +987,8 @@ struct IntegrationRegistryInner { extra_js_module_ids: Vec<&'static str>, // Preparers from every builder, named or not, in registration order. request_preparers: Vec, + // Finalizers from every builder, named or not, in registration order. + response_finalizers: Vec, // Geo modules the running registrations supply, each with the name // `[geo] module` selects it by, in registration order. Declaring one does // not activate it. @@ -923,6 +1034,7 @@ impl Default for IntegrationRegistryInner { request_filters: Vec::new(), extra_js_module_ids: Vec::new(), request_preparers: Vec::new(), + response_finalizers: Vec::new(), geo_modules: Vec::new(), ec_modules: Vec::new(), device_modules: Vec::new(), @@ -1420,6 +1532,9 @@ impl IntegrationRegistry { if let Some(prepare) = builder.prepare_request() { inner.request_preparers.push(prepare); } + if let Some(finalize) = builder.finalize_response() { + inner.response_finalizers.push(finalize); + } // Only a builder whose module a section selects is built, so an // integration runs exactly when an operator selects it, whatever @@ -1667,7 +1782,9 @@ impl IntegrationRegistry { /// /// Preparers run whether or not their integration runs, so an /// integration can sanitize its own reserved query or cookie in a - /// deployment that has it switched off. + /// deployment that has it switched off. They run once for a request, so + /// a caller further along the request path can call this again and be + /// sure the request is prepared without running any of them twice. /// /// # Errors /// @@ -1677,12 +1794,38 @@ impl IntegrationRegistry { settings: &Settings, request: &mut Request, ) -> Result<(), Report> { + if request.extensions().get::().is_some() { + return Ok(()); + } for prepare in &self.inner.request_preparers { prepare(settings, request)?; } + request.extensions_mut().insert(RequestPrepared); Ok(()) } + /// Marks `request` as one the preparers have run for, for a test that + /// hands a handler the request an adapter would have prepared. + #[cfg(test)] + pub(crate) fn mark_prepared_for_tests(request: &mut Request) { + request.extensions_mut().insert(RequestPrepared); + } + + /// Runs every registered integration's response finalizer, in + /// registration order, on the response the page path is about to return. + /// + /// Each is handed what the module's request hooks left for this request, + /// and one with nothing left for it has nothing to do. + pub fn finalize_response( + &self, + request_state: &IntegrationRequestState, + response: &mut Response, + ) { + for finalize in &self.inner.response_finalizers { + finalize(request_state, response); + } + } + /// Run pre-routing request filters. /// /// Request header mutations are applied immediately so later filters and @@ -1843,6 +1986,17 @@ impl IntegrationRegistry { inserts } + /// Collect HTML snippets for insertion straight after the main script + /// bundle. + #[must_use] + pub fn after_bundle_inserts(&self, ctx: &IntegrationHtmlContext<'_>) -> Vec { + self.inner + .head_injectors + .iter() + .flat_map(|injector| injector.after_bundle_inserts(ctx)) + .collect() + } + /// Collect static attributes for the publisher TSJS bundle tag. #[must_use] pub fn tsjs_script_tag_attributes(&self) -> Vec<(&'static str, &'static str)> { @@ -2126,6 +2280,7 @@ impl IntegrationRegistry { head_injectors: Vec::new(), request_filters: Vec::new(), request_preparers: Vec::new(), + response_finalizers: Vec::new(), deferred_js_ids: Vec::new(), disabled_js_ids: Vec::new(), extra_js_module_ids: Vec::new(), @@ -2168,6 +2323,7 @@ impl IntegrationRegistry { head_injectors, request_filters: Vec::new(), request_preparers: Vec::new(), + response_finalizers: Vec::new(), deferred_js_ids: Vec::new(), disabled_js_ids: Vec::new(), extra_js_module_ids: Vec::new(), @@ -2206,6 +2362,7 @@ impl IntegrationRegistry { head_injectors: Vec::new(), request_filters, request_preparers: Vec::new(), + response_finalizers: Vec::new(), deferred_js_ids: Vec::new(), disabled_js_ids: Vec::new(), extra_js_module_ids: Vec::new(), @@ -2284,6 +2441,7 @@ impl IntegrationRegistry { head_injectors: Vec::new(), request_filters: Vec::new(), request_preparers: Vec::new(), + response_finalizers: Vec::new(), deferred_js_ids: Vec::new(), disabled_js_ids: Vec::new(), extra_js_module_ids: Vec::new(), @@ -2346,6 +2504,182 @@ pub(crate) mod test_support { Ok(true) } + /// A stand-in for an integration that acts on one request, for core's + /// own tests of the page path. + /// + /// Its preparer runs for every request, as any preparer does, and strips + /// the query `fixture_request=1`. Where that query arrived on a document + /// navigation it also leaves a mark for the page path. Selected, the + /// stand-in's head injector reads the mark from the document's state and + /// writes one script at the start of `` and the tag of its + /// standalone module after the bundle. Its response finalizer sets a + /// cookie on the response to a marked request, and its builder declares + /// that it reads the auction token. + pub(crate) mod request_fixture { + use std::sync::Arc; + + use edgezero_core::body::Body as EdgeBody; + use error_stack::Report; + use http::{HeaderValue, Method, Request, Response, Uri, header}; + + use crate::error::TrustedServerError; + use crate::integrations::registry::{ + CarriedJsModule, IntegrationHeadInjector, IntegrationHtmlContext, + IntegrationRegistration, IntegrationRequestState, + }; + use crate::integrations::{CORE_SOURCE, IntegrationBuilder}; + use crate::settings::Settings; + use crate::tsjs_bundle::JsModulePart; + + /// The integration id the stand-in registers under. + pub(crate) const ID: &str = "request_fixture"; + /// The name a test's settings select the stand-in by, in `[testing]`. + pub(crate) const MODULE: &str = "testing.request-fixture"; + /// The query a navigation asks the stand-in to act with. + pub(crate) const QUERY: &str = "fixture_request=1"; + /// What the stand-in's head insert sets for a marked request. + pub(crate) const HEAD_FLAG: &str = "window.__ts_request_fixture=true;"; + /// The file of the stand-in's standalone module, which the document + /// of a marked request loads after the bundle. + pub(crate) const MODULE_FILE: &str = "tsjs-request_fixture.min.js"; + /// The cookie the finalizer sets on the response to a marked request. + pub(crate) const COOKIE: &str = "ts-request-fixture=1; Path=/"; + + const JS: &str = "(function(){window.__ts_request_fixture_loaded=1;})();"; + // SHA-256 of JS, hex. The registry refuses a literal that is not. + const JS_SHA256: &str = "7160e730ce9301ed134c84fa13605d7cd979cf929679f1c3176ea88b1989477e"; + + /// The builder core's test build lists beside its own. + pub(crate) const BUILDER: IntegrationBuilder = + IntegrationBuilder::new(ID, CORE_SOURCE, register, validate) + .with_module_name(MODULE) + .with_request_preparer(prepare) + .with_response_finalizer(finalize) + .with_auction_token(); + + #[derive(Debug, serde::Deserialize, validator::Validate)] + #[serde(deny_unknown_fields)] + struct FixtureSettings {} + + impl crate::settings::IntegrationConfig for FixtureSettings {} + + /// What the preparer leaves for the page path. + #[derive(Debug, Clone, Copy)] + pub(crate) struct Mark; + + /// Leaves the mark on `request`, as the preparer does for the query. + pub(crate) fn mark(request: &mut Request) { + IntegrationRequestState::insert(request, ID, Mark); + } + + /// The request state of a marked request. + pub(crate) fn marked() -> IntegrationRequestState { + let mut state = IntegrationRequestState::default(); + state.set(ID, Mark); + state + } + + fn prepare( + _settings: &Settings, + request: &mut Request, + ) -> Result<(), Report> { + let query = request.uri().query().unwrap_or_default(); + if !query.split('&').any(|pair| pair == QUERY) { + return Ok(()); + } + let retained = query + .split('&') + .filter(|pair| *pair != QUERY) + .collect::>() + .join("&"); + let mut path_and_query = request.uri().path().to_owned(); + if !retained.is_empty() { + path_and_query.push('?'); + path_and_query.push_str(&retained); + } + let mut parts = request.uri().clone().into_parts(); + parts.path_and_query = Some( + path_and_query + .parse() + .expect("should keep a valid path and query"), + ); + *request.uri_mut() = Uri::from_parts(parts).expect("should keep a valid URI"); + + if request.method() == Method::GET && crate::http_util::is_navigation_request(request) { + mark(request); + } + Ok(()) + } + + fn finalize(request_state: &IntegrationRequestState, response: &mut Response) { + if request_state.get::(ID).is_some() { + response + .headers_mut() + .append(header::SET_COOKIE, HeaderValue::from_static(COOKIE)); + } + } + + struct Head; + + impl Head { + fn marked(ctx: &IntegrationHtmlContext<'_>) -> bool { + ctx.document_state.get::(ID).is_some() + } + } + + impl IntegrationHeadInjector for Head { + fn integration_id(&self) -> &'static str { + ID + } + + fn head_inserts(&self, ctx: &IntegrationHtmlContext<'_>) -> Vec { + if !Self::marked(ctx) { + return Vec::new(); + } + vec![format!("")] + } + + fn after_bundle_inserts(&self, ctx: &IntegrationHtmlContext<'_>) -> Vec { + if !Self::marked(ctx) { + return Vec::new(); + } + let module = JsModulePart { + id: ID, + source: JS, + sha256: JS_SHA256, + }; + vec![format!( + "", + crate::tsjs::tsjs_single_module_script_src(&module) + )] + } + } + + fn register( + settings: &Settings, + ) -> Result, Report> { + let Some(_config) = settings.module_config::(MODULE)? else { + return Ok(None); + }; + Ok(Some( + IntegrationRegistration::builder(ID) + .with_js_module(CarriedJsModule { + source: JS, + sha256: JS_SHA256, + }) + .with_standalone_js() + .with_head_injector(Arc::new(Head)) + .build(), + )) + } + + fn validate(settings: &Settings) -> Result> { + settings + .module_config::(MODULE) + .map(|config| config.is_some()) + } + } + /// A stand-in for an integration that tags a page, for core's own tests /// of the HTML processor and the JavaScript asset proxy. /// @@ -4351,6 +4685,224 @@ mod tests { ); } + /// Appends to a header on each run, so a test can count the runs on the + /// request itself. + fn counting_preparer( + _settings: &Settings, + request: &mut Request, + ) -> Result<(), Report> { + request + .headers_mut() + .append("x-probe-prepared", HeaderValue::from_static("1")); + Ok(()) + } + + #[test] + fn prepare_request_runs_the_preparers_once_for_a_request() { + let settings = crate::test_support::tests::create_test_settings(); + let extra = [crate::integrations::IntegrationBuilder::new( + "probe-counting", + "seam-probe", + never_registering_builder, + validate_nothing, + ) + .with_module_name("testing.probe-counting") + .with_request_preparer(counting_preparer)]; + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should build registry with a request preparer"); + let mut request = plain_request(); + + registry + .prepare_request(&settings, &mut request) + .expect("should prepare the request"); + registry + .prepare_request(&settings, &mut request) + .expect("should accept a request that is already prepared"); + + assert_eq!( + request.headers().get_all("x-probe-prepared").iter().count(), + 1, + "should run a preparer once however often the request is prepared" + ); + + let mut next = plain_request(); + registry + .prepare_request(&settings, &mut next) + .expect("should prepare the next request"); + assert_eq!( + next.headers().get_all("x-probe-prepared").iter().count(), + 1, + "should prepare each request" + ); + } + + /// What the second probe's request hook leaves for its finalizer. + #[derive(Debug, PartialEq)] + struct ProbeNote(&'static str); + + fn first_finalizer(_state: &IntegrationRequestState, response: &mut Response) { + response + .headers_mut() + .append("x-probe-finalized", HeaderValue::from_static("first")); + } + + fn second_finalizer(state: &IntegrationRequestState, response: &mut Response) { + let value = match state.get::("probe-second") { + Some(_) => "second-with-its-note", + None => "second", + }; + response + .headers_mut() + .append("x-probe-finalized", HeaderValue::from_static(value)); + } + + #[test] + fn finalize_response_runs_every_finalizer_in_registration_order_with_the_request_state() { + let settings = crate::test_support::tests::create_test_settings(); + let extra = [ + crate::integrations::IntegrationBuilder::new( + "probe-first", + "seam-probe", + never_registering_builder, + validate_nothing, + ) + .with_module_name("testing.probe-first") + .with_response_finalizer(first_finalizer), + crate::integrations::IntegrationBuilder::new( + "probe-second", + "seam-probe", + never_registering_builder, + validate_nothing, + ) + .with_module_name("testing.probe-second") + .with_response_finalizer(second_finalizer), + ]; + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should build registry with response finalizers"); + let finalized = |response: &Response| { + response + .headers() + .get_all("x-probe-finalized") + .iter() + .map(|value| value.to_str().expect("should be text").to_owned()) + .collect::>() + }; + + let mut request = plain_request(); + IntegrationRequestState::insert(&mut request, "probe-second", ProbeNote("left")); + let mut response = Response::builder() + .body(EdgeBody::empty()) + .expect("should build response"); + registry.finalize_response(&IntegrationRequestState::of(&request), &mut response); + + assert_eq!( + finalized(&response), + vec!["first", "second-with-its-note"], + "should run both finalizers in registration order, neither integration \ + running, and hand each what was left on the request" + ); + + let mut unmarked = Response::builder() + .body(EdgeBody::empty()) + .expect("should build response"); + registry.finalize_response(&IntegrationRequestState::default(), &mut unmarked); + assert_eq!( + finalized(&unmarked), + vec!["first", "second"], + "should hand a finalizer no state for a request nothing was left on" + ); + } + + #[test] + fn what_a_module_leaves_on_a_request_reaches_a_document_when_it_is_seeded() { + let mut request = plain_request(); + assert!( + IntegrationRequestState::of(&request).is_empty(), + "should hold nothing for a request no module left a value on" + ); + + IntegrationRequestState::insert(&mut request, "probe", ProbeNote("first")); + IntegrationRequestState::insert(&mut request, "probe", ProbeNote("second")); + IntegrationRequestState::insert(&mut request, "other", ProbeNote("other")); + let state = IntegrationRequestState::of(&request); + + assert!(!state.is_empty(), "should hold what was left"); + assert_eq!( + state.get::("probe").as_deref(), + Some(&ProbeNote("second")), + "should keep the last value of one type left for an integration" + ); + assert!( + state.get::("absent").is_none(), + "should hold nothing for an integration that left nothing" + ); + + let document = IntegrationDocumentState::default(); + assert!( + document.get::("probe").is_none(), + "should start a document with none of the request's values" + ); + state.seed(&document); + assert_eq!( + document.get::("probe").as_deref(), + Some(&ProbeNote("second")), + "should copy the value into the document's state" + ); + assert_eq!( + document.get::("other").as_deref(), + Some(&ProbeNote("other")), + "should copy every integration's value" + ); + } + + #[test] + fn the_request_stand_in_marks_a_navigation_and_strips_its_query() { + use super::test_support::request_fixture; + + let settings = crate::test_support::tests::create_test_settings(); + let registry = IntegrationRegistry::new(&settings).expect("should build registry"); + let mut navigation = Request::builder() + .method(Method::GET) + .uri("https://publisher.example.com/article?fixture_request=1&keep=yes") + .header("sec-fetch-dest", "document") + .body(EdgeBody::empty()) + .expect("should build request"); + let mut subresource = Request::builder() + .method(Method::GET) + .uri("https://publisher.example.com/data.json?fixture_request=1") + .header("sec-fetch-dest", "empty") + .body(EdgeBody::empty()) + .expect("should build request"); + + registry + .prepare_request(&settings, &mut navigation) + .expect("should prepare the navigation"); + registry + .prepare_request(&settings, &mut subresource) + .expect("should prepare the subresource request"); + + assert_eq!( + navigation.uri().query(), + Some("keep=yes"), + "should strip the stand-in's query and keep the rest" + ); + assert!( + IntegrationRequestState::of(&navigation) + .get::(request_fixture::ID) + .is_some(), + "should leave the mark on a document navigation, selected or not" + ); + assert_eq!( + subresource.uri().query(), + None, + "should strip the stand-in's query from any request" + ); + assert!( + IntegrationRequestState::of(&subresource).is_empty(), + "should leave no mark on a request for something other than a document" + ); + } + #[test] fn prepare_request_runs_the_built_in_gpt_diagnostics_preparer() { // The adapters call the registry rather than naming GPT diagnostics, so diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 920853e2b8..32d106e127 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -68,8 +68,8 @@ use crate::ec::{EcContext, EidSyncSource}; use crate::error::TrustedServerError; use crate::html_processor::BodyCloseInjection; use crate::http_util::{RequestInfo, is_navigation_request, serve_static_with_etag}; -use crate::integrations::IntegrationRegistry; use crate::integrations::aps::{APS_RENDERER_BID_ID_KEY, APS_RENDERER_TYPE}; +use crate::integrations::{IntegrationRegistry, IntegrationRequestState}; use crate::permissions::PermissionState; use crate::platform::{ GeoInfo, PlatformBackendSpec, PlatformHttpRequest, RuntimeServices, @@ -639,9 +639,8 @@ struct ProcessResponseParams<'a> { permissions_script: Option<&'a str>, ad_slots_script: Option<&'a str>, ad_bids_state: &'a Arc>>, - suppress_datadome_client_side_tag: bool, - gpt_diagnostics: - Option<&'a crate::integrations::gpt_diagnostics::GptDiagnosticsRequestDecision>, + /// See [`HtmlStreamProcessorParams::request_state`]. + request_state: &'a IntegrationRequestState, /// See [`HtmlStreamProcessorParams::shared_template_authorized`]. shared_template_authorized: bool, /// See [`HtmlStreamProcessorParams::csp_nonce_observed`]. @@ -683,8 +682,7 @@ impl PublisherBodyProcessor { permissions_script: permissions_script_for(params, settings), ad_slots_script: params.ad_slots_script.as_deref().map(str::to_string), ad_bids_state: Arc::clone(params.ad_bids_state.script_cell()), - suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, - gpt_diagnostics: params.gpt_diagnostics.clone(), + request_state: params.request_state.clone(), shared_template_authorized: params.template_cache_key.is_some(), csp_nonce_observed: params.csp_nonce_observed.clone(), deferred_inline_marker: inline_seam_token @@ -768,8 +766,7 @@ fn process_response_streaming( permissions_script: params.permissions_script.map(str::to_string), ad_slots_script: params.ad_slots_script.map(str::to_string), ad_bids_state: params.ad_bids_state.clone(), - suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, - gpt_diagnostics: params.gpt_diagnostics.cloned(), + request_state: params.request_state.clone(), shared_template_authorized: params.shared_template_authorized, csp_nonce_observed: params.csp_nonce_observed.cloned(), deferred_inline_marker: None, @@ -1328,8 +1325,9 @@ struct HtmlStreamProcessorParams<'a> { permissions_script: Option, ad_slots_script: Option, ad_bids_state: Arc>>, - suppress_datadome_client_side_tag: bool, - gpt_diagnostics: Option, + /// What modules left on the request for their page hooks. See + /// [`template_request_state`]. + request_state: IntegrationRequestState, /// Whether a shared template was authorized for this response. /// /// Carried rather than re-derived so both seams see the same answer. See @@ -1341,26 +1339,22 @@ struct HtmlStreamProcessorParams<'a> { deferred_inline_marker: Option, } -/// The diagnostics decision the template may carry. +/// The request state a document's page hooks may read. /// -/// Diagnostics is request-scoped — activated by a cookie or query parameter, and -/// documented as an immutable per-request decision — so it must not reach a shared -/// template. +/// What a module leaves on a request is made for that one request, so it +/// must not reach a template that is stored and served to other readers. /// -/// It does not leak today even without this gate, but only by coincidence: -/// `requires_private_no_store()` is a strict superset of the conditions under which -/// a script is emitted, and that stamp lands before the template cache gate reads response -/// headers, so the gate refuses. Two independent conditions that happen to align, -/// with nothing enforcing the relationship. This makes the guarantee explicit; -/// `requires_private_no_store_is_a_superset_of_injection` keeps the coincidence as a -/// backstop if this gate is ever removed. -pub(crate) fn template_gpt_diagnostics( +/// [`handle_publisher_request`] already keeps a request that carries any off +/// the shared template path, so a shared mode never sees one today. This +/// makes the same guarantee where the document is built, so that neither +/// place depends on the other. +pub(crate) fn template_request_state( mode: AssemblyMode, - decision: Option, -) -> Option { + request_state: IntegrationRequestState, +) -> IntegrationRequestState { match mode { - AssemblyMode::Inline => decision, - AssemblyMode::Esi => None, + AssemblyMode::Inline => request_state, + AssemblyMode::Esi => IntegrationRequestState::default(), } } @@ -1538,7 +1532,7 @@ fn create_html_stream_processor( _ => body_close_injection(assembly_mode, params.ad_slots_script.is_some()), }; - let gpt_diagnostics = template_gpt_diagnostics(assembly_mode, params.gpt_diagnostics); + let request_state = template_request_state(assembly_mode, params.request_state); // Only a response that can be stored has a consumer for the observation, so the // handlers are not registered for ordinary inline traffic. @@ -1550,10 +1544,9 @@ fn create_html_stream_processor( let config = config .with_permissions_script(params.permissions_script) .with_ad_state(params.ad_slots_script, params.ad_bids_state) - .with_gpt_diagnostics(gpt_diagnostics) + .with_request_state(request_state) .with_body_close(body_close) - .with_csp_nonce_observer(csp_nonce_observed) - .with_datadome_client_tag_suppression(params.suppress_datadome_client_side_tag); + .with_csp_nonce_observer(csp_nonce_observed); Ok(create_html_processor(config)) } @@ -1752,11 +1745,8 @@ pub struct OwnedProcessResponseParams { pub(crate) dispatched_auction: Option, /// Price granularity used to bucket bids when building `tsjs.bids`. pub(crate) price_granularity: PriceGranularity, - /// Whether to omit Trusted Server's automatic `DataDome` client-side tag. - pub(crate) suppress_datadome_client_side_tag: bool, - /// Request-scoped conditional diagnostics delivery decision. - pub(crate) gpt_diagnostics: - Option, + /// What modules left on the request for their page hooks. + pub(crate) request_state: IntegrationRequestState, /// Set by the transform when the document carries a response-bound CSP nonce. /// /// `None` wherever no transform runs. Recorded by the HTML parser rather than @@ -2202,8 +2192,9 @@ fn build_template_assembly_params( auction_request: None, dispatched_auction: None, price_granularity, - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + // A request that carries any keeps to the origin path, so none reaches a + // document served from a template. + request_state: IntegrationRequestState::default(), } } @@ -2967,8 +2958,7 @@ pub fn stream_publisher_body( permissions_script: permissions_script.as_deref(), ad_slots_script: params.ad_slots_script.as_deref(), ad_bids_state: params.ad_bids_state.script_cell(), - suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, - gpt_diagnostics: params.gpt_diagnostics.as_ref(), + request_state: ¶ms.request_state, shared_template_authorized: params.template_cache_key.is_some(), csp_nonce_observed: params.csp_nonce_observed.as_ref(), }; @@ -3073,8 +3063,7 @@ pub async fn stream_publisher_body_async( permissions_script: permissions_script_for(params, settings), ad_slots_script: params.ad_slots_script.as_deref().map(str::to_string), ad_bids_state: Arc::clone(params.ad_bids_state.script_cell()), - suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, - gpt_diagnostics: params.gpt_diagnostics.clone(), + request_state: params.request_state.clone(), shared_template_authorized: params.template_cache_key.is_some(), csp_nonce_observed: params.csp_nonce_observed.clone(), deferred_inline_marker: inline_seam_token @@ -4256,7 +4245,7 @@ async fn collect_non_html_auction( let auction_id = telemetry .auction_request .as_ref() - .and_then(|_| diagnostics_auction_id(settings)); + .and_then(|_| auction_token(orchestrator)); let placeholder = adserver_placeholder_request(); let result = orchestrator .collect_dispatched_auction( @@ -4310,7 +4299,7 @@ async fn collect_stream_auction( let auction_id = telemetry .auction_request .as_ref() - .and_then(|_| diagnostics_auction_id(settings)); + .and_then(|_| auction_token(orchestrator)); log::info!("body_close_hold_loop: collecting dispatched auction before held body tail"); let placeholder = adserver_placeholder_request(); let collect_ctx = make_collect_context(settings, services, &placeholder); @@ -4510,10 +4499,12 @@ pub async fn handle_publisher_request( log::debug!("Proxying request to publisher_origin"); - // Adapter fallbacks prepare this before EC/cookie handling. Keep this - // idempotent call as a direct-handler safety net and for focused tests. - let gpt_diagnostics = - crate::integrations::gpt_diagnostics::prepare_request(settings, &mut req)?; + // Adapters prepare the request before EC and cookie handling, and the + // registry prepares one request once, so this covers a caller that + // reaches the handler directly and changes nothing for the others. + integration_registry.prepare_request(settings, &mut req)?; + // What the modules' request hooks left for this request's document. + let request_state = IntegrationRequestState::of(&req); // Prebid.js requests are not intercepted here anymore. The HTML processor removes // publisher-supplied Prebid scripts; the unified TSJS bundle includes Prebid.js when enabled. @@ -4753,18 +4744,11 @@ pub async fn handle_publisher_request( // for a range or a conditional response must not be handed a full document // synthesized from a template shared with other readers, whatever the origin is then // asked for on their behalf. + // A request a module left state on gets a document made for it alone, which + // a template shared with other readers cannot be. let reader_requires_origin = request_bypasses_template_cache(req.headers()) - || gpt_diagnostics.requires_private_no_store() + || !request_state.is_empty() || personalization_requires_origin; - // The only DataDome-specific signal core still reads. It decides nothing about caching - // or the origin path, because it travels to `HtmlProcessorConfig` so DataDome's own - // head injection can leave its client-side tag out, and it moves out of core with - // the DataDome integration. Read from the request rather than derived from the - // neutral personalization marker above, so the two concerns stay separate. - let suppress_datadome_client_side_tag = req - .extensions() - .get::() - .is_some(); let reader_compression = negotiate_reader_compression(req.headers()); let reader_supports_assembly = reader_compression.is_ok(); // A failed negotiation bypasses template cache below, so this value is used only on an @@ -4790,7 +4774,7 @@ pub async fn handle_publisher_request( // request directives, so `If-Match`, `If-Unmodified-Since` and a `no-store` reader // still disqualify, stripped or not. let request_requires_origin = request_bypasses_template_cache(req.headers()) - || gpt_diagnostics.requires_private_no_store() + || !request_state.is_empty() || personalization_requires_origin; let method_is_cacheable = req.method() == Method::GET; @@ -5449,10 +5433,13 @@ pub async fn handle_publisher_request( } } } + // A document that modules' request state is copied into is that reader's + // alone, whether or not the module that left it also marked the response + // personalized. apply_personalized_response_cache_privacy( &mut response, &request_method, - response_is_personalized, + response_is_personalized || !request_state.is_empty(), &origin_content_type, ); apply_publisher_asset_cache_policy( @@ -5463,7 +5450,7 @@ pub async fn handle_publisher_request( &mut response, )?; - crate::integrations::gpt_diagnostics::finalize_response(&gpt_diagnostics, &mut response); + integration_registry.finalize_response(&request_state, &mut response); let content_type = response .headers() @@ -5582,12 +5569,11 @@ pub async fn handle_publisher_request( permissions_json, ad_slots_script: ad_slots_script.clone(), ad_bids_state: ad_bids_state.clone(), - suppress_datadome_client_side_tag, + request_state, auction_observation, auction_request: auction_request_for_telemetry, dispatched_auction, price_granularity, - gpt_diagnostics: Some(gpt_diagnostics), }), }) } @@ -5721,7 +5707,8 @@ pub(crate) fn build_auction_request( } } -/// Mint the browser-visible auction correlation token for GPT diagnostics. +/// Mint the browser-visible token that lets a browser module tell one +/// auction from the next. /// /// The token is freshly generated per auction and carries no user identity. /// [`AuctionRequest::id`] must never be used here: for a consented visitor it is @@ -5729,10 +5716,11 @@ pub(crate) fn build_auction_request( /// EC identifier to any script on the page, and — being stable per visitor — it /// could not distinguish one auction from the next either. /// -/// Returns `None` unless the GPT diagnostics integration runs, since -/// nothing else consumes the value. -fn diagnostics_auction_id(settings: &Settings) -> Option { - crate::integrations::gpt_diagnostics::runs(settings) +/// Returns `None` unless a module that runs declared that it reads the token, +/// since nothing else consumes the value. +fn auction_token(orchestrator: &AuctionOrchestrator) -> Option { + orchestrator + .publishes_auction_token() .then(|| format!("ts-auc-{}", uuid::Uuid::new_v4().simple())) } @@ -7308,7 +7296,7 @@ pub async fn handle_page_bids( { Ok(result) => { let winning_bids = result.winning_bids.clone(); - let auction_id = diagnostics_auction_id(settings); + let auction_id = auction_token(auction.orchestrator); let bid_map = build_bid_map_with_auction_id( &winning_bids, co_config.price_granularity, @@ -7883,9 +7871,7 @@ mod tests { .header(header::USER_AGENT, "scheduling-browser") .body(EdgeBody::empty()) .expect("should build navigation request"); - request - .extensions_mut() - .insert(crate::integrations::datadome::DataDomeClientTagSuppressed); + crate::integrations::datadome::suppress_client_tag(&mut request); request } @@ -9236,8 +9222,7 @@ mod tests { auction_request: None, dispatched_auction: None, price_granularity: Default::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), } } @@ -9296,12 +9281,11 @@ mod tests { .header("sec-fetch-dest", "document") .body(EdgeBody::empty()) .expect("should build activation request"); - let decision = - crate::integrations::gpt_diagnostics::prepare_request(&settings, &mut request) - .expect("should prepare diagnostics request"); + crate::integrations::gpt_diagnostics::prepare_request(&settings, &mut request) + .expect("should prepare diagnostics request"); let mut params = make_stream_params(&settings, ""); params.content_type = "text/html".to_owned(); - params.gpt_diagnostics = Some(decision); + params.request_state = IntegrationRequestState::of(&request); let mut output = Vec::new(); stream_publisher_body( @@ -9482,7 +9466,7 @@ mod tests { use crate::creative_opportunities::AssemblyMode; use crate::html_processor::{HtmlProcessorConfig, create_html_processor}; use crate::integrations::IntegrationRegistry; - use crate::integrations::gpt_diagnostics::GptDiagnosticsRequestDecision; + use crate::integrations::registry_test_support::request_fixture; const DOCUMENT: &[u8] = b"t

content

"; @@ -9493,8 +9477,8 @@ mod tests { struct RequestShape { /// Folds in consent, bot classification, prefetch and the kill switch. ad_stack_ran: bool, - /// Cookie- or query-activated. - diagnostics_active: bool, + /// A module left state on the request for its page hooks. + module_state_left: bool, /// A resolved auction, present only when one was dispatched. bids_available: bool, } @@ -9502,17 +9486,24 @@ mod tests { /// Build the config exactly as `create_html_stream_processor` does, so a /// drift between a decision and its use is caught rather than hidden. fn render(mode: AssemblyMode, shape: RequestShape) -> String { - let settings = settings_with_slots(); + let mut settings = settings_with_slots(); + settings.proxy.allowed_domains = + vec!["*.example".to_string(), "*.example.com".to_string()]; + // The stand-in is what writes a marked request's state into a + // document, so it runs for every shape. + settings.select_module("testing", request_fixture::MODULE); let slots = [slot()]; let ad_slots_script = template_ad_slots_script(mode, shape.ad_stack_ran, &settings, &slots, "/"); let body_close = body_close_injection(mode, ad_slots_script.is_some()); - let gpt_diagnostics = template_gpt_diagnostics( + let request_state = template_request_state( mode, - shape - .diagnostics_active - .then(GptDiagnosticsRequestDecision::active_for_tests), + if shape.module_state_left { + request_fixture::marked() + } else { + IntegrationRequestState::default() + }, ); let ad_bids_state = @@ -9525,14 +9516,14 @@ mod tests { origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), - integrations: IntegrationRegistry::empty_for_tests(), + integrations: IntegrationRegistry::new(&settings) + .expect("should build a registry that runs the stand-in"), permissions_script: template_permissions_script(mode, &permissions_json_fixture()), ad_slots_script, ad_bids_state, max_buffered_body_bytes: 16 * 1024 * 1024, - gpt_diagnostics, + request_state, body_close, - suppress_datadome_client_side_tag: false, }; let mut processor = create_html_processor(config); @@ -9545,11 +9536,11 @@ mod tests { fn every_shape() -> Vec { let mut shapes = Vec::new(); for ad_stack_ran in [false, true] { - for diagnostics_active in [false, true] { + for module_state_left in [false, true] { for bids_available in [false, true] { shapes.push(RequestShape { ad_stack_ran, - diagnostics_active, + module_state_left, bids_available, }); } @@ -9607,7 +9598,7 @@ mod tests { mode, RequestShape { ad_stack_ran: true, - diagnostics_active: true, + module_state_left: true, bids_available: true, }, ); @@ -9615,8 +9606,8 @@ mod tests { ".adSlots", ".bids=", "permissions", - "__tsjs_gpt_diagnostics_active", - "history.replaceState", + request_fixture::HEAD_FLAG, + request_fixture::MODULE_FILE, ] { assert!( !rendered.contains(forbidden), @@ -9632,7 +9623,7 @@ mod tests { AssemblyMode::Inline, RequestShape { ad_stack_ran: true, - diagnostics_active: false, + module_state_left: false, bids_available: true, }, ); @@ -9658,6 +9649,49 @@ mod tests { ); } + #[test] + fn an_inline_document_carries_what_a_module_left_on_its_request() { + // The shared-mode test above would pass if a module's state reached + // no document at all, so this shows where it does. + let rendered = render( + AssemblyMode::Inline, + RequestShape { + ad_stack_ran: false, + module_state_left: true, + bids_available: false, + }, + ); + + let flag = rendered + .find(request_fixture::HEAD_FLAG) + .expect("should write the module's head insert for a marked request"); + let bundle = rendered + .find("id=\"trustedserver-js\"") + .expect("should inject the tsjs bundle into an inline head"); + let module = rendered + .find(request_fixture::MODULE_FILE) + .expect("should load the module's script for a marked request"); + assert!( + flag < bundle && bundle < module, + "the head insert should precede the bundle and the after-bundle \ + insert should follow it:\n{rendered}" + ); + + let unmarked = render( + AssemblyMode::Inline, + RequestShape { + ad_stack_ran: false, + module_state_left: false, + bids_available: false, + }, + ); + assert!( + !unmarked.contains(request_fixture::HEAD_FLAG) + && !unmarked.contains(request_fixture::MODULE_FILE), + "a request no module left state on should get neither:\n{unmarked}" + ); + } + #[test] fn inline_documents_with_no_ad_stack_still_carry_the_permission_state() { // Arrange / Act: the ad stack is skipped for bots, prefetches and @@ -9666,7 +9700,7 @@ mod tests { AssemblyMode::Inline, RequestShape { ad_stack_ran: false, - diagnostics_active: false, + module_state_left: false, bids_available: false, }, ); @@ -9691,7 +9725,7 @@ mod tests { AssemblyMode::Inline, RequestShape { ad_stack_ran: true, - diagnostics_active: false, + module_state_left: false, bids_available: true, }, ); @@ -9699,7 +9733,7 @@ mod tests { AssemblyMode::Inline, RequestShape { ad_stack_ran: false, - diagnostics_active: false, + module_state_left: false, bids_available: false, }, ); @@ -10683,14 +10717,11 @@ mod tests { request } - /// A navigation carrying an **active** diagnostics decision, the way the real one - /// arrives: in the request extensions, set from a per-reader cookie or query - /// parameter. - fn diagnostics_navigation_request() -> Request { + /// A navigation a module left state on for its page hooks, the way a + /// real one arrives: on the request, left by a preparer or a filter. + fn marked_navigation_request() -> Request { let mut request = navigation_request(); - request.extensions_mut().insert( - crate::integrations::gpt_diagnostics::GptDiagnosticsRequestDecision::active_for_tests(), - ); + crate::integrations::registry_test_support::request_fixture::mark(&mut request); request } @@ -13726,13 +13757,11 @@ mod tests { request } - // Exercise raw fields at the prepared-request boundary. Ordinary diagnostics - // preparation removes invalid fields/empty pairs before generic cookie handling. + // Exercise raw fields at the prepared-request boundary. A module's preparer + // may remove invalid fields and empty pairs before generic cookie handling. fn prepared_cookie_policy_request(fields: &[&[u8]]) -> Request { let mut request = cookie_policy_request(fields); - request.extensions_mut().insert( - crate::integrations::gpt_diagnostics::GptDiagnosticsRequestDecision::default(), - ); + IntegrationRegistry::mark_prepared_for_tests(&mut request); request } @@ -14750,24 +14779,19 @@ mod tests { } #[tokio::test] - async fn an_active_diagnostics_request_never_stores_a_template() { - // An independent review reintroduced a diagnostics leak scoped to - // A request-private diagnostics mutation could otherwise leak through a - // shared template. `requires_private_no_store()` is a - // strict superset of the condition under which diagnostics markup is - // emitted, and that stamp lands *before* the template cache gate reads response headers, - // so such a request never stores a template at all. - // - // That is a coincidence between two independent conditions, and the whole - // protection rests on it. This pins the consequence directly, so the - // relationship is checked rather than merely reasoned about. + async fn a_request_a_module_left_state_on_never_stores_a_template() { + // What a module leaves on a request is written into that request's + // document, so a template stored from it would hand one reader's + // document to every later one. The request is kept off the shared + // path before the origin is asked, and this pins the consequence + // directly rather than leaving it to be reasoned about. let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); let settings = Arc::new(settings_with_mode("esi")); let services = services(Arc::clone(&stub), Arc::clone(&cache)); queue_shareable_html(&stub); - let _ = run(&settings, &services, diagnostics_navigation_request()).await; + let _ = run(&settings, &services, marked_navigation_request()).await; assert!( cache @@ -14775,38 +14799,57 @@ mod tests { .lock() .expect("should lock entries") .is_empty(), - "a reader running diagnostics must not contribute to a shared cache" + "a request a module left state on must not contribute to a shared cache" ); } #[tokio::test] - async fn active_diagnostics_bypass_an_already_warm_template() { + async fn a_request_a_module_left_state_on_bypasses_an_already_warm_template() { let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); - let settings = Arc::new(settings_with_mode("esi")); + let mut raw = settings_with_mode("esi"); + raw.select_module( + "testing", + crate::integrations::registry_test_support::request_fixture::MODULE, + ); + let settings = Arc::new(raw); let services = services(Arc::clone(&stub), Arc::clone(&cache)); queue_shareable_html(&stub); queue_shareable_html(&stub); let _ = body_of(run(&settings, &services, navigation_request()).await).await; - let response = run(&settings, &services, diagnostics_navigation_request()).await; + let response = run(&settings, &services, marked_navigation_request()).await; + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("no-store, private"), + "a document a module's request state is written into is one reader's" + ); + assert!( + response.headers().contains_key(header::SET_COOKIE), + "the module's response finalizer should run on the page path" + ); let document = String::from_utf8(body_of(response).await) - .expect("diagnostics document should be UTF-8"); + .expect("the marked request's document should be UTF-8"); assert_eq!( stub.recorded_request_uris().len(), 2, - "request-private diagnostics must reach the origin even when an ordinary \ - shared template is warm" + "a request a module left state on must reach the origin even when an \ + ordinary shared template is warm" ); assert_eq!( cache.lookups.lock().expect("should lock lookups").len(), 1, - "the diagnostics request must not consult template cache at all" + "the marked request must not consult template cache at all" ); assert!( - document.contains("__tsjs_gpt_diagnostics_active"), - "origin fallback must retain the request-private diagnostics bootstrap" + document.contains( + crate::integrations::registry_test_support::request_fixture::HEAD_FLAG + ), + "origin fallback must keep what the module writes for the request" ); } @@ -14840,9 +14883,7 @@ mod tests { suppressed_request .headers_mut() .insert("sec-fetch-dest", HeaderValue::from_static("script")); - suppressed_request - .extensions_mut() - .insert(crate::integrations::datadome::DataDomeClientTagSuppressed); + crate::integrations::datadome::suppress_client_tag(&mut suppressed_request); suppressed_request .extensions_mut() .insert(crate::response_privacy::PersonalizedResponse); @@ -18161,8 +18202,7 @@ mod tests { .header(header::IF_RANGE, "\"cached-page\"") .body(EdgeBody::empty()) .expect("should build conditional request"); - req.extensions_mut() - .insert(crate::integrations::datadome::DataDomeClientTagSuppressed); + crate::integrations::datadome::suppress_client_tag(&mut req); req.extensions_mut() .insert(crate::response_privacy::PersonalizedResponse); @@ -18211,8 +18251,7 @@ mod tests { .header(header::IF_RANGE, "\"cached-frame\"") .body(EdgeBody::empty()) .expect("should build conditional iframe request"); - req.extensions_mut() - .insert(crate::integrations::datadome::DataDomeClientTagSuppressed); + crate::integrations::datadome::suppress_client_tag(&mut req); req.extensions_mut() .insert(crate::response_privacy::PersonalizedResponse); @@ -18261,8 +18300,7 @@ mod tests { .header(header::IF_RANGE, "\"cached-video\"") .body(EdgeBody::empty()) .expect("should build conditional subresource request"); - req.extensions_mut() - .insert(crate::integrations::datadome::DataDomeClientTagSuppressed); + crate::integrations::datadome::suppress_client_tag(&mut req); req.extensions_mut() .insert(crate::response_privacy::PersonalizedResponse); @@ -18502,7 +18540,12 @@ mod tests { .expect("should create integration registry with DataDome"); let mut params = make_stream_params(&settings, "identity"); params.content_type = "text/html; charset=utf-8".to_string(); - params.suppress_datadome_client_side_tag = true; + let mut suppressed = IntegrationRequestState::default(); + suppressed.set( + crate::integrations::datadome::DATADOME_INTEGRATION_ID, + crate::integrations::datadome::DataDomeClientTagSuppressed, + ); + params.request_state = suppressed; let mut output = Vec::new(); stream_publisher_body( @@ -20300,8 +20343,7 @@ mod tests { auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let mut output = Vec::new(); @@ -20360,8 +20402,7 @@ mod tests { auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let mut output = Vec::new(); @@ -20409,8 +20450,7 @@ mod tests { auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let body = EdgeBody::from_stream(futures::stream::iter(vec![Ok::<_, io::Error>( bytes::Bytes::from_static(b"live"), @@ -20538,8 +20578,7 @@ mod tests { auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let body = EdgeBody::stream(futures::stream::iter(vec![ bytes::Bytes::from_static(b"body{background:url('https://origin.example.com/"), @@ -20605,8 +20644,7 @@ mod tests { auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let compressed = gzip_encode(b"body{background:url('https://origin.example.com/asset.png')}"); @@ -20675,8 +20713,7 @@ mod tests { auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let compressed = deflate_encode(b"body{background:url('https://origin.example.com/asset.png')}"); @@ -20745,8 +20782,7 @@ mod tests { auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let compressed = brotli_encode(b"body{background:url('https://origin.example.com/asset.png')}"); @@ -20815,8 +20851,7 @@ mod tests { auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let compressed = brotli_encode(b"body{background:url('https://origin.example.com/asset.png')}"); @@ -20865,8 +20900,7 @@ mod tests { auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), } } @@ -21097,8 +21131,7 @@ mod tests { 10, )), price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let body = EdgeBody::stream(futures::stream::iter(vec![ bytes::Bytes::from_static(b"hello"), @@ -21175,8 +21208,7 @@ mod tests { 10, )), price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; // The `` that triggers bid injection lives in the SECOND gzip // member. `flate2::read::GzDecoder` decodes only the first member, so @@ -21252,8 +21284,7 @@ mod tests { 10, )), price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let body = EdgeBody::stream(futures::stream::iter(vec![bytes::Bytes::from_static( b"body{background:url('https://origin.example.com/asset.png')}", @@ -21323,8 +21354,7 @@ mod tests { auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let publisher_response = PublisherResponse::Stream { response, @@ -21482,8 +21512,7 @@ mod tests { auction_request: dispatched_auction.as_ref().map(|_| test_auction_request()), dispatched_auction, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), } } @@ -22162,8 +22191,7 @@ mod tests { 10, )), price_granularity: PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), } }; let make_stream_response = || PublisherResponse::Stream { @@ -22356,8 +22384,7 @@ mod tests { 10, )), price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let publisher_response = PublisherResponse::Stream { response, @@ -22435,8 +22462,7 @@ mod tests { auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let mut output = Vec::new(); @@ -22497,8 +22523,7 @@ mod tests { auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let bogus_body = EdgeBody::from(b"not gzip".to_vec()); @@ -22610,8 +22635,7 @@ mod tests { auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let mut output = Vec::new(); stream_publisher_body(body, &mut output, ¶ms, &settings, ®istry) @@ -22672,8 +22696,7 @@ mod tests { auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let mut output = Vec::new(); @@ -22706,8 +22729,8 @@ mod tests { #[cfg(test)] mod creative_opportunities_tests { use super::super::{ - AdBidsState, MatchedSlotsContext, build_ad_slots_script, build_auction_request, - build_bid_map, build_bids_script, diagnostics_auction_id, html_escape_for_script, + AdBidsState, MatchedSlotsContext, auction_token, build_ad_slots_script, + build_auction_request, build_bid_map, build_bids_script, html_escape_for_script, write_bids_to_state, }; use crate::auction::types::{Bid, BidRenderer, MediaType}; @@ -22977,25 +23000,28 @@ mod tests { } /// Guards the browser-visible token every auction path shares: it must - /// be fresh per auction and absent unless diagnostics can consume it. + /// be fresh per auction and absent unless a module reads it. #[test] - fn diagnostics_auction_id_is_fresh_and_gated() { - let mut settings = test_settings(); + fn auction_token_is_fresh_and_made_only_where_a_module_reads_it() { + let settings = test_settings(); + let orchestrator = crate::auction::orchestrator::AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); assert_eq!( - diagnostics_auction_id(&settings), + auction_token(&orchestrator), None, - "no token should be minted without the diagnostics integration" + "no token should be minted where no module reads one" ); - settings.select_module("ad-tag", "ad-tag.google.diagnostics"); + let orchestrator = orchestrator.publishing_auction_token(); let first = - diagnostics_auction_id(&settings).expect("enabled diagnostics should mint a token"); + auction_token(&orchestrator).expect("should mint a token where a module reads one"); let second = - diagnostics_auction_id(&settings).expect("enabled diagnostics should mint a token"); + auction_token(&orchestrator).expect("should mint a token where a module reads one"); assert!( first.starts_with("ts-auc-"), - "token should use the diagnostics prefix, got `{first}`" + "token should use the auction token prefix, got `{first}`" ); assert_ne!(first, second, "each auction should mint its own token"); } @@ -24702,6 +24728,13 @@ mod tests { captured_request, winning_bid, })); + // The plan compiled for a deployment publishes a token when a + // selected module reads one, and the stand-in is such a module. + if settings + .selects_module(crate::integrations::registry_test_support::request_fixture::MODULE) + { + orchestrator = orchestrator.publishing_auction_token(); + } orchestrator } @@ -24710,7 +24743,10 @@ mod tests { let mut settings = settings_with_co(); settings.demand = crate::auction::test_support::demand_named(&[AUCTION_ID_TEST_PROVIDER]); - settings.select_module("ad-tag", "ad-tag.google.diagnostics"); + settings.select_module( + "testing", + crate::integrations::registry_test_support::request_fixture::MODULE, + ); let slots = article_slot(); let winning_stub = Arc::new(StubHttpClient::new()); winning_stub.push_response(200, b"winner".to_vec()); @@ -24765,7 +24801,7 @@ mod tests { .to_string(); assert!( winning_auction_id.starts_with("ts-auc-"), - "page-bids should expose a freshly minted diagnostics token, got `{winning_auction_id}`" + "page-bids should expose a freshly minted auction token, got `{winning_auction_id}`" ); assert_ne!( winning_auction_id, auction_request.id, @@ -24815,10 +24851,10 @@ mod tests { } /// The browser-visible auction ID is minted per auction and only for - /// deployments that run the diagnostics integration, so it can neither + /// deployments that run a module that reads it, so it can neither /// carry EC identity across auctions nor reach pages that ignore it. #[tokio::test] - async fn page_bids_auction_id_is_per_auction_and_gated_on_diagnostics() { + async fn page_bids_auction_id_is_per_auction_and_made_only_where_a_module_reads_it() { async fn winning_auction_id(settings: &Settings) -> Option { let slots = article_slot(); let stub = Arc::new(StubHttpClient::new()); @@ -24864,24 +24900,30 @@ mod tests { let mut settings = settings_with_co(); settings.demand = crate::auction::test_support::demand_named(&[AUCTION_ID_TEST_PROVIDER]); - settings.select_module("ad-tag", "ad-tag.google.diagnostics"); + settings.select_module( + "testing", + crate::integrations::registry_test_support::request_fixture::MODULE, + ); let first = winning_auction_id(&settings) .await - .expect("first auction should expose a diagnostics token"); + .expect("first auction should expose an auction token"); let second = winning_auction_id(&settings) .await - .expect("second auction should expose a diagnostics token"); + .expect("second auction should expose an auction token"); assert_ne!( first, second, "each auction for the same visitor should mint its own token" ); - settings.remove_module("ad-tag", "ad-tag.google.diagnostics"); + settings.remove_module( + "testing", + crate::integrations::registry_test_support::request_fixture::MODULE, + ); assert_eq!( winning_auction_id(&settings).await, None, - "no auction metadata should reach the page without the diagnostics integration" + "no auction metadata should reach the page where no module reads it" ); } diff --git a/docs/guide/integration-guide.md b/docs/guide/integration-guide.md index 767b9c0d49..0ec28d1967 100644 --- a/docs/guide/integration-guide.md +++ b/docs/guide/integration-guide.md @@ -239,7 +239,9 @@ points at the functions that do the work. | Build function | Reads `Settings` and returns the registration. It is called only when a section selects the module | | Validate function | The integration's own deploy rules. It runs for every builder, selected or not | | `.with_module_name(".")` | The name a section selects the module by, which is the crate's path under `crates/` with `.` between the parts. `module_name!()` derives it from the crate's folder | -| `.with_request_preparer(...)` | Optional. Runs before routing, whether or not the module is selected | +| `.with_request_preparer(...)` | Optional. Runs before routing, once for a request, whether or not the module is selected | +| `.with_response_finalizer(...)` | Optional. Finishes the response the page path returns, from what the module's request hooks left for the request | +| `.with_auction_token()` | Optional. Declares that the module's browser script reads the token an auction publishes with its winning bids, which is made only when a selected module reads one | | `.with_demand(...)` and `.with_adserver(...)` | Optional. Registers an auction implementation that `[demand]` or `[ad-server]` can name. A crate that supplies nothing else starts from `IntegrationBuilder::implementations(...)` | ```rust @@ -266,7 +268,7 @@ builder the integrations in core use. | Declaration | What it does | | ----------------------------------------------------- | ----------------------------------------------------------------------------------- | | `.with_proxy(...)` | Routes the paths the proxy declares | -| `.with_head_injector(...)` | Emits markup at the start of `` | +| `.with_head_injector(...)` | Emits markup at the start of `` and, if it chooses, after the script bundle | | `.with_attribute_rewriter(...)` | Rewrites attribute values in publisher HTML | | `.with_script_rewriter(...)` | Rewrites inline script contents | | `.with_html_stream_processor(...)` | Works on the document as it streams | @@ -307,6 +309,34 @@ startup. The message lists the modules of that type the deployment runs, and says when the name is a module no section selects or one that supplies no module of that type. +### Acting on one request + +A module that decides something about one request and acts on it later +carries the decision in the request's `IntegrationRequestState`. A request +preparer or a request filter leaves a value under the integration's id. + +```rust +IntegrationRequestState::insert(request, EXAMPLE_ID, ExampleDecision::default()); +``` + +When the request produces an HTML document, each value is copied into the +document's state before parsing starts, so the head injector, the rewriters +and the stream processors read it with +`ctx.document_state.get::(EXAMPLE_ID)`. The same values are +handed to the response finalizer the builder declared, which runs on the +response the page path returns. + +A request that carries any value gets a document made for it alone. The +document is fetched from the origin, never read from a shared template and +never stored as one, and an HTML response with a body is sent +`private, no-store`. So a module leaves a value only for a request it will +act on, and an ordinary request stays on the shared path. + +A head injector has two places to write. `head_inserts` writes at the start +of ``, before the script bundle, and `after_bundle_inserts` writes +straight after the bundle, for a script that needs the bundle to have run and +has to run before the page's own scripts. + ### How an adapter composes it in The Axum, Cloudflare and Spin adapters take the builders as an argument, so no From 4f8fd2a591e93cb7b4b71e33f372a183b43a5795 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 8 Oct 2026 02:34:31 +0100 Subject: [PATCH 096/124] The Google Publisher Tags diagnostics module moves out of core The diagnostics module is `ad-tag.google.diagnostics`, the second module of the crate at `crates/ad-tag/google`, and it now lives there beside the Google Publisher Tags module. The stock list offers it last, where its hooks ran before, and the settings that select it do not change. It prepares the request, writes its two scripts and finishes the response through the hooks any module has, so core calls nothing of its own. Its tests went with it, and four were added for what it now does through those hooks: that only a decision which acts is left for the page path, that the finalizer acts only on such a decision, and that a document gets both scripts in order when diagnostics is active and neither when it is not. The test that the built-in list attached its request preparer is now a test that the stock list does. Core's tests that used the module as their example use core's stand-in: the HTML processor's test of what is written around the bundle, the registry's two tests of a standalone browser module, and the page path's tests of a query a module acts on, a warm template, an inactive ad stack and a standalone module's own path. The Fastly adapter's test that a late filter cannot make such a response public marks the response the way a module's finalizer does and names no module. One of core's template cache tests depended on this module without saying so. The module's preparer writes the Cookie header of every request out again as it strips its own cookie, dropping a field it cannot read, and the test expected a request with such a field to be stored. The stand-in now reserves a cookie and does the same, so the test still shows that the cache classifies a request as a preparer left it. A stock build behaves as before, because the stock list still runs the module's preparer for every request. The page path's two request classifiers the module uses, for a prefetch and for a crawler, are public. The changelog's list of moved modules had been garbled by the full stop in Next.js and reads correctly again. --- CHANGELOG.md | 2 +- Cargo.lock | 2 + .../google/src/diagnostics.rs} | 226 +++++++++++++++--- crates/ad-tag/google/src/lib.rs | 5 + .../trusted-server-adapter-fastly/src/lib.rs | 63 +---- .../trusted-server-core/src/html_processor.rs | 64 ++--- .../src/integrations/mod.rs | 2 - .../src/integrations/registry.rs | 102 ++++---- crates/trusted-server-core/src/publisher.rs | 171 +++++++------ crates/trusted-server-modules/Cargo.toml | 2 + crates/trusted-server-modules/src/lib.rs | 42 ++++ 11 files changed, 440 insertions(+), 241 deletions(-) rename crates/{trusted-server-core/src/integrations/gpt_diagnostics.rs => ad-tag/google/src/diagnostics.rs} (76%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 43775ba7ae..d9c1cb64d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- Integration modules are moving out of `trusted-server-core` into crates of their own under `crates///`, each named by that folder, so `crates/cmp/osano` is the module `cmp.osano`. The settings that select and configure a module do not change. `crates/trusted-server-modules` lists the modules a stock build ships from crates of their own, in the order their hooks run. Every adapter builds its state and loads its settings with that list, and `ts config validate` and `ts config push` run each stock module's own rules. Code that named a moved module under `trusted_server_core::integrations` names its crate instead, such as `trusted_server_cmp_osano`. Moved so far: Osano, Lockr, Permutive, Sourcepoint, Didomi, Google Tag Manager, Testlight, Next.js and Google Publisher Tags.js and Google Publisher Tags.js.js. +- Integration modules are moving out of `trusted-server-core` into crates of their own under `crates///`, each named by that folder, so `crates/cmp/osano` is the module `cmp.osano`. The settings that select and configure a module do not change. `crates/trusted-server-modules` lists the modules a stock build ships from crates of their own, in the order their hooks run. Every adapter builds its state and loads its settings with that list, and `ts config validate` and `ts config push` run each stock module's own rules. Code that named a moved module under `trusted_server_core::integrations` names its crate instead, such as `trusted_server_cmp_osano`. Moved so far: Osano, Lockr, Permutive, Sourcepoint, Didomi, Google Tag Manager, Testlight, Next.js, Google Publisher Tags and its diagnostics. - The Fastly adapter is a library with a thin binary over it. A deployment that ships a vendor crate writes a binary of its own that calls `trusted_server_adapter_fastly::run_with(vec![...])` with that crate's integration builder, in place of editing the adapter. The builders are composed with the built-in ones, and a builder runs only when the settings select its module. The Wasm artifact keeps its name, `trusted-server-adapter-fastly.wasm`. - **Breaking:** `[integrations]` is gone. Each page integration is a module selected in the section of its type, with its settings in the table at its name: `[cmp] module = "didomi"` (or `"sourcepoint"` or `"osano"`), `[tag] modules = ["google-tag-manager"]`, `[ad-tag] modules = ["google", "google.diagnostics"]`, `[bot-protection] module = "datadome"`, `[identity] module = "lockr"`, `[audience] module = "permutive"`, `[framework] module = "nextjs"`, `[auction] modules = ["prebid"]` (with `"testing.testlight"` for the test module) and `[proxy] modules = ["js_asset_proxy"]`, so `[integrations.gpt]` is `[ad-tag.google]` and `[integrations.prebid]` is `[auction.prebid]`. A module's name is `.`, which is its crate's path under `crates/` once it has a crate of its own and a constant in core until then, and it is written in its section with the type left off. A section that selects nothing, a table its section does not select, and a name no module in the deployment supplies are each refused at startup. An `[integrations]` or `[integration]` table is refused with directions. Environment overlays name a section as written, so `TRUSTED_SERVER__INTEGRATIONS__GPT__GAM_ATTRIBUTION_ENABLED` is `TRUSTED_SERVER__AD-TAG__GOOGLE__GAM_ATTRIBUTION_ENABLED`. - **Breaking:** Every pluggable component now shares one configuration syntax. Each selectable type is a top-level table named for the job, a `module` key inside it selects what runs, or `modules` for a type that runs several, and a `[.]` table holds that name's settings when it has any. The types are `ec`, `geo`, `device`, `permission-signal`, `demand` and `ad-server`, and the section of each page integration type, such as `cmp` or `tag`. A module from a crate is named by its folder below `crates/`, such as `permission-signal.gpc`, and within its own table the type folder may be left off. A `demand` or `ad-server` name is snake_case, because it may be a label of your own. A settings table its type's selector does not name refuses startup, so a block left behind after a module is switched off is caught rather than sitting unread, and every module rejects settings it does not know, so a misspelled key fails rather than being ignored. A table's name is the implementation unless the table carries an `implementation` line, and a demand table always carries one, naming the implementation by its module path (`auction-protocol.openrtb`, `auction.prebid-server` or `auction.aps`), which is how two Prebid Servers run side by side under names of their own. The ad server implementation is `ad-server.mock`, written `mock` in `[ad-server]`. A `demand` or `ad-server` endpoint must be HTTPS, or HTTP to `127.0.0.1`, `::1` or `localhost`. The word mediator is gone. It is "ad server" in prose and `ad-server` in configuration, `[debug.auction_html_comment_options] include_mediator_response` is now `include_adserver_response`, and the auction response metadata value `parallel_mediation` is now `parallel_adserver`. `auction-protocol.openrtb`, `auction.prebid-server`, `auction.aps` and `ad-server.mock` supply implementations only, so a page integration's section that names one is refused. The full rules and what is checked at each gate are in [Configuration Rules](docs/guide/configuration-rules.md). Migrate as follows: diff --git a/Cargo.lock b/Cargo.lock index 136a64f377..be7b3907b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5785,6 +5785,8 @@ dependencies = [ name = "trusted-server-modules" version = "0.1.0" dependencies = [ + "edgezero-core", + "http", "serde_json", "trusted-server-ad-tag-google", "trusted-server-audience-permutive", diff --git a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs b/crates/ad-tag/google/src/diagnostics.rs similarity index 76% rename from crates/trusted-server-core/src/integrations/gpt_diagnostics.rs rename to crates/ad-tag/google/src/diagnostics.rs index 3748252382..3299a90584 100644 --- a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs +++ b/crates/ad-tag/google/src/diagnostics.rs @@ -5,26 +5,24 @@ //! active documents load the module synchronously without adding diagnostics to //! the ordinary unified bundle. +use std::sync::Arc; + +use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt}; use http::{HeaderValue, Method, Request, Response, Uri, header, uri::PathAndQuery}; use serde::Deserialize; use validator::Validate; -use edgezero_core::body::Body as EdgeBody; - -use std::sync::Arc; - -use crate::error::TrustedServerError; -use crate::http_util::is_navigation_request; -use crate::response_privacy::enforce_terminal_private_cache_privacy; -use crate::settings::{IntegrationConfig, Settings}; -use crate::tsjs; -use crate::tsjs_bundle::JsModulePart; - -use super::{ - IntegrationHeadInjector, IntegrationHtmlContext, IntegrationRegistration, +use trusted_server_core::error::TrustedServerError; +use trusted_server_core::http_util::is_navigation_request; +use trusted_server_core::integrations::{ + IntegrationBuilder, IntegrationHeadInjector, IntegrationHtmlContext, IntegrationRegistration, IntegrationRequestState, }; +use trusted_server_core::response_privacy::enforce_terminal_private_cache_privacy; +use trusted_server_core::settings::{IntegrationConfig, Settings}; +use trusted_server_core::tsjs; +use trusted_server_core::tsjs_bundle::JsModulePart; /// Stable integration identifier. pub const GPT_DIAGNOSTICS_INTEGRATION_ID: &str = "gpt_diagnostics"; @@ -32,18 +30,22 @@ pub const GPT_DIAGNOSTICS_INTEGRATION_ID: &str = "gpt_diagnostics"; /// The name this module is selected by, in `[ad-tag]`. pub const MODULE: &str = "ad-tag.google.diagnostics"; -/// The builder the registry runs when a section selects [`MODULE`]. -pub(crate) const BUILDER: crate::integrations::IntegrationBuilder = - crate::integrations::IntegrationBuilder::new( +/// The builder a deployment hands to an adapter, which the registry runs when +/// a section selects [`MODULE`]. +#[must_use] +pub fn builder() -> IntegrationBuilder { + IntegrationBuilder::new( GPT_DIAGNOSTICS_INTEGRATION_ID, - crate::integrations::CORE_SOURCE, + env!("CARGO_PKG_NAME"), register, validate, ) .with_module_name(MODULE) .with_request_preparer(prepare_request_hook) .with_response_finalizer(finalize_response_hook) - .with_auction_token(); + .with_auction_token() +} + /// Reserved activation query parameter. pub const GPT_DIAGNOSTICS_QUERY: &str = "ts_console"; /// Host-only browser-session activation cookie. @@ -319,8 +321,8 @@ pub fn prepare_request( let cookie_state = console_cookie_state(request); let eligible_navigation = request.method() == Method::GET && is_navigation_request(request) - && !crate::publisher::is_prefetch_request(request) - && !crate::publisher::is_bot_user_agent(request); + && !trusted_server_core::publisher::is_prefetch_request(request) + && !trusted_server_core::publisher::is_bot_user_agent(request); sanitize_console_cookie(request); if had_reserved_query { @@ -505,9 +507,9 @@ fn replace_path_and_query( #[cfg(test)] mod tests { use super::*; - use crate::integrations::IntegrationRegistry; - use crate::test_support::tests::create_test_settings; use serde_json::json; + use trusted_server_core::integrations::IntegrationRegistry; + use trusted_server_core::test_support::tests::create_test_settings; fn settings() -> Settings { let mut settings = create_test_settings(); @@ -531,11 +533,8 @@ mod tests { #[test] fn register_excludes_diagnostics_from_unified_and_deferred_bundles() { let settings = settings(); - let plan = std::sync::Arc::new( - crate::auction::compile_auction_plan(&settings).expect("should compile auction plan"), - ); - let registry = - IntegrationRegistry::with_plan(&settings, plan).expect("should build registry"); + let registry = IntegrationRegistry::with_registrations(&settings, &[builder()]) + .expect("should build registry"); assert!(registry.integration_runs(GPT_DIAGNOSTICS_INTEGRATION_ID)); assert!( @@ -709,7 +708,7 @@ mod tests { assert!( response .extensions() - .get::() + .get::() .is_some(), "should mark request-scoped diagnostics HTML for terminal re-enforcement" ); @@ -736,7 +735,7 @@ mod tests { assert!( response .extensions() - .get::() + .get::() .is_none(), "should not mark an untouched response terminal-private" ); @@ -759,4 +758,173 @@ mod tests { let error_text = format!("{error:?}"); assert!(error_text.contains("typo") || error_text.contains("unknown field")); } + + #[test] + fn module_constant_names_the_crate_folder_and_the_module_within_it() { + assert_eq!( + MODULE, + format!("{}.diagnostics", trusted_server_core::module_name!()), + "should be named by the folder this crate lives in" + ); + } + + #[test] + fn only_a_decision_that_acts_is_left_for_the_page_path() { + let mut ordinary = navigation("https://publisher.example/page", None); + prepare_request(&settings(), &mut ordinary).expect("should prepare"); + assert!( + IntegrationRequestState::of(&ordinary).is_empty(), + "an ordinary request should stay on the shared template path" + ); + + let mut activating = navigation("https://publisher.example/page?ts_console=1", None); + let decision = prepare_request(&settings(), &mut activating).expect("should prepare"); + assert_eq!( + IntegrationRequestState::of(&activating) + .get::(GPT_DIAGNOSTICS_INTEGRATION_ID) + .as_deref(), + Some(&decision), + "should leave the decision that acts for the page path" + ); + } + + #[test] + fn the_finalizer_acts_only_on_a_decision_left_for_the_page_path() { + let mut untouched = Response::builder() + .header(header::CACHE_CONTROL, "public, max-age=60") + .body(EdgeBody::empty()) + .expect("should build response"); + finalize_response_hook(&IntegrationRequestState::default(), &mut untouched); + assert_eq!( + untouched.headers()[header::CACHE_CONTROL], + "public, max-age=60", + "should leave a response alone when nothing was left for it" + ); + assert!( + !untouched.headers().contains_key(header::SET_COOKIE), + "should set no cookie when nothing was left for it" + ); + + let mut request = navigation("https://publisher.example/?ts_console=1", None); + prepare_request(&settings(), &mut request).expect("should prepare"); + let mut response = Response::builder() + .header(header::CACHE_CONTROL, "public, max-age=60") + .body(EdgeBody::empty()) + .expect("should build response"); + finalize_response_hook(&IntegrationRequestState::of(&request), &mut response); + assert_eq!( + response.headers()[header::CACHE_CONTROL], + "no-store, private", + "should make the activating response private" + ); + assert_eq!( + response.headers()[header::SET_COOKIE], + SET_CONSOLE_COOKIE, + "should establish the session" + ); + } + + #[test] + fn an_active_document_loads_the_module_after_the_unified_bundle_once() { + use trusted_server_core::html_processor::{HtmlProcessorConfig, create_html_processor}; + use trusted_server_core::streaming_processor::StreamProcessor as _; + + let settings = settings(); + let registry = IntegrationRegistry::with_registrations(&settings, &[builder()]) + .expect("should build registry"); + let mut request = navigation("https://publisher.example/page?ts_console=1", None); + let decision = prepare_request(&settings, &mut request).expect("should prepare"); + assert!( + decision.active(), + "should activate diagnostics for the query" + ); + let config = HtmlProcessorConfig::from_settings( + &settings, + ®istry, + "origin.example.com", + "publisher.example", + "https", + ) + .with_request_state(IntegrationRequestState::of(&request)); + let mut processor = create_html_processor(config); + + let output = processor + .process_chunk( + b"Test", + true, + ) + .expect("should process HTML"); + let processed = String::from_utf8(output).expect("should produce valid UTF-8"); + let bootstrap_marker = "__tsjs_gpt_diagnostics_active"; + let bundle_marker = "id=\"trustedserver-js\""; + let diagnostics_marker = "tsjs-gpt_diagnostics.min.js"; + + assert_eq!( + processed.matches(bootstrap_marker).count(), + 1, + "should inject the diagnostics bootstrap once" + ); + assert_eq!( + processed.matches(bundle_marker).count(), + 1, + "should inject the immediate TSJS bundle once" + ); + assert_eq!( + processed.matches(diagnostics_marker).count(), + 1, + "should inject one standalone diagnostics module" + ); + let bootstrap_index = processed + .find(bootstrap_marker) + .expect("should include diagnostics bootstrap"); + let bundle_index = processed + .find(bundle_marker) + .expect("should include immediate TSJS bundle"); + let diagnostics_index = processed + .find(diagnostics_marker) + .expect("should include standalone diagnostics module"); + assert!( + bootstrap_index < bundle_index, + "should activate before core executes" + ); + assert!( + bundle_index < diagnostics_index, + "should load diagnostics after core" + ); + } + + #[test] + fn an_inactive_document_gets_neither_script() { + use trusted_server_core::html_processor::{HtmlProcessorConfig, create_html_processor}; + use trusted_server_core::streaming_processor::StreamProcessor as _; + + let settings = settings(); + let registry = IntegrationRegistry::with_registrations(&settings, &[builder()]) + .expect("should build registry"); + let mut request = navigation("https://publisher.example/page", None); + prepare_request(&settings, &mut request).expect("should prepare"); + let config = HtmlProcessorConfig::from_settings( + &settings, + ®istry, + "origin.example.com", + "publisher.example", + "https", + ) + .with_request_state(IntegrationRequestState::of(&request)); + let mut processor = create_html_processor(config); + + let output = processor + .process_chunk( + b"Test", + true, + ) + .expect("should process HTML"); + let processed = String::from_utf8(output).expect("should produce valid UTF-8"); + + assert!( + !processed.contains("__tsjs_gpt_diagnostics_active") + && !processed.contains("tsjs-gpt_diagnostics.min.js"), + "should write nothing into a document diagnostics is not active for: {processed}" + ); + } } diff --git a/crates/ad-tag/google/src/lib.rs b/crates/ad-tag/google/src/lib.rs index b328539b44..3472280ea9 100644 --- a/crates/ad-tag/google/src/lib.rs +++ b/crates/ad-tag/google/src/lib.rs @@ -31,6 +31,9 @@ //! `content"#; + + fn process_document(suppress: bool) -> String { + use trusted_server_core::html_processor::{HtmlProcessorConfig, create_html_processor}; + use trusted_server_core::integrations::{IntegrationRegistry, IntegrationRequestState}; + use trusted_server_core::streaming_processor::StreamProcessor as _; + + let mut settings = create_test_settings(); + settings + .insert_module_config( + "bot-protection", + MODULE, + &serde_json::json!({ "client_side_key": "test-client-key" }), + ) + .expect("should configure DataDome integration"); + let registry = IntegrationRegistry::with_registrations(&settings, &[builder()]) + .expect("should create integration registry with DataDome"); + let mut request = http::Request::builder() + .uri("https://test.example.com/") + .body(EdgeBody::empty()) + .expect("should build request"); + if suppress { + suppress_client_tag(&mut request); + } + let config = HtmlProcessorConfig::from_settings( + &settings, + ®istry, + "origin.example.com", + "test.example.com", + "https", + ) + .with_request_state(IntegrationRequestState::of(&request)); + let mut processor = create_html_processor(config); + + let output = processor + .process_chunk(PUBLISHER_TAGGED_DOCUMENT, true) + .expect("should process HTML"); + String::from_utf8(output).expect("should produce UTF-8 HTML") + } + + #[test] + fn a_suppressed_document_keeps_the_publisher_s_own_tag_and_rewrites_it() { + let html = process_document(true); + + assert!( + !html.contains("window.ddjskey"), + "should omit the DataDome client configuration" + ); + assert!( + html.contains("id=\"publisher-datadome\""), + "should preserve the publisher-originated DataDome tag" + ); + assert!( + html.contains("src=\"/integrations/datadome/tags.js\""), + "should rewrite the publisher-originated DataDome tag" + ); + assert!( + !html.contains("https://js.datadome.co/tags.js"), + "should remove the original third-party DataDome URL" + ); + assert_eq!( + html.matches("/integrations/datadome/tags.js").count(), + 1, + "should leave exactly one publisher-originated DataDome tag" + ); + } + + #[test] + fn an_ordinary_document_gets_the_client_configuration() { + let html = process_document(false); + + assert!( + html.contains("window.ddjskey"), + "should write the DataDome client configuration when nothing suppresses it" + ); + } + + /// Holds the two keys the settings name, and refuses a key that is + /// unused or is itself a resolved secret, so a lookup that should not + /// happen fails. + struct KeyStore; + + impl trusted_server_core::platform::PlatformSecretStore for KeyStore { + fn get_bytes( + &self, + _store_name: &trusted_server_core::platform::StoreName, + key: &str, + ) -> Result, Report> { + let value = match key { + "datadome-server-key" => "resolved-datadome-server-key", + "datadome-bypass-key" => "resolved-datadome-bypass-credential-32-bytes", + "unit-test-proxy-secret" => "unit-test-proxy-secret-32-bytes-ok", + key if key.starts_with("unused-") || key.starts_with("resolved-") => { + return Err(Report::new( + trusted_server_core::platform::PlatformError::SecretStore, + )); + } + other => other, + }; + Ok(value.as_bytes().to_vec()) + } + + fn create( + &self, + _store_id: &trusted_server_core::platform::StoreId, + _name: &str, + _value: &str, + ) -> Result<(), Report> { + Ok(()) + } + + fn delete( + &self, + _store_id: &trusted_server_core::platform::StoreId, + _name: &str, + ) -> Result<(), Report> { + Ok(()) + } + } + + /// Loads settings carrying `table` as this module's, the way a deployment + /// that ships this crate loads them. + fn load_with_table(table: &serde_json::Value) -> DataDomeConfig { + let mut settings = create_test_settings(); + settings + .insert_module_config("bot-protection", MODULE, table) + .expect("should insert DataDome's table"); + let data = serde_json::to_value(&settings).expect("should serialize settings to JSON"); + let envelope = edgezero_core::blob_envelope::BlobEnvelope::new( + data, + "2026-01-01T00:00:00Z".to_string(), + ); + let envelope = serde_json::to_string(&envelope).expect("should serialize envelope"); + + trusted_server_core::config_payload::settings_from_config_blob_with( + &envelope, + &KeyStore, + &trusted_server_core::platform::StoreName::from("ts_secrets"), + &[builder()], + ) + .expect("should load the settings") + .module_config::(MODULE) + .expect("should parse DataDome's table") + .expect("should select DataDome") + } + + #[test] + fn the_settings_load_looks_up_both_secrets_when_both_are_in_use() { + let config = load_with_table(&serde_json::json!({ + "enable_protection": true, + "server_side_key_secret_name": "datadome-server-key", + "protection_test_bypass": { + "enabled": true, + "credential_secret_name": "datadome-bypass-key", + }, + })); + + assert_eq!( + config + .server_side_key_secret_name + .as_ref() + .map(Redacted::expose) + .map(String::as_str), + Some("resolved-datadome-server-key"), + "should hold the server-side key where the table named it" + ); + assert_eq!( + config + .protection_test_bypass + .as_ref() + .and_then(|bypass| bypass.credential_secret_name.as_ref()) + .map(Redacted::expose) + .map(String::as_str), + Some("resolved-datadome-bypass-credential-32-bytes"), + "should hold the bypass credential where the table named it" + ); + } + + #[test] + fn the_settings_load_clears_the_secrets_protection_does_not_use() { + let config = load_with_table(&serde_json::json!({ + "enable_protection": false, + "server_side_key_secret_name": "unused-datadome-key", + "protection_test_bypass": { + "enabled": false, + "credential_secret_name": "unused-bypass-key", + }, + })); + + assert!( + config.server_side_key_secret_name.is_none(), + "should clear a server-side key name protection does not use" + ); + assert!( + config + .protection_test_bypass + .as_ref() + .is_some_and(|bypass| bypass.credential_secret_name.is_none()), + "should clear a bypass credential name the bypass does not use" + ); + } + + #[test] + fn the_settings_load_clears_a_bypass_credential_when_only_protection_is_on() { + let config = load_with_table(&serde_json::json!({ + "enable_protection": true, + "server_side_key_secret_name": "datadome-server-key", + "protection_test_bypass": { + "enabled": false, + "credential_secret_name": "unused-bypass-key", + }, + })); + + assert!( + config.server_side_key_secret_name.is_some(), + "should keep the key protection uses" + ); + assert!( + config + .protection_test_bypass + .as_ref() + .is_some_and(|bypass| bypass.credential_secret_name.is_none()), + "should clear a credential name a disabled bypass does not use" + ); + } + + #[test] + fn deploy_validation_rejects_an_invalid_test_bypass() { + for (enable_protection, name, expected_message) in [ + (false, "datadome_test_bypass", "requires enable_protection"), + (true, "", "credential_secret_name"), + ] { + let mut settings = create_test_settings(); + settings + .insert_module_config( + "bot-protection", + MODULE, + &serde_json::json!({ + "enable_protection": enable_protection, + "server_side_key_secret_name": "datadome_server_side_key", + "protection_test_bypass": { + "enabled": true, + "credential_secret_name": name, + }, + }), + ) + .expect("should insert DataDome config"); + + let err = trusted_server_core::config::validate_settings_for_deploy_with( + &settings, + &[builder()], + ) + .expect_err("should reject invalid DataDome test bypass"); + assert!( + format!("{err:?}").contains(expected_message), + "error should mention the invalid bypass setting: {err:?}" + ); + } + } + + #[test] + fn a_removed_setting_is_rejected() { + let mut settings = create_test_settings(); + settings + .insert_module_config( + "bot-protection", + MODULE, + &serde_json::json!({ "account_id": "removed-value" }), + ) + .expect("should insert the removed DataDome field"); + + let error = settings + .module_config::(MODULE) + .expect_err("should reject the removed DataDome field"); + + assert!( + format!("{error:?}").contains("account_id"), + "should identify the removed field: {error:?}" + ); + } } diff --git a/crates/trusted-server-core/src/integrations/datadome/protection.rs b/crates/bot-protection/datadome/src/protection.rs similarity index 97% rename from crates/trusted-server-core/src/integrations/datadome/protection.rs rename to crates/bot-protection/datadome/src/protection.rs index 3239848729..17c8760a2f 100644 --- a/crates/trusted-server-core/src/integrations/datadome/protection.rs +++ b/crates/bot-protection/datadome/src/protection.rs @@ -8,13 +8,13 @@ use sha2::{Digest as _, Sha256}; use subtle::ConstantTimeEq as _; use url::Url; -use crate::error::TrustedServerError; -use crate::http_util::is_navigation_request; -use crate::integrations::{ +use trusted_server_core::error::TrustedServerError; +use trusted_server_core::http_util::is_navigation_request; +use trusted_server_core::integrations::{ HeaderMutation, RequestFilterDecision, RequestFilterEffects, RequestFilterInput, }; -use crate::platform::{PlatformBackendSpec, PlatformHttpRequest, RuntimeServices}; -use crate::redacted::Redacted; +use trusted_server_core::platform::{PlatformBackendSpec, PlatformHttpRequest, RuntimeServices}; +use trusted_server_core::redacted::Redacted; use super::DataDomeIntegration; use super::protection_scope::{ @@ -51,7 +51,7 @@ impl DataDomeIntegration { input .request .extensions_mut() - .insert(crate::response_privacy::PersonalizedResponse); + .insert(trusted_server_core::response_privacy::PersonalizedResponse); log_protection_test_bypass(&input); return RequestFilterDecision::Continue(RequestFilterEffects::default()); } @@ -172,7 +172,7 @@ impl DataDomeIntegration { input .request .extensions_mut() - .insert(crate::response_privacy::PersonalizedResponse); + .insert(trusted_server_core::response_privacy::PersonalizedResponse); } log_protection_skip(input, &rule_id, reason, suppress_client_tag); return false; @@ -824,17 +824,17 @@ mod tests { use std::net::{IpAddr, Ipv4Addr}; use std::sync::{Arc, Mutex}; - use crate::integrations::datadome::{ + use crate::{ DataDomeConfig, ProtectionExclusionRuleConfig, ProtectionMatcherConfig, ProtectionTestBypassConfig, }; - use crate::platform::GeoInfo; - use crate::platform::test_support::{ + use trusted_server_core::platform::GeoInfo; + use trusted_server_core::platform::test_support::{ HashMapConfigStore, HashMapSecretStore, NoopConfigStore, NoopSecretStore, StubHttpClient, build_services_with_config_and_secret, build_services_with_config_and_secret_and_client_ip, build_services_with_secret_and_http_client, noop_services_with_client_ip, }; - use crate::settings::Settings; + use trusted_server_core::settings::Settings; use super::*; @@ -869,16 +869,22 @@ mod tests { let _guard = FASTLY_IS_STAGING_ENV_LOCK .lock() .expect("should lock staging environment test guard"); - temp_env::with_var(crate::constants::ENV_FASTLY_IS_STAGING, Some("1"), || { - futures::executor::block_on(integration.filter_protection_request(RequestFilterInput { - settings, - services, - request, - geo_info: None, - permissions: None, - is_integration_route: false, - })) - }) + temp_env::with_var( + trusted_server_core::constants::ENV_FASTLY_IS_STAGING, + Some("1"), + || { + futures::executor::block_on(integration.filter_protection_request( + RequestFilterInput { + settings, + services, + request, + geo_info: None, + permissions: None, + is_integration_route: false, + }, + )) + }, + ) } fn filter_marks_request( @@ -928,7 +934,7 @@ mod tests { } fn has_client_tag_suppression_marker(request: &Request) -> bool { - crate::integrations::IntegrationRequestState::of(request) + trusted_server_core::integrations::IntegrationRequestState::of(request) .get::(super::super::DATADOME_INTEGRATION_ID) .is_some() } @@ -940,7 +946,7 @@ mod tests { fn has_personalized_response_marker(request: &Request) -> bool { request .extensions() - .get::() + .get::() .is_some() } @@ -1116,7 +1122,7 @@ mod tests { .lock() .expect("should lock staging environment test guard"); let decision = temp_env::with_var( - crate::constants::ENV_FASTLY_IS_STAGING, + trusted_server_core::constants::ENV_FASTLY_IS_STAGING, None::<&str>, || { futures::executor::block_on(integration.filter_protection_request( diff --git a/crates/trusted-server-core/src/integrations/datadome/protection_scope.rs b/crates/bot-protection/datadome/src/protection_scope.rs similarity index 96% rename from crates/trusted-server-core/src/integrations/datadome/protection_scope.rs rename to crates/bot-protection/datadome/src/protection_scope.rs index baf10bd2b3..e64f82f6f9 100644 --- a/crates/trusted-server-core/src/integrations/datadome/protection_scope.rs +++ b/crates/bot-protection/datadome/src/protection_scope.rs @@ -8,8 +8,8 @@ use error_stack::Report; use regex::Regex; use serde::Deserialize; -use crate::error::TrustedServerError; -use crate::platform::{RuntimeServices, StoreName}; +use trusted_server_core::error::TrustedServerError; +use trusted_server_core::platform::{RuntimeServices, StoreName}; use super::{DATADOME_INTEGRATION_ID, DataDomeConfig}; @@ -37,7 +37,10 @@ pub struct ProtectionExclusionRuleConfig { #[serde(default = "default_enabled_rule")] pub enabled: bool, /// Optional methods this rule applies to. Empty means every method. - #[serde(default, deserialize_with = "crate::settings::vec_from_seq_or_map")] + #[serde( + default, + deserialize_with = "trusted_server_core::settings::vec_from_seq_or_map" + )] pub methods: Vec, /// Matcher-specific rule configuration. #[serde(flatten)] @@ -50,32 +53,32 @@ pub struct ProtectionExclusionRuleConfig { pub enum ProtectionMatcherConfig { /// Match exact request paths. PathExact { - #[serde(deserialize_with = "crate::settings::vec_from_seq_or_map")] + #[serde(deserialize_with = "trusted_server_core::settings::vec_from_seq_or_map")] paths: Vec, }, /// Match request path prefixes. PathPrefix { - #[serde(deserialize_with = "crate::settings::vec_from_seq_or_map")] + #[serde(deserialize_with = "trusted_server_core::settings::vec_from_seq_or_map")] prefixes: Vec, }, /// Match request paths with one or more regexes. PathRegex { - #[serde(deserialize_with = "crate::settings::vec_from_seq_or_map")] + #[serde(deserialize_with = "trusted_server_core::settings::vec_from_seq_or_map")] patterns: Vec, }, /// Match when any query parameter has a non-empty value. QueryParamNonEmpty { - #[serde(deserialize_with = "crate::settings::vec_from_seq_or_map")] + #[serde(deserialize_with = "trusted_server_core::settings::vec_from_seq_or_map")] names: Vec, }, /// Match client autonomous system numbers. Asn { - #[serde(deserialize_with = "crate::settings::vec_from_seq_or_map")] + #[serde(deserialize_with = "trusted_server_core::settings::vec_from_seq_or_map")] values: Vec, }, /// Match client IP CIDRs configured inline. IpCidr { - #[serde(deserialize_with = "crate::settings::vec_from_seq_or_map")] + #[serde(deserialize_with = "trusted_server_core::settings::vec_from_seq_or_map")] cidrs: Vec, }, /// Match client IP CIDRs loaded from Config Store. @@ -720,7 +723,7 @@ mod tests { use std::collections::HashMap; use std::net::{Ipv4Addr, Ipv6Addr}; - use crate::platform::test_support::{ + use trusted_server_core::platform::test_support::{ HashMapConfigStore, NoopSecretStore, build_services_with_config_and_secret, }; @@ -797,7 +800,7 @@ mod tests { let mut config = config_with_protection(); config.protection_excluded_methods = vec!["OPTIONS".to_string(), "FASTLYPURGE".to_string()]; let scope = ProtectionScope::compile(&config).expect("should compile scope"); - let services = crate::platform::test_support::noop_services(); + let services = trusted_server_core::platform::test_support::noop_services(); let decision = scope.evaluate(&facts("FASTLYPURGE", "/page", None, None, None), &services); @@ -816,7 +819,7 @@ mod tests { let mut config = config_with_protection(); config.protection_excluded_asns = vec![19750, 209366]; let scope = ProtectionScope::compile(&config).expect("should compile scope"); - let services = crate::platform::test_support::noop_services(); + let services = trusted_server_core::platform::test_support::noop_services(); let decision = scope.evaluate(&facts("GET", "/page", None, None, Some(19750)), &services); @@ -835,7 +838,7 @@ mod tests { let mut config = config_with_protection(); config.protection_excluded_ip_cidrs = vec!["198.51.100.0/24".to_string()]; let scope = ProtectionScope::compile(&config).expect("should compile scope"); - let services = crate::platform::test_support::noop_services(); + let services = trusted_server_core::platform::test_support::noop_services(); let decision = scope.evaluate( &facts( @@ -960,7 +963,7 @@ mod tests { }, }]; let scope = ProtectionScope::compile(&config).expect("should compile scope"); - let services = crate::platform::test_support::noop_services(); + let services = trusted_server_core::platform::test_support::noop_services(); assert!(matches!( scope.evaluate(&facts("GET", "/app.JSON", None, None, None), &services), @@ -982,7 +985,7 @@ mod tests { config.protection_excluded_methods = vec!["GET".to_string()]; config.protection_excluded_ip_cidrs = vec!["192.0.2.0/24".to_string()]; let scope = ProtectionScope::compile(&config).expect("should compile scope"); - let services = crate::platform::test_support::noop_services(); + let services = trusted_server_core::platform::test_support::noop_services(); let decision = scope.evaluate( &facts( @@ -1029,7 +1032,7 @@ mod tests { }, ]; let scope = ProtectionScope::compile(&config).expect("should compile scope"); - let services = crate::platform::test_support::noop_services(); + let services = trusted_server_core::platform::test_support::noop_services(); let decision = scope.evaluate( &facts( @@ -1066,7 +1069,7 @@ mod tests { }, }]; let scope = ProtectionScope::compile(&config).expect("should compile scope"); - let services = crate::platform::test_support::noop_services(); + let services = trusted_server_core::platform::test_support::noop_services(); let decision = scope.evaluate( &facts( @@ -1101,7 +1104,7 @@ mod tests { }, }]; let scope = ProtectionScope::compile(&config).expect("should compile scope"); - let services = crate::platform::test_support::noop_services(); + let services = trusted_server_core::platform::test_support::noop_services(); assert!(matches!( scope.evaluate( diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index 81dde44a2b..872ebf2cc8 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -1001,27 +1001,6 @@ formats = [{ width = 300, height = 250 }] true, ), ("proxy.asset_routes[*].auth.session_token".to_owned(), true), - // What a module declares in its own table follows, at the - // short name its section selects it by and at its full name. - ( - "bot-protection.datadome.server_side_key_secret_name".to_owned(), - true, - ), - ( - "bot-protection.bot-protection.datadome.server_side_key_secret_name" - .to_owned(), - true, - ), - ( - "bot-protection.datadome.protection_test_bypass.credential_secret_name" - .to_owned(), - true, - ), - ( - "bot-protection.bot-protection.datadome.protection_test_bypass.credential_secret_name" - .to_owned(), - true, - ), ], "should expose the native EdgeZero secret metadata contract" ); @@ -1068,19 +1047,6 @@ formats = [{ width = 300, height = 250 }] fn legacy_static_secret_store_selectors_are_accepted_but_not_serialized() { let mut settings = valid_settings(); settings.tinybird.secret_store = Some("legacy-tinybird-store".to_string()); - settings - .insert_module_config( - "bot-protection", - "bot-protection.datadome", - &serde_json::json!({ - "server_side_key_secret_store": "legacy-datadome-store", - "protection_test_bypass": { - "enabled": false, - "credential_secret_store": "legacy-bypass-store", - }, - }), - ) - .expect("should insert legacy DataDome selectors"); let mut route = ProxyAssetRoute::new( "/assets/", "https://examplebucket.s3.us-east-1.amazonaws.com", @@ -1098,12 +1064,7 @@ formats = [{ width = 300, height = 250 }] settings.normalize_deserialized(); let serialized = serde_json::to_string(&settings).expect("should serialize settings"); - for legacy_store in [ - "legacy-tinybird-store", - "legacy-datadome-store", - "legacy-bypass-store", - "legacy-s3-store", - ] { + for legacy_store in ["legacy-tinybird-store", "legacy-s3-store"] { assert!( !serialized.contains(legacy_store), "serialized config should omit deprecated selector {legacy_store}" @@ -1118,19 +1079,19 @@ formats = [{ width = 300, height = 250 }] Some(Redacted::new("resolved-tinybird-secret".to_string())); settings .insert_module_config( - "bot-protection", - "bot-protection.datadome", + "testing", + "testing.example", &serde_json::json!({ - "server_side_key_secret_name": "resolved-datadome-secret", + "key_name": "resolved-module-secret", }), ) - .expect("should insert resolved DataDome config"); + .expect("should insert a module's table holding a resolved secret"); let debug = format!("{settings:?}"); assert!(!debug.contains("resolved-tinybird-secret")); - assert!(!debug.contains("resolved-datadome-secret")); - assert!(debug.contains("datadome")); + assert!(!debug.contains("resolved-module-secret")); + assert!(debug.contains("example")); } #[test] @@ -1827,37 +1788,84 @@ password = "production-admin-password-32-bytes" ); } + fn module_key_is_in_use(table: &serde_json::Map) -> bool { + table.get("lock").and_then(serde_json::Value::as_bool) == Some(true) + } + + const MODULE_SECRETS: &[crate::integrations::ModuleSecretSetting] = + &[crate::integrations::ModuleSecretSetting { + path: &["spare", "key_name"], + in_use: module_key_is_in_use, + }]; + + /// A builder a deployment added, whose module names a secret in its own + /// table. + fn module_with_a_secret() -> IntegrationBuilder { + IntegrationBuilder::new( + "probe", + "example-crate", + crate::integrations::registry_test_support::probe_registration, + crate::integrations::registry_test_support::validate_nothing, + ) + .with_module_name("testing.probe") + .with_secret_settings(MODULE_SECRETS) + } + #[test] - fn deploy_validation_rejects_invalid_datadome_test_bypass() { - for (enable_protection, name, expected_message) in [ - (false, "datadome_test_bypass", "requires enable_protection"), - (true, "", "credential_secret_name"), + fn deploy_validation_refuses_a_module_s_secret_setting_in_use_that_names_no_key() { + for (table, expected) in [ + ( + serde_json::json!({ "lock": true }), + Some("testing.probe.spare.key_name"), + ), + ( + serde_json::json!({ "lock": true, "spare": { "key_name": " " } }), + Some("testing.probe.spare.key_name"), + ), + ( + serde_json::json!({ "lock": true, "spare": { "key_name": "module_key" } }), + None, + ), + // A setting the table does not put to use is not checked. + (serde_json::json!({ "lock": false }), None), ] { let mut settings = valid_settings(); settings - .insert_module_config( - "bot-protection", - "bot-protection.datadome", - &serde_json::json!({ - "enable_protection": enable_protection, - "server_side_key_secret_name": "datadome_server_side_key", - "protection_test_bypass": { - "enabled": true, - "credential_secret_name": name, - }, - }), - ) - .expect("should insert DataDome config"); - - let err = validate_settings_for_deploy(&settings) - .expect_err("should reject invalid DataDome test bypass"); - assert!( - format!("{err:?}").contains(expected_message), - "error should mention the invalid bypass setting: {err:?}" - ); + .insert_module_config("testing", "testing.probe", &table) + .expect("should insert the module's table"); + + let result = validate_settings_for_deploy_with(&settings, &[module_with_a_secret()]); + + match expected { + Some(path) => { + let error = result.expect_err("should refuse a setting that names no key"); + assert!( + format!("{error:?}").contains(path), + "should name the setting for {table}: {error:?}" + ); + } + None => result.unwrap_or_else(|error| { + panic!("should accept {table}: {error:?}"); + }), + } } } + #[test] + fn deploy_validation_checks_no_secret_setting_without_the_module_s_builder() { + let mut settings = valid_settings(); + settings + .insert_module_config( + "testing", + "testing.probe", + &serde_json::json!({ "lock": true }), + ) + .expect("should insert the module's table"); + + validate_settings_for_deploy(&settings) + .expect("should check nothing for a module that declared nothing to this validation"); + } + #[test] fn validate_rejects_invalid_js_asset_proxy_assets() { let mut settings = valid_settings(); diff --git a/crates/trusted-server-core/src/config_payload.rs b/crates/trusted-server-core/src/config_payload.rs index a439d4b7fc..ece4789725 100644 --- a/crates/trusted-server-core/src/config_payload.rs +++ b/crates/trusted-server-core/src/config_payload.rs @@ -198,8 +198,6 @@ mod tests { let value = match key { "unit-test-proxy-secret" => "unit-test-proxy-secret-32-bytes-ok", "tinybird-token-key" => "resolved-tinybird-token", - "datadome-server-key" => "resolved-datadome-server-key", - "datadome-bypass-key" => "resolved-datadome-bypass-credential-32-bytes", "access_key_id" | "s3-access-key" => "AKIAIOSFODNN7EXAMPLE", "secret_access_key" | "s3-secret-key" => "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", "s3-session-key" => "resolved-session-token", @@ -546,20 +544,6 @@ mod tests { original.tinybird.api_host = "api.example.com".to_string(); original.tinybird.auction_token_secret = Some(Redacted::new("tinybird-token-key".to_string())); - original - .insert_module_config( - "bot-protection", - "bot-protection.datadome", - &serde_json::json!({ - "enable_protection": true, - "server_side_key_secret_name": "datadome-server-key", - "protection_test_bypass": { - "enabled": true, - "credential_secret_name": "datadome-bypass-key", - }, - }), - ) - .expect("should configure DataDome references"); let mut route = ProxyAssetRoute::new( "/assets/", "https://examplebucket.s3.us-east-1.amazonaws.com", @@ -594,32 +578,6 @@ mod tests { .map(String::as_str), Some("resolved-tinybird-token") ); - let datadome = reconstructed - .module_config::( - crate::integrations::datadome::MODULE, - ) - .expect("should parse DataDome config") - .expect("should enable DataDome"); - assert_eq!( - datadome - .server_side_key_secret_name - .as_ref() - .map(Redacted::expose) - .map(String::as_str), - Some("resolved-datadome-server-key") - ); - let bypass = datadome - .protection_test_bypass - .as_ref() - .expect("should configure bypass"); - assert_eq!( - bypass - .credential_secret_name - .as_ref() - .map(Redacted::expose) - .map(String::as_str), - Some("resolved-datadome-bypass-credential-32-bytes") - ); let auth = reconstructed.proxy.asset_routes[0] .auth .as_ref() @@ -949,20 +907,6 @@ mod tests { let mut original = test_settings(); original.tinybird.auction_token_secret = Some(Redacted::new("unused-tinybird-key".to_string())); - original - .insert_module_config( - "bot-protection", - "bot-protection.datadome", - &serde_json::json!({ - "enable_protection": false, - "server_side_key_secret_name": "unused-datadome-key", - "protection_test_bypass": { - "enabled": false, - "credential_secret_name": "unused-bypass-key", - }, - }), - ) - .expect("should configure inactive references"); original .ec .partners @@ -977,19 +921,6 @@ mod tests { assert!(reconstructed.tinybird.auction_token_secret.is_none()); assert!(reconstructed.ec.partners[0].ts_pull_token.is_none()); - let datadome = reconstructed - .module_config::( - crate::integrations::datadome::MODULE, - ) - .expect("should parse inactive DataDome config") - .expect("client-side DataDome remains enabled"); - assert!(datadome.server_side_key_secret_name.is_none()); - assert!( - datadome - .protection_test_bypass - .as_ref() - .is_some_and(|bypass| bypass.credential_secret_name.is_none()) - ); } /// A table for a module its section does not select is refused, and its @@ -997,41 +928,35 @@ mod tests { /// the table's own fault rather than a secret-store failure that follows /// from it. #[test] - fn an_unselected_datadome_table_is_refused_without_resolving_its_secrets() { + fn an_unselected_module_s_table_is_refused_without_resolving_its_secrets() { let original = test_settings(); let mut data = serde_json::to_value(&original).expect("should serialize settings to JSON"); data.as_object_mut() .expect("settings should serialize as an object") .insert( - "bot-protection".to_owned(), + "testing".to_owned(), serde_json::json!({ - "datadome": { - "enable_protection": true, - "server_side_key_secret_name": "unused-datadome-key", - "protection_test_bypass": { - "enabled": true, - "credential_secret_name": "unused-bypass-key", - }, - }, + "probe": { "lock": true, "key_name": "unused-module-key" }, }), ); let envelope = BlobEnvelope::new(data, "2026-01-01T00:00:00Z".to_string()); let envelope = serde_json::to_string(&envelope).expect("should serialize envelope"); - let error = settings_from_config_blob( + let error = settings_from_config_blob_with( &envelope, - &UnifiedSecretStore, - &StoreName::from("ts_secrets"), + &ModuleSecretStore, + &StoreName::from("trusted_server_secrets"), + &[module_with_a_secret()], ) .expect_err("should refuse a table its section does not select"); let rendered = format!("{error:?}"); assert!( - rendered.contains("[bot-protection] selects no module"), + rendered.contains("[testing] selects no module"), "should name the section and what it is missing: {rendered}" ); assert!( - !rendered.contains("unused-datadome-key"), + !rendered.contains("unused-module-key"), "should not have tried to resolve the stale secret reference: {rendered}" ); } diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index 90dfa19edd..c77b678b10 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -1077,66 +1077,6 @@ mod tests { assert_eq!(config.request_scheme, "https"); } - #[test] - fn suppressed_datadome_tag_preserves_and_rewrites_publisher_tag() { - let mut settings = create_test_settings(); - settings - .insert_module_config( - "bot-protection", - "bot-protection.datadome", - &json!({ - "client_side_key": "test-client-key", - }), - ) - .expect("should configure DataDome integration"); - let registry = IntegrationRegistry::new(&settings) - .expect("should create integration registry with DataDome"); - let mut suppressed = IntegrationRequestState::default(); - suppressed.set( - crate::integrations::datadome::DATADOME_INTEGRATION_ID, - crate::integrations::datadome::DataDomeClientTagSuppressed, - ); - let config = HtmlProcessorConfig::from_settings( - &settings, - ®istry, - "origin.example.com", - "test.example.com", - "https", - ) - .with_request_state(suppressed); - let mut processor = create_html_processor(config); - - let output = processor - .process_chunk( - br#"content"#, - true, - ) - .expect("should process HTML"); - let html = String::from_utf8(output).expect("should produce UTF-8 HTML"); - - assert!( - !html.contains("window.ddjskey"), - "should omit the DataDome client configuration" - ); - assert!( - html.contains("id=\"publisher-datadome\""), - "should preserve the publisher-originated DataDome tag" - ); - assert!( - html.contains("src=\"/integrations/datadome/tags.js\""), - "should rewrite the publisher-originated DataDome tag" - ); - assert!( - !html.contains("https://js.datadome.co/tags.js"), - "should remove the original third-party DataDome URL" - ); - assert_eq!( - html.matches("/integrations/datadome/tags.js").count(), - 1, - "should leave exactly one publisher-originated DataDome tag" - ); - } - #[test] fn test_real_publisher_html() { // Test with publisher HTML from test_publisher.html diff --git a/crates/trusted-server-core/src/integrations/mod.rs b/crates/trusted-server-core/src/integrations/mod.rs index 0e7ae6ecae..a2c51348cb 100644 --- a/crates/trusted-server-core/src/integrations/mod.rs +++ b/crates/trusted-server-core/src/integrations/mod.rs @@ -15,7 +15,6 @@ use crate::settings::Settings; pub mod adserver_mock; pub mod aps; -pub mod datadome; pub mod js_asset_proxy; pub mod openrtb; pub mod prebid; @@ -661,7 +660,6 @@ const BUILT_IN_BUILDERS: &[IntegrationBuilder] = &[ // tests. #[cfg(test)] registry_test_support::request_fixture::BUILDER, - datadome::BUILDER, // Implementations `[demand]` and `[ad-server]` can name. None of them is // a module a section selects. IntegrationBuilder::implementations(openrtb::MODULE, CORE_SOURCE).with_demand(&openrtb::DEMAND), diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index e6b40c49f5..fd41285e9a 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -2511,7 +2511,10 @@ pub(crate) mod test_support { /// the query `fixture_request=1`. Where that query arrived on a document /// navigation it also leaves a mark for the page path. It strips the /// cookie it reserves too, and writes the Cookie header out again as it - /// does, the way a module that reserves a cookie does. Selected, the + /// does, the way a module that reserves a cookie does. Selected, its + /// request filter leaves the same mark on a request that carries its + /// header, which is the route a module that decides in its filter takes. + /// Selected, the /// stand-in's head injector reads the mark from the document's state and /// writes one script at the start of `` and the tag of its /// standalone module after the bundle. Its response finalizer sets a @@ -2527,7 +2530,8 @@ pub(crate) mod test_support { use crate::error::TrustedServerError; use crate::integrations::registry::{ CarriedJsModule, IntegrationHeadInjector, IntegrationHtmlContext, - IntegrationRegistration, IntegrationRequestState, + IntegrationRegistration, IntegrationRequestFilter, IntegrationRequestState, + RequestFilterDecision, RequestFilterEffects, RequestFilterInput, }; use crate::integrations::{CORE_SOURCE, IntegrationBuilder}; use crate::settings::Settings; @@ -2548,6 +2552,9 @@ pub(crate) mod test_support { pub(crate) const COOKIE: &str = "ts-request-fixture=1; Path=/"; /// The name of that cookie, which the preparer strips from a request. pub(crate) const COOKIE_NAME: &str = "ts-request-fixture"; + /// The request header that has the stand-in's request filter leave + /// the mark. + pub(crate) const FILTER_HEADER: &str = "x-ts-request-fixture"; const JS: &str = "(function(){window.__ts_request_fixture_loaded=1;})();"; // SHA-256 of JS, hex. The registry refuses a literal that is not. @@ -2657,6 +2664,27 @@ pub(crate) mod test_support { } } + struct Filter; + + #[async_trait::async_trait(?Send)] + impl IntegrationRequestFilter for Filter { + fn integration_id(&self) -> &'static str { + ID + } + + async fn filter_request( + &self, + input: RequestFilterInput<'_>, + ) -> Result> { + if input.request.headers().contains_key(FILTER_HEADER) { + mark(input.request); + } + Ok(RequestFilterDecision::Continue( + RequestFilterEffects::default(), + )) + } + } + struct Head; impl Head { @@ -2707,6 +2735,7 @@ pub(crate) mod test_support { }) .with_standalone_js() .with_head_injector(Arc::new(Head)) + .with_request_filter(Arc::new(Filter)) .build(), )) } @@ -3232,7 +3261,7 @@ mod tests { ) -> Result> { input.request.extensions_mut().insert(RequestAnnotation); Ok(RequestFilterDecision::Continue(RequestFilterEffects { - request_headers: vec![HeaderMutation::set("x-datadome-isbot", "1")], + request_headers: vec![HeaderMutation::set("x-probe-isbot", "1")], response_headers: vec![HeaderMutation::set("x-dd-b", "allowed")], })) } @@ -3469,10 +3498,10 @@ mod tests { assert_eq!( req.headers() - .get("x-datadome-isbot") + .get("x-probe-isbot") .and_then(|value| value.to_str().ok()), Some("1"), - "should apply DataDome-style request enrichment before routing" + "should apply a filter's request enrichment before routing" ); assert!( req.extensions().get::().is_some(), diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 2d976e0a63..132943671e 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -6275,8 +6275,8 @@ pub(crate) enum TemplateCacheBypassReason { /// concern rather than a hypothetical one. #[display("request carried Authorization")] AuthorizedRequest, - /// Not a 200. This is also what covers a `DataDome` block, which replaces the - /// document with a `403` (`integrations/datadome/protection.rs:778`). + /// Not a 200. This is also what covers a bot protection module's block, which + /// replaces the document with a `403`. #[display("status was not 200 OK")] NonOkStatus, /// Not HTML, so there is no template to transform. @@ -7687,8 +7687,7 @@ mod tests { }; use crate::permissions::{Permission, PermissionSet}; use crate::platform::test_support::{ - NoopSecretStore, StubHttpClient, build_services_with_http_client, - build_services_with_secret_http_client_and_client_ip, noop_services, + StubHttpClient, build_services_with_http_client, noop_services, noop_services_with_telemetry_sink, }; use crate::test_support::tests::{crate_test_settings_str, create_test_settings}; @@ -7822,7 +7821,7 @@ mod tests { response_is_private: bool, auction_preserved_client_snapshot: bool, origin_kv_auction_order: bool, - datadome_tag_suppressed: bool, + module_state_written: bool, ok: bool, } @@ -7879,7 +7878,7 @@ mod tests { .header(header::USER_AGENT, "scheduling-browser") .body(EdgeBody::empty()) .expect("should build navigation request"); - crate::integrations::datadome::suppress_client_tag(&mut request); + crate::integrations::registry_test_support::request_fixture::mark(&mut request); request } @@ -7892,15 +7891,10 @@ mod tests { let mut settings = Settings::from_toml(&toml).expect("should parse scheduling settings"); settings.demand = crate::auction::test_support::demand_named(&[SCHEDULING_PROVIDER]); settings.proxy.allowed_domains = vec!["*.example".to_owned(), "*.example.com".to_owned()]; - settings - .insert_module_config( - "bot-protection", - "bot-protection.datadome", - &serde_json::json!({ - "client_side_key": "scheduling-test-key", - }), - ) - .expect("should configure DataDome integration"); + settings.select_module( + "testing", + crate::integrations::registry_test_support::request_fixture::MODULE, + ); settings } @@ -7937,9 +7931,11 @@ mod tests { http.set_streaming_responses_supported(streaming_responses); http.set_pending_streaming_responses_supported(pending_streaming_responses); if queue_origin { + // The document has a head, so what a module writes there for the + // request has somewhere to go. http.push_response_with_headers( 200, - b"ok".to_vec(), + b"ok".to_vec(), vec![("content-type", "text/html; charset=utf-8")], ); } @@ -8049,9 +8045,9 @@ mod tests { let origin_kv_auction_order = captured_auction.as_ref().is_some_and(|captured| { captured.http_calls_at_dispatch == 1 && captured.lookups_at_dispatch == 1 }); - let datadome_tag_suppressed = if let Ok(response) = result { + let module_state_written = if let Ok(response) = result { let registry = IntegrationRegistry::new(&settings) - .expect("should create integration registry with DataDome"); + .expect("should create integration registry with the stand-in"); buffer_publisher_response_async( response, &Method::GET, @@ -8065,7 +8061,9 @@ mod tests { .and_then(|response| response.into_body().into_bytes()) .and_then(|body| String::from_utf8(body.to_vec()).ok()) .is_some_and(|html| { - !html.contains("window.ddjskey") && !html.contains("/integrations/datadome/tags.js") + html.contains( + crate::integrations::registry_test_support::request_fixture::HEAD_FLAG, + ) }) } else { false @@ -8081,7 +8079,7 @@ mod tests { response_is_private, auction_preserved_client_snapshot, origin_kv_auction_order, - datadome_tag_suppressed, + module_state_written, ok, } } @@ -8279,7 +8277,7 @@ mod tests { ); assert!( outcome.response_is_private, - "DataDome-suppressed pending HTML should remain private" + "pending HTML a module wrote into should remain private" ); assert!( outcome.origin_kv_auction_order, @@ -8290,8 +8288,8 @@ mod tests { "auction dispatch should retain the original client URI and headers" ); assert!( - outcome.datadome_tag_suppressed, - "configured DataDome client injection should remain suppressed" + outcome.module_state_written, + "what a module left on the request should reach the pending document" ); } @@ -14858,64 +14856,43 @@ mod tests { } #[tokio::test] - async fn datadome_suppressed_request_bypasses_a_warm_shared_template() { + async fn a_personalized_request_bypasses_a_warm_shared_template() { let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); - let mut raw = settings_with_mode("esi"); - raw.insert_module_config( - "bot-protection", - "bot-protection.datadome", - &serde_json::json!({ - "client_side_key": "test-client-key", - }), - ) - .expect("should configure DataDome integration"); - let settings = Arc::new(raw); + let settings = Arc::new(settings_with_mode("esi")); let services = services(Arc::clone(&stub), Arc::clone(&cache)); queue_shareable_html(&stub); queue_shareable_html(&stub); - let ordinary = run(&settings, &services, navigation_request()).await; - let ordinary_document = String::from_utf8(body_of(ordinary).await) - .expect("ordinary document should be UTF-8"); - assert!( - ordinary_document.contains("/integrations/datadome/tags.js"), - "the warm template fixture must contain the ordinary DataDome tag" - ); + let _ = body_of(run(&settings, &services, navigation_request()).await).await; - let mut suppressed_request = navigation_request(); - suppressed_request + let mut personalized_request = navigation_request(); + personalized_request .headers_mut() .insert("sec-fetch-dest", HeaderValue::from_static("script")); - crate::integrations::datadome::suppress_client_tag(&mut suppressed_request); - suppressed_request + personalized_request .extensions_mut() .insert(crate::response_privacy::PersonalizedResponse); - let suppressed = run(&settings, &services, suppressed_request).await; + let personalized = run(&settings, &services, personalized_request).await; assert_eq!( - suppressed + personalized .headers() .get(HEADER_X_TS_TEMPLATE_CACHE) .and_then(|value| value.to_str().ok()), Some("bypass-request"), - "request-scoped tag suppression must not read a shared template" + "a personalized request must not read a shared template" ); - let suppressed_document = String::from_utf8(body_of(suppressed).await) - .expect("suppressed document should be UTF-8"); + let _ = body_of(personalized).await; assert_eq!( stub.recorded_request_uris().len(), 2, - "the suppressed navigation must reach the origin even when template cache is warm" + "the personalized request must reach the origin even when template cache is warm" ); assert_eq!( cache.lookups.lock().expect("should lock lookups").len(), 1, - "the suppressed request must bypass template cache before lookup" - ); - assert!( - !suppressed_document.contains("/integrations/datadome/tags.js"), - "the request-scoped suppression decision must survive origin processing" + "the personalized request must bypass template cache before lookup" ); } @@ -16417,10 +16394,9 @@ mod tests { } #[test] - fn a_datadome_block_is_refused_by_the_status_check() { - // DataDome replaces the document with a 403 - // (`integrations/datadome/protection.rs:778`). There is no separate - // marker to detect, and none is needed. + fn a_bot_protection_block_is_refused_by_the_status_check() { + // A bot protection module replaces the document with a 403. There + // is no separate marker to detect, and none is needed. assert_eq!( template_cache_bypass_reason( AssemblyMode::Esi, @@ -18215,7 +18191,7 @@ mod tests { } #[tokio::test] - async fn suppressed_navigation_removes_conditional_and_range_headers() { + async fn personalized_navigation_removes_conditional_and_range_headers() { let settings = create_test_settings(); let stub = Arc::new(StubHttpClient::new()); stub.push_response_with_headers( @@ -18237,7 +18213,6 @@ mod tests { .header(header::IF_RANGE, "\"cached-page\"") .body(EdgeBody::empty()) .expect("should build conditional request"); - crate::integrations::datadome::suppress_client_tag(&mut req); req.extensions_mut() .insert(crate::response_privacy::PersonalizedResponse); @@ -18258,13 +18233,13 @@ mod tests { headers .iter() .all(|(name, _)| !name.eq_ignore_ascii_case(header_name.as_str())), - "suppressed navigations must not forward {header_name}" + "personalized navigations must not forward {header_name}" ); } } #[tokio::test] - async fn suppressed_iframe_removes_conditional_and_range_headers() { + async fn personalized_iframe_removes_conditional_and_range_headers() { let settings = create_test_settings(); let stub = Arc::new(StubHttpClient::new()); stub.push_response_with_headers( @@ -18286,7 +18261,6 @@ mod tests { .header(header::IF_RANGE, "\"cached-frame\"") .body(EdgeBody::empty()) .expect("should build conditional iframe request"); - crate::integrations::datadome::suppress_client_tag(&mut req); req.extensions_mut() .insert(crate::response_privacy::PersonalizedResponse); @@ -18307,13 +18281,13 @@ mod tests { headers .iter() .all(|(name, _)| !name.eq_ignore_ascii_case(header_name.as_str())), - "suppressed iframe documents must not forward {header_name}" + "personalized iframe documents must not forward {header_name}" ); } } #[tokio::test] - async fn suppressed_subresource_preserves_conditional_and_range_headers() { + async fn personalized_subresource_preserves_conditional_and_range_headers() { let settings = create_test_settings(); let stub = Arc::new(StubHttpClient::new()); stub.push_response_with_headers( @@ -18335,7 +18309,6 @@ mod tests { .header(header::IF_RANGE, "\"cached-video\"") .body(EdgeBody::empty()) .expect("should build conditional subresource request"); - crate::integrations::datadome::suppress_client_tag(&mut req); req.extensions_mut() .insert(crate::response_privacy::PersonalizedResponse); @@ -18358,7 +18331,7 @@ mod tests { .find(|(name, _)| name.eq_ignore_ascii_case(header_name.as_str())) .map(|(_, value)| value.as_str()), Some(expected), - "suppressed subresources should preserve {header_name}" + "personalized subresources should preserve {header_name}" ); } } @@ -18486,38 +18459,32 @@ mod tests { } #[tokio::test] - async fn datadome_filter_marker_survives_into_publisher_html_pipeline() { + async fn what_a_request_filter_leaves_survives_into_the_publisher_html_pipeline() { let mut settings = create_test_settings(); - settings - .insert_module_config( - "bot-protection", - "bot-protection.datadome", - &serde_json::json!({ - "enable_protection": true, - "server_side_key_secret_name": "server-side-key", - "protection_excluded_ip_cidrs": ["192.0.2.0/24"], - "client_side_key": "test-client-key", - }), - ) - .expect("should configure DataDome integration"); + settings.select_module( + "testing", + crate::integrations::registry_test_support::request_fixture::MODULE, + ); let registry = IntegrationRegistry::new(&settings) - .expect("should create integration registry with DataDome"); + .expect("should create integration registry with the stand-in"); let stub = Arc::new(StubHttpClient::new()); stub.push_response_with_headers( 200, b"content".to_vec(), vec![("content-type", "text/html; charset=utf-8")], ); - let services = build_services_with_secret_http_client_and_client_ip( - NoopSecretStore, - Arc::clone(&stub) as Arc, - Some("192.0.2.10".parse().expect("should parse client IP")), + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc ); let mut req = HttpRequest::builder() .method(Method::GET) .uri("https://publisher.example/page") .header(header::HOST, "publisher.example") .header("sec-fetch-dest", "document") + .header( + crate::integrations::registry_test_support::request_fixture::FILTER_HEADER, + "1", + ) .body(EdgeBody::empty()) .expect("should build request"); @@ -18530,7 +18497,7 @@ mod tests { permissions: None, }) .await - .expect("should run DataDome filter"); + .expect("should run the stand-in's filter"); assert!(matches!( filter_outcome, crate::integrations::RequestFilterRegistryOutcome::Continue(_) @@ -18550,8 +18517,10 @@ mod tests { .expect("should buffer publisher response"); let html = response_body_string(response); - assert!(!html.contains("window.ddjskey")); - assert!(!html.contains("/integrations/datadome/tags.js")); + assert!( + html.contains(crate::integrations::registry_test_support::request_fixture::HEAD_FLAG), + "what the filter left on the request should reach the document" + ); assert_eq!( stub.recorded_backend_names().len(), 1, @@ -18560,51 +18529,7 @@ mod tests { } #[test] - fn suppressed_datadome_tag_reaches_publisher_html_pipeline() { - let mut settings = create_test_settings(); - settings - .insert_module_config( - "bot-protection", - "bot-protection.datadome", - &serde_json::json!({ - "client_side_key": "test-client-key", - }), - ) - .expect("should configure DataDome integration"); - let registry = IntegrationRegistry::new(&settings) - .expect("should create integration registry with DataDome"); - let mut params = make_stream_params(&settings, "identity"); - params.content_type = "text/html; charset=utf-8".to_string(); - let mut suppressed = IntegrationRequestState::default(); - suppressed.set( - crate::integrations::datadome::DATADOME_INTEGRATION_ID, - crate::integrations::datadome::DataDomeClientTagSuppressed, - ); - params.request_state = suppressed; - let mut output = Vec::new(); - - stream_publisher_body( - EdgeBody::from(b"content".to_vec()), - &mut output, - ¶ms, - &settings, - ®istry, - ) - .expect("should process suppressed HTML"); - - let html = String::from_utf8(output).expect("should produce UTF-8 HTML"); - assert!( - !html.contains("window.ddjskey"), - "publisher processing should omit the DataDome client configuration" - ); - assert!( - !html.contains("/integrations/datadome/tags.js"), - "publisher processing should omit the DataDome client tag URL" - ); - } - - #[test] - fn suppressed_datadome_html_is_private_and_not_shared_cached() { + fn personalized_html_is_private_and_not_shared_cached() { let mut response = Response::builder() .status(StatusCode::OK) .header(header::CACHE_CONTROL, "public, max-age=600") @@ -18630,31 +18555,31 @@ mod tests { .get(header::CACHE_CONTROL) .and_then(|value| value.to_str().ok()), Some("no-store, private"), - "suppressed HTML should be private and non-storable" + "personalized HTML should be private and non-storable" ); assert!( response.headers().get("surrogate-control").is_none(), - "suppressed HTML should not retain Surrogate-Control" + "personalized HTML should not retain Surrogate-Control" ); assert!( response.headers().get("fastly-surrogate-control").is_none(), - "suppressed HTML should not retain Fastly-Surrogate-Control" + "personalized HTML should not retain Fastly-Surrogate-Control" ); assert!( response .headers() .get("cloudflare-cdn-cache-control") .is_none(), - "suppressed HTML should not retain Cloudflare-CDN-Cache-Control" + "personalized HTML should not retain Cloudflare-CDN-Cache-Control" ); assert!( response.headers().get("cdn-cache-control").is_none(), - "suppressed HTML should not retain CDN-Cache-Control" + "personalized HTML should not retain CDN-Cache-Control" ); for header_name in [header::ETAG, header::LAST_MODIFIED] { assert!( !response.headers().contains_key(&header_name), - "suppressed HTML should not retain {header_name}" + "personalized HTML should not retain {header_name}" ); } @@ -18675,12 +18600,12 @@ mod tests { .get(header::CACHE_CONTROL) .and_then(|value| value.to_str().ok()), Some("no-store, private"), - "suppressed HTML should use the exact synthesized-HTML policy" + "personalized HTML should use the exact synthesized-HTML policy" ); } #[test] - fn datadome_cache_privacy_does_not_change_non_html_or_unsuppressed_responses() { + fn personalized_cache_privacy_does_not_change_non_html_or_ordinary_responses() { let mut response = Response::builder() .status(StatusCode::OK) .header(header::CACHE_CONTROL, "public, max-age=600") @@ -18700,7 +18625,7 @@ mod tests { .get(header::CACHE_CONTROL) .and_then(|value| value.to_str().ok()), Some("public, max-age=600"), - "unsuppressed HTML should retain its existing cache policy" + "ordinary HTML should retain its existing cache policy" ); super::apply_personalized_response_cache_privacy( @@ -18768,7 +18693,7 @@ mod tests { } #[test] - fn core_reads_the_neutral_marker_not_the_datadome_type() { + fn the_neutral_marker_alone_marks_a_response_personalized() { let mut personalized = HttpRequest::builder() .method(Method::GET) .uri("https://publisher.example/page") diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 31cc744d39..942625da4c 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -3981,7 +3981,6 @@ impl Settings { self.image_optimizer.normalize(); self.debug.auction_html_comment_options.normalize(); self.tinybird.normalize(); - self.remove_legacy_static_secret_store_selectors(); self.consent.validate(); } @@ -4565,32 +4564,6 @@ impl Settings { } Ok(()) } - - /// Drops the `DataDome` secret-store selectors a previous release read, - /// from the module's table when it is configured. - fn remove_legacy_static_secret_store_selectors(&mut self) { - let Some(datadome) = self - .sections - .0 - .get_mut("bot-protection") - .and_then(|section| section.settings_of_mut("datadome")) - else { - return; - }; - - let mut removed = datadome.remove("server_side_key_secret_store").is_some(); - if let Some(bypass) = datadome - .get_mut("protection_test_bypass") - .and_then(JsonValue::as_object_mut) - { - removed |= bypass.remove("credential_secret_store").is_some(); - } - if removed { - log::warn!( - "DataDome secret-store selectors are deprecated and ignored; static credentials resolve through the default secret store" - ); - } - } } fn validate_publisher_domain(value: &str) -> Result<(), ValidationError> { @@ -5213,7 +5186,7 @@ module = \"none\"", const CANARY_S3_SESSION_TOKEN: &str = "CANARY-S3-SESSION-TOKEN-0123456789"; const CANARY_TINYBIRD_AUCTION_TOKEN: &str = "CANARY-TINYBIRD-AUCTION-TOKEN-0123456789"; const CANARY_TINYBIRD_ACCESS_TOKEN: &str = "CANARY-TINYBIRD-ACCESS-TOKEN-0123456789"; - const CANARY_DATADOME_SERVER_SIDE_KEY: &str = "CANARY-DATADOME-SERVER-SIDE-KEY-0123456789"; + const CANARY_MODULE_KEY: &str = "CANARY-MODULE-TABLE-KEY-0123456789"; let mut settings = create_test_settings(); @@ -5273,17 +5246,17 @@ module = \"none\"", }; // A module's table is opaque JSON, and the section's hand-written - // `Debug` impl is the only thing keeping resolved DataDome - // credentials out of this output, so pin it here. + // `Debug` impl is the only thing keeping a secret a module's table + // holds out of this output, so pin it here. settings .insert_module_config( - "bot-protection", - crate::integrations::datadome::MODULE, + "testing", + "testing.example", &json!({ - "server_side_key_secret_name": CANARY_DATADOME_SERVER_SIDE_KEY, + "key_name": CANARY_MODULE_KEY, }), ) - .expect("should insert datadome integration config"); + .expect("should insert a module's table"); let debug = format!("{settings:?}"); @@ -5327,10 +5300,7 @@ module = \"none\"", CANARY_TINYBIRD_AUCTION_TOKEN, ), ("tinybird.access_token_secret", CANARY_TINYBIRD_ACCESS_TOKEN), - ( - "bot-protection.datadome.server_side_key_secret_name", - CANARY_DATADOME_SERVER_SIDE_KEY, - ), + ("testing.example.key_name", CANARY_MODULE_KEY), ]; for (field, canary) in canaries { @@ -8437,7 +8407,7 @@ source_domain = "partner.example.com" #[test] fn removed_integration_fields_are_rejected() { - use crate::integrations::{datadome, prebid}; + use crate::integrations::prebid; let mut settings = create_test_settings(); settings @@ -8454,22 +8424,6 @@ source_domain = "partner.example.com" format!("{error:?}").contains("server_url"), "should identify the removed field: {error:?}" ); - - let mut settings = create_test_settings(); - settings - .insert_module_config( - "bot-protection", - datadome::MODULE, - &json!({ "account_id": "removed-value" }), - ) - .expect("should insert the removed DataDome field"); - let error = settings - .module_config::(datadome::MODULE) - .expect_err("should reject the removed DataDome field"); - assert!( - format!("{error:?}").contains("account_id"), - "should identify the removed field: {error:?}" - ); } /// A table written for a module its section does not select is refused, diff --git a/crates/trusted-server-modules/Cargo.toml b/crates/trusted-server-modules/Cargo.toml index 9b5e11cd2d..2d888f3a18 100644 --- a/crates/trusted-server-modules/Cargo.toml +++ b/crates/trusted-server-modules/Cargo.toml @@ -16,6 +16,7 @@ workspace = true [dependencies] trusted-server-ad-tag-google = { workspace = true } trusted-server-audience-permutive = { workspace = true } +trusted-server-bot-protection-datadome = { workspace = true } trusted-server-cmp-didomi = { workspace = true } trusted-server-cmp-osano = { workspace = true } trusted-server-cmp-sourcepoint = { workspace = true } diff --git a/crates/trusted-server-modules/src/lib.rs b/crates/trusted-server-modules/src/lib.rs index a321558ca4..1ed3e935a9 100644 --- a/crates/trusted-server-modules/src/lib.rs +++ b/crates/trusted-server-modules/src/lib.rs @@ -28,6 +28,7 @@ pub fn builders() -> Vec { trusted_server_cmp_sourcepoint::builder(), trusted_server_cmp_osano::builder(), trusted_server_tag_google_tag_manager::builder(), + trusted_server_bot_protection_datadome::builder(), trusted_server_ad_tag_google::builder(), trusted_server_ad_tag_google::diagnostics::builder(), ] @@ -66,6 +67,7 @@ mod tests { "cmp.sourcepoint", "cmp.osano", "tag.google-tag-manager", + "bot-protection.datadome", "ad-tag.google", "ad-tag.google.diagnostics", ], @@ -264,4 +266,30 @@ mod tests { "should strip the reserved diagnostics cookie and keep the rest" ); } + + /// The `ts` tool registers the stock builders, so the secret settings a + /// stock module declares are among the leaves a push treats as naming a + /// key. + #[test] + fn a_stock_module_s_secret_settings_are_listed_once_the_list_is_registered() { + use edgezero_core::app_config::AppConfigMeta as _; + use trusted_server_core::config::{TrustedServerAppConfig, register_deploy_integrations}; + + register_deploy_integrations(builders()); + + let paths = TrustedServerAppConfig::secret_fields() + .iter() + .map(edgezero_core::app_config::SecretField::dotted_path) + .collect::>(); + + for expected in [ + "bot-protection.datadome.server_side_key_secret_name", + "bot-protection.datadome.protection_test_bypass.credential_secret_name", + ] { + assert!( + paths.iter().any(|path| path == expected), + "should list `{expected}` among {paths:?}" + ); + } + } } diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 408175ac62..032a6c9d6c 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -109,26 +109,26 @@ diff; it does not move inline values to a secret store. The following table records the independent lifecycle, key-identity, serialization, runtime, and secret axes for every exceptional field: -| Path | Lifecycle | Key identity | Serialization | Runtime | Secret handling | -| ------------------------------------------------------------ | ---------- | ----------------------------------- | ------------- | -------------------- | ------------------------ | -| `AssetOriginAuth.s3_sig_v4` | deprecated | alias of `AssetOriginAuth.s3_sigv4` | skipped | deserialization only | none | -| `DataDomeConfig.server_side_key_secret_name` | canonical | canonical | serialized | active | store resolved | -| `DataDomeConfig.server_side_key_secret_store` | deprecated | canonical | skipped | normalized away | none | -| `DataDomeProtectionTestBypassConfig.credential_secret_name` | canonical | canonical | serialized | active | store resolved | -| `DataDomeProtectionTestBypassConfig.credential_secret_store` | deprecated | canonical | skipped | normalized away | none | -| `Ec.passphrase` | canonical | canonical | serialized | active | store resolved | -| `EcPartner.api_token` | canonical | canonical | serialized | active | store resolved | -| `EcPartner.ts_pull_token` | canonical | canonical | serialized | active | store resolved | -| `Handler.password` | canonical | canonical | serialized | active | store resolved | -| `Publisher.proxy_secret` | canonical | canonical | serialized | active | store resolved | -| `S3SigV4AuthConfig.access_key_id` | canonical | canonical | serialized | active | store resolved | -| `S3SigV4AuthConfig.secret_access_key` | canonical | canonical | serialized | active | store resolved | -| `S3SigV4AuthConfig.secret_store` | deprecated | canonical | skipped | normalized away | none | -| `S3SigV4AuthConfig.session_token` | canonical | canonical | serialized | active | store resolved | -| `TinybirdSettings.access_token_secret` | deprecated | canonical | skipped | normalized away | accepted, then discarded | -| `TinybirdSettings.auction_token_secret` | canonical | canonical | serialized | active | store resolved | -| `TinybirdSettings.secret_store` | deprecated | canonical | skipped | normalized away | none | -| `TrustedClientIpConfig.shared_secret` | canonical | canonical | serialized | active | deliberately inline | +| Path | Lifecycle | Key identity | Serialization | Runtime | Secret handling | +| ------------------------------------------------------------ | ---------- | ----------------------------------- | ------------- | ----------------------- | ------------------------ | +| `AssetOriginAuth.s3_sig_v4` | deprecated | alias of `AssetOriginAuth.s3_sigv4` | skipped | deserialization only | none | +| `DataDomeConfig.server_side_key_secret_name` | canonical | canonical | serialized | active | store resolved | +| `DataDomeConfig.server_side_key_secret_store` | deprecated | canonical | serialized | discarded by the module | none | +| `DataDomeProtectionTestBypassConfig.credential_secret_name` | canonical | canonical | serialized | active | store resolved | +| `DataDomeProtectionTestBypassConfig.credential_secret_store` | deprecated | canonical | serialized | discarded by the module | none | +| `Ec.passphrase` | canonical | canonical | serialized | active | store resolved | +| `EcPartner.api_token` | canonical | canonical | serialized | active | store resolved | +| `EcPartner.ts_pull_token` | canonical | canonical | serialized | active | store resolved | +| `Handler.password` | canonical | canonical | serialized | active | store resolved | +| `Publisher.proxy_secret` | canonical | canonical | serialized | active | store resolved | +| `S3SigV4AuthConfig.access_key_id` | canonical | canonical | serialized | active | store resolved | +| `S3SigV4AuthConfig.secret_access_key` | canonical | canonical | serialized | active | store resolved | +| `S3SigV4AuthConfig.secret_store` | deprecated | canonical | skipped | normalized away | none | +| `S3SigV4AuthConfig.session_token` | canonical | canonical | serialized | active | store resolved | +| `TinybirdSettings.access_token_secret` | deprecated | canonical | skipped | normalized away | accepted, then discarded | +| `TinybirdSettings.auction_token_secret` | canonical | canonical | serialized | active | store resolved | +| `TinybirdSettings.secret_store` | deprecated | canonical | skipped | normalized away | none | +| `TrustedClientIpConfig.shared_secret` | canonical | canonical | serialized | active | deliberately inline | Prepare an initial reference-based deployment in this order: From e08ef734d93c1bcbe9a550fce3d43cacb486e502 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 8 Oct 2026 03:10:18 +0100 Subject: [PATCH 099/124] The audit generator asks the stock list where a module is selected `ts audit` turns the integrations it finds on a page into the section that selects each one's module and the name written there. It took each module's name from `trusted_server_core::integrations`, so it stopped compiling when the first module left core, and no local check showed that because the `ts` tool cannot be built on the machine this branch was made on. The stock list crate now answers the question by integration id from the builders it already lists, with a test of every id the generator knows, and the generator asks it. The generator names no module of a vendor's. Prebid is the one exception, because it is registered through the auction plan and no builder carries its name, and it stays in core for now. --- .../src/commands/audit/generate/mod.rs | 31 +++++------- crates/trusted-server-modules/src/lib.rs | 47 ++++++++++++++++++- 2 files changed, 58 insertions(+), 20 deletions(-) diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs index ea5477a779..9590c2bda9 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -668,25 +668,18 @@ fn select_js_asset_proxy(draft: &str) -> CliResult { /// is written under there, or `None` for one no module in this build /// configures. fn module_section(id: &str) -> Option<(&'static str, &'static str)> { - use trusted_server_core::integrations as modules; - let (section, name) = match id { - "datadome" => ("bot-protection", modules::datadome::MODULE), - "didomi" => ("cmp", modules::didomi::MODULE), - "sourcepoint" => ("cmp", modules::sourcepoint::MODULE), - "osano" => ("cmp", modules::osano::MODULE), - "lockr" => ("identity", modules::lockr::MODULE), - "permutive" => ("audience", modules::permutive::MODULE), - "nextjs" => ("framework", modules::nextjs::MODULE), - "gpt" => ("ad-tag", modules::gpt::MODULE), - "google_tag_manager" => ("tag", modules::google_tag_manager::MODULE), - "prebid" => ("auction", modules::prebid::MODULE), - "testlight" => ("auction", modules::testlight::MODULE), - _ => return None, - }; - Some(( - section, - trusted_server_core::module_name::short_form(section, name), - )) + // Prebid is registered through the auction plan, so no builder in the + // stock list carries its module's name. + if id == "prebid" { + return Some(( + "auction", + trusted_server_core::module_name::short_form( + "auction", + trusted_server_core::integrations::prebid::MODULE, + ), + )); + } + trusted_server_modules::selection_of(id) } fn select_js_asset_proxy_candidates( diff --git a/crates/trusted-server-modules/src/lib.rs b/crates/trusted-server-modules/src/lib.rs index 1ed3e935a9..ade47e7e21 100644 --- a/crates/trusted-server-modules/src/lib.rs +++ b/crates/trusted-server-modules/src/lib.rs @@ -43,11 +43,27 @@ pub fn builders_with(extra: &[IntegrationBuilder]) -> Vec { all } +/// The section that selects the stock module with the integration id `id`, +/// and the name the module is written under there. +/// +/// A tool that knows an integration by its id, as the `ts` audit does from +/// what it finds on a page, writes a configuration with this and names no +/// module itself. +#[must_use] +pub fn selection_of(id: &str) -> Option<(&'static str, &'static str)> { + let builder = builders().into_iter().find(|builder| builder.id() == id)?; + let section = builder.section()?; + Some(( + section, + trusted_server_core::module_name::short_form(section, builder.module_name()?), + )) +} + #[cfg(test)] mod tests { use trusted_server_core::integrations::IntegrationBuilder; - use super::{builders, builders_with}; + use super::{builders, builders_with, selection_of}; #[test] fn stock_modules_are_offered_in_hook_order() { @@ -267,6 +283,35 @@ mod tests { ); } + #[test] + fn a_stock_module_s_selection_is_found_by_its_integration_id() { + for (id, section, written) in [ + ("datadome", "bot-protection", "datadome"), + ("didomi", "cmp", "didomi"), + ("sourcepoint", "cmp", "sourcepoint"), + ("osano", "cmp", "osano"), + ("lockr", "identity", "lockr"), + ("permutive", "audience", "permutive"), + ("nextjs", "framework", "nextjs"), + ("gpt", "ad-tag", "google"), + ("google_tag_manager", "tag", "google-tag-manager"), + // Testlight says `[auction]` selects it, where its full name is + // written. + ("testlight", "auction", "testing.testlight"), + ] { + assert_eq!( + selection_of(id), + Some((section, written)), + "should find where `{id}` is selected" + ); + } + assert_eq!( + selection_of("no-such-integration"), + None, + "should find nothing for an id no stock module has" + ); + } + /// The `ts` tool registers the stock builders, so the secret settings a /// stock module declares are among the leaves a push treats as naming a /// key. From d31e511f1a5a5fbc30c2daec119a90c6bc949d89 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 8 Oct 2026 03:27:39 +0100 Subject: [PATCH 100/124] A module registers and validates against the auction plan Prebid and APS each register a page integration that depends on the compiled auction plan, so the registry called both by name before it ran any builder, recorded Prebid's id by hand, and listed Prebid's module name among the ones a section may select. Deploy validation called Prebid's two checks by name for the same reason. Neither module can leave core while core does that. A builder now registers from the plan with `with_plan_registration` and checks its settings against the plan with `with_plan_validator`. A registration from the plan is made whether or not a section selects the module, because the function decides from the plan and the settings whether the module runs, and its hooks go ahead of every module a section selects, which is where Prebid's and APS's ran. Prebid is a builder like any other, selected in `[auction]`, and APS's implementation builder carries its registration. The registry and the deploy check name neither. The page path read the identifier a renderer picks its bid by with APS's type tag and APS's payload key. The code that builds a renderer descriptor now states which payload key holds that identifier, with `picking_bid_by`, and the page path asks the descriptor. A descriptor built without the statement has no such identifier whatever its payload holds, so a key of the same name under another renderer's tag is still not read, which a test pins. The statement is no part of what is sent to a page. --- CHANGELOG.md | 1 + .../trusted-server-core/src/auction/types.rs | 45 ++++++ crates/trusted-server-core/src/config.rs | 69 +++++--- .../src/integrations/aps.rs | 22 ++- .../src/integrations/mod.rs | 68 +++++++- .../src/integrations/prebid.rs | 43 +++++ .../src/integrations/registry.rs | 147 +++++++++++++++--- crates/trusted-server-core/src/publisher.rs | 21 +-- docs/guide/integration-guide.md | 26 ++-- 9 files changed, 364 insertions(+), 78 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a45f304b3c..f672962acd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,6 +64,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- A builder registers its module from the compiled auction plan with `with_plan_registration` and checks its settings against the plan with `with_plan_validator`, so a module whose page support follows what the plan selects is not named in `trusted-server-core`. Prebid and APS register this way. A renderer descriptor carries its builder's statement of which payload key holds the identifier the renderer picks its bid by, made with `BidRenderer::picking_bid_by`, and the page's `hb_adid` is read through that statement. - A module declares the settings in its own table that hold the name of a secret, with `with_secret_settings` on its builder, each with a rule for when the table puts it to use. A setting in use is looked up in the default secret store as the settings load and has to name a key for deploy validation to pass, and one not in use is cleared and never looked up. DataDome's server-side key and its test bypass credential are declared this way, and `trusted-server-core` no longer names them. - A module can act on one request from start to finish without `trusted-server-core` naming it. A request preparer or a request filter leaves a value in the request's `IntegrationRequestState`, which is copied into the document's state for the module's page hooks and handed to the response finalizer the builder declares with `with_response_finalizer`. A request that carries any value keeps to the origin path, so its document is never read from or stored as a shared template, and its HTML response is sent `private, no-store`. A head injector can also write straight after the script bundle with `after_bundle_inserts`, and a builder declares with `with_auction_token` that its browser script reads the token an auction publishes with its winning bids. The registry runs the request preparers once for a request. - Added the `[auction].rewrite_creatives` (default `true`) and `[auction].sanitize_creatives` (default `false`) options. `rewrite_creatives` rewrites winning-bid adm to first-party endpoints across `POST /auction` and publisher SSAT/page-bids delivery (proxy/click URL conversion, bidder `` removal; creative TSJS injection on `POST /auction` only). Enabling `sanitize_creatives` strips executable markup from winning-bid adm before delivery. diff --git a/crates/trusted-server-core/src/auction/types.rs b/crates/trusted-server-core/src/auction/types.rs index 6ad13f73c6..74e1f72e38 100644 --- a/crates/trusted-server-core/src/auction/types.rs +++ b/crates/trusted-server-core/src/auction/types.rs @@ -202,6 +202,11 @@ pub struct BidRenderer { renderer_type: String, #[serde(flatten)] payload: serde_json::Map, + /// The payload key the code that built the descriptor said holds the + /// identifier the renderer picks its bid by. A statement about the + /// payload and no part of what is sent. + #[serde(skip)] + bid_id_key: Option<&'static str>, } impl BidRenderer { @@ -243,6 +248,7 @@ impl BidRenderer { Ok(Self { renderer_type: renderer_type.to_string(), payload, + bid_id_key: None, }) } @@ -338,6 +344,45 @@ impl BidRenderer { } self.payload.get(key) } + + /// States that the payload key `key` holds the identifier this renderer + /// picks its bid by. + /// + /// The statement is the builder's own about its payload. A descriptor + /// built without it has no such identifier, whatever its payload holds, + /// so a key of the same name under another renderer's type tag is not + /// read. + #[must_use] + pub fn picking_bid_by(mut self, key: &'static str) -> Self { + self.bid_id_key = Some(key); + self + } + + /// The identifier the renderer picks its bid by, when the code that built + /// the descriptor said which payload key holds it. + /// + /// # Examples + /// + /// ``` + /// use serde_json::json; + /// use trusted_server_core::auction::types::BidRenderer; + /// + /// let payload = json!({ "bidId": "fictional-bid-id" }); + /// let unstated = BidRenderer::new("example", payload.clone()) + /// .expect("should accept an object payload"); + /// assert_eq!(unstated.bid_id(), None); + /// + /// let stated = BidRenderer::new("example", payload) + /// .expect("should accept an object payload") + /// .picking_bid_by("bidId"); + /// assert_eq!(stated.bid_id(), Some("fictional-bid-id")); + /// ``` + #[must_use] + pub fn bid_id(&self) -> Option<&str> { + self.payload + .get(self.bid_id_key?) + .and_then(serde_json::Value::as_str) + } } /// Individual bid from a provider. diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index 872ebf2cc8..92f2c33a47 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -17,7 +17,7 @@ use crate::ec::module::{HMAC_MODULE_KEY, HOST_SIGNALS_MODULE_KEY}; use crate::ec::registry::PartnerRegistry; use crate::error::TrustedServerError; -use crate::integrations::{IntegrationBuilder, prebid}; +use crate::integrations::IntegrationBuilder; use crate::settings::{AssetOriginAuth, Ec, MODULE_IMPLEMENTATION_KEY, Settings}; const DEPLOY_VALIDATION_FIELD: &str = "trusted_server"; @@ -413,14 +413,12 @@ pub fn validate_settings_for_runtime_with( Ok(()) } -/// Validates every integration block against the compiled auction plan. -/// -/// Prebid, APS and the ad server mock are auction plan providers rather than -/// builders, so they are checked here by name, and a Prebid browser bidder is -/// checked against the providers the plan carries. Every builder then -/// validates its own block, the built-in ones first and then +/// Validates every integration block, the built-in ones first and then /// `extra_integrations`. /// +/// Each builder validates its own block, and one with a rule that depends on +/// what the auction plan selects then checks its block against the plan. +/// /// # Errors /// /// Returns [`TrustedServerError`] when any integration block fails its @@ -430,25 +428,15 @@ fn validate_integration_blocks( plan: &crate::auction::AuctionPlan, extra_integrations: &[IntegrationBuilder], ) -> Result<(), Report> { - validate_prebid(settings, plan)?; for builder in crate::integrations::all_builders(extra_integrations) { builder.validate(settings)?; + if let Some(validate) = builder.plan_validator() { + validate(settings, plan)?; + } } Ok(()) } -fn validate_prebid( - settings: &Settings, - plan: &crate::auction::AuctionPlan, -) -> Result<(), Report> { - let Some(config) = settings.module_config::(prebid::MODULE)? - else { - return Ok(()); - }; - prebid::validate_browser_config_for_startup(&config, &settings.proxy.allowed_domains)?; - prebid::validate_browser_bidder_ownership(&config, plan) -} - fn validate_non_secret_deploy_placeholders( settings: &Settings, ) -> Result<(), Report> { @@ -1851,6 +1839,47 @@ password = "production-admin-password-32-bytes" } } + const PLAN_RULE_MESSAGE: &str = "probe module refuses an enabled auction"; + + fn refuse_an_enabled_auction( + _settings: &Settings, + plan: &crate::auction::AuctionPlan, + ) -> Result<(), Report> { + if plan.enabled() { + return Err(Report::new(TrustedServerError::Configuration { + message: PLAN_RULE_MESSAGE.to_owned(), + })); + } + Ok(()) + } + + #[test] + fn deploy_validation_runs_a_builder_s_check_against_the_auction_plan() { + let extra = [IntegrationBuilder::new( + "probe", + "example-crate", + crate::integrations::registry_test_support::probe_registration, + crate::integrations::registry_test_support::validate_nothing, + ) + .with_module_name("testing.probe") + .with_plan_validator(refuse_an_enabled_auction)]; + let mut settings = valid_settings(); + + settings.auction.enabled = false; + validate_settings_for_deploy_with(&settings, &extra) + .expect("should accept a plan the module's rule allows"); + + settings.auction.enabled = true; + let error = validate_settings_for_deploy_with(&settings, &extra) + .expect_err("should refuse a plan the module's rule does not allow"); + assert!( + error.to_string().contains(PLAN_RULE_MESSAGE), + "should keep the module's message intact: {error:?}" + ); + validate_settings_for_deploy(&settings) + .expect("should apply no such rule without the module's builder"); + } + #[test] fn deploy_validation_checks_no_secret_setting_without_the_module_s_builder() { let mut settings = valid_settings(); diff --git a/crates/trusted-server-core/src/integrations/aps.rs b/crates/trusted-server-core/src/integrations/aps.rs index 494a3d9791..368a75e3be 100644 --- a/crates/trusted-server-core/src/integrations/aps.rs +++ b/crates/trusted-server-core/src/integrations/aps.rs @@ -805,7 +805,9 @@ fn planned_aps_renderer( width: input.width, height: input.height, }; - BidRenderer::from_typed(APS_RENDERER_TYPE, &descriptor).ok() + BidRenderer::from_typed(APS_RENDERER_TYPE, &descriptor) + .ok() + .map(|renderer| renderer.picking_bid_by(APS_RENDERER_BID_ID_KEY)) } fn planned_aps_valid_creative_url(value: &str, publisher_domain: &str) -> bool { @@ -1522,7 +1524,7 @@ impl ApsAuctionProvider { height: input.height, }; match BidRenderer::from_typed(APS_RENDERER_TYPE, &descriptor) { - Ok(renderer) => Some(renderer), + Ok(renderer) => Some(renderer.picking_bid_by(APS_RENDERER_BID_ID_KEY)), Err(error) => { log::warn!( "Dropping APS bid '{}': its renderer descriptor could not be built: {error:?}", @@ -2079,6 +2081,7 @@ impl IntegrationHeadInjector for ApsRendererIntegration { /// Returns an error when two selected APS sources set different rendering /// modes. pub fn register_for_plan( + _settings: &Settings, plan: &crate::auction::AuctionPlan, ) -> Result, Report> { let mut selected: Option<(&str, ApsRenderingMode)> = None; @@ -3230,7 +3233,7 @@ mod tests { #[test] fn a_selected_aps_source_registers_the_trusted_server_renderer() { - let registration = register_for_plan(&aps_plan(&[None])) + let registration = register_for_plan(&create_test_settings(), &aps_plan(&[None])) .expect("should register APS renderer support") .expect("should return APS renderer registration"); @@ -3261,7 +3264,7 @@ mod tests { .expect("should compile an empty plan"); assert!( - register_for_plan(&plan) + register_for_plan(&create_test_settings(), &plan) .expect("should evaluate renderer registration") .is_none(), "a plan with no APS source should register no renderer" @@ -3270,9 +3273,12 @@ mod tests { #[test] fn publisher_native_rendering_drops_the_renderer_route_and_marks_the_bundle_tag() { - let registration = register_for_plan(&aps_plan(&[Some("publisher_native")])) - .expect("should register APS renderer support") - .expect("should return APS renderer registration"); + let registration = register_for_plan( + &create_test_settings(), + &aps_plan(&[Some("publisher_native")]), + ) + .expect("should register APS renderer support") + .expect("should return APS renderer registration"); assert!( registration.proxies.is_empty(), "publisher-native rendering should not register the static renderer" @@ -3307,7 +3313,7 @@ mod tests { #[test] fn two_aps_sources_that_disagree_on_rendering_are_refused() { let plan = aps_plan(&[None, Some("publisher_native")]); - let error = match register_for_plan(&plan) { + let error = match register_for_plan(&create_test_settings(), &plan) { Ok(_) => panic!("should refuse two rendering modes"), Err(error) => error, }; diff --git a/crates/trusted-server-core/src/integrations/mod.rs b/crates/trusted-server-core/src/integrations/mod.rs index a2c51348cb..5a5ecd684e 100644 --- a/crates/trusted-server-core/src/integrations/mod.rs +++ b/crates/trusted-server-core/src/integrations/mod.rs @@ -8,6 +8,7 @@ use futures::StreamExt as _; use http::{Request, Response}; use url::Url; +use crate::auction::AuctionPlan; use crate::auction::demand::{AdServerImplementation, DemandImplementation}; use crate::error::TrustedServerError; use crate::platform::{DEFAULT_FIRST_BYTE_TIMEOUT, PlatformBackendSpec, RuntimeServices}; @@ -335,6 +336,26 @@ pub type IntegrationPrepareRequestFn = /// does, and has nothing to do for a request its module left nothing on. pub type IntegrationFinalizeResponseFn = fn(&IntegrationRequestState, &mut Response); +/// Builds a module's registration from the settings and the compiled auction +/// plan, or `None` when the two give it nothing to register. +/// +/// Runs for every builder that has one, whether or not a section selects the +/// module, because a module registered this way follows what the plan +/// selects and decides for itself whether it runs. +pub type IntegrationPlanRegistrationFn = + fn( + &Settings, + &AuctionPlan, + ) -> Result, Report>; + +/// Checks a module's configuration against the compiled auction plan, for a +/// rule that depends on what the plan selects. +/// +/// Runs for every builder that has one, when a deployment is validated and +/// as the settings load. +pub type IntegrationPlanValidateFn = + fn(&Settings, &AuctionPlan) -> Result<(), Report>; + /// A setting in a module's own table that holds the name of a secret. /// /// The name is looked up in the default secret store as the settings load, @@ -400,6 +421,8 @@ pub struct IntegrationBuilder { validate: IntegrationValidateFn, prepare_request: Option, finalize_response: Option, + plan_registration: Option, + plan_validator: Option, reads_auction_token: bool, secret_settings: &'static [ModuleSecretSetting], supplies_integration: bool, @@ -438,6 +461,8 @@ impl IntegrationBuilder { validate, prepare_request: None, finalize_response: None, + plan_registration: None, + plan_validator: None, reads_auction_token: false, secret_settings: &[], supplies_integration: true, @@ -463,6 +488,8 @@ impl IntegrationBuilder { validate: nothing_to_validate, prepare_request: None, finalize_response: None, + plan_registration: None, + plan_validator: None, reads_auction_token: false, secret_settings: &[], supplies_integration: false, @@ -529,6 +556,27 @@ impl IntegrationBuilder { self } + /// Registers the module from the compiled auction plan as well as the + /// settings, for a module whose page support follows what the plan + /// selects. + /// + /// The function runs whether or not a section selects the module, and + /// what it registers has its hooks run ahead of the modules the sections + /// select. + #[must_use] + pub const fn with_plan_registration(mut self, register: IntegrationPlanRegistrationFn) -> Self { + self.plan_registration = Some(register); + self + } + + /// Attaches a check of the module's configuration against the compiled + /// auction plan. + #[must_use] + pub const fn with_plan_validator(mut self, validate: IntegrationPlanValidateFn) -> Self { + self.plan_validator = Some(validate); + self + } + /// Declares that the module's browser script reads the token an auction /// publishes with its winning bids. /// @@ -642,12 +690,25 @@ impl IntegrationBuilder { pub(crate) fn finalize_response(&self) -> Option { self.finalize_response } + + /// The registration from the auction plan, when one is attached. + pub(crate) fn plan_registration(&self) -> Option { + self.plan_registration + } + + /// The check against the auction plan, when one is attached. + pub(crate) fn plan_validator(&self) -> Option { + self.plan_validator + } } /// The built-in integrations, in hook order. const BUILT_IN_BUILDERS: &[IntegrationBuilder] = &[ - // This must remain first: attribute rewriters chain replacements and - // short-circuit removals. + // Prebid is registered from the auction plan, which puts its hooks ahead + // of every section's module whatever its place here. + prebid::BUILDER, + // This must remain the first module a section selects: attribute + // rewriters chain replacements and short-circuit removals. js_asset_proxy::BUILDER, // A stand-in for an integration that streams, which core's own tests // select where they need one. @@ -666,7 +727,8 @@ const BUILT_IN_BUILDERS: &[IntegrationBuilder] = &[ IntegrationBuilder::implementations(prebid_server::MODULE, CORE_SOURCE) .with_demand(&prebid_server::DEMAND), IntegrationBuilder::implementations(aps::APS_INTEGRATION_ID, CORE_SOURCE) - .with_demand(&aps::DEMAND), + .with_demand(&aps::DEMAND) + .with_plan_registration(aps::register_for_plan), IntegrationBuilder::implementations(adserver_mock::MODULE, CORE_SOURCE) .with_adserver(&adserver_mock::ADSERVER), ]; diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index cc7bc4ef2d..c4606092bb 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -67,6 +67,49 @@ pub(crate) const PREBID_INTEGRATION_ID: &str = "prebid"; /// The name this module is selected by, in `[auction]`. pub const MODULE: &str = "auction.prebid"; + +/// The builder that offers Prebid: selected in `[auction]`, registered from +/// the auction plan and checked against it. +pub(crate) const BUILDER: crate::integrations::IntegrationBuilder = + crate::integrations::IntegrationBuilder::new( + PREBID_INTEGRATION_ID, + crate::integrations::CORE_SOURCE, + registered_from_the_plan, + validate, + ) + .with_module_name(MODULE) + .with_plan_registration(register_for_plan) + .with_plan_validator(validate_against_plan); + +/// Prebid's registration needs the auction plan, so [`register_for_plan`] +/// makes it and the settings alone add nothing. +fn registered_from_the_plan( + _settings: &Settings, +) -> Result, Report> { + Ok(None) +} + +/// Validates the browser configuration for deployment and reports whether +/// `[auction]` selects Prebid. +fn validate(settings: &Settings) -> Result> { + let Some(config) = settings.module_config::(MODULE)? else { + return Ok(false); + }; + validate_browser_config_for_startup(&config, &settings.proxy.allowed_domains)?; + Ok(true) +} + +/// Refuses a bidder the browser configuration keeps client-side that the +/// plan also runs server-side. +fn validate_against_plan( + settings: &Settings, + plan: &AuctionPlan, +) -> Result<(), Report> { + let Some(config) = settings.module_config::(MODULE)? else { + return Ok(()); + }; + validate_browser_bidder_ownership(&config, plan) +} const PREBID_BUNDLE_ROUTE: &str = "/integrations/prebid/bundle.js"; const PREBID_BUNDLE_CONTENT_TYPE: &str = "application/javascript; charset=utf-8"; const PREBID_BUNDLE_IMMUTABLE_CACHE_CONTROL: &str = "public, max-age=31536000, immutable"; diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index fd41285e9a..3403301fb0 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -1373,11 +1373,8 @@ fn check_section_selections( settings: &Settings, extra: &[crate::integrations::IntegrationBuilder], ) -> Result<(), Report> { - // Prebid registers through the auction plan rather than a builder, and is - // selected in `[auction]` all the same. let offered: Vec<&'static str> = crate::integrations::all_builders(extra) .filter_map(|builder| builder.module_name()) - .chain([crate::integrations::prebid::MODULE]) .collect(); for (section, modules) in settings.module_sections() { for written in modules.selected() { @@ -1494,24 +1491,10 @@ impl IntegrationRegistry { extra: &[crate::integrations::IntegrationBuilder], ) -> Result> { let mut inner = IntegrationRegistryInner::default(); - // Prebid registers through the auction plan rather than through a - // builder, but its id is core's all the same. Recording it with the - // builders refuses an outside builder that claims it, and lets a - // selector naming it while it does not run report it as registered but - // not running. - inner.builder_ids.push(( - crate::integrations::prebid::PREBID_INTEGRATION_ID, - crate::integrations::CORE_SOURCE, - )); - // The plan-backed auction providers register first, so opening the - // builder table leaves every existing hook order unchanged. - let mut registrations: Vec = [ - crate::integrations::prebid::register_for_plan(settings, &plan)?, - crate::integrations::aps::register_for_plan(&plan)?, - ] - .into_iter() - .flatten() - .collect(); + // What a builder registers from the auction plan goes first, so its + // hooks run ahead of every module a section selects. + let mut plan_registrations: Vec = Vec::new(); + let mut registrations: Vec = Vec::new(); for builder in crate::integrations::all_builders(extra) { if let Some((_, first_source)) = @@ -1535,6 +1518,19 @@ impl IntegrationRegistry { if let Some(finalize) = builder.finalize_response() { inner.response_finalizers.push(finalize); } + // A registration from the plan is made whatever the sections + // select, because the function decides from the plan and the + // settings whether the module runs. + if let Some(register) = builder.plan_registration() + && let Some(registration) = register(settings, &plan)? + { + debug_assert_eq!( + registration.integration_id, + builder.id(), + "integration builder ID should match registration ID" + ); + plan_registrations.push(registration); + } // Only a builder whose module a section selects is built, so an // integration runs exactly when an operator selects it, whatever @@ -1569,7 +1565,7 @@ impl IntegrationRegistry { // The geo, Edge Cookie and device module names taken so far, with the // integration that supplies each. let mut claimed = Vec::new(); - for registration in registrations { + for registration in plan_registrations.into_iter().chain(registrations) { inner .running_integration_ids .push(registration.integration_id); @@ -4763,6 +4759,113 @@ mod tests { ); } + /// Writes one fixed insert, so a test can read the order hooks ran in. + struct FixedHeadInsert { + id: &'static str, + insert: &'static str, + } + + impl IntegrationHeadInjector for FixedHeadInsert { + fn integration_id(&self) -> &'static str { + self.id + } + + fn head_inserts(&self, _ctx: &IntegrationHtmlContext<'_>) -> Vec { + vec![self.insert.to_owned()] + } + } + + const SECTION_INSERT: &str = ""; + const PLAN_INSERT: &str = ""; + + fn section_probe_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("probe-section") + .with_head_injector(Arc::new(FixedHeadInsert { + id: "probe-section", + insert: SECTION_INSERT, + })) + .build(), + )) + } + + /// Registers when the plan's auction is enabled, whatever the sections + /// select. + fn plan_probe_registration( + _settings: &Settings, + plan: &AuctionPlan, + ) -> Result, Report> { + Ok(plan.enabled().then(|| { + IntegrationRegistration::builder("probe-plan") + .with_head_injector(Arc::new(FixedHeadInsert { + id: "probe-plan", + insert: PLAN_INSERT, + })) + .build() + })) + } + + #[test] + fn a_registration_from_the_plan_runs_unselected_and_ahead_of_a_section_s_module() { + let mut settings = settings_naming("testing.probe-section"); + // The section's module is listed first, and what the plan registers + // still goes ahead of it. + let extra = [ + crate::integrations::IntegrationBuilder::new( + "probe-section", + "seam-probe", + section_probe_registration, + validate_nothing, + ) + .with_module_name("testing.probe-section"), + crate::integrations::IntegrationBuilder::new( + "probe-plan", + "seam-probe", + never_registering_builder, + validate_nothing, + ) + .with_module_name("testing.probe-plan") + .with_plan_registration(plan_probe_registration), + ]; + let probe_inserts = |registry: &IntegrationRegistry| { + let document_state = IntegrationDocumentState::default(); + registry + .head_inserts(&IntegrationHtmlContext { + request_host: "publisher.example.com", + request_scheme: "https", + origin_host: "origin.example.com", + document_state: &document_state, + }) + .into_iter() + .filter(|insert| insert == SECTION_INSERT || insert == PLAN_INSERT) + .collect::>() + }; + + settings.auction.enabled = false; + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should build registry"); + assert!( + !registry.integration_runs("probe-plan"), + "should register nothing where the function finds nothing in the plan" + ); + assert_eq!(probe_inserts(®istry), vec![SECTION_INSERT]); + + settings.auction.enabled = true; + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should build registry"); + assert!( + registry.integration_runs("probe-plan"), + "should register from the plan a module no section selects" + ); + assert_eq!( + probe_inserts(®istry), + vec![PLAN_INSERT, SECTION_INSERT], + "should run the hooks of what the plan registered ahead of a section's module" + ); + } + /// Appends to a header on each run, so a test can count the runs on the /// request itself. fn counting_preparer( diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 132943671e..e7c4d66fd4 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -49,7 +49,7 @@ use crate::auction::telemetry::{ emit_auction_events_best_effort_lazy, }; use crate::auction::types::{ - AuctionContext, AuctionRequest, Bid, DeviceInfo, PublisherInfo, SiteInfo, UserInfo, + AuctionContext, AuctionRequest, Bid, BidRenderer, DeviceInfo, PublisherInfo, SiteInfo, UserInfo, }; use crate::cache_policy::{ CachePolicy, EdgeCacheHeader, cache_control_headers_are_private_or_no_store, @@ -68,7 +68,6 @@ use crate::ec::{EcContext, EidSyncSource}; use crate::error::TrustedServerError; use crate::html_processor::BodyCloseInjection; use crate::http_util::{RequestInfo, is_navigation_request, serve_static_with_etag}; -use crate::integrations::aps::{APS_RENDERER_BID_ID_KEY, APS_RENDERER_TYPE}; use crate::integrations::{IntegrationRegistry, IntegrationRequestState}; use crate::permissions::PermissionState; use crate::platform::{ @@ -5854,13 +5853,7 @@ pub(crate) fn build_bid_map_with_auction_id( // whole descriptor, which would clone a payload carrying a // base64 encoding of a creative envelope of up to 256 KB, once // per bid per page view. - let renderer_bid_id = bid - .renderer - .as_ref() - .and_then(|renderer| { - renderer.payload_field(APS_RENDERER_TYPE, APS_RENDERER_BID_ID_KEY) - }) - .and_then(serde_json::Value::as_str); + let renderer_bid_id = bid.renderer.as_ref().and_then(BidRenderer::bid_id); let hb_adid = non_empty(bid.cache_id.as_deref()) .or_else(|| non_empty(renderer_bid_id)) .or_else(|| non_empty(bid.ad_id.as_deref())) @@ -23890,7 +23883,8 @@ mod tests { height: 250, }, ) - .expect("should build APS renderer descriptor"), + .expect("should build APS renderer descriptor") + .picking_bid_by(crate::integrations::aps::APS_RENDERER_BID_ID_KEY), ); let winning_bids = HashMap::from([("atf_sidebar_ad".to_string(), bid)]); @@ -23937,7 +23931,8 @@ mod tests { height: 250, }, ) - .expect("should build APS renderer descriptor"), + .expect("should build APS renderer descriptor") + .picking_bid_by(crate::integrations::aps::APS_RENDERER_BID_ID_KEY), ); let winning_bids = HashMap::from([("atf_sidebar_ad".to_string(), bid)]); @@ -23953,7 +23948,7 @@ mod tests { } #[test] - fn bid_map_ignores_a_renderer_bid_id_carried_under_another_type_tag() { + fn bid_map_ignores_a_renderer_bid_id_its_builder_did_not_state() { let mut settings = test_settings(); settings.auction.sanitize_creatives = true; let mut bid = make_bid("atf_sidebar_ad", 1.50, "example", "ad-id-value", "", ""); @@ -23971,7 +23966,7 @@ mod tests { assert_eq!( obj["hb_adid"], "ad-id-value", - "should ignore a bidId carried under a tag that is not the APS renderer" + "should ignore a bidId the code that built the renderer did not state it picks its bid by" ); } diff --git a/docs/guide/integration-guide.md b/docs/guide/integration-guide.md index 6ca826f11b..c677d1c951 100644 --- a/docs/guide/integration-guide.md +++ b/docs/guide/integration-guide.md @@ -232,18 +232,20 @@ through the Axum adapter, and the Fastly adapter carries the same round trip. The crate hands out an `IntegrationBuilder`, which names the integration and points at the functions that do the work. -| Part | Purpose | -| --------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Id | Names the integration in its routes under `/integrations//`, in its browser module and in diagnostics | -| Source | The crate or package name, reported when two builders claim one id, so an operator can tell which crates collided | -| Build function | Reads `Settings` and returns the registration. It is called only when a section selects the module | -| Validate function | The integration's own deploy rules. It runs for every builder, selected or not | -| `.with_module_name(".")` | The name a section selects the module by, which is the crate's path under `crates/` with `.` between the parts. `module_name!()` derives it from the crate's folder | -| `.with_request_preparer(...)` | Optional. Runs before routing, once for a request, whether or not the module is selected | -| `.with_response_finalizer(...)` | Optional. Finishes the response the page path returns, from what the module's request hooks left for the request | -| `.with_auction_token()` | Optional. Declares that the module's browser script reads the token an auction publishes with its winning bids, which is made only when a selected module reads one | -| `.with_secret_settings(...)` | Optional. Lists the settings in the module's own table that hold the name of a secret, so each is looked up as the settings load | -| `.with_demand(...)` and `.with_adserver(...)` | Optional. Registers an auction implementation that `[demand]` or `[ad-server]` can name. A crate that supplies nothing else starts from `IntegrationBuilder::implementations(...)` | +| Part | Purpose | +| --------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Id | Names the integration in its routes under `/integrations//`, in its browser module and in diagnostics | +| Source | The crate or package name, reported when two builders claim one id, so an operator can tell which crates collided | +| Build function | Reads `Settings` and returns the registration. It is called only when a section selects the module | +| Validate function | The integration's own deploy rules. It runs for every builder, selected or not | +| `.with_module_name(".")` | The name a section selects the module by, which is the crate's path under `crates/` with `.` between the parts. `module_name!()` derives it from the crate's folder | +| `.with_request_preparer(...)` | Optional. Runs before routing, once for a request, whether or not the module is selected | +| `.with_response_finalizer(...)` | Optional. Finishes the response the page path returns, from what the module's request hooks left for the request | +| `.with_auction_token()` | Optional. Declares that the module's browser script reads the token an auction publishes with its winning bids, which is made only when a selected module reads one | +| `.with_secret_settings(...)` | Optional. Lists the settings in the module's own table that hold the name of a secret, so each is looked up as the settings load | +| `.with_plan_registration(...)` | Optional. Registers the module from the compiled auction plan, for a module whose page support follows what the plan selects. It runs whether or not a section selects the module, and its hooks go first | +| `.with_plan_validator(...)` | Optional. Checks the module's settings against the compiled auction plan, when a deployment is validated and as the settings load | +| `.with_demand(...)` and `.with_adserver(...)` | Optional. Registers an auction implementation that `[demand]` or `[ad-server]` can name. A crate that supplies nothing else starts from `IntegrationBuilder::implementations(...)` | ```rust pub fn module_name() -> &'static str { From 549cbb7dc657388b0226a7d7d4be961b241877ae Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 8 Oct 2026 03:56:02 +0100 Subject: [PATCH 101/124] The mock ad server moves out of core The demonstration ad server is the implementation `ad-server.mock`, and it now lives in `crates/ad-server/mock`. The stock list offers it, so `[ad-server] module = "mock"` selects it as before and its settings do not change. Its 22 tests went with it. Core's tests of the ad server seam selected the mock because it was the one ad server there was. They select a stand-in compiled for core's tests only, `ad-server.fixture`, which reads an endpoint and a timeout, refuses any other setting and answers every request with no bid. That covers the plan compiler's tests of how a name resolves within `[ad-server]` and how a table's settings are checked, the registry's refusal of an auction implementation named in a section, and three tests that built a mock provider only because they needed a provider. One orchestrator test ran APS through the mock ad server and checked that the winner kept its seat, its bidder name and its renderer. It is a test of the two together, so it is in the stock list crate, where both are on offer, and it drives the orchestrator through its public interface. The bid dimension helpers the mock ad server admits a bid with are public, so an auction implementation in a crate of its own applies the rules core's response reader applies. --- .cargo/config.toml | 12 +- CHANGELOG.md | 2 +- Cargo.lock | 18 +++ Cargo.toml | 2 + crates/ad-server/mock/Cargo.toml | 33 ++++ .../mock/src/lib.rs} | 54 +++++-- crates/trusted-server-core/src/auction/mod.rs | 14 +- .../src/auction/openrtb.rs | 32 +++- .../src/auction/orchestrator.rs | 88 +---------- .../trusted-server-core/src/auction/plan.rs | 22 +-- .../src/auction/test_support.rs | 107 +++++++++++++ crates/trusted-server-core/src/config.rs | 8 +- .../trusted-server-core/src/config_payload.rs | 8 +- .../src/integrations/mod.rs | 11 +- .../src/integrations/registry.rs | 2 +- crates/trusted-server-modules/Cargo.toml | 2 + crates/trusted-server-modules/src/lib.rs | 141 +++++++++++++++++- 17 files changed, 410 insertions(+), 146 deletions(-) create mode 100644 crates/ad-server/mock/Cargo.toml rename crates/{trusted-server-core/src/integrations/adserver_mock.rs => ad-server/mock/src/lib.rs} (97%) diff --git a/.cargo/config.toml b/.cargo/config.toml index a28b490a32..bce0907ddf 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -26,10 +26,10 @@ test_details = "test --target aarch64-apple-darwin" # native crate needs no change here. Axum (native), Cloudflare # (wasm32-unknown-unknown), Spin, the CLI (native), and integration-tests # (native) are simply not listed. -build-fastly = "build -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" -check-fastly = "check -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" -clippy-fastly = "clippy -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-js -p trusted-server-openrtb --all-targets --all-features --target wasm32-wasip1 -- -D warnings" -test-fastly = "test -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" +build-fastly = "build -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" +check-fastly = "check -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" +clippy-fastly = "clippy -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-js -p trusted-server-openrtb --all-targets --all-features --target wasm32-wasip1 -- -D warnings" +test-fastly = "test -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" # Feature-on counterpart of `test-fastly`. `test-fastly` does NOT pass # --all-features, so without this the reusable-sandbox code is linted by # `clippy-fastly` but its tests never execute. @@ -40,8 +40,8 @@ test-fastly-reuse = "test -p trusted-server-adapter-fastly --target wasm32-wasip # run with the Axum adapter in CI. build-axum = "build -p trusted-server-adapter-axum" check-axum = "check -p trusted-server-adapter-axum" -clippy-axum = "clippy -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome --all-targets --all-features -- -D warnings" -test-axum = "test -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome" +clippy-axum = "clippy -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock --all-targets --all-features -- -D warnings" +test-axum = "test -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock" # --- Cloudflare adapter (native host + wasm32-unknown-unknown) --- # Build/check target the WASM runtime (requires the `cloudflare` feature); diff --git a/CHANGELOG.md b/CHANGELOG.md index f672962acd..d1876e157e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - A pushed configuration keeps DataDome's two deprecated secret store selectors, `server_side_key_secret_store` and `protection_test_bypass.credential_secret_store`, where they were removed as the settings were read. DataDome still accepts and discards both, so a configuration that carries one loads as before. -- Integration modules are moving out of `trusted-server-core` into crates of their own under `crates///`, each named by that folder, so `crates/cmp/osano` is the module `cmp.osano`. The settings that select and configure a module do not change. `crates/trusted-server-modules` lists the modules a stock build ships from crates of their own, in the order their hooks run. Every adapter builds its state and loads its settings with that list, and `ts config validate` and `ts config push` run each stock module's own rules. Code that named a moved module under `trusted_server_core::integrations` names its crate instead, such as `trusted_server_cmp_osano`. Moved so far: Osano, Lockr, Permutive, Sourcepoint, Didomi, Google Tag Manager, Testlight, Next.js, Google Publisher Tags and its diagnostics, and DataDome. +- Integration modules are moving out of `trusted-server-core` into crates of their own under `crates///`, each named by that folder, so `crates/cmp/osano` is the module `cmp.osano`. The settings that select and configure a module do not change. `crates/trusted-server-modules` lists the modules a stock build ships from crates of their own, in the order their hooks run. Every adapter builds its state and loads its settings with that list, and `ts config validate` and `ts config push` run each stock module's own rules. Code that named a moved module under `trusted_server_core::integrations` names its crate instead, such as `trusted_server_cmp_osano`. Moved so far: Osano, Lockr, Permutive, Sourcepoint, Didomi, Google Tag Manager, Testlight, Next.js, Google Publisher Tags and its diagnostics, DataDome and the mock ad server. - The Fastly adapter is a library with a thin binary over it. A deployment that ships a vendor crate writes a binary of its own that calls `trusted_server_adapter_fastly::run_with(vec![...])` with that crate's integration builder, in place of editing the adapter. The builders are composed with the built-in ones, and a builder runs only when the settings select its module. The Wasm artifact keeps its name, `trusted-server-adapter-fastly.wasm`. - **Breaking:** `[integrations]` is gone. Each page integration is a module selected in the section of its type, with its settings in the table at its name: `[cmp] module = "didomi"` (or `"sourcepoint"` or `"osano"`), `[tag] modules = ["google-tag-manager"]`, `[ad-tag] modules = ["google", "google.diagnostics"]`, `[bot-protection] module = "datadome"`, `[identity] module = "lockr"`, `[audience] module = "permutive"`, `[framework] module = "nextjs"`, `[auction] modules = ["prebid"]` (with `"testing.testlight"` for the test module) and `[proxy] modules = ["js_asset_proxy"]`, so `[integrations.gpt]` is `[ad-tag.google]` and `[integrations.prebid]` is `[auction.prebid]`. A module's name is `.`, which is its crate's path under `crates/` once it has a crate of its own and a constant in core until then, and it is written in its section with the type left off. A section that selects nothing, a table its section does not select, and a name no module in the deployment supplies are each refused at startup. An `[integrations]` or `[integration]` table is refused with directions. Environment overlays name a section as written, so `TRUSTED_SERVER__INTEGRATIONS__GPT__GAM_ATTRIBUTION_ENABLED` is `TRUSTED_SERVER__AD-TAG__GOOGLE__GAM_ATTRIBUTION_ENABLED`. - **Breaking:** Every pluggable component now shares one configuration syntax. Each selectable type is a top-level table named for the job, a `module` key inside it selects what runs, or `modules` for a type that runs several, and a `[.]` table holds that name's settings when it has any. The types are `ec`, `geo`, `device`, `permission-signal`, `demand` and `ad-server`, and the section of each page integration type, such as `cmp` or `tag`. A module from a crate is named by its folder below `crates/`, such as `permission-signal.gpc`, and within its own table the type folder may be left off. A `demand` or `ad-server` name is snake_case, because it may be a label of your own. A settings table its type's selector does not name refuses startup, so a block left behind after a module is switched off is caught rather than sitting unread, and every module rejects settings it does not know, so a misspelled key fails rather than being ignored. A table's name is the implementation unless the table carries an `implementation` line, and a demand table always carries one, naming the implementation by its module path (`auction-protocol.openrtb`, `auction.prebid-server` or `auction.aps`), which is how two Prebid Servers run side by side under names of their own. The ad server implementation is `ad-server.mock`, written `mock` in `[ad-server]`. A `demand` or `ad-server` endpoint must be HTTPS, or HTTP to `127.0.0.1`, `::1` or `localhost`. The word mediator is gone. It is "ad server" in prose and `ad-server` in configuration, `[debug.auction_html_comment_options] include_mediator_response` is now `include_adserver_response`, and the auction response metadata value `parallel_mediation` is now `parallel_adserver`. `auction-protocol.openrtb`, `auction.prebid-server`, `auction.aps` and `ad-server.mock` supply implementations only, so a page integration's section that names one is refused. The full rules and what is checked at each gate are in [Configuration Rules](docs/guide/configuration-rules.md). Migrate as follows: diff --git a/Cargo.lock b/Cargo.lock index 399742e5eb..e98a47e605 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5378,6 +5378,22 @@ dependencies = [ "once_cell", ] +[[package]] +name = "trusted-server-ad-server-mock" +version = "0.1.0" +dependencies = [ + "async-trait", + "edgezero-core", + "error-stack", + "http", + "log", + "serde", + "serde_json", + "trusted-server-core", + "url", + "validator", +] + [[package]] name = "trusted-server-ad-tag-google" version = "0.1.0" @@ -5812,6 +5828,8 @@ dependencies = [ "edgezero-core", "http", "serde_json", + "tokio", + "trusted-server-ad-server-mock", "trusted-server-ad-tag-google", "trusted-server-audience-permutive", "trusted-server-bot-protection-datadome", diff --git a/Cargo.toml b/Cargo.toml index ed99f7be99..0e4a19fb62 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ members = [ "crates/framework/nextjs", "crates/ad-tag/google", "crates/bot-protection/datadome", + "crates/ad-server/mock", "crates/trusted-server-adapter-axum", "crates/trusted-server-adapter-cloudflare", "crates/trusted-server-adapter-fastly", @@ -154,6 +155,7 @@ trusted-server-testing-testlight = { path = "crates/testing/testlight" } trusted-server-framework-nextjs = { path = "crates/framework/nextjs" } trusted-server-ad-tag-google = { path = "crates/ad-tag/google" } trusted-server-bot-protection-datadome = { path = "crates/bot-protection/datadome" } +trusted-server-ad-server-mock = { path = "crates/ad-server/mock" } trusted-server-js = { path = "crates/trusted-server-js" } trusted-server-modules = { path = "crates/trusted-server-modules" } trusted-server-openrtb = { path = "crates/trusted-server-openrtb" } diff --git a/crates/ad-server/mock/Cargo.toml b/crates/ad-server/mock/Cargo.toml new file mode 100644 index 0000000000..99076d0eec --- /dev/null +++ b/crates/ad-server/mock/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "trusted-server-ad-server-mock" +description = "The demonstration ad server, which asks mocktioneer's decision endpoint to pick the winning bid, as an ad server module." +authors = { workspace = true } +edition = { workspace = true } +license = { workspace = true } +publish = { workspace = true } +version = { workspace = true } + +[lib] +doctest = false + +[lints] +workspace = true + +[dependencies] +async-trait = { workspace = true } +edgezero-core = { workspace = true } +error-stack = { workspace = true } +http = { workspace = true } +log = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +trusted-server-core = { workspace = true } +url = { workspace = true } +validator = { workspace = true } + +# The visible owner of this vendor crate, the way Prebid.js requires a named +# maintainer of every adapter. No vendor has not yet adopted this crate, so the +# Trusted Server maintainers own it until a vendor owner steps up. +[package.metadata.maintainers] +owner = "Trusted Server maintainers" +status = "seeking vendor owner" diff --git a/crates/trusted-server-core/src/integrations/adserver_mock.rs b/crates/ad-server/mock/src/lib.rs similarity index 97% rename from crates/trusted-server-core/src/integrations/adserver_mock.rs rename to crates/ad-server/mock/src/lib.rs index ee7b48ccc4..0edb977eb2 100644 --- a/crates/trusted-server-core/src/integrations/adserver_mock.rs +++ b/crates/ad-server/mock/src/lib.rs @@ -4,6 +4,18 @@ //! price, so a deployment can exercise the ad server seam without a real ad //! server. +#![cfg_attr( + test, + allow( + clippy::print_stdout, + clippy::print_stderr, + clippy::panic, + clippy::dbg_macro, + clippy::unwrap_used, + reason = "tests use direct diagnostics and panic-on-failure helpers" + ) +)] + use async_trait::async_trait; use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt}; @@ -15,22 +27,22 @@ use std::sync::Arc; use std::time::Duration; use validator::Validate; -use crate::auction::context::{ContextQueryParams, build_url_with_context_params}; -use crate::auction::demand::AdServerImplementation; -use crate::auction::openrtb::{ +use trusted_server_core::auction::context::{ContextQueryParams, build_url_with_context_params}; +use trusted_server_core::auction::demand::AdServerImplementation; +use trusted_server_core::auction::openrtb::{ BidDimensionIndex, BidRejectionReason, build_bid_dimension_index_from_slots, parse_optional_bid_dimension, resolve_bid_dimensions, }; -use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; -use crate::auction::types::{ +use trusted_server_core::auction::provider::{AuctionProvider, ProviderRequestOutcome}; +use trusted_server_core::auction::types::{ AuctionContext, AuctionRequest, AuctionResponse, Bid, BidStatus, MediaType, }; -use crate::error::TrustedServerError; -use crate::integrations::{ +use trusted_server_core::error::TrustedServerError; +use trusted_server_core::integrations::{ UPSTREAM_RTB_MAX_RESPONSE_BYTES, collect_response_bounded, ensure_integration_backend_with_timeout, predict_integration_backend_name, }; -use crate::platform::{PlatformHttpRequest, PlatformResponse, RuntimeServices}; +use trusted_server_core::platform::{PlatformHttpRequest, PlatformResponse, RuntimeServices}; // ============================================================================ // Configuration @@ -43,6 +55,17 @@ pub const MODULE: &str = "ad-server.mock"; /// The demonstration ad server implementation. pub static ADSERVER: AdServerImplementation = AdServerImplementation { id: MODULE, build }; +/// The builder a deployment hands to an adapter, which offers this ad server +/// to `[ad-server]`. +#[must_use] +pub fn builder() -> trusted_server_core::integrations::IntegrationBuilder { + trusted_server_core::integrations::IntegrationBuilder::implementations( + MODULE, + env!("CARGO_PKG_NAME"), + ) + .with_adserver(&ADSERVER) +} + fn build( name: &str, settings: &serde_json::Map, @@ -609,8 +632,8 @@ impl AuctionProvider for AdServerMockProvider { #[cfg(test)] mod tests { use super::*; - use crate::auction::context::ContextValue; - use crate::auction::types::*; + use trusted_server_core::auction::context::ContextValue; + use trusted_server_core::auction::types::*; fn create_test_auction_request() -> AuctionRequest { AuctionRequest { @@ -1245,7 +1268,7 @@ mod tests { #[test] fn test_adserver_request_includes_consent() { - use crate::consent::ConsentContext; + use trusted_server_core::consent::ConsentContext; let config = AdServerMockSettings { endpoint: "http://localhost:6767/mediate".to_string(), @@ -1803,4 +1826,13 @@ mod tests { "http://localhost:6767/adserver/mediate?debug=true&permutive=123%2Cadv" ); } + + #[test] + fn module_constant_is_the_crate_folder() { + assert_eq!( + super::MODULE, + trusted_server_core::module_name!(), + "should be named by the folder this crate lives in" + ); + } } diff --git a/crates/trusted-server-core/src/auction/mod.rs b/crates/trusted-server-core/src/auction/mod.rs index 5e0f73c002..7d792deeda 100644 --- a/crates/trusted-server-core/src/auction/mod.rs +++ b/crates/trusted-server-core/src/auction/mod.rs @@ -17,7 +17,7 @@ pub mod context; pub mod demand; pub mod endpoints; pub mod formats; -pub(crate) mod openrtb; +pub mod openrtb; pub mod orchestrator; pub mod plan; pub mod provider; @@ -257,13 +257,7 @@ mod plan_sharing_tests { _settings: &Map, ) -> Result, Report> { Ok(Arc::new( - crate::integrations::adserver_mock::AdServerMockProvider::new( - name, - crate::integrations::adserver_mock::AdServerMockSettings { - endpoint: "https://external.example/mediate".to_string(), - ..Default::default() - }, - ), + crate::auction::test_support::adserver_fixture::FixtureAdServer::new(name, 500), )) } @@ -301,7 +295,7 @@ mod plan_sharing_tests { fn the_selected_ad_server_is_built_from_its_own_table() { let mut settings = create_test_settings(); settings.adserver = adserver( - "mock", + "fixture", Map::from_iter([( "endpoint".to_string(), json!("https://adserver.example/mediate"), @@ -315,7 +309,7 @@ mod plan_sharing_tests { #[test] fn an_ad_server_table_with_no_endpoint_fails_the_plan() { let mut settings = create_test_settings(); - settings.adserver = adserver("mock", Map::new()); + settings.adserver = adserver("fixture", Map::new()); let error = compile_auction_plan(&settings) .expect_err("should refuse an ad server with no endpoint"); assert!( diff --git a/crates/trusted-server-core/src/auction/openrtb.rs b/crates/trusted-server-core/src/auction/openrtb.rs index 9be112251c..555e6912dc 100644 --- a/crates/trusted-server-core/src/auction/openrtb.rs +++ b/crates/trusted-server-core/src/auction/openrtb.rs @@ -29,11 +29,20 @@ const DEFAULT_CURRENCY: &str = "USD"; const TRUSTED_SERVER_EXT_KEY: &str = "trusted_server"; /// Fixed reasons why an upstream bid failed response admission. +/// +/// Public, with the dimension helpers below, so an auction implementation in +/// a crate of its own admits a bid by the rules core's own response reader +/// applies. #[derive(Debug, Clone, Copy, Eq, Ord, PartialEq, PartialOrd)] -pub(crate) enum BidRejectionReason { +pub enum BidRejectionReason { + /// The bid is malformed. InvalidBid, + /// The bid names an impression the request did not carry. UnrequestedImpression, + /// The bid's dimensions match no format the impression asked for. DimensionMismatch, + /// The bid's dimensions match more than one format the impression asked + /// for. AmbiguousDimensions, } @@ -105,7 +114,7 @@ impl DimensionMatch { } /// Precomputed dimension resolutions for one requested impression. -pub(crate) struct SlotBidDimensions { +pub struct SlotBidDimensions { exact: BTreeSet<(u32, u32)>, by_width: BTreeMap, by_height: BTreeMap, @@ -148,7 +157,7 @@ impl SlotBidDimensions { } /// Precomputed requested banner dimensions keyed by impression ID. -pub(crate) type BidDimensionIndex = BTreeMap; +pub type BidDimensionIndex = BTreeMap; /// Build the requested-dimension index once for one provider response. pub(crate) fn build_bid_dimension_index(input: &ProviderAuctionInput) -> BidDimensionIndex { @@ -158,7 +167,8 @@ pub(crate) fn build_bid_dimension_index(input: &ProviderAuctionInput) -> BidDime /// Build the requested-dimension index from plain [`AdSlot`]s. /// /// The first slot wins when several share an ID. -pub(crate) fn build_bid_dimension_index_from_slots<'a>( +#[must_use] +pub fn build_bid_dimension_index_from_slots<'a>( slots: impl IntoIterator, ) -> BidDimensionIndex { let mut index = BidDimensionIndex::new(); @@ -171,7 +181,12 @@ pub(crate) fn build_bid_dimension_index_from_slots<'a>( } /// Parse an optional positive `OpenRTB` bid dimension. -pub(crate) fn parse_optional_bid_dimension( +/// +/// # Errors +/// +/// Returns [`BidRejectionReason::InvalidBid`] when the value is present and is +/// not a positive whole number that fits. +pub fn parse_optional_bid_dimension( value: &Value, key: &str, ) -> Result, BidRejectionReason> { @@ -199,7 +214,12 @@ pub(crate) fn parse_optional_bid_dimension( } /// Validate explicit dimensions or infer them from one matching banner format. -pub(crate) fn resolve_bid_dimensions( +/// +/// # Errors +/// +/// Returns the reason the bid is refused: the impression was not requested, +/// or the dimensions match none or more than one of its formats. +pub fn resolve_bid_dimensions( dimensions_by_slot: &BidDimensionIndex, slot_id: &str, width: Option, diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 8639525342..8191e0ddbb 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -2598,7 +2598,6 @@ mod tests { BidStatus, MediaType, PublisherInfo, UserInfo, }; use crate::error::TrustedServerError; - use crate::integrations::adserver_mock::{AdServerMockProvider, AdServerMockSettings}; use crate::integrations::aps::{APS_RENDERER_TYPE, ApsRendererV1, ApsTagType}; use crate::platform::test_support::{ StubHttpClient, build_services_with_backend_and_http_client, @@ -3740,8 +3739,8 @@ mod tests { } } - /// Ad server whose context-aware parse restores `nurl`/`ad_id` (mirroring - /// `adserver_mock`), while its context-free parse does not. Lets a test prove + /// Ad server whose context-aware parse restores `nurl`/`ad_id`, as one that + /// keeps them out of its request does, while its context-free parse does not. Lets a test prove /// the synchronous ad server decision path calls `parse_response_with_context`. struct CacheRestoringAdServer; @@ -5111,9 +5110,9 @@ mod tests { let mut adserver_only = plan_config(Vec::new()); adserver_only.timeout_ms = 49; adserver_only.adserver = crate::provider_table::ProviderChoice::new( - Some("mock".to_string()), + Some("fixture".to_string()), BTreeMap::from([( - "mock".to_string(), + "fixture".to_string(), serde_json::Map::from_iter([( "endpoint".to_string(), serde_json::json!("https://adserver.example/mediate"), @@ -6276,81 +6275,6 @@ mod tests { ); } - #[tokio::test] - async fn planned_aps_mock_adserver_preserves_three_identities_and_renderer() { - let http = Arc::new(StubHttpClient::new()); - http.push_response( - 200, - serde_json::to_vec(&serde_json::json!({ - "seatbid": [{"seat": "upstream-seat", "bid": [{ - "id": "aps-bid", "impid": "fictional-slot", "price": 2.0, - "w": 300, "h": 250, - "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"} - }]}] - })) - .expect("should serialize APS response"), - ); - http.push_response( - 200, - serde_json::to_vec(&serde_json::json!({ - "seatbid": [{"seat": "aps_instance", "bid": [{ - "id": "adserver-aps", "impid": "fictional-slot", "price": 2.0, - "adm": "ignored", "w": 300, "h": 250, "crid": "aps-creative" - }]}] - })) - .expect("should serialize adserver response"), - ); - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, - ); - let plan = - AuctionPlan::compile(planned_aps_config()).expect("should compile planned APS auction"); - let adserver = AdServerMockProvider::new( - "adserver_mock", - AdServerMockSettings { - endpoint: "https://adserver.example/mediate".to_string(), - timeout_ms: 500, - ..AdServerMockSettings::default() - }, - ); - let orchestrator = AuctionOrchestratorHarness::new(plan, Some(Arc::new(adserver))); - let request = planned_request(); - let settings = create_test_settings(); - let inbound = http::Request::new(edgezero_core::body::Body::empty()); - let context = AuctionContext { - settings: &settings, - request: &inbound, - timeout_ms: 777, - transport_timeout_ms: 777, - provider_responses: None, - services: &services, - }; - - let result = orchestrator - .run_auction(&request, &context) - .await - .expect("should decide planned APS bid"); - - let provider_bid = &result.provider_responses[0].bids[0]; - assert_eq!(result.provider_responses[0].provider, "aps_instance"); - assert_eq!(provider_bid.returned_seat.as_deref(), Some("upstream-seat")); - assert_eq!(provider_bid.bidder, "aps"); - let winner = &result.winning_bids["fictional-slot"]; - assert_eq!(winner.returned_seat.as_deref(), Some("upstream-seat")); - assert_eq!(winner.bidder, "aps"); - assert!(winner.renderer.is_some()); - assert!(winner.creative.is_none()); - assert_eq!( - result - .adserver_response - .as_ref() - .map(|response| response.provider.as_str()), - Some("adserver_mock") - ); - } - #[tokio::test] async fn planned_aps_transport_omits_accept_header() { let http = Arc::new(StubHttpClient::new()); @@ -7322,9 +7246,9 @@ mod tests { ); let mut config = planned_config(&[("provider", RoutingMode::AllEligible)], false); config.adserver = crate::provider_table::ProviderChoice::new( - Some("mock".to_string()), + Some("fixture".to_string()), BTreeMap::from([( - "mock".to_string(), + "fixture".to_string(), serde_json::Map::from_iter([( "endpoint".to_string(), serde_json::json!("https://adserver.example/mediate"), diff --git a/crates/trusted-server-core/src/auction/plan.rs b/crates/trusted-server-core/src/auction/plan.rs index bf78b40da4..9786ac81ab 100644 --- a/crates/trusted-server-core/src/auction/plan.rs +++ b/crates/trusted-server-core/src/auction/plan.rs @@ -944,19 +944,19 @@ mod tests { fn an_ad_server_named_by_its_short_form_needs_no_implementation_line() { let mut raw = one_source(); raw.adserver = ProviderChoice::new( - Some("mock".to_string()), + Some("fixture".to_string()), BTreeMap::from([( - "mock".to_string(), + "fixture".to_string(), Map::from_iter([( ENDPOINT_KEY.to_string(), json!("https://adserver.example/mediate"), )]), )]), ); - let plan = AuctionPlan::compile(raw).expect("should resolve `mock` within [ad-server]"); + let plan = AuctionPlan::compile(raw).expect("should resolve `fixture` within [ad-server]"); assert_eq!( plan.adserver().map(|adserver| adserver.implementation.id), - Some("ad-server.mock") + Some("ad-server.fixture") ); } @@ -1457,9 +1457,9 @@ mod tests { secret_store_id: "example-secret-store".to_string(), }); raw.adserver = ProviderChoice::new( - Some("mock".to_string()), + Some("fixture".to_string()), BTreeMap::from([( - "mock".to_string(), + "fixture".to_string(), Map::from_iter([( ENDPOINT_KEY.to_string(), json!("http://127.0.0.1:6767/adserver/mediate"), @@ -1471,7 +1471,7 @@ mod tests { assert!(plan.signing_enabled()); assert_eq!( plan.adserver().map(|adserver| adserver.id.as_str()), - Some("mock") + Some("fixture") ); let mut invalid = config(Vec::new()); @@ -1493,7 +1493,7 @@ mod tests { BTreeMap::from([( "house".to_string(), Map::from_iter([ - (IMPLEMENTATION_KEY.to_string(), json!("mock")), + (IMPLEMENTATION_KEY.to_string(), json!("fixture")), ( ENDPOINT_KEY.to_string(), json!("https://adserver.example/mediate"), @@ -1505,13 +1505,13 @@ mod tests { let plan = AuctionPlan::compile(raw).expect("should compile a named ad server"); let adserver = plan.adserver().expect("should select an ad server"); assert_eq!(adserver.id.as_str(), "house"); - assert_eq!(adserver.implementation.id, "ad-server.mock"); + assert_eq!(adserver.implementation.id, "ad-server.fixture"); let mut unknown_setting = one_source(); unknown_setting.adserver = ProviderChoice::new( - Some("mock".to_string()), + Some("fixture".to_string()), BTreeMap::from([( - "mock".to_string(), + "fixture".to_string(), Map::from_iter([ ( ENDPOINT_KEY.to_string(), diff --git a/crates/trusted-server-core/src/auction/test_support.rs b/crates/trusted-server-core/src/auction/test_support.rs index ec049ca6f5..a9fa74f7ea 100644 --- a/crates/trusted-server-core/src/auction/test_support.rs +++ b/crates/trusted-server-core/src/auction/test_support.rs @@ -205,3 +205,110 @@ pub(crate) fn legacy_auction_config(settings: &Settings) -> crate::auction::Auct .collect(); config } + +/// A stand-in ad server implementation for core's own tests of the ad server +/// seam, selected with `[ad-server] module = "fixture"`. +/// +/// It reads an `endpoint`, which has to be a URL, and an optional +/// `timeout_ms` from its table and refuses any other setting, as an +/// implementation of a vendor's does. Its provider answers every request +/// with no bid and calls nothing. +pub(crate) mod adserver_fixture { + use std::sync::Arc; + + use async_trait::async_trait; + use error_stack::Report; + use serde::Deserialize; + + use crate::auction::demand::AdServerImplementation; + use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; + use crate::auction::types::{AuctionContext, AuctionRequest, AuctionResponse}; + use crate::error::TrustedServerError; + use crate::platform::PlatformResponse; + + /// The name the stand-in is selected by, which `[ad-server]` shortens to + /// `fixture`. + pub(crate) const MODULE: &str = "ad-server.fixture"; + + /// The stand-in ad server implementation. + pub(crate) static ADSERVER: AdServerImplementation = + AdServerImplementation { id: MODULE, build }; + + #[derive(Debug, Deserialize)] + #[serde(deny_unknown_fields)] + struct FixtureSettings { + endpoint: String, + #[serde(default = "default_timeout_ms")] + timeout_ms: u32, + } + + fn default_timeout_ms() -> u32 { + 500 + } + + fn build( + name: &str, + settings: &serde_json::Map, + ) -> Result, Report> { + let settings: FixtureSettings = serde_json::from_value(serde_json::Value::Object( + settings.clone(), + )) + .map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: format!("invalid `{MODULE}` settings: {error}"), + }) + })?; + url::Url::parse(&settings.endpoint).map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: format!("invalid `{MODULE}` endpoint: {error}"), + }) + })?; + Ok(Arc::new(FixtureAdServer::new(name, settings.timeout_ms))) + } + + /// The stand-in's provider. + pub(crate) struct FixtureAdServer { + name: String, + timeout_ms: u32, + } + + impl FixtureAdServer { + /// A provider called `name`. + pub(crate) fn new(name: &str, timeout_ms: u32) -> Self { + Self { + name: name.to_owned(), + timeout_ms, + } + } + } + + #[async_trait(?Send)] + impl AuctionProvider for FixtureAdServer { + fn provider_name(&self) -> &str { + &self.name + } + + async fn request_bids( + &self, + _request: &AuctionRequest, + _context: &AuctionContext<'_>, + ) -> Result> { + Ok(ProviderRequestOutcome::Immediate(AuctionResponse::no_bid( + self.name.clone(), + 0, + ))) + } + + async fn parse_response( + &self, + _response: PlatformResponse, + response_time_ms: u64, + ) -> Result> { + Ok(AuctionResponse::no_bid(self.name.clone(), response_time_ms)) + } + + fn timeout_ms(&self) -> u32 { + self.timeout_ms + } + } +} diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index 92f2c33a47..7cc28c45e1 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -1508,13 +1508,7 @@ password = "production-admin-password-32-bytes" Report, > { Ok(std::sync::Arc::new( - crate::integrations::adserver_mock::AdServerMockProvider::new( - name, - crate::integrations::adserver_mock::AdServerMockSettings { - endpoint: "https://external.example/mediate".to_string(), - ..Default::default() - }, - ), + crate::auction::test_support::adserver_fixture::FixtureAdServer::new(name, 500), )) } diff --git a/crates/trusted-server-core/src/config_payload.rs b/crates/trusted-server-core/src/config_payload.rs index ece4789725..094372d268 100644 --- a/crates/trusted-server-core/src/config_payload.rs +++ b/crates/trusted-server-core/src/config_payload.rs @@ -296,13 +296,7 @@ mod tests { Report, > { Ok(std::sync::Arc::new( - crate::integrations::adserver_mock::AdServerMockProvider::new( - name, - crate::integrations::adserver_mock::AdServerMockSettings { - endpoint: "https://external.example/mediate".to_string(), - ..Default::default() - }, - ), + crate::auction::test_support::adserver_fixture::FixtureAdServer::new(name, 500), )) } diff --git a/crates/trusted-server-core/src/integrations/mod.rs b/crates/trusted-server-core/src/integrations/mod.rs index 5a5ecd684e..c91506bc74 100644 --- a/crates/trusted-server-core/src/integrations/mod.rs +++ b/crates/trusted-server-core/src/integrations/mod.rs @@ -14,7 +14,6 @@ use crate::error::TrustedServerError; use crate::platform::{DEFAULT_FIRST_BYTE_TIMEOUT, PlatformBackendSpec, RuntimeServices}; use crate::settings::Settings; -pub mod adserver_mock; pub mod aps; pub mod js_asset_proxy; pub mod openrtb; @@ -729,8 +728,14 @@ const BUILT_IN_BUILDERS: &[IntegrationBuilder] = &[ IntegrationBuilder::implementations(aps::APS_INTEGRATION_ID, CORE_SOURCE) .with_demand(&aps::DEMAND) .with_plan_registration(aps::register_for_plan), - IntegrationBuilder::implementations(adserver_mock::MODULE, CORE_SOURCE) - .with_adserver(&adserver_mock::ADSERVER), + // A stand-in ad server, which core's own tests select where they need + // one. + #[cfg(test)] + IntegrationBuilder::implementations( + crate::auction::test_support::adserver_fixture::MODULE, + CORE_SOURCE, + ) + .with_adserver(&crate::auction::test_support::adserver_fixture::ADSERVER), ]; /// The built-in integration builders, in hook order. diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index 3403301fb0..d236e6c96b 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -5397,7 +5397,7 @@ mod tests { for (section, name) in [ ("auction", "prebid-server"), ("auction", "aps"), - ("auction", "ad-server.mock"), + ("auction", "ad-server.fixture"), ("auction-protocol", "openrtb"), ] { let mut settings = crate::test_support::tests::create_test_settings(); diff --git a/crates/trusted-server-modules/Cargo.toml b/crates/trusted-server-modules/Cargo.toml index 2d888f3a18..2da46012ec 100644 --- a/crates/trusted-server-modules/Cargo.toml +++ b/crates/trusted-server-modules/Cargo.toml @@ -14,6 +14,7 @@ doctest = false workspace = true [dependencies] +trusted-server-ad-server-mock = { workspace = true } trusted-server-ad-tag-google = { workspace = true } trusted-server-audience-permutive = { workspace = true } trusted-server-bot-protection-datadome = { workspace = true } @@ -30,4 +31,5 @@ trusted-server-testing-testlight = { workspace = true } edgezero-core = { workspace = true } http = { workspace = true } serde_json = { workspace = true } +tokio = { workspace = true } trusted-server-core = { workspace = true, features = ["test-utils"] } diff --git a/crates/trusted-server-modules/src/lib.rs b/crates/trusted-server-modules/src/lib.rs index ade47e7e21..049d55b767 100644 --- a/crates/trusted-server-modules/src/lib.rs +++ b/crates/trusted-server-modules/src/lib.rs @@ -31,6 +31,7 @@ pub fn builders() -> Vec { trusted_server_bot_protection_datadome::builder(), trusted_server_ad_tag_google::builder(), trusted_server_ad_tag_google::diagnostics::builder(), + trusted_server_ad_server_mock::builder(), ] } @@ -67,9 +68,17 @@ mod tests { #[test] fn stock_modules_are_offered_in_hook_order() { + // A module is named by what its section selects, and an + // implementation by what `[demand]` or `[ad-server]` names. let names: Vec<&str> = builders() .iter() - .filter_map(IntegrationBuilder::module_name) + .map(|builder| { + builder + .module_name() + .or_else(|| builder.demand().map(|demand| demand.id)) + .or_else(|| builder.adserver().map(|adserver| adserver.id)) + .unwrap_or_else(|| builder.id()) + }) .collect(); assert_eq!( @@ -86,6 +95,7 @@ mod tests { "bot-protection.datadome", "ad-tag.google", "ad-tag.google.diagnostics", + "ad-server.mock", ], "should offer the stock modules in the order their hooks run" ); @@ -337,4 +347,133 @@ mod tests { ); } } + + /// The mock ad server decides between the bids APS returned, and the + /// winner keeps the seat the exchange returned, APS's own bidder name and + /// its renderer. + #[tokio::test] + async fn the_mock_ad_server_keeps_an_aps_winner_s_identities_and_renderer() { + use std::collections::{BTreeMap, HashMap}; + use std::sync::Arc; + + use trusted_server_ad_server_mock::{AdServerMockProvider, AdServerMockSettings}; + use trusted_server_core::auction::orchestrator::AuctionOrchestrator; + use trusted_server_core::auction::types::{ + AdFormat, AdSlot, AuctionContext, AuctionRequest, MediaType, PublisherInfo, UserInfo, + }; + use trusted_server_core::platform::test_support::{ + StubHttpClient, build_services_with_http_client, + }; + use trusted_server_core::provider_table::ProviderList; + use trusted_server_core::test_support::tests::create_test_settings; + + let http = Arc::new(StubHttpClient::new()); + http.push_response( + 200, + serde_json::to_vec(&serde_json::json!({ + "seatbid": [{"seat": "upstream-seat", "bid": [{ + "id": "aps-bid", "impid": "fictional-slot", "price": 2.0, + "w": 300, "h": 250, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"} + }]}] + })) + .expect("should serialize APS response"), + ); + http.push_response( + 200, + serde_json::to_vec(&serde_json::json!({ + "seatbid": [{"seat": "aps_instance", "bid": [{ + "id": "adserver-aps", "impid": "fictional-slot", "price": 2.0, + "adm": "ignored", "w": 300, "h": 250, "crid": "aps-creative" + }]}] + })) + .expect("should serialize adserver response"), + ); + let services = build_services_with_http_client(Arc::clone(&http) as Arc<_>); + + let mut settings = create_test_settings(); + settings.auction.enabled = true; + settings.auction.timeout_ms = 777; + let serde_json::Value::Object(aps) = serde_json::json!({ + "implementation": "auction.aps", + "endpoint": "https://aps.example/e/pb/bid", + "account_id": "example-account", + "timeout_ms": 1000, + "routing": "all_eligible", + }) else { + panic!("should build the APS table"); + }; + settings.demand = ProviderList::new( + vec!["aps_instance".to_string()], + BTreeMap::from([("aps_instance".to_string(), aps)]), + ); + let plan = trusted_server_core::auction::compile_auction_plan_with(&settings, &builders()) + .expect("should compile the planned APS auction"); + let adserver = AdServerMockProvider::new( + "adserver_mock", + AdServerMockSettings { + endpoint: "https://adserver.example/mediate".to_string(), + timeout_ms: 500, + ..AdServerMockSettings::default() + }, + ); + let orchestrator = AuctionOrchestrator::from_plan(Arc::new(plan), Some(Arc::new(adserver))); + let request = AuctionRequest { + id: "fictional-auction".to_string(), + slots: vec![AdSlot { + id: "fictional-slot".to_string(), + formats: vec![AdFormat { + media_type: MediaType::Banner, + width: 300, + height: 250, + }], + floor_price: Some(1.0), + targeting: HashMap::new(), + bidders: HashMap::new(), + }], + publisher: PublisherInfo { + domain: "publisher.example".to_string(), + page_url: Some("https://publisher.example/article".to_string()), + }, + user: UserInfo { + id: None, + consent: None, + eids: None, + }, + device: None, + site: None, + context: HashMap::new(), + }; + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should decide planned APS bid"); + + let provider_bid = &result.provider_responses[0].bids[0]; + assert_eq!(result.provider_responses[0].provider, "aps_instance"); + assert_eq!(provider_bid.returned_seat.as_deref(), Some("upstream-seat")); + assert_eq!(provider_bid.bidder, "aps"); + let winner = &result.winning_bids["fictional-slot"]; + assert_eq!(winner.returned_seat.as_deref(), Some("upstream-seat")); + assert_eq!(winner.bidder, "aps"); + assert!(winner.renderer.is_some()); + assert!(winner.creative.is_none()); + assert_eq!( + result + .adserver_response + .as_ref() + .map(|response| response.provider.as_str()), + Some("adserver_mock") + ); + } } From a1339c9f18266533cf63c2829b9c8da89309a8e2 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 8 Oct 2026 04:41:03 +0100 Subject: [PATCH 102/124] APS moves out of core The Amazon Publisher Services demand implementation is `auction.aps`, and it now lives in `crates/auction/aps`. The stock list offers it, so a `[demand.]` table with `implementation = "auction.aps"` selects it as before and its settings do not change. Its builder registers the page support the APS renderer needs when the auction plan selects an APS source. Its 38 tests went with it, and 24 more came from core, where APS had been the example the engine was tested with: - three tests of the renderer descriptor APS writes, - eight that run an APS source through the orchestrator and the response reader, - four of the plan compiler and the page registration, - six of the request APS builds, with both golden requests carried byte for byte, - three of the deploy check of APS's own settings. Those tests run against core's engine from APS's crate, so core offers what they need under its `test-utils` feature. That is the helpers that build an auction context, a demand table and a plan, two drivers that build a request for a plan's first source and read a response as that source, and the backend double in the platform test doubles. Core's own tests of routing, the plan compiler, the request builder, the response format and the page's bid map use core's plain OpenRTB source, or a renderer descriptor of the test's own tagged `example`, in place of APS. `ERROR_TYPE_HTTP_STATUS` and `ignored_bidder_params_count` are public, so an implementation in a crate of its own reports a failed call and counts ignored bidder parameters the way core does. --- .cargo/config.toml | 12 +- CHANGELOG.md | 2 +- Cargo.lock | 20 + Cargo.toml | 2 + crates/auction/aps/Cargo.toml | 39 + .../aps.rs => auction/aps/src/lib.rs} | 1284 ++++++++++++++++- .../src/auction/formats.rs | 84 +- crates/trusted-server-core/src/auction/mod.rs | 47 +- .../src/auction/openrtb.rs | 12 +- .../src/auction/openrtb/tests.rs | 122 +- .../src/auction/orchestrator.rs | 755 ++-------- .../trusted-server-core/src/auction/plan.rs | 59 +- .../src/auction/provider.rs | 2 +- .../src/auction/routing.rs | 53 +- .../src/auction/test_support.rs | 201 ++- .../trusted-server-core/src/auction/types.rs | 278 ++-- crates/trusted-server-core/src/config.rs | 82 -- .../src/integrations/mod.rs | 4 - .../src/integrations/registry.rs | 48 +- .../src/platform/test_support.rs | 76 + crates/trusted-server-core/src/publisher.rs | 69 +- crates/trusted-server-modules/Cargo.toml | 1 + crates/trusted-server-modules/src/lib.rs | 2 + docs/guide/integrations/aps.md | 2 +- 24 files changed, 1910 insertions(+), 1346 deletions(-) create mode 100644 crates/auction/aps/Cargo.toml rename crates/{trusted-server-core/src/integrations/aps.rs => auction/aps/src/lib.rs} (69%) diff --git a/.cargo/config.toml b/.cargo/config.toml index bce0907ddf..f10b342025 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -26,10 +26,10 @@ test_details = "test --target aarch64-apple-darwin" # native crate needs no change here. Axum (native), Cloudflare # (wasm32-unknown-unknown), Spin, the CLI (native), and integration-tests # (native) are simply not listed. -build-fastly = "build -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" -check-fastly = "check -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" -clippy-fastly = "clippy -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-js -p trusted-server-openrtb --all-targets --all-features --target wasm32-wasip1 -- -D warnings" -test-fastly = "test -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" +build-fastly = "build -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" +check-fastly = "check -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" +clippy-fastly = "clippy -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-js -p trusted-server-openrtb --all-targets --all-features --target wasm32-wasip1 -- -D warnings" +test-fastly = "test -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" # Feature-on counterpart of `test-fastly`. `test-fastly` does NOT pass # --all-features, so without this the reusable-sandbox code is linted by # `clippy-fastly` but its tests never execute. @@ -40,8 +40,8 @@ test-fastly-reuse = "test -p trusted-server-adapter-fastly --target wasm32-wasip # run with the Axum adapter in CI. build-axum = "build -p trusted-server-adapter-axum" check-axum = "check -p trusted-server-adapter-axum" -clippy-axum = "clippy -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock --all-targets --all-features -- -D warnings" -test-axum = "test -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock" +clippy-axum = "clippy -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps --all-targets --all-features -- -D warnings" +test-axum = "test -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps" # --- Cloudflare adapter (native host + wasm32-unknown-unknown) --- # Build/check target the WASM runtime (requires the `cloudflare` feature); diff --git a/CHANGELOG.md b/CHANGELOG.md index d1876e157e..dc36d09f0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - A pushed configuration keeps DataDome's two deprecated secret store selectors, `server_side_key_secret_store` and `protection_test_bypass.credential_secret_store`, where they were removed as the settings were read. DataDome still accepts and discards both, so a configuration that carries one loads as before. -- Integration modules are moving out of `trusted-server-core` into crates of their own under `crates///`, each named by that folder, so `crates/cmp/osano` is the module `cmp.osano`. The settings that select and configure a module do not change. `crates/trusted-server-modules` lists the modules a stock build ships from crates of their own, in the order their hooks run. Every adapter builds its state and loads its settings with that list, and `ts config validate` and `ts config push` run each stock module's own rules. Code that named a moved module under `trusted_server_core::integrations` names its crate instead, such as `trusted_server_cmp_osano`. Moved so far: Osano, Lockr, Permutive, Sourcepoint, Didomi, Google Tag Manager, Testlight, Next.js, Google Publisher Tags and its diagnostics, DataDome and the mock ad server. +- Integration modules are moving out of `trusted-server-core` into crates of their own under `crates///`, each named by that folder, so `crates/cmp/osano` is the module `cmp.osano`. The settings that select and configure a module do not change. `crates/trusted-server-modules` lists the modules a stock build ships from crates of their own, in the order their hooks run. Every adapter builds its state and loads its settings with that list, and `ts config validate` and `ts config push` run each stock module's own rules. Code that named a moved module under `trusted_server_core::integrations` names its crate instead, such as `trusted_server_cmp_osano`. Moved so far: Osano, Lockr, Permutive, Sourcepoint, Didomi, Google Tag Manager, Testlight, Next.js, Google Publisher Tags and its diagnostics, DataDome, the mock ad server and APS. - The Fastly adapter is a library with a thin binary over it. A deployment that ships a vendor crate writes a binary of its own that calls `trusted_server_adapter_fastly::run_with(vec![...])` with that crate's integration builder, in place of editing the adapter. The builders are composed with the built-in ones, and a builder runs only when the settings select its module. The Wasm artifact keeps its name, `trusted-server-adapter-fastly.wasm`. - **Breaking:** `[integrations]` is gone. Each page integration is a module selected in the section of its type, with its settings in the table at its name: `[cmp] module = "didomi"` (or `"sourcepoint"` or `"osano"`), `[tag] modules = ["google-tag-manager"]`, `[ad-tag] modules = ["google", "google.diagnostics"]`, `[bot-protection] module = "datadome"`, `[identity] module = "lockr"`, `[audience] module = "permutive"`, `[framework] module = "nextjs"`, `[auction] modules = ["prebid"]` (with `"testing.testlight"` for the test module) and `[proxy] modules = ["js_asset_proxy"]`, so `[integrations.gpt]` is `[ad-tag.google]` and `[integrations.prebid]` is `[auction.prebid]`. A module's name is `.`, which is its crate's path under `crates/` once it has a crate of its own and a constant in core until then, and it is written in its section with the type left off. A section that selects nothing, a table its section does not select, and a name no module in the deployment supplies are each refused at startup. An `[integrations]` or `[integration]` table is refused with directions. Environment overlays name a section as written, so `TRUSTED_SERVER__INTEGRATIONS__GPT__GAM_ATTRIBUTION_ENABLED` is `TRUSTED_SERVER__AD-TAG__GOOGLE__GAM_ATTRIBUTION_ENABLED`. - **Breaking:** Every pluggable component now shares one configuration syntax. Each selectable type is a top-level table named for the job, a `module` key inside it selects what runs, or `modules` for a type that runs several, and a `[.]` table holds that name's settings when it has any. The types are `ec`, `geo`, `device`, `permission-signal`, `demand` and `ad-server`, and the section of each page integration type, such as `cmp` or `tag`. A module from a crate is named by its folder below `crates/`, such as `permission-signal.gpc`, and within its own table the type folder may be left off. A `demand` or `ad-server` name is snake_case, because it may be a label of your own. A settings table its type's selector does not name refuses startup, so a block left behind after a module is switched off is caught rather than sitting unread, and every module rejects settings it does not know, so a misspelled key fails rather than being ignored. A table's name is the implementation unless the table carries an `implementation` line, and a demand table always carries one, naming the implementation by its module path (`auction-protocol.openrtb`, `auction.prebid-server` or `auction.aps`), which is how two Prebid Servers run side by side under names of their own. The ad server implementation is `ad-server.mock`, written `mock` in `[ad-server]`. A `demand` or `ad-server` endpoint must be HTTPS, or HTTP to `127.0.0.1`, `::1` or `localhost`. The word mediator is gone. It is "ad server" in prose and `ad-server` in configuration, `[debug.auction_html_comment_options] include_mediator_response` is now `include_adserver_response`, and the auction response metadata value `parallel_mediation` is now `parallel_adserver`. `auction-protocol.openrtb`, `auction.prebid-server`, `auction.aps` and `ad-server.mock` supply implementations only, so a page integration's section that names one is refused. The full rules and what is checked at each gate are in [Configuration Rules](docs/guide/configuration-rules.md). Migrate as follows: diff --git a/Cargo.lock b/Cargo.lock index e98a47e605..fb837b8258 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5530,6 +5530,25 @@ dependencies = [ "trusted-server-permission-signal-us-privacy", ] +[[package]] +name = "trusted-server-auction-aps" +version = "0.1.0" +dependencies = [ + "async-trait", + "base64", + "edgezero-core", + "error-stack", + "futures", + "http", + "log", + "serde", + "serde_json", + "tokio", + "trusted-server-core", + "url", + "validator", +] + [[package]] name = "trusted-server-audience-permutive" version = "0.1.0" @@ -5831,6 +5850,7 @@ dependencies = [ "tokio", "trusted-server-ad-server-mock", "trusted-server-ad-tag-google", + "trusted-server-auction-aps", "trusted-server-audience-permutive", "trusted-server-bot-protection-datadome", "trusted-server-cmp-didomi", diff --git a/Cargo.toml b/Cargo.toml index 0e4a19fb62..55984a9d46 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,7 @@ members = [ "crates/ad-tag/google", "crates/bot-protection/datadome", "crates/ad-server/mock", + "crates/auction/aps", "crates/trusted-server-adapter-axum", "crates/trusted-server-adapter-cloudflare", "crates/trusted-server-adapter-fastly", @@ -156,6 +157,7 @@ trusted-server-framework-nextjs = { path = "crates/framework/nextjs" } trusted-server-ad-tag-google = { path = "crates/ad-tag/google" } trusted-server-bot-protection-datadome = { path = "crates/bot-protection/datadome" } trusted-server-ad-server-mock = { path = "crates/ad-server/mock" } +trusted-server-auction-aps = { path = "crates/auction/aps" } trusted-server-js = { path = "crates/trusted-server-js" } trusted-server-modules = { path = "crates/trusted-server-modules" } trusted-server-openrtb = { path = "crates/trusted-server-openrtb" } diff --git a/crates/auction/aps/Cargo.toml b/crates/auction/aps/Cargo.toml new file mode 100644 index 0000000000..7156f62a3e --- /dev/null +++ b/crates/auction/aps/Cargo.toml @@ -0,0 +1,39 @@ +[package] +name = "trusted-server-auction-aps" +description = "Amazon Publisher Services as a demand implementation, with the page support its renderer needs." +authors = { workspace = true } +edition = { workspace = true } +license = { workspace = true } +publish = { workspace = true } +version = { workspace = true } + +[lib] +doctest = false + +[lints] +workspace = true + +[dependencies] +async-trait = { workspace = true } +base64 = { workspace = true } +edgezero-core = { workspace = true } +error-stack = { workspace = true } +http = { workspace = true } +log = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +trusted-server-core = { workspace = true } +url = { workspace = true } +validator = { workspace = true } + +[dev-dependencies] +futures = { workspace = true } +tokio = { workspace = true } +trusted-server-core = { workspace = true, features = ["test-utils"] } + +# The visible owner of this vendor crate, the way Prebid.js requires a named +# maintainer of every adapter. Amazon has not yet adopted this crate, so the +# Trusted Server maintainers own it until a vendor owner steps up. +[package.metadata.maintainers] +owner = "Trusted Server maintainers" +status = "seeking vendor owner" diff --git a/crates/trusted-server-core/src/integrations/aps.rs b/crates/auction/aps/src/lib.rs similarity index 69% rename from crates/trusted-server-core/src/integrations/aps.rs rename to crates/auction/aps/src/lib.rs index 368a75e3be..24c44c8bdc 100644 --- a/crates/trusted-server-core/src/integrations/aps.rs +++ b/crates/auction/aps/src/lib.rs @@ -1,5 +1,17 @@ //! Amazon Publisher Services (APS/TAM) `OpenRTB` integration. +#![cfg_attr( + test, + allow( + clippy::print_stdout, + clippy::print_stderr, + clippy::panic, + clippy::dbg_macro, + clippy::unwrap_used, + reason = "tests use direct diagnostics and panic-on-failure helpers" + ) +)] + use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::Arc; #[cfg(test)] @@ -19,40 +31,53 @@ use url::Url; use validator::Validate; use validator::ValidationError; -use crate::auction::demand::{ +use trusted_server_core::auction::demand::{ CONSERVATIVE_LANGUAGE_MAX_BYTES, CompiledDemand, DemandFieldPolicy, DemandImplementation, - DemandResponse, DemandTimeoutDefault, RegsPolicy, RequestExtensions, + DemandResponse, DemandTimeoutDefault, ProviderAuctionInput, RegsPolicy, RequestExtensions, }; -use crate::auction::openrtb::ignored_bidder_params_count; -use crate::auction::orchestrator::ERROR_TYPE_HTTP_STATUS; +use trusted_server_core::auction::openrtb::ignored_bidder_params_count; +use trusted_server_core::auction::orchestrator::ERROR_TYPE_HTTP_STATUS; #[cfg(test)] -use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; -use crate::auction::routing::ProviderAuctionInput; +use trusted_server_core::auction::provider::{AuctionProvider, ProviderRequestOutcome}; #[cfg(test)] -use crate::auction::types::{AdSlot, AuctionContext, AuctionRequest}; -use crate::auction::types::{AuctionResponse, Bid, BidRenderer, MediaType}; -use crate::error::TrustedServerError; -use crate::integrations::{ +use trusted_server_core::auction::types::{AdSlot, AuctionContext, AuctionRequest}; +use trusted_server_core::auction::types::{AuctionResponse, Bid, BidRenderer, MediaType}; +use trusted_server_core::error::TrustedServerError; +use trusted_server_core::integrations::{ IntegrationEndpoint, IntegrationHeadInjector, IntegrationHtmlContext, IntegrationProxy, IntegrationRegistration, UPSTREAM_RTB_MAX_RESPONSE_BYTES, collect_response_bounded, }; #[cfg(test)] -use crate::integrations::{ +use trusted_server_core::integrations::{ ensure_integration_backend_with_timeout, predict_integration_backend_name, }; #[cfg(test)] -use crate::openrtb::ToExt; +use trusted_server_core::openrtb::ToExt; #[cfg(test)] -use crate::openrtb::{ +use trusted_server_core::openrtb::{ Banner, Device, Format, Geo, Imp, OpenRtbRequest, Publisher, Regs, RegsExt, Site, User, UserExt, to_openrtb_i32, }; #[cfg(test)] -use crate::platform::PlatformHttpRequest; -use crate::platform::{PlatformResponse, RuntimeServices}; -use crate::settings::Settings; +use trusted_server_core::platform::PlatformHttpRequest; +use trusted_server_core::platform::{PlatformResponse, RuntimeServices}; +use trusted_server_core::settings::Settings; pub(crate) const APS_INTEGRATION_ID: &str = "aps"; + +/// The builder a deployment hands to an adapter. It offers APS to `[demand]` +/// and registers the page support its renderer needs when the auction plan +/// selects an APS source. +#[must_use] +pub fn builder() -> trusted_server_core::integrations::IntegrationBuilder { + trusted_server_core::integrations::IntegrationBuilder::implementations( + APS_INTEGRATION_ID, + env!("CARGO_PKG_NAME"), + ) + .with_demand(&DEMAND) + .with_plan_registration(register_for_plan) +} + /// The name an `implementation` line gives this implementation, its module /// path. pub const MODULE: &str = "auction.aps"; @@ -1178,7 +1203,7 @@ impl ApsAuctionProvider { Self { config } } - fn build_regs(consent: Option<&crate::consent::ConsentContext>) -> Option { + fn build_regs(consent: Option<&trusted_server_core::consent::ConsentContext>) -> Option { let consent = consent?; let ext = RegsExt { gdpr: Some(u8::from(consent.gdpr_applies)), @@ -2082,7 +2107,7 @@ impl IntegrationHeadInjector for ApsRendererIntegration { /// modes. pub fn register_for_plan( _settings: &Settings, - plan: &crate::auction::AuctionPlan, + plan: &trusted_server_core::auction::AuctionPlan, ) -> Result, Report> { let mut selected: Option<(&str, ApsRenderingMode)> = None; for provider in plan.providers() { @@ -2123,16 +2148,16 @@ pub fn register_for_plan( #[cfg(test)] mod tests { use super::*; - use crate::auction::test_support::canonical_parity_auction_request; - use crate::auction::types::{ + use serde_json::json; + use trusted_server_core::auction::test_support::canonical_parity_auction_request; + use trusted_server_core::auction::types::{ AdFormat, AdSlot, AuctionContext, AuctionRequest, BidStatus, PublisherInfo, UserInfo, }; - use crate::integrations::IntegrationDocumentState; - use crate::platform::test_support::{ + use trusted_server_core::integrations::IntegrationDocumentState; + use trusted_server_core::platform::test_support::{ StubHttpClient, build_services_with_http_client, noop_services, }; - use crate::test_support::tests::create_test_settings; - use serde_json::json; + use trusted_server_core::test_support::tests::create_test_settings; fn config() -> LegacyApsProviderConfig { LegacyApsProviderConfig { @@ -2745,7 +2770,7 @@ mod tests { let stub = Arc::new(StubHttpClient::new()); stub.push_response(200, br#"{"seatbid":[]}"#.to_vec()); let services = build_services_with_http_client( - Arc::clone(&stub) as Arc + Arc::clone(&stub) as Arc ); let settings = create_test_settings(); let downstream = http::Request::builder() @@ -3213,13 +3238,16 @@ mod tests { assert_eq!(response.status(), StatusCode::NOT_FOUND); } - fn aps_plan(rendering_modes: &[Option<&str>]) -> crate::auction::AuctionPlan { + fn aps_plan(rendering_modes: &[Option<&str>]) -> trusted_server_core::auction::AuctionPlan { let tables = rendering_modes .iter() .enumerate() .map(|(index, mode)| { - let mut table = - crate::auction::test_support::demand_table(MODULE, &default_endpoint()); + let mut table = trusted_server_core::auction::test_support::demand_table( + MODULE, + &default_endpoint(), + ); + table.insert("account_id".to_string(), json!("example-account")); table.insert("routing".to_string(), json!("all_eligible")); if let Some(mode) = mode { table.insert("rendering_mode".to_string(), json!(mode)); @@ -3227,8 +3255,10 @@ mod tests { (if index == 0 { "aps_main" } else { "aps_second" }, table) }) .collect::>(); - crate::auction::AuctionPlan::compile(crate::auction::test_support::plan_config(tables)) - .expect("should compile APS plan") + trusted_server_core::auction::AuctionPlan::compile( + trusted_server_core::auction::test_support::plan_config_with(tables, &[builder()]), + ) + .expect("should compile APS plan") } #[test] @@ -3258,9 +3288,9 @@ mod tests { #[test] fn no_aps_source_registers_nothing() { - let plan = crate::auction::AuctionPlan::compile(crate::auction::test_support::plan_config( - Vec::new(), - )) + let plan = trusted_server_core::auction::AuctionPlan::compile( + trusted_server_core::auction::test_support::plan_config(Vec::new()), + ) .expect("should compile an empty plan"); assert!( @@ -3417,4 +3447,1192 @@ mod tests { "should read the same value as deserializing the whole descriptor" ); } + + #[test] + fn aps_renderer_serializes_to_versioned_camel_case_contract() { + let renderer = BidRenderer::from_typed( + APS_RENDERER_TYPE, + &ApsRendererV1 { + version: 1, + account_id: "example-account-id".to_string(), + bid_id: "fictional-bid-id".to_string(), + creative_id: Some("fictional-creative-id".to_string()), + tag_type: ApsTagType::Iframe, + creative_url: "https://creative.example/render".to_string(), + aax_response: "base64-data".to_string(), + width: 300, + height: 250, + }, + ) + .expect("should build APS renderer descriptor"); + + let serialized = serde_json::to_value(&renderer).expect("should serialize renderer"); + + assert_eq!( + serialized, + json!({ + "type": "aps", + "version": 1, + "accountId": "example-account-id", + "bidId": "fictional-bid-id", + "creativeId": "fictional-creative-id", + "tagType": "iframe", + "creativeUrl": "https://creative.example/render", + "aaxResponse": "base64-data", + "width": 300, + "height": 250 + }), + "should match renderer wire contract" + ); + } + + #[test] + fn aps_renderer_omits_absent_creative_id() { + let renderer = BidRenderer::from_typed( + APS_RENDERER_TYPE, + &ApsRendererV1 { + version: 1, + account_id: "example-account-id".to_string(), + bid_id: "fictional-bid-id".to_string(), + creative_id: None, + tag_type: ApsTagType::Iframe, + creative_url: "https://creative.example/render".to_string(), + aax_response: "base64-data".to_string(), + width: 300, + height: 250, + }, + ) + .expect("should build APS renderer descriptor"); + + let serialized = serde_json::to_value(&renderer).expect("should serialize renderer"); + + assert!( + serialized.get("creativeId").is_none(), + "should omit absent creative ID" + ); + } + + /// Rewrites every object in `value` with its keys in sorted order, so + /// serializing the result gives one fixed key order. + /// + /// `serde_json::Map` is a `BTreeMap`, which serializes keys in sorted + /// order, only while the crate's `preserve_order` feature is off. With the + /// feature on it is an `IndexMap` and the order follows insertion instead. + /// Nothing in this crate asks for the feature, but Cargo unifies features + /// across everything built for one target, and `trusted-server-cli` pulls + /// it in through `edgezero-cli` and then `handlebars`. A maintainer + /// running `cargo test --workspace --target ` therefore builds this + /// crate with `preserve_order` on, and a test that pinned insertion order + /// would fail there for no reason. Sorting both sides removes the + /// dependence on which map `serde_json` was built with. + fn with_sorted_keys(value: &serde_json::Value) -> serde_json::Value { + match value { + serde_json::Value::Object(map) => { + let mut keys = map.keys().collect::>(); + keys.sort_unstable(); + let mut sorted = serde_json::Map::with_capacity(keys.len()); + for key in keys { + let child = map.get(key).expect("should find a key the map just listed"); + sorted.insert(key.clone(), with_sorted_keys(child)); + } + serde_json::Value::Object(sorted) + } + serde_json::Value::Array(items) => { + serde_json::Value::Array(items.iter().map(with_sorted_keys).collect()) + } + scalar => scalar.clone(), + } + } + + #[test] + fn the_open_renderer_serializes_to_the_same_bytes_as_the_aps_variant_did() { + // Literal strings captured from the closed-enum form before this + // change, through the same `serde_json::to_value` path production + // uses: `BidExt::to_ext` for the OpenRTB response extension, and + // `build_bid_map` for `window.tsjs.bids`. + // + // Both sides go through `with_sorted_keys` first, because the key + // order `serde_json` emits is not ours to pin, and that function + // explains why. Sorting settles the order without weakening what is + // pinned, since two objects serialize to the same sorted bytes only + // when they carry exactly the same keys with exactly the same values. + let full = BidRenderer::from_typed( + APS_RENDERER_TYPE, + &ApsRendererV1 { + version: 1, + account_id: "example-account-id".to_string(), + bid_id: "fictional-bid-id".to_string(), + creative_id: Some("fictional-creative-id".to_string()), + tag_type: ApsTagType::Iframe, + creative_url: "https://creative.example/render".to_string(), + aax_response: "base64-data".to_string(), + width: 300, + height: 250, + }, + ) + .expect("should build APS renderer descriptor"); + let absent = BidRenderer::from_typed( + APS_RENDERER_TYPE, + &ApsRendererV1 { + version: 1, + account_id: "example-account-id".to_string(), + bid_id: "fictional-bid-id".to_string(), + creative_id: None, + tag_type: ApsTagType::Script, + creative_url: "https://creative.example/render".to_string(), + aax_response: "base64-data".to_string(), + width: 300, + height: 250, + }, + ) + .expect("should build APS renderer descriptor"); + + let full_bytes = serde_json::to_string(&with_sorted_keys( + &serde_json::to_value(&full).expect("should convert renderer to a JSON value"), + )) + .expect("should serialize renderer"); + let absent_bytes = serde_json::to_string(&with_sorted_keys( + &serde_json::to_value(&absent).expect("should convert renderer to a JSON value"), + )) + .expect("should serialize renderer"); + + assert_eq!( + full_bytes, + "{\"aaxResponse\":\"base64-data\",\"accountId\":\"example-account-id\",\"bidId\":\"fictional-bid-id\",\"creativeId\":\"fictional-creative-id\",\"creativeUrl\":\"https://creative.example/render\",\"height\":250,\"tagType\":\"iframe\",\"type\":\"aps\",\"version\":1,\"width\":300}", + "should serialize to the bytes the closed enum produced" + ); + assert_eq!( + absent_bytes, + "{\"aaxResponse\":\"base64-data\",\"accountId\":\"example-account-id\",\"bidId\":\"fictional-bid-id\",\"creativeUrl\":\"https://creative.example/render\",\"height\":250,\"tagType\":\"script\",\"type\":\"aps\",\"version\":1,\"width\":300}", + "should serialize to the bytes the closed enum produced with no creative ID" + ); + } + + #[test] + fn module_constant_is_the_crate_folder() { + assert_eq!( + super::MODULE, + trusted_server_core::module_name!(), + "should be named by the folder this crate lives in" + ); + } +} + +/// APS run through core's auction engine: the orchestrator, and the reading +/// of an APS response as the driver hands it over. +#[cfg(test)] +mod engine_tests { + use std::collections::HashMap; + use std::sync::Arc; + + use base64::Engine as _; + + use trusted_server_core::auction::orchestrator::AuctionOrchestrator; + use trusted_server_core::auction::plan::{AuctionPlan, AuctionPlanConfig, NotificationConfig}; + use trusted_server_core::auction::test_support::{ + build_for_first_source, canonical_parity_auction_request, demand_table, + deterministic_signer, golden_inbound_request, golden_plan_config, parse_as_first_source, + plan_config_with, + }; + use trusted_server_core::auction::types::{ + AdFormat, AdSlot, AuctionContext, AuctionRequest, BidStatus, MediaType, PublisherInfo, + UserInfo, + }; + use trusted_server_core::platform::test_support::{ + NamingBackend, StubHttpClient, build_services_with_backend_and_http_client, + }; + use trusted_server_core::platform::{BackendNamingPolicy, PlatformResponse}; + use trusted_server_core::test_support::tests::create_test_settings; + + use super::{APS_RENDERER_TYPE, ApsRendererV1, MODULE, builder}; + + /// A plan configuration of APS sources, each with the settings and the + /// notification policy given. + fn aps_instances_config( + providers: &[(&str, serde_json::Value, NotificationConfig)], + ) -> AuctionPlanConfig { + let tables = providers + .iter() + .map(|(id, settings, notifications)| { + let mut entry = demand_table(MODULE, "https://aps.example/e/pb/bid"); + entry.insert("timeout_ms".to_string(), serde_json::json!(1_000)); + entry.insert("routing".to_string(), serde_json::json!("all_eligible")); + entry.insert( + "notifications".to_string(), + serde_json::to_value(notifications).expect("should serialize notifications"), + ); + if let serde_json::Value::Object(settings) = settings { + entry.extend(settings.clone()); + } + (*id, entry) + }) + .collect::>(); + let mut config = plan_config_with(tables, &[builder()]); + config.timeout_ms = 777; + config + } + + fn aps_config() -> AuctionPlanConfig { + aps_instances_config(&[( + "aps_instance", + serde_json::json!({"account_id": "example-account"}), + NotificationConfig::default(), + )]) + } + + fn planned_request() -> AuctionRequest { + AuctionRequest { + id: "fictional-auction".to_string(), + slots: vec![AdSlot { + id: "fictional-slot".to_string(), + formats: vec![AdFormat { + media_type: MediaType::Banner, + width: 300, + height: 250, + }], + floor_price: Some(1.0), + targeting: HashMap::new(), + bidders: HashMap::new(), + }], + publisher: PublisherInfo { + domain: "publisher.example".to_string(), + page_url: Some("https://publisher.example/article".to_string()), + }, + user: UserInfo { + id: None, + consent: None, + eids: None, + }, + device: None, + site: None, + context: HashMap::new(), + } + } + + /// One APS source as a `[demand.]` table, with nothing set that APS + /// defaults. + fn bare_aps_table() -> serde_json::Map { + let mut table = demand_table(MODULE, "https://aps.example/e/pb/bid"); + table.insert( + "account_id".to_string(), + serde_json::json!("example-account"), + ); + table + } + + fn compile_one( + table: serde_json::Map, + ) -> Result> + { + AuctionPlan::compile(plan_config_with(vec![("aps_one", table)], &[builder()])) + } + + #[test] + fn the_plan_compiles_an_aps_source_with_its_own_defaults() { + let plan = compile_one(bare_aps_table()).expect("should compile an APS source"); + + assert!( + plan.has_implementation(MODULE), + "a compiled plan should say it selected APS" + ); + let source = &plan.providers()[0]; + assert!( + source.demand.as_any().is::(), + "the APS source should compile its own settings" + ); + assert_eq!( + source.timeout_ms, 800, + "should default to the timeout APS declares" + ); + assert_eq!( + source.endpoint.as_str(), + "https://aps.example/e/pb/bid", + "should keep the endpoint as written" + ); + } + + #[test] + fn the_plan_refuses_what_aps_does_not_accept() { + let mut legacy_path = bare_aps_table(); + legacy_path.insert( + "endpoint".to_string(), + serde_json::json!("https://aps.example/e/dtb/bid"), + ); + assert!( + compile_one(legacy_path).is_err(), + "should refuse the legacy APS path" + ); + + let mut domain_alone = bare_aps_table(); + domain_alone.insert( + "inventory_domain".to_string(), + serde_json::json!("publisher.example"), + ); + assert!( + compile_one(domain_alone).is_err(), + "should refuse an APS inventory domain without its page origin" + ); + + let mut without_account = bare_aps_table(); + without_account.remove("account_id"); + assert!( + compile_one(without_account).is_err(), + "should refuse an APS source with no account" + ); + } + + #[test] + fn an_aps_demand_source_registers_the_renderer_with_no_integration_table() { + use trusted_server_core::auction::test_support::demand_selection; + use trusted_server_core::integrations::IntegrationRegistry; + + let mut settings = create_test_settings(); + let mut table = bare_aps_table(); + table.insert("routing".to_string(), serde_json::json!("all_eligible")); + settings.demand = demand_selection(vec![("aps_main", table)]); + let plan = Arc::new( + trusted_server_core::auction::compile_auction_plan_with(&settings, &[builder()]) + .expect("should compile APS plan"), + ); + let registry = + IntegrationRegistry::with_plan_and_registrations(&settings, plan, &[builder()]) + .expect("should build APS renderer registry"); + + assert!(registry.has_route(&http::Method::GET, "/integrations/aps/renderer")); + } + + #[test] + fn two_aps_sources_that_disagree_on_rendering_are_refused() { + use trusted_server_core::auction::test_support::demand_selection; + use trusted_server_core::integrations::IntegrationRegistry; + + let mut settings = create_test_settings(); + let mut publisher_native = bare_aps_table(); + publisher_native.insert( + "rendering_mode".to_string(), + serde_json::json!("publisher_native"), + ); + settings.demand = demand_selection(vec![ + ("aps_one", bare_aps_table()), + ("aps_two", publisher_native), + ]); + let plan = Arc::new( + trusted_server_core::auction::compile_auction_plan_with(&settings, &[builder()]) + .expect("should compile APS plan"), + ); + let error = + match IntegrationRegistry::with_plan_and_registrations(&settings, plan, &[builder()]) { + Ok(_) => panic!("should refuse two rendering modes"), + Err(error) => error, + }; + assert!( + error.to_string().contains("rendering_mode"), + "should name the setting that disagrees: {error:?}" + ); + } + + /// The request the driver builds for one APS source over the canonical + /// request, the way the goldens were captured. + fn golden_request( + settings: serde_json::Value, + signer: Option<&trusted_server_core::request_signing::RequestSigner>, + ) -> trusted_server_core::openrtb::OpenRtbRequest { + let plan = AuctionPlan::compile(golden_plan_config(MODULE, settings, true, &[builder()])) + .expect("should compile plan"); + build_for_first_source( + &plan, + canonical_parity_auction_request(), + &golden_inbound_request(), + 321, + signer, + ) + .expect("should build request") + .expect("should retain impression") + } + + /// The request the driver builds for one APS source over `request`. + fn request_for( + settings: serde_json::Value, + request: AuctionRequest, + accept_language: Option<&str>, + ) -> trusted_server_core::openrtb::OpenRtbRequest { + let plan = AuctionPlan::compile(golden_plan_config(MODULE, settings, true, &[builder()])) + .expect("should compile plan"); + let mut inbound = http::Request::builder().uri("https://publisher.example/auction"); + if let Some(language) = accept_language { + inbound = inbound.header(http::header::ACCEPT_LANGUAGE, language); + } + let inbound = inbound + .body(edgezero_core::body::Body::empty()) + .expect("should build inbound request"); + build_for_first_source(&plan, request, &inbound, 321, None) + .expect("should build request") + .expect("should retain impression") + } + + #[test] + fn consent_fields_keep_an_empty_admitted_context() { + use trusted_server_core::consent::ConsentContext; + use trusted_server_core::consent::jurisdiction::Jurisdiction; + + let cases = [ + ("empty", ConsentContext::default()), + ( + "gdpr", + ConsentContext { + gdpr_applies: true, + raw_tc_string: Some("tc-string".to_string()), + jurisdiction: Jurisdiction::Gdpr, + ..Default::default() + }, + ), + ( + "unknown-gpc", + ConsentContext { + gpc: true, + jurisdiction: Jurisdiction::Unknown, + ..Default::default() + }, + ), + ( + "nonregulated-gpc", + ConsentContext { + gpc: true, + jurisdiction: Jurisdiction::NonRegulated, + ..Default::default() + }, + ), + ( + "usp-gpp", + ConsentContext { + raw_us_privacy: Some("1YNN".to_string()), + raw_gpp_string: Some("gpp-string".to_string()), + gpp_section_ids: Some(vec![7, 8]), + jurisdiction: Jurisdiction::NonRegulated, + ..Default::default() + }, + ), + ]; + for (name, consent) in cases { + let mut canonical = canonical_parity_auction_request(); + canonical.user.consent = Some(consent.clone()); + let value = serde_json::to_value(request_for( + serde_json::json!({"account_id": "example-account-id"}), + canonical, + None, + )) + .expect("should serialize request"); + let regs = value + .get("regs") + .expect("APS should preserve empty admitted context"); + assert_eq!( + regs["gdpr"], + serde_json::json!(u8::from(consent.gdpr_applies)), + "{name}" + ); + assert!( + !value.to_string().contains("1YYY"), + "must never synthesize USP from GPC" + ); + if name == "usp-gpp" { + assert_eq!(regs["us_privacy"], "1YNN"); + assert_eq!(regs["gpp"], "gpp-string"); + assert_eq!(regs["gpp_sid"], serde_json::json!([7, 8])); + assert_eq!(regs["ext"]["us_privacy"], "1YNN"); + assert_eq!(regs["ext"]["gpp"], "gpp-string"); + assert_eq!(regs["ext"]["gpp_sid"], serde_json::json!([7, 8])); + } + } + } + + #[test] + fn language_is_kept_only_within_the_conservative_limit() { + let settings = serde_json::json!({"account_id": "example-account-id"}); + + let long = request_for( + settings.clone(), + canonical_parity_auction_request(), + Some("abcdefghijk"), + ); + assert_eq!( + long.device.and_then(|device| device.language).as_deref(), + None, + "should drop a language tag over the limit" + ); + + let ordinary = request_for( + settings, + canonical_parity_auction_request(), + Some("en-US,en;q=0.9"), + ); + assert_eq!( + ordinary + .device + .and_then(|device| device.language) + .as_deref(), + Some("en") + ); + } + + #[test] + fn inventory_identity_and_page_fallback_preserve_legacy_policy() { + let mut request = canonical_parity_auction_request(); + request.publisher.domain = "deployment.example".to_string(); + request.publisher.page_url = + Some("https://deployment.example/news/story?edition=fictional#section".to_string()); + let built = request_for( + serde_json::json!({ + "account_id": "example-account-id", + "inventory_domain": "publisher.example", + "inventory_page_origin": "https://www.publisher.example" + }), + request, + None, + ); + let site = built.site.expect("should include APS site"); + assert_eq!(site.domain.as_deref(), Some("publisher.example")); + assert_eq!( + site.page.as_deref(), + Some("https://www.publisher.example/news/story?edition=fictional") + ); + assert_eq!( + site.publisher + .and_then(|publisher| publisher.domain) + .as_deref(), + Some("publisher.example") + ); + + for unsafe_page in [ + "https://user:password@publisher.example/private", + "data:text/html,fictional", + ] { + let mut request = canonical_parity_auction_request(); + request.publisher.page_url = Some(unsafe_page.to_string()); + let built = request_for( + serde_json::json!({"account_id":"example-account-id"}), + request, + None, + ); + assert_eq!( + built.site.and_then(|site| site.page).as_deref(), + Some("https://publisher.example"), + "unsafe page should fall back to publisher domain" + ); + } + } + + #[test] + fn the_driver_request_matches_the_exact_golden() { + let request = golden_request( + serde_json::json!({"account_id": "example-account-id"}), + None, + ); + assert_eq!( + serde_json::to_string(&request).expect("should serialize APS driver request"), + r#"{"id":"fictional-auction","imp":[{"id":"fictional-slot","banner":{"format":[{"w":300,"h":250},{"w":728,"h":90}],"w":300,"h":250,"topframe":0},"bidfloor":1.0,"bidfloorcur":"USD","secure":1}],"site":{"domain":"publisher.example","page":"https://publisher.example/article","publisher":{"domain":"publisher.example"}},"device":{"geo":{"type":2,"country":"US","region":"CA","metro":"501","city":"Example City"},"dnt":1,"ua":"Fictional Browser","ip":"192.0.2.10","language":"en"},"user":{"id":"fictional-user","consent":"fictional-tcf","ext":{"consent":"fictional-tcf","eids":[{"source":"identity.example","uids":[{"atype":1,"id":"fictional-uid"}]}]}},"tmax":321,"cur":["USD"],"regs":{"gdpr":1,"us_privacy":"1YNN","gpp":"fictional-gpp","gpp_sid":[2,6],"ext":{"gdpr":1,"gpp":"fictional-gpp","gpp_sid":[2,6],"us_privacy":"1YNN"}},"ext":{"account":"example-account-id","sdk":{"source":"prebid","version":"2.2.0"}}}"#, + "should preserve APS parity differences" + ); + } + + #[test] + fn signing_is_applied_after_the_request_is_built_and_sets_every_owned_key() { + let settings = serde_json::json!({"account_id": "example-account-id"}); + let unsigned = serde_json::to_value(golden_request(settings.clone(), None)) + .expect("should serialize unsigned request"); + assert!( + unsigned["ext"].get("trusted_server").is_none(), + "should omit the extension when unsigned" + ); + + let signer = deterministic_signer(); + let signed = serde_json::to_value(golden_request(settings, Some(&signer))) + .expect("should serialize signed request"); + let extension = &signed["ext"]["trusted_server"]; + assert_eq!(extension["version"], "1.1", "should set signing version"); + assert_eq!(extension["kid"], "fictional-kid", "should set key ID"); + assert_eq!( + extension["request_host"], "publisher.example", + "should set host" + ); + assert_eq!(extension["request_scheme"], "https", "should set scheme"); + assert_eq!( + extension["ts"], 1_706_900_000_u64, + "should set explicit time" + ); + assert!( + extension["signature"] + .as_str() + .is_some_and(|value| !value.is_empty()), + "should set signature" + ); + } + + #[test] + fn the_signed_driver_request_matches_the_exact_golden() { + let signer = deterministic_signer(); + let request = golden_request( + serde_json::json!({"account_id": "example-account-id"}), + Some(&signer), + ); + assert_eq!( + serde_json::to_string(&request).expect("should serialize signed request"), + r#"{"id":"fictional-auction","imp":[{"id":"fictional-slot","banner":{"format":[{"w":300,"h":250},{"w":728,"h":90}],"w":300,"h":250,"topframe":0},"bidfloor":1.0,"bidfloorcur":"USD","secure":1}],"site":{"domain":"publisher.example","page":"https://publisher.example/article","publisher":{"domain":"publisher.example"}},"device":{"geo":{"type":2,"country":"US","region":"CA","metro":"501","city":"Example City"},"dnt":1,"ua":"Fictional Browser","ip":"192.0.2.10","language":"en"},"user":{"id":"fictional-user","consent":"fictional-tcf","ext":{"consent":"fictional-tcf","eids":[{"source":"identity.example","uids":[{"atype":1,"id":"fictional-uid"}]}]}},"tmax":321,"cur":["USD"],"regs":{"gdpr":1,"us_privacy":"1YNN","gpp":"fictional-gpp","gpp_sid":[2,6],"ext":{"gdpr":1,"gpp":"fictional-gpp","gpp_sid":[2,6],"us_privacy":"1YNN"}},"ext":{"account":"example-account-id","sdk":{"source":"prebid","version":"2.2.0"},"trusted_server":{"kid":"fictional-kid","request_host":"publisher.example","request_scheme":"https","signature":"LU_JUIA1BT80ShZNjSa4PIF5T-uMjEeodwKrV_6bXgh0hi1SYVtCKn9g_DTW62krmjCOFgoFYPHsu6L0nAcuDg","ts":1706900000,"version":"1.1"}}}"#, + "signed wire fixture should stay exact" + ); + } + + /// Settings whose `[demand]` selects one APS source with `account_id` + /// and the settings `extra` adds. + fn settings_with_aps_source( + account_id: &str, + extra: &[(&str, serde_json::Value)], + ) -> trusted_server_core::settings::Settings { + let mut settings = create_test_settings(); + let mut table = demand_table(MODULE, "https://aps.example.com/e/pb/bid"); + table.insert("account_id".to_string(), serde_json::json!(account_id)); + for (key, value) in extra { + table.insert((*key).to_string(), value.clone()); + } + settings.demand = + trusted_server_core::auction::test_support::demand_selection(vec![("aps_main", table)]); + settings + } + + fn validate_for_deploy( + settings: &trusted_server_core::settings::Settings, + ) -> Result<(), error_stack::Report> { + trusted_server_core::config::validate_settings_for_deploy_with(settings, &[builder()]) + } + + #[test] + fn deploy_validation_rejects_a_blank_account_id() { + for (label, account_id) in [("empty", ""), ("whitespace-only", " ")] { + let err = validate_for_deploy(&settings_with_aps_source(account_id, &[])) + .expect_err("should reject blank APS account_id"); + + assert!( + format!("{err:?}").contains("account_id"), + "should mention the APS account_id for {label}: {err:?}" + ); + } + } + + #[test] + fn deploy_validation_normalizes_a_padded_account_id() { + validate_for_deploy(&settings_with_aps_source(" example-account ", &[])) + .expect("should accept a padded APS account_id after trimming it"); + } + + #[test] + fn deploy_validation_rejects_a_setting_aps_does_not_know() { + let settings = + settings_with_aps_source("example-account", &[("enabled", serde_json::json!(false))]); + + let error = validate_for_deploy(&settings) + .expect_err("should reject a setting the APS implementation does not know"); + let rendered = format!("{error:?}"); + assert!( + rendered.contains("aps_main"), + "should identify the demand source: {rendered}" + ); + assert!( + rendered.contains("enabled"), + "should identify the setting it does not know: {rendered}" + ); + } + + #[tokio::test] + async fn planned_aps_transport_omits_accept_header() { + let http = Arc::new(StubHttpClient::new()); + http.push_response(400, Vec::new()); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Fastly)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = AuctionPlan::compile(aps_config()).expect("should compile planned APS auction"); + let orchestrator = AuctionOrchestrator::from_plan(Arc::new(plan), None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::builder() + .uri("https://publisher.example/auction") + .body(edgezero_core::body::Body::empty()) + .expect("should build inbound request"); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + orchestrator + .run_auction(&request, &context) + .await + .expect("should execute planned APS auction"); + + let headers = http.recorded_request_headers(); + assert_eq!(headers.len(), 1); + assert!( + headers[0].iter().all(|(name, _)| name != "accept"), + "planned APS transport must not add Accept beyond legacy headers" + ); + } + + #[tokio::test] + async fn planned_aps_profile_normalizes_renderer_reduction_and_metadata() { + let http = Arc::new(StubHttpClient::new()); + http.push_response_with_headers( + 200, + serde_json::to_vec(&serde_json::json!({ + "cur": "USD", + "seatbid": [ + {"seat": "returned-seat", "bid": [ + {"id": "z-high", "impid": "fictional-slot", "price": 2.0, "w": 300, "h": 250, + "nurl": "https://notice.example/win", "burl": "https://notice.example/bill", + "crid": "fictional-creative", "adomain": ["advertiser.example"], + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, + {"id": "a-high", "impid": "fictional-slot", "price": 2.0, "w": 300, "h": 250, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, + {"id": "bad-script", "impid": "fictional-slot", "price": 9.0, "w": 300, "h": 250, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "script"}}, + {"id": "bad-domain", "impid": "fictional-slot", "price": 8.0, "w": 300, "h": 250, + "ext": {"creativeurl": "https://publisher.example/render", "tagtype": "iframe"}}, + {"id": "bad-credentials", "impid": "fictional-slot", "price": 8.0, "w": 300, "h": 250, + "ext": {"creativeurl": "https://user:password@creative.example/render", "tagtype": "iframe"}}, + {"id": "bad-imp", "impid": "unknown-slot", "price": 8.0, "w": 300, "h": 250, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, + {"id": "bad-dimensions", "impid": "fictional-slot", "price": 8.0, "w": 320, "h": 50, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, + {"id": "bad-price", "impid": "fictional-slot", "price": "high", "w": 300, "h": 250, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, + {"id": "bad-mtype", "impid": "fictional-slot", "price": 8.0, "mtype": 2, "w": 300, "h": 250, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, + {"id": "bad-tag", "impid": "fictional-slot", "price": 8.0, "w": 300, "h": 250, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "native"}}, + {"id": "bad-crid", "impid": "fictional-slot", "price": 8.0, "w": 300, "h": 250, + "crid": "x".repeat(1025), + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, + {"impid": "fictional-slot", "price": 8.0, "w": 300, "h": 250, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}} + ]}, + {"seat": 7, "bid": "bad-shape"} + ] + })) + .expect("should serialize APS profile response"), + vec![ + ("content-type", "application/json"), + ("authorization", "fictional-secret"), + ], + ); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Fastly)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = AuctionPlan::compile(aps_instances_config(&[( + "aps_instance", + serde_json::json!({"account_id": "example-account", "debug": true}), + NotificationConfig { + suppress_all: false, + suppress_seats: vec!["returned-seat".to_string()], + }, + )])) + .expect("should compile planned APS profile"); + let orchestrator = AuctionOrchestrator::from_plan(Arc::new(plan), None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should execute planned APS profile"); + + let response = &result.provider_responses[0]; + assert_eq!(response.provider, "aps_instance"); + assert_eq!(response.status, BidStatus::Success); + assert_eq!( + response.bids.len(), + 1, + "should retain one bid per impression" + ); + let bid = &response.bids[0]; + assert_eq!(bid.bidder, "aps"); + assert_eq!(bid.returned_seat.as_deref(), Some("returned-seat")); + assert_eq!( + bid.bid_id.as_deref(), + Some("a-high"), + "lexical ID should break equal-price tie" + ); + assert!(bid.creative.is_none()); + assert!( + bid.nurl.is_none() && bid.burl.is_none(), + "APS must discard notification URLs" + ); + let renderer = bid + .renderer + .as_ref() + .and_then(|renderer| renderer.payload_as::(APS_RENDERER_TYPE)) + .expect("should construct typed APS renderer"); + assert_eq!(renderer.account_id, "example-account"); + let decoded = base64::engine::general_purpose::STANDARD + .decode(&renderer.aax_response) + .expect("should decode minimized APS response"); + assert_eq!( + serde_json::from_slice::(&decoded) + .expect("should parse minimized APS response"), + serde_json::json!({"seatbid":[{"bid":[{ + "id":"a-high","price":2.0,"w":300,"h":250, + "ext":{"creativeurl":"https://creative.example/render","tagtype":"iframe"} + }]}]}) + ); + assert_eq!(response.metadata["seatbid_count"], 2); + assert_eq!(response.metadata["accepted_bid_count"], 1); + assert_eq!(response.metadata["dropped_bid_count"], 12); + for reason in [ + "lost_to_higher_bid", + "script_rendering_disabled", + "unknown_impid", + "invalid_dimensions", + "invalid_price", + "unsupported_media_type", + "unsupported_tagtype", + "creative_id_too_large", + "missing_render_source", + "empty_seatbid_bids", + ] { + assert_eq!(response.metadata["drop_reasons"][reason], 1, "{reason}"); + } + assert_eq!( + response.metadata["drop_reasons"]["invalid_creative_url"], 2, + "same-publisher and credentialed URLs should both be rejected" + ); + assert_eq!( + response.metadata["routing"]["unused_bidder_params_count"], + 0 + ); + let debug = &response.metadata["debug"]["httpcalls"]["aps"][0]; + assert_eq!(debug["uri"], "https://aps.example/e/pb/bid"); + assert_eq!( + debug["responseheaders"], + serde_json::json!({"content-type": ["application/json"]}), + "async stub should preserve allowlisted response headers" + ); + assert!( + debug["requestbody"] + .as_str() + .is_some_and(|body| body.contains("example-account")) + ); + assert!(debug["requestheaders"].get("authorization").is_none()); + assert!(debug["responseheaders"].get("authorization").is_none()); + } + + #[tokio::test] + async fn two_planned_aps_instances_correlate_independently() { + let http = Arc::new(StubHttpClient::new()); + for (seat, id, price) in [("seat-a", "bid-a", 1.0), ("seat-b", "bid-b", 2.0)] { + http.push_response( + 200, + serde_json::to_vec(&serde_json::json!({"seatbid":[{"seat":seat,"bid":[{ + "id":id,"impid":"fictional-slot","price":price,"w":300,"h":250, + "ext":{"creativeurl":"https://creative.example/render","tagtype":"iframe"} + }]}]})) + .expect("should serialize APS instance response"), + ); + } + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Fastly)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = AuctionPlan::compile(aps_instances_config(&[ + ( + "aps_a", + serde_json::json!({"account_id":"account-a"}), + NotificationConfig::default(), + ), + ( + "aps_b", + serde_json::json!({"account_id":"account-b"}), + NotificationConfig::default(), + ), + ])) + .expect("should compile two APS instances"); + let orchestrator = AuctionOrchestrator::from_plan(Arc::new(plan), None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should execute two APS instances"); + + assert_eq!(result.provider_responses.len(), 2); + assert_eq!(result.provider_responses[0].provider, "aps_a"); + assert_eq!( + result.provider_responses[0].bids[0].bid_id.as_deref(), + Some("bid-a") + ); + assert_eq!(result.provider_responses[1].provider, "aps_b"); + assert_eq!( + result.provider_responses[1].bids[0].bid_id.as_deref(), + Some("bid-b") + ); + assert_eq!(http.recorded_request_bodies().len(), 2); + assert_eq!( + result.winning_bids["fictional-slot"].bid_id.as_deref(), + Some("bid-b"), + "global ranking should remain orchestrator-owned" + ); + let specs = backend.specs.lock().expect("should lock specs"); + assert_eq!(specs.len(), 2); + assert_ne!(specs[0].discriminator, specs[1].discriminator); + } + + #[tokio::test] + async fn planned_aps_returned_seat_accepts_only_valid_nonempty_strings() { + let plan = AuctionPlan::compile(aps_config()).expect("should compile APS plan"); + for (seat, expected) in [ + (serde_json::Value::Null, None), + (serde_json::json!(7), None), + (serde_json::json!(""), None), + (serde_json::json!("exact-seat"), Some("exact-seat")), + ] { + let response = PlatformResponse::new( + edgezero_core::http::response_builder() + .status(200) + .body(edgezero_core::body::Body::from( + serde_json::to_vec(&serde_json::json!({"seatbid":[{"seat":seat,"bid":[{ + "id":"bid","impid":"fictional-slot","price":1.0,"w":300,"h":250, + "nurl":"https://notice.example/win","burl":"https://notice.example/bill", + "ext":{"creativeurl":"https://creative.example/render","tagtype":"iframe"} + }]}]})) + .expect("should serialize seat identity response"), + )) + .expect("should build seat identity response"), + ); + let parsed = parse_as_first_source(&plan, planned_request(), response, 4) + .await + .expect("should parse seat identity response"); + assert_eq!(parsed.bids[0].returned_seat.as_deref(), expected); + assert!(parsed.bids[0].nurl.is_none() && parsed.bids[0].burl.is_none()); + } + } + + #[tokio::test] + async fn planned_aps_response_status_shape_and_currency_matrix() { + let plan = AuctionPlan::compile(aps_config()).expect("should compile APS plan"); + let cases = [ + (204, Vec::new(), BidStatus::NoBid, None, None), + (400, Vec::new(), BidStatus::Error, None, Some("http_status")), + ( + 200, + b"not-json".to_vec(), + BidStatus::Error, + Some("unexpected_response_shape"), + Some("parse_response"), + ), + ( + 200, + b"[]".to_vec(), + BidStatus::Error, + Some("unexpected_response_shape"), + Some("parse_response"), + ), + ( + 200, + br#"{"contextual":true}"#.to_vec(), + BidStatus::Error, + Some("unexpected_response_shape"), + Some("parse_response"), + ), + ( + 200, + br#"{"cur":"EUR","seatbid":[]}"#.to_vec(), + BidStatus::NoBid, + Some("unsupported_currency"), + None, + ), + ]; + for (status, body, expected, reason, error_type) in cases { + let response = PlatformResponse::new( + edgezero_core::http::response_builder() + .status(status) + .body(edgezero_core::body::Body::from(body)) + .expect("should build APS matrix response"), + ); + let parsed = parse_as_first_source(&plan, planned_request(), response, 4) + .await + .expect("should classify APS matrix response"); + assert_eq!(parsed.status, expected, "status {status}"); + if let Some(reason) = reason { + assert_eq!( + parsed.metadata["drop_reasons"][reason], 1, + "status {status}" + ); + } + if let Some(error_type) = error_type { + assert_eq!(parsed.metadata["error_type"], error_type, "status {status}"); + } + } + } + + #[tokio::test] + async fn planned_aps_script_opt_in_matches_shared_renderer_fixture() { + let plan = AuctionPlan::compile(aps_instances_config(&[( + "aps_instance", + serde_json::json!({ + "account_id":"example-account-id", + "allow_script_creatives":true + }), + NotificationConfig::default(), + )])) + .expect("should compile script-enabled APS plan"); + let response = PlatformResponse::new( + edgezero_core::http::response_builder() + .status(200) + .body(edgezero_core::body::Body::from( + serde_json::to_vec(&serde_json::json!({"seatbid":[{"bid":[{ + "id":"fictional-selected-bid-id","impid":"fictional-slot","price":1.23, + "w":300,"h":250,"crid":"fictional-creative", + "ext":{"creativeurl":"https://creative.example/render","tagtype":"iframe"} + },{ + "id":"script-bid","impid":"fictional-slot","price":1.0, + "w":300,"h":250, + "ext":{"creativeurl":"https://creative.example/script","tagtype":"script"} + }]}]})) + .expect("should serialize APS renderer fixture response"), + )) + .expect("should build APS renderer fixture response"), + ); + + let parsed = parse_as_first_source(&plan, planned_request(), response, 3) + .await + .expect("should parse APS renderer fixture response"); + + assert_eq!(parsed.status, BidStatus::Success); + assert_eq!( + parsed.metadata["drop_reasons"]["lost_to_higher_bid"], 1, + "enabled script creative should be eligible before reduction" + ); + let renderer = parsed.bids[0] + .renderer + .as_ref() + .and_then(|renderer| renderer.payload_as::(APS_RENDERER_TYPE)) + .expect("should construct APS renderer"); + let decoded = base64::engine::general_purpose::STANDARD + .decode(&renderer.aax_response) + .expect("should decode APS fixture envelope"); + let fixture: serde_json::Value = serde_json::from_str(include_str!( + "../../../trusted-server-js/lib/test/fixtures/aps-renderer-v1.json" + )) + .expect("should parse shared APS renderer fixture"); + assert_eq!( + serde_json::from_slice::(&decoded) + .expect("should parse decoded APS renderer"), + fixture + ); + } + + #[tokio::test] + async fn planned_aps_debug_response_headers_are_allowlisted() { + let plan = AuctionPlan::compile(aps_instances_config(&[( + "aps_instance", + serde_json::json!({"account_id":"example-account","debug":true}), + NotificationConfig::default(), + )])) + .expect("should compile debug APS plan"); + let response = PlatformResponse::new( + edgezero_core::http::response_builder() + .status(200) + .header("content-type", "application/json") + .header("authorization", "fictional-secret") + .body(edgezero_core::body::Body::from("{}")) + .expect("should build debug APS response"), + ); + + let parsed = parse_as_first_source(&plan, planned_request(), response, 3) + .await + .expect("should parse debug APS response"); + + let headers = &parsed.metadata["debug"]["httpcalls"]["aps"][0]["responseheaders"]; + assert_eq!( + headers, + &serde_json::json!({"content-type":["application/json"]}) + ); + assert!(headers.get("authorization").is_none()); + } + + #[tokio::test] + async fn planned_routing_count_survives_a_bounded_body_failure() { + let (profile, provider_id) = ("aps", "aps_instance"); + let http = Arc::new(StubHttpClient::new()); + http.push_response(200, vec![b'x'; 1024 * 1024 + 1]); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let mut config = aps_config(); + config.bidders.insert( + "example-bidder" + .parse() + .expect("should parse fictional bidder ID"), + trusted_server_core::auction::plan::BidderRouteConfig { + module: provider_id + .parse() + .expect("should parse fictional provider ID"), + }, + ); + let plan = AuctionPlan::compile(config).expect("should compile bounded-body plan"); + let orchestrator = AuctionOrchestrator::from_plan(Arc::new(plan), None); + let mut request = planned_request(); + request.slots[0].bidders.insert( + "example-bidder".to_string(), + serde_json::json!({"private": "value"}), + ); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should materialize bounded-body failure"); + let response = &result.provider_responses[0]; + assert_eq!(response.status, BidStatus::Error, "{profile}"); + assert_eq!( + response.metadata["routing"]["unused_bidder_params_count"], 1, + "{profile} bounded-body failure should retain the input-derived count" + ); + let routing = serde_json::to_string(&response.metadata["routing"]) + .expect("should serialize routing metadata"); + assert!(!routing.contains("example-bidder") && !routing.contains("private")); + } } diff --git a/crates/trusted-server-core/src/auction/formats.rs b/crates/trusted-server-core/src/auction/formats.rs index 3d7a11d600..ce1b01684e 100644 --- a/crates/trusted-server-core/src/auction/formats.rs +++ b/crates/trusted-server-core/src/auction/formats.rs @@ -559,7 +559,6 @@ mod tests { use crate::auction::routing::route_auction; use crate::auction::test_support::{demand_table, plan_config}; use crate::auction::types::{AuctionResponse, Bid, BidRenderer, BidStatus}; - use crate::integrations::aps::{APS_RENDERER_TYPE, ApsRendererV1, ApsTagType}; use crate::openrtb::{Eid, Uid}; use crate::platform::test_support::noop_services; use crate::test_support::tests::create_test_settings; @@ -1521,26 +1520,21 @@ mod tests { rejected.creative = Some("".to_string()); let unpriced = make_bid("unpriced", "invalid", None); let ordinary = make_bid("ordinary", "appnexus", Some(2.75)); - let mut renderer = make_bid("renderer", "aps", Some(2.5)); + let mut renderer = make_bid("renderer", "example", Some(2.5)); renderer.creative = Some("".to_string()); renderer.bid_id = Some("upstream-renderer-bid".to_string()); renderer.creative_id = None; renderer.renderer = Some( - BidRenderer::from_typed( - APS_RENDERER_TYPE, - &ApsRendererV1 { - version: 1, - account_id: "example-account".to_string(), - bid_id: "upstream-renderer-bid".to_string(), - creative_id: None, - tag_type: ApsTagType::Iframe, - creative_url: "https://creative.example/render".to_string(), - aax_response: "fictional-base64".to_string(), - width: 300, - height: 250, - }, + BidRenderer::new( + "example", + json!({ + "version": 1, + "bidId": "upstream-renderer-bid", + "creativeUrl": "https://creative.example/render", + "envelope": "fictional-base64", + }), ) - .expect("the APS renderer payload should be a JSON object"), + .expect("should build the example renderer descriptor"), ); let result = OrchestrationResult { provider_responses: vec![], @@ -1628,23 +1622,18 @@ mod tests { fn convert_to_openrtb_response_prefers_creative_when_both_render_sources_exist() { let settings = make_settings(); let auction_request = make_auction_request(); - let mut bid = make_bid("div-gpt-top", "aps", Some(2.75)); + let mut bid = make_bid("div-gpt-top", "example", Some(2.75)); bid.renderer = Some( - BidRenderer::from_typed( - APS_RENDERER_TYPE, - &ApsRendererV1 { - version: 1, - account_id: "example-account".to_string(), - bid_id: "fictional-bid".to_string(), - creative_id: None, - tag_type: ApsTagType::Iframe, - creative_url: "https://creative.example/render".to_string(), - aax_response: "fictional-base64".to_string(), - width: 300, - height: 250, - }, + BidRenderer::new( + "example", + json!({ + "version": 1, + "bidId": "fictional-bid", + "creativeUrl": "https://creative.example/render", + "envelope": "fictional-base64", + }), ) - .expect("the APS renderer payload should be a JSON object"), + .expect("should build the example renderer descriptor"), ); let result = make_result(bid); @@ -1667,35 +1656,30 @@ mod tests { } #[test] - fn convert_to_openrtb_response_emits_typed_aps_renderer_without_adm() { + fn convert_to_openrtb_response_emits_a_typed_renderer_without_adm() { let settings = make_settings(); let auction_request = make_auction_request(); - let mut bid = make_bid("div-gpt-top", "aps", Some(2.75)); + let mut bid = make_bid("div-gpt-top", "example", Some(2.75)); bid.creative = None; bid.bid_id = Some("fictional-bid".to_string()); bid.ad_id = Some("fictional-ad".to_string()); bid.creative_id = Some("fictional-creative".to_string()); bid.renderer = Some( - BidRenderer::from_typed( - APS_RENDERER_TYPE, - &ApsRendererV1 { - version: 1, - account_id: "example-account".to_string(), - bid_id: "fictional-bid".to_string(), - creative_id: Some("fictional-creative".to_string()), - tag_type: ApsTagType::Iframe, - creative_url: "https://creative.example/render".to_string(), - aax_response: "fictional-base64".to_string(), - width: 300, - height: 250, - }, + BidRenderer::new( + "example", + json!({ + "version": 1, + "bidId": "fictional-bid", + "creativeUrl": "https://creative.example/render", + "envelope": "fictional-base64", + }), ) - .expect("the APS renderer payload should be a JSON object"), + .expect("should build the example renderer descriptor"), ); let result = make_result(bid); let response = convert_to_openrtb_response(&result, &settings, &auction_request, false) - .expect("should convert APS renderer bid"); + .expect("should convert a typed renderer bid"); let json = response_json(response); let bid = json["seatbid"][0]["bid"][0] .as_object() @@ -1706,12 +1690,12 @@ mod tests { "should omit adm for renderer bids" ); assert_eq!(bid["id"], json!("fictional-bid")); - assert_eq!(json["seatbid"][0]["seat"], json!("aps")); + assert_eq!(json["seatbid"][0]["seat"], json!("example")); assert_eq!(bid["adid"], json!("fictional-ad")); assert_eq!(bid["crid"], json!("fictional-creative")); assert_eq!( bid["ext"]["trusted_server"]["renderer"]["type"], - json!("aps") + json!("example") ); assert_eq!( bid["ext"]["trusted_server"]["renderer"]["bidId"], diff --git a/crates/trusted-server-core/src/auction/mod.rs b/crates/trusted-server-core/src/auction/mod.rs index 7d792deeda..f97d074264 100644 --- a/crates/trusted-server-core/src/auction/mod.rs +++ b/crates/trusted-server-core/src/auction/mod.rs @@ -23,8 +23,13 @@ pub mod plan; pub mod provider; pub(crate) mod routing; pub mod telemetry; -#[cfg(test)] -pub(crate) mod test_support; +#[cfg(any(test, feature = "test-utils"))] +#[allow( + clippy::must_use_candidate, + clippy::missing_panics_doc, + reason = "test helpers, offered to other crates' tests under the test-utils feature" +)] +pub mod test_support; pub mod types; pub use config::AuctionConfig; @@ -204,7 +209,7 @@ mod auction_token_tests { #[cfg(test)] mod plan_sharing_tests { use super::*; - use crate::auction::test_support::{demand_named, demand_selection, demand_table}; + use crate::auction::test_support::demand_named; use crate::integrations::IntegrationRegistry; use crate::provider_table::ProviderChoice; use crate::test_support::tests::create_test_settings; @@ -339,40 +344,4 @@ mod plan_sharing_tests { ); } } - - #[test] - fn an_aps_demand_source_registers_the_renderer_with_no_integration_table() { - let mut settings = create_test_settings(); - let mut table = demand_table("auction.aps", "https://aps.example/e/pb/bid"); - table.insert("routing".to_string(), json!("all_eligible")); - settings.demand = demand_selection(vec![("aps_main", table)]); - let plan = Arc::new(compile_auction_plan(&settings).expect("should compile APS plan")); - let registry = IntegrationRegistry::with_plan(&settings, plan) - .expect("should build APS renderer registry"); - - assert!(registry.has_route(&http::Method::GET, "/integrations/aps/renderer")); - } - - #[test] - fn two_aps_sources_that_disagree_on_rendering_are_refused() { - let mut settings = create_test_settings(); - let mut publisher_native = demand_table("auction.aps", "https://aps.example/e/pb/bid"); - publisher_native.insert("rendering_mode".to_string(), json!("publisher_native")); - settings.demand = demand_selection(vec![ - ( - "aps_one", - demand_table("auction.aps", "https://aps.example/e/pb/bid"), - ), - ("aps_two", publisher_native), - ]); - let plan = Arc::new(compile_auction_plan(&settings).expect("should compile APS plan")); - let error = match IntegrationRegistry::with_plan(&settings, plan) { - Ok(_) => panic!("should refuse two rendering modes"), - Err(error) => error, - }; - assert!( - error.to_string().contains("rendering_mode"), - "should name the setting that disagrees: {error:?}" - ); - } } diff --git a/crates/trusted-server-core/src/auction/openrtb.rs b/crates/trusted-server-core/src/auction/openrtb.rs index 555e6912dc..168eb26f1b 100644 --- a/crates/trusted-server-core/src/auction/openrtb.rs +++ b/crates/trusted-server-core/src/auction/openrtb.rs @@ -776,8 +776,11 @@ pub(crate) fn unused_bidder_params_count( } /// Count routed bidder params for a demand source known to ignore them. +/// +/// Public so an implementation that takes no bidder parameters reports the +/// ones it was routed the way core's own reader does. #[must_use] -pub(crate) fn ignored_bidder_params_count(input: &ProviderAuctionInput) -> u32 { +pub fn ignored_bidder_params_count(input: &ProviderAuctionInput) -> u32 { saturating_bidder_param_counts(input.slots().iter().map(|slot| slot.bidder_params().len())) } @@ -814,13 +817,8 @@ mod routing_metadata_tests { for (implementation, expected) in [ ("auction.prebid-server", 0), ("auction-protocol.openrtb", 1), - ("auction.aps", 1), ] { - let endpoint = if implementation == "auction.aps" { - "https://aps.example/e/pb/bid" - } else { - "https://provider.example/openrtb" - }; + let endpoint = "https://provider.example/openrtb"; let provider_id = ProviderId::from_str("fictional_provider").expect("should parse provider ID"); let mut config = plan_config(vec![( diff --git a/crates/trusted-server-core/src/auction/openrtb/tests.rs b/crates/trusted-server-core/src/auction/openrtb/tests.rs index 65cea93da3..c7007e7f78 100644 --- a/crates/trusted-server-core/src/auction/openrtb/tests.rs +++ b/crates/trusted-server-core/src/auction/openrtb/tests.rs @@ -164,7 +164,7 @@ fn deterministic_signer() -> RequestSigner { } #[test] -fn consent_matrix_preserves_pbs_standard_and_aps_policies() { +fn consent_matrix_preserves_pbs_and_standard_policies() { let cases = [ ("empty", ConsentContext::default()), ( @@ -204,30 +204,18 @@ fn consent_matrix_preserves_pbs_standard_and_aps_policies() { ), ]; for (name, consent) in cases { - for implementation in [ - "auction-protocol.openrtb", - "auction.prebid-server", - "auction.aps", - ] { + for implementation in ["auction-protocol.openrtb", "auction.prebid-server"] { let mut canonical = canonical_parity_auction_request(); canonical.user.consent = Some(consent.clone()); - let config = if implementation == "auction.aps" { - json!({"account_id": "example-account-id"}) - } else { - json!({}) - }; - let value = - serde_json::to_value(build_with_request(implementation, config, canonical, None)) - .expect("should serialize request"); + let value = serde_json::to_value(build_with_request( + implementation, + json!({}), + canonical, + None, + )) + .expect("should serialize request"); let regs = value.get("regs"); - if implementation == "auction.aps" { - let regs = regs.expect("APS should preserve empty admitted context"); - assert_eq!( - regs["gdpr"], - json!(u8::from(consent.gdpr_applies)), - "{name}" - ); - } else if name == "empty" { + if name == "empty" { assert!(regs.is_none(), "{implementation} should omit empty regs"); } else { let regs = regs.expect("should emit actionable regs"); @@ -296,17 +284,11 @@ fn language_limits_are_profile_specific() { let language = "abcdefghijk"; for (implementation, expected) in [ ("auction.prebid-server", Some(language)), - ("auction.aps", None), ("auction-protocol.openrtb", None), ] { - let config = if implementation == "auction.aps" { - json!({"account_id": "example-account-id"}) - } else { - json!({}) - }; let request = build_with_request( implementation, - config, + json!({}), canonical_parity_auction_request(), Some(language), ); @@ -315,19 +297,10 @@ fn language_limits_are_profile_specific() { expected ); } - for implementation in [ - "auction.prebid-server", - "auction.aps", - "auction-protocol.openrtb", - ] { - let config = if implementation == "auction.aps" { - json!({"account_id": "example-account-id"}) - } else { - json!({}) - }; + for implementation in ["auction.prebid-server", "auction-protocol.openrtb"] { let request = build_with_request( implementation, - config, + json!({}), canonical_parity_auction_request(), Some("en-US,en;q=0.9"), ); @@ -772,69 +745,6 @@ fn pbs_driver_exact_golden_preserves_profile_policy() { ); } -#[test] -fn aps_inventory_identity_and_page_fallback_preserve_legacy_policy() { - let mut request = canonical_parity_auction_request(); - request.publisher.domain = "deployment.example".to_string(); - request.publisher.page_url = - Some("https://deployment.example/news/story?edition=fictional#section".to_string()); - let built = build_with_request( - "auction.aps", - json!({ - "account_id": "example-account-id", - "inventory_domain": "publisher.example", - "inventory_page_origin": "https://www.publisher.example" - }), - request, - None, - ); - let site = built.site.expect("should include APS site"); - assert_eq!(site.domain.as_deref(), Some("publisher.example")); - assert_eq!( - site.page.as_deref(), - Some("https://www.publisher.example/news/story?edition=fictional") - ); - assert_eq!( - site.publisher - .and_then(|publisher| publisher.domain) - .as_deref(), - Some("publisher.example") - ); - - for unsafe_page in [ - "https://user:password@publisher.example/private", - "data:text/html,fictional", - ] { - let mut request = canonical_parity_auction_request(); - request.publisher.page_url = Some(unsafe_page.to_string()); - let built = build_with_request( - "auction.aps", - json!({"account_id":"example-account-id"}), - request, - None, - ); - assert_eq!( - built.site.and_then(|site| site.page).as_deref(), - Some("https://publisher.example"), - "unsafe page should fall back to publisher domain" - ); - } -} - -#[test] -fn aps_driver_exact_golden_preserves_profile_policy() { - let request = build( - "auction.aps", - json!({"account_id": "example-account-id"}), - None, - ); - assert_eq!( - serde_json::to_string(&request).expect("should serialize APS driver request"), - r#"{"id":"fictional-auction","imp":[{"id":"fictional-slot","banner":{"format":[{"w":300,"h":250},{"w":728,"h":90}],"w":300,"h":250,"topframe":0},"bidfloor":1.0,"bidfloorcur":"USD","secure":1}],"site":{"domain":"publisher.example","page":"https://publisher.example/article","publisher":{"domain":"publisher.example"}},"device":{"geo":{"type":2,"country":"US","region":"CA","metro":"501","city":"Example City"},"dnt":1,"ua":"Fictional Browser","ip":"192.0.2.10","language":"en"},"user":{"id":"fictional-user","consent":"fictional-tcf","ext":{"consent":"fictional-tcf","eids":[{"source":"identity.example","uids":[{"atype":1,"id":"fictional-uid"}]}]}},"tmax":321,"cur":["USD"],"regs":{"gdpr":1,"us_privacy":"1YNN","gpp":"fictional-gpp","gpp_sid":[2,6],"ext":{"gdpr":1,"gpp":"fictional-gpp","gpp_sid":[2,6],"us_privacy":"1YNN"}},"ext":{"account":"example-account-id","sdk":{"source":"prebid","version":"2.2.0"}}}"#, - "should preserve APS parity differences" - ); -} - #[test] fn signing_finalization_is_after_profiles_and_asserts_every_owned_key() { let signer = deterministic_signer(); @@ -844,7 +754,6 @@ fn signing_finalization_is_after_profiles_and_asserts_every_owned_key() { json!({"request_ext": {"fictional": true}}), ), ("auction.prebid-server", json!({})), - ("auction.aps", json!({"account_id": "example-account-id"})), ] { let unsigned = serde_json::to_value(build(implementation, config.clone(), None)) .expect("should serialize unsigned request"); @@ -899,11 +808,6 @@ fn signed_profiles_and_unsigned_standard_have_exact_full_goldens() { json!({}), r#"{"id":"fictional-auction","imp":[{"id":"fictional-slot","banner":{"format":[{"w":300,"h":250},{"w":728,"h":90}]},"tagid":"fictional-slot","bidfloor":1.0,"bidfloorcur":"USD","secure":1,"ext":{"prebid":{"bidder":{"exampleBidder":{"placement":"fictional-placement"}}}}}],"site":{"domain":"publisher.example","page":"https://publisher.example/article","ref":"https://referrer.example/story?fictional=1","publisher":{"domain":"publisher.example"}},"device":{"geo":{"lat":12.34,"lon":56.78,"type":2,"country":"US","region":"CA","metro":"501","city":"Example City"},"dnt":1,"ua":"Fictional Browser","ip":"192.0.2.10","language":"en"},"user":{"id":"fictional-user","consent":"fictional-tcf","ext":{"ConsentedProvidersSettings":{"consented_providers":"fictional-ac"},"consent":"fictional-tcf","eids":[{"source":"identity.example","uids":[{"atype":1,"id":"fictional-uid"}]}]}},"tmax":321,"cur":["USD"],"regs":{"gdpr":1,"us_privacy":"1YNN","gpp":"fictional-gpp","gpp_sid":[2,6],"ext":{"gdpr":1,"gpp":"fictional-gpp","gpp_sid":[2,6],"us_privacy":"1YNN"}},"ext":{"prebid":{},"trusted_server":{"kid":"fictional-kid","request_host":"publisher.example","request_scheme":"https","signature":"LU_JUIA1BT80ShZNjSa4PIF5T-uMjEeodwKrV_6bXgh0hi1SYVtCKn9g_DTW62krmjCOFgoFYPHsu6L0nAcuDg","ts":1706900000,"version":"1.1"}}}"#, ), - ( - "auction.aps", - json!({"account_id": "example-account-id"}), - r#"{"id":"fictional-auction","imp":[{"id":"fictional-slot","banner":{"format":[{"w":300,"h":250},{"w":728,"h":90}],"w":300,"h":250,"topframe":0},"bidfloor":1.0,"bidfloorcur":"USD","secure":1}],"site":{"domain":"publisher.example","page":"https://publisher.example/article","publisher":{"domain":"publisher.example"}},"device":{"geo":{"type":2,"country":"US","region":"CA","metro":"501","city":"Example City"},"dnt":1,"ua":"Fictional Browser","ip":"192.0.2.10","language":"en"},"user":{"id":"fictional-user","consent":"fictional-tcf","ext":{"consent":"fictional-tcf","eids":[{"source":"identity.example","uids":[{"atype":1,"id":"fictional-uid"}]}]}},"tmax":321,"cur":["USD"],"regs":{"gdpr":1,"us_privacy":"1YNN","gpp":"fictional-gpp","gpp_sid":[2,6],"ext":{"gdpr":1,"gpp":"fictional-gpp","gpp_sid":[2,6],"us_privacy":"1YNN"}},"ext":{"account":"example-account-id","sdk":{"source":"prebid","version":"2.2.0"},"trusted_server":{"kid":"fictional-kid","request_host":"publisher.example","request_scheme":"https","signature":"LU_JUIA1BT80ShZNjSa4PIF5T-uMjEeodwKrV_6bXgh0hi1SYVtCKn9g_DTW62krmjCOFgoFYPHsu6L0nAcuDg","ts":1706900000,"version":"1.1"}}}"#, - ), ]; for (implementation, config, expected) in cases { assert_eq!( diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 8191e0ddbb..7618648a56 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -148,9 +148,9 @@ pub(crate) const ERROR_TYPE_TRANSPORT: &str = "transport"; pub(crate) const ERROR_TYPE_TIMEOUT: &str = "timeout"; /// A non-2xx HTTP status from an upstream SSP (e.g. a PBS 4xx/5xx). Distinct /// from [`ERROR_TYPE_TRANSPORT`] (a connection-level failure) so telemetry can -/// bucket it separately. `pub(crate)` so producers such as the prebid provider -/// tag errors with the exact value the telemetry layer recognises. -pub(crate) const ERROR_TYPE_HTTP_STATUS: &str = "http_status"; +/// bucket it separately. Public so a producer in a crate of its own tags +/// errors with the exact value the telemetry layer recognises. +pub const ERROR_TYPE_HTTP_STATUS: &str = "http_status"; /// Every server-owned `error_type` classification. /// @@ -2582,7 +2582,6 @@ impl OrchestrationResult { mod tests { use std::time::Duration; - use base64::Engine as _; use web_time::Instant; use crate::auction::config::AuctionConfig; @@ -2598,9 +2597,8 @@ mod tests { BidStatus, MediaType, PublisherInfo, UserInfo, }; use crate::error::TrustedServerError; - use crate::integrations::aps::{APS_RENDERER_TYPE, ApsRendererV1, ApsTagType}; use crate::platform::test_support::{ - StubHttpClient, build_services_with_backend_and_http_client, + NamingBackend, StubHttpClient, build_services_with_backend_and_http_client, build_services_with_http_client, noop_services, }; use crate::platform::{ @@ -2611,7 +2609,7 @@ mod tests { }; use crate::test_support::tests::crate_test_settings_str; use error_stack::{Report, ResultExt}; - use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; + use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; @@ -2692,37 +2690,6 @@ mod tests { config } - fn planned_aps_config() -> AuctionPlanConfig { - planned_aps_instances_config(&[( - "aps_instance", - serde_json::json!({"account_id": "example-account"}), - NotificationConfig::default(), - )]) - } - - fn planned_aps_instances_config( - providers: &[(&str, serde_json::Value, NotificationConfig)], - ) -> AuctionPlanConfig { - let tables = providers - .iter() - .map(|(id, settings, notifications)| { - ( - *id, - planned_table( - "auction.aps", - "https://aps.example/e/pb/bid", - RoutingMode::AllEligible, - settings, - notifications, - ), - ) - }) - .collect::>(); - let mut config = plan_config(tables); - config.timeout_ms = 777; - config - } - fn planned_request() -> AuctionRequest { AuctionRequest { id: "fictional-auction".to_string(), @@ -3012,71 +2979,6 @@ mod tests { } } - struct NamingBackend { - policy: BackendNamingPolicy, - predicted: AtomicUsize, - ensured: AtomicUsize, - specs: Mutex>, - fail_ensure_for: Mutex>, - } - - impl NamingBackend { - fn new(policy: BackendNamingPolicy) -> Self { - Self { - policy, - predicted: AtomicUsize::new(0), - ensured: AtomicUsize::new(0), - specs: Mutex::new(Vec::new()), - fail_ensure_for: Mutex::new(HashSet::new()), - } - } - - fn fail_ensure_for(&self, provider_id: &str) { - self.fail_ensure_for - .lock() - .expect("should lock failing provider IDs") - .insert(provider_id.to_string()); - } - - fn name(&self, spec: &PlatformBackendSpec) -> Result> { - self.policy - .predict(spec) - .map(|prediction| prediction.name) - .change_context(PlatformError::Backend) - } - } - - impl PlatformBackend for NamingBackend { - fn naming_policy(&self) -> BackendNamingPolicy { - self.policy - } - - fn predict_name( - &self, - spec: &PlatformBackendSpec, - ) -> Result> { - self.predicted.fetch_add(1, Ordering::Relaxed); - self.name(spec) - } - - fn ensure(&self, spec: &PlatformBackendSpec) -> Result> { - self.ensured.fetch_add(1, Ordering::Relaxed); - if spec.discriminator.as_deref().is_some_and(|provider_id| { - self.fail_ensure_for - .lock() - .expect("should lock failing provider IDs") - .contains(provider_id) - }) { - return Err(Report::new(PlatformError::Backend)); - } - self.specs - .lock() - .expect("should lock planned backend specs") - .push(spec.clone()); - self.name(spec) - } - } - struct ZeroCanonicalBackend { predicted: AtomicUsize, ensured: AtomicUsize, @@ -3745,22 +3647,17 @@ mod tests { struct CacheRestoringAdServer; fn auction_bid(bidder: &str, price: f64) -> Bid { - let renderer = (bidder == "aps").then(|| { - BidRenderer::from_typed( - APS_RENDERER_TYPE, - &ApsRendererV1 { - version: 1, - account_id: "example-account".to_string(), - bid_id: "aps-selected-bid".to_string(), - creative_id: None, - tag_type: ApsTagType::Iframe, - creative_url: "https://creative.example/render".to_string(), - aax_response: "fictional-base64".to_string(), - width: 300, - height: 250, - }, + let renderer = (bidder == "typed").then(|| { + BidRenderer::new( + "example", + serde_json::json!({ + "version": 1, + "bidId": "typed-selected-bid", + "creativeUrl": "https://creative.example/render", + "envelope": "fictional-base64", + }), ) - .expect("the APS renderer payload should be a JSON object") + .expect("should build the example renderer descriptor") }); Bid { slot_id: "slot-1".to_string(), @@ -3776,7 +3673,7 @@ mod tests { height: 250, nurl: None, burl: None, - bid_id: (bidder == "aps").then(|| "aps-selected-bid".to_string()), + bid_id: (bidder == "typed").then(|| "typed-selected-bid".to_string()), ad_id: None, creative_id: None, renderer, @@ -5954,7 +5851,8 @@ mod tests { async fn planned_prebid_stored_intent_filters_wire_demand_and_skips_empty_transports() { for inline_providers in 0..=2 { let http = Arc::new(StubHttpClient::new()); - // Providers launch in ID order: APS, then only PBS instances with usable demand. + // Providers launch in ID order: the source that takes every slot, then + // only the PBS instances with usable demand. http.push_response(204, Vec::new()); for index in 0..inline_providers { http.push_response( @@ -5991,18 +5889,26 @@ mod tests { NotificationConfig::default(), ), ]); - let aps = planned_aps_config(); + let all_eligible = + planned_config(&[("all_eligible_source", RoutingMode::AllEligible)], false); let mut tables = config.demand.tables().clone(); - tables.extend(aps.demand.tables().clone()); + tables.extend(all_eligible.demand.tables().clone()); let mut selected = config .demand .selected() .into_iter() .map(str::to_owned) .collect::>(); - selected.extend(aps.demand.selected().into_iter().map(str::to_owned)); + selected.extend( + all_eligible + .demand + .selected() + .into_iter() + .map(str::to_owned), + ); // Sources launch in the order they are selected, and this test reads - // the APS request first, so the selection is in name order. + // the all-eligible source's request first, so the selection is in + // name order. selected.sort(); config.demand = crate::provider_table::ProviderList::new(selected, tables); for (bidder, provider) in [("alpha", "pbs_a"), ("beta", "pbs_b")] { @@ -6063,10 +5969,10 @@ mod tests { 1 + inline_providers, "should never transport an empty PBS request" ); - let aps: serde_json::Value = - serde_json::from_slice(&bodies[0]).expect("should parse APS wire request"); + let all_eligible_wire: serde_json::Value = serde_json::from_slice(&bodies[0]) + .expect("should parse the all-eligible source's wire request"); assert_eq!( - aps["imp"] + all_eligible_wire["imp"] .as_array() .expect("should have impressions") .len(), @@ -6275,380 +6181,6 @@ mod tests { ); } - #[tokio::test] - async fn planned_aps_transport_omits_accept_header() { - let http = Arc::new(StubHttpClient::new()); - http.push_response(400, Vec::new()); - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Fastly)); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, - ); - let plan = - AuctionPlan::compile(planned_aps_config()).expect("should compile planned APS auction"); - let orchestrator = AuctionOrchestratorHarness::new(plan, None); - let request = planned_request(); - let settings = create_test_settings(); - let inbound = http::Request::builder() - .uri("https://publisher.example/auction") - .body(edgezero_core::body::Body::empty()) - .expect("should build inbound request"); - let context = AuctionContext { - settings: &settings, - request: &inbound, - timeout_ms: 777, - transport_timeout_ms: 777, - provider_responses: None, - services: &services, - }; - - orchestrator - .run_auction(&request, &context) - .await - .expect("should execute planned APS auction"); - - let headers = http.recorded_request_headers(); - assert_eq!(headers.len(), 1); - assert!( - headers[0].iter().all(|(name, _)| name != "accept"), - "planned APS transport must not add Accept beyond legacy headers" - ); - } - - #[tokio::test] - async fn planned_aps_profile_normalizes_renderer_reduction_and_metadata() { - let http = Arc::new(StubHttpClient::new()); - http.push_response_with_headers( - 200, - serde_json::to_vec(&serde_json::json!({ - "cur": "USD", - "seatbid": [ - {"seat": "returned-seat", "bid": [ - {"id": "z-high", "impid": "fictional-slot", "price": 2.0, "w": 300, "h": 250, - "nurl": "https://notice.example/win", "burl": "https://notice.example/bill", - "crid": "fictional-creative", "adomain": ["advertiser.example"], - "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, - {"id": "a-high", "impid": "fictional-slot", "price": 2.0, "w": 300, "h": 250, - "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, - {"id": "bad-script", "impid": "fictional-slot", "price": 9.0, "w": 300, "h": 250, - "ext": {"creativeurl": "https://creative.example/render", "tagtype": "script"}}, - {"id": "bad-domain", "impid": "fictional-slot", "price": 8.0, "w": 300, "h": 250, - "ext": {"creativeurl": "https://publisher.example/render", "tagtype": "iframe"}}, - {"id": "bad-credentials", "impid": "fictional-slot", "price": 8.0, "w": 300, "h": 250, - "ext": {"creativeurl": "https://user:password@creative.example/render", "tagtype": "iframe"}}, - {"id": "bad-imp", "impid": "unknown-slot", "price": 8.0, "w": 300, "h": 250, - "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, - {"id": "bad-dimensions", "impid": "fictional-slot", "price": 8.0, "w": 320, "h": 50, - "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, - {"id": "bad-price", "impid": "fictional-slot", "price": "high", "w": 300, "h": 250, - "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, - {"id": "bad-mtype", "impid": "fictional-slot", "price": 8.0, "mtype": 2, "w": 300, "h": 250, - "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, - {"id": "bad-tag", "impid": "fictional-slot", "price": 8.0, "w": 300, "h": 250, - "ext": {"creativeurl": "https://creative.example/render", "tagtype": "native"}}, - {"id": "bad-crid", "impid": "fictional-slot", "price": 8.0, "w": 300, "h": 250, - "crid": "x".repeat(1025), - "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, - {"impid": "fictional-slot", "price": 8.0, "w": 300, "h": 250, - "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}} - ]}, - {"seat": 7, "bid": "bad-shape"} - ] - })) - .expect("should serialize APS profile response"), - vec![ - ("content-type", "application/json"), - ("authorization", "fictional-secret"), - ], - ); - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Fastly)); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, - ); - let plan = AuctionPlan::compile(planned_aps_instances_config(&[( - "aps_instance", - serde_json::json!({"account_id": "example-account", "debug": true}), - NotificationConfig { - suppress_all: false, - suppress_seats: vec!["returned-seat".to_string()], - }, - )])) - .expect("should compile planned APS profile"); - let orchestrator = AuctionOrchestratorHarness::new(plan, None); - let request = planned_request(); - let settings = create_test_settings(); - let inbound = http::Request::new(edgezero_core::body::Body::empty()); - let context = AuctionContext { - settings: &settings, - request: &inbound, - timeout_ms: 777, - transport_timeout_ms: 777, - provider_responses: None, - services: &services, - }; - - let result = orchestrator - .run_auction(&request, &context) - .await - .expect("should execute planned APS profile"); - - let response = &result.provider_responses[0]; - assert_eq!(response.provider, "aps_instance"); - assert_eq!(response.status, BidStatus::Success); - assert_eq!( - response.bids.len(), - 1, - "should retain one bid per impression" - ); - let bid = &response.bids[0]; - assert_eq!(bid.bidder, "aps"); - assert_eq!(bid.returned_seat.as_deref(), Some("returned-seat")); - assert_eq!( - bid.bid_id.as_deref(), - Some("a-high"), - "lexical ID should break equal-price tie" - ); - assert!(bid.creative.is_none()); - assert!( - bid.nurl.is_none() && bid.burl.is_none(), - "APS must discard notification URLs" - ); - let renderer = bid - .renderer - .as_ref() - .and_then(|renderer| renderer.payload_as::(APS_RENDERER_TYPE)) - .expect("should construct typed APS renderer"); - assert_eq!(renderer.account_id, "example-account"); - let decoded = base64::engine::general_purpose::STANDARD - .decode(&renderer.aax_response) - .expect("should decode minimized APS response"); - assert_eq!( - serde_json::from_slice::(&decoded) - .expect("should parse minimized APS response"), - serde_json::json!({"seatbid":[{"bid":[{ - "id":"a-high","price":2.0,"w":300,"h":250, - "ext":{"creativeurl":"https://creative.example/render","tagtype":"iframe"} - }]}]}) - ); - assert_eq!(response.metadata["seatbid_count"], 2); - assert_eq!(response.metadata["accepted_bid_count"], 1); - assert_eq!(response.metadata["dropped_bid_count"], 12); - for reason in [ - "lost_to_higher_bid", - "script_rendering_disabled", - "unknown_impid", - "invalid_dimensions", - "invalid_price", - "unsupported_media_type", - "unsupported_tagtype", - "creative_id_too_large", - "missing_render_source", - "empty_seatbid_bids", - ] { - assert_eq!(response.metadata["drop_reasons"][reason], 1, "{reason}"); - } - assert_eq!( - response.metadata["drop_reasons"]["invalid_creative_url"], 2, - "same-publisher and credentialed URLs should both be rejected" - ); - assert_eq!( - response.metadata["routing"]["unused_bidder_params_count"], - 0 - ); - let debug = &response.metadata["debug"]["httpcalls"]["aps"][0]; - assert_eq!(debug["uri"], "https://aps.example/e/pb/bid"); - assert_eq!( - debug["responseheaders"], - serde_json::json!({"content-type": ["application/json"]}), - "async stub should preserve allowlisted response headers" - ); - assert!( - debug["requestbody"] - .as_str() - .is_some_and(|body| body.contains("example-account")) - ); - assert!(debug["requestheaders"].get("authorization").is_none()); - assert!(debug["responseheaders"].get("authorization").is_none()); - } - - #[tokio::test] - async fn two_planned_aps_instances_correlate_independently() { - let http = Arc::new(StubHttpClient::new()); - for (seat, id, price) in [("seat-a", "bid-a", 1.0), ("seat-b", "bid-b", 2.0)] { - http.push_response( - 200, - serde_json::to_vec(&serde_json::json!({"seatbid":[{"seat":seat,"bid":[{ - "id":id,"impid":"fictional-slot","price":price,"w":300,"h":250, - "ext":{"creativeurl":"https://creative.example/render","tagtype":"iframe"} - }]}]})) - .expect("should serialize APS instance response"), - ); - } - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Fastly)); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, - ); - let plan = AuctionPlan::compile(planned_aps_instances_config(&[ - ( - "aps_a", - serde_json::json!({"account_id":"account-a"}), - NotificationConfig::default(), - ), - ( - "aps_b", - serde_json::json!({"account_id":"account-b"}), - NotificationConfig::default(), - ), - ])) - .expect("should compile two APS instances"); - let orchestrator = AuctionOrchestratorHarness::new(plan, None); - let request = planned_request(); - let settings = create_test_settings(); - let inbound = http::Request::new(edgezero_core::body::Body::empty()); - let context = AuctionContext { - settings: &settings, - request: &inbound, - timeout_ms: 777, - transport_timeout_ms: 777, - provider_responses: None, - services: &services, - }; - - let result = orchestrator - .run_auction(&request, &context) - .await - .expect("should execute two APS instances"); - - assert_eq!(result.provider_responses.len(), 2); - assert_eq!(result.provider_responses[0].provider, "aps_a"); - assert_eq!( - result.provider_responses[0].bids[0].bid_id.as_deref(), - Some("bid-a") - ); - assert_eq!(result.provider_responses[1].provider, "aps_b"); - assert_eq!( - result.provider_responses[1].bids[0].bid_id.as_deref(), - Some("bid-b") - ); - assert_eq!(http.recorded_request_bodies().len(), 2); - assert_eq!( - result.winning_bids["fictional-slot"].bid_id.as_deref(), - Some("bid-b"), - "global ranking should remain orchestrator-owned" - ); - let specs = backend.specs.lock().expect("should lock specs"); - assert_eq!(specs.len(), 2); - assert_ne!(specs[0].discriminator, specs[1].discriminator); - } - - #[tokio::test] - async fn planned_aps_returned_seat_accepts_only_valid_nonempty_strings() { - let plan = AuctionPlan::compile(planned_aps_config()).expect("should compile APS plan"); - let routed = route_auction( - planned_request(), - &http::Request::new(edgezero_core::body::Body::empty()), - &plan, - None, - ); - let provider = GenericOpenRtbProvider::new(plan.providers()[0].clone()); - for (seat, expected) in [ - (serde_json::Value::Null, None), - (serde_json::json!(7), None), - (serde_json::json!(""), None), - (serde_json::json!("exact-seat"), Some("exact-seat")), - ] { - let state = provider.parse_state_for_test(routed.inputs()[0].clone()); - let response = PlatformResponse::new( - edgezero_core::http::response_builder() - .status(200) - .body(edgezero_core::body::Body::from( - serde_json::to_vec(&serde_json::json!({"seatbid":[{"seat":seat,"bid":[{ - "id":"bid","impid":"fictional-slot","price":1.0,"w":300,"h":250, - "nurl":"https://notice.example/win","burl":"https://notice.example/bill", - "ext":{"creativeurl":"https://creative.example/render","tagtype":"iframe"} - }]}]})) - .expect("should serialize seat identity response"), - )) - .expect("should build seat identity response"), - ); - let parsed = provider - .parse_response_with_state(response, 4, Some(state.as_ref())) - .await - .expect("should parse seat identity response"); - assert_eq!(parsed.bids[0].returned_seat.as_deref(), expected); - assert!(parsed.bids[0].nurl.is_none() && parsed.bids[0].burl.is_none()); - } - } - - #[tokio::test] - async fn planned_aps_response_status_shape_and_currency_matrix() { - let plan = AuctionPlan::compile(planned_aps_config()).expect("should compile APS plan"); - let routed = route_auction( - planned_request(), - &http::Request::new(edgezero_core::body::Body::empty()), - &plan, - None, - ); - let provider = GenericOpenRtbProvider::new(plan.providers()[0].clone()); - let cases = [ - (204, Vec::new(), BidStatus::NoBid, None, None), - (400, Vec::new(), BidStatus::Error, None, Some("http_status")), - ( - 200, - b"not-json".to_vec(), - BidStatus::Error, - Some("unexpected_response_shape"), - Some("parse_response"), - ), - ( - 200, - b"[]".to_vec(), - BidStatus::Error, - Some("unexpected_response_shape"), - Some("parse_response"), - ), - ( - 200, - br#"{"contextual":true}"#.to_vec(), - BidStatus::Error, - Some("unexpected_response_shape"), - Some("parse_response"), - ), - ( - 200, - br#"{"cur":"EUR","seatbid":[]}"#.to_vec(), - BidStatus::NoBid, - Some("unsupported_currency"), - None, - ), - ]; - for (status, body, expected, reason, error_type) in cases { - let state = provider.parse_state_for_test(routed.inputs()[0].clone()); - let response = PlatformResponse::new( - edgezero_core::http::response_builder() - .status(status) - .body(edgezero_core::body::Body::from(body)) - .expect("should build APS matrix response"), - ); - let parsed = provider - .parse_response_with_state(response, 4, Some(state.as_ref())) - .await - .expect("should classify APS matrix response"); - assert_eq!(parsed.status, expected, "status {status}"); - if let Some(reason) = reason { - assert_eq!( - parsed.metadata["drop_reasons"][reason], 1, - "status {status}" - ); - } - if let Some(error_type) = error_type { - assert_eq!(parsed.metadata["error_type"], error_type, "status {status}"); - } - } - } - #[tokio::test] async fn planned_provider_outcome_matrix_has_fixed_count_only_routing_metadata() { let standard_plan = AuctionPlan::compile(planned_config( @@ -6722,110 +6254,6 @@ mod tests { } } - #[tokio::test] - async fn planned_aps_script_opt_in_matches_shared_renderer_fixture() { - let plan = AuctionPlan::compile(planned_aps_instances_config(&[( - "aps_instance", - serde_json::json!({ - "account_id":"example-account-id", - "allow_script_creatives":true - }), - NotificationConfig::default(), - )])) - .expect("should compile script-enabled APS plan"); - let routed = route_auction( - planned_request(), - &http::Request::new(edgezero_core::body::Body::empty()), - &plan, - None, - ); - let provider = GenericOpenRtbProvider::new(plan.providers()[0].clone()); - let state = provider.parse_state_for_test(routed.inputs()[0].clone()); - let response = PlatformResponse::new( - edgezero_core::http::response_builder() - .status(200) - .body(edgezero_core::body::Body::from( - serde_json::to_vec(&serde_json::json!({"seatbid":[{"bid":[{ - "id":"fictional-selected-bid-id","impid":"fictional-slot","price":1.23, - "w":300,"h":250,"crid":"fictional-creative", - "ext":{"creativeurl":"https://creative.example/render","tagtype":"iframe"} - },{ - "id":"script-bid","impid":"fictional-slot","price":1.0, - "w":300,"h":250, - "ext":{"creativeurl":"https://creative.example/script","tagtype":"script"} - }]}]})) - .expect("should serialize APS renderer fixture response"), - )) - .expect("should build APS renderer fixture response"), - ); - - let parsed = provider - .parse_response_with_state(response, 3, Some(state.as_ref())) - .await - .expect("should parse APS renderer fixture response"); - - assert_eq!(parsed.status, BidStatus::Success); - assert_eq!( - parsed.metadata["drop_reasons"]["lost_to_higher_bid"], 1, - "enabled script creative should be eligible before reduction" - ); - let renderer = parsed.bids[0] - .renderer - .as_ref() - .and_then(|renderer| renderer.payload_as::(APS_RENDERER_TYPE)) - .expect("should construct APS renderer"); - let decoded = base64::engine::general_purpose::STANDARD - .decode(&renderer.aax_response) - .expect("should decode APS fixture envelope"); - let fixture: serde_json::Value = serde_json::from_str(include_str!( - "../../../trusted-server-js/lib/test/fixtures/aps-renderer-v1.json" - )) - .expect("should parse shared APS renderer fixture"); - assert_eq!( - serde_json::from_slice::(&decoded) - .expect("should parse decoded APS renderer"), - fixture - ); - } - - #[tokio::test] - async fn planned_aps_debug_response_headers_are_allowlisted() { - let plan = AuctionPlan::compile(planned_aps_instances_config(&[( - "aps_instance", - serde_json::json!({"account_id":"example-account","debug":true}), - NotificationConfig::default(), - )])) - .expect("should compile debug APS plan"); - let routed = route_auction( - planned_request(), - &http::Request::new(edgezero_core::body::Body::empty()), - &plan, - None, - ); - let provider = GenericOpenRtbProvider::new(plan.providers()[0].clone()); - let state = provider.parse_state_for_test(routed.inputs()[0].clone()); - let response = PlatformResponse::new( - edgezero_core::http::response_builder() - .status(200) - .header("content-type", "application/json") - .header("authorization", "fictional-secret") - .body(edgezero_core::body::Body::from("{}")) - .expect("should build debug APS response"), - ); - - let parsed = provider - .parse_response_with_state(response, 3, Some(state.as_ref())) - .await - .expect("should parse debug APS response"); - - let headers = &parsed.metadata["debug"]["httpcalls"]["aps"][0]["responseheaders"]; - assert_eq!( - headers, - &serde_json::json!({"content-type":["application/json"]}) - ); - assert!(headers.get("authorization").is_none()); - } - #[tokio::test] async fn planned_standard_instances_have_distinct_backends_and_independent_results() { let http = Arc::new(StubHttpClient::new()); @@ -7449,62 +6877,57 @@ mod tests { } #[tokio::test] - async fn planned_routing_count_survives_standard_and_aps_bounded_body_failures() { - for (profile, provider_id) in [("standard", "standard"), ("aps", "aps_instance")] { - let http = Arc::new(StubHttpClient::new()); - http.push_response(200, vec![b'x'; 1024 * 1024 + 1]); - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, - ); - let mut config = if profile == "standard" { - planned_config(&[(provider_id, RoutingMode::AllEligible)], false) - } else { - planned_aps_config() - }; - config.bidders.insert( - "example-bidder" + async fn planned_routing_count_survives_a_bounded_body_failure() { + let (profile, provider_id) = ("standard", "standard"); + let http = Arc::new(StubHttpClient::new()); + http.push_response(200, vec![b'x'; 1024 * 1024 + 1]); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let mut config = planned_config(&[(provider_id, RoutingMode::AllEligible)], false); + config.bidders.insert( + "example-bidder" + .parse() + .expect("should parse fictional bidder ID"), + crate::auction::plan::BidderRouteConfig { + module: provider_id .parse() - .expect("should parse fictional bidder ID"), - crate::auction::plan::BidderRouteConfig { - module: provider_id - .parse() - .expect("should parse fictional provider ID"), - }, - ); - let plan = AuctionPlan::compile(config).expect("should compile bounded-body plan"); - let orchestrator = AuctionOrchestratorHarness::new(plan, None); - let mut request = planned_request(); - request.slots[0].bidders.insert( - "example-bidder".to_string(), - serde_json::json!({"private": "value"}), - ); - let settings = create_test_settings(); - let inbound = http::Request::new(edgezero_core::body::Body::empty()); - let context = AuctionContext { - settings: &settings, - request: &inbound, - timeout_ms: 777, - transport_timeout_ms: 777, - provider_responses: None, - services: &services, - }; + .expect("should parse fictional provider ID"), + }, + ); + let plan = AuctionPlan::compile(config).expect("should compile bounded-body plan"); + let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let mut request = planned_request(); + request.slots[0].bidders.insert( + "example-bidder".to_string(), + serde_json::json!({"private": "value"}), + ); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; - let result = orchestrator - .run_auction(&request, &context) - .await - .expect("should materialize bounded-body failure"); - let response = &result.provider_responses[0]; - assert_eq!(response.status, BidStatus::Error, "{profile}"); - assert_eq!( - response.metadata["routing"]["unused_bidder_params_count"], 1, - "{profile} bounded-body failure should retain the input-derived count" - ); - let routing = serde_json::to_string(&response.metadata["routing"]) - .expect("should serialize routing metadata"); - assert!(!routing.contains("example-bidder") && !routing.contains("private")); - } + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should materialize bounded-body failure"); + let response = &result.provider_responses[0]; + assert_eq!(response.status, BidStatus::Error, "{profile}"); + assert_eq!( + response.metadata["routing"]["unused_bidder_params_count"], 1, + "{profile} bounded-body failure should retain the input-derived count" + ); + let routing = serde_json::to_string(&response.metadata["routing"]) + .expect("should serialize routing metadata"); + assert!(!routing.contains("example-bidder") && !routing.contains("private")); } #[tokio::test] @@ -7917,26 +7340,28 @@ mod tests { } #[test] - fn decoded_aps_bid_competes_directly_by_cpm() { + fn a_bid_with_a_typed_renderer_competes_directly_by_cpm() { let orchestrator = AuctionOrchestrator::new(AuctionConfig::default()); let floor_prices = HashMap::new(); let response = |provider: &str, bid: Bid| AuctionResponse::success(provider, vec![bid], 1); - let aps_wins = orchestrator.select_winning_bids( + let typed_wins = orchestrator.select_winning_bids( &[ - response("aps", auction_bid("aps", 2.0)), + response("typed", auction_bid("typed", 2.0)), response("ordinary", auction_bid("ordinary", 1.0)), ], &floor_prices, ); - let winner = aps_wins.get("slot-1").expect("should select APS bid"); - assert_eq!(winner.bidder, "aps"); + let winner = typed_wins + .get("slot-1") + .expect("should select the typed renderer bid"); + assert_eq!(winner.bidder, "typed"); assert!(winner.renderer.is_some()); assert!(winner.creative.is_none()); let ordinary_wins = orchestrator.select_winning_bids( &[ - response("aps", auction_bid("aps", 2.0)), + response("typed", auction_bid("typed", 2.0)), response("ordinary", auction_bid("ordinary", 3.0)), ], &floor_prices, diff --git a/crates/trusted-server-core/src/auction/plan.rs b/crates/trusted-server-core/src/auction/plan.rs index 9786ac81ab..991ebad080 100644 --- a/crates/trusted-server-core/src/auction/plan.rs +++ b/crates/trusted-server-core/src/auction/plan.rs @@ -764,7 +764,6 @@ fn compile_notifications( #[cfg(test)] mod tests { use super::*; - use crate::integrations::aps::ApsDemand; use crate::integrations::openrtb::OpenRtbDemand; use crate::integrations::prebid_server::PrebidServerDemand; use crate::provider_table::IMPLEMENTATION_KEY; @@ -786,21 +785,13 @@ mod tests { /// One `[demand.]` table naming an implementation. fn table(implementation: &str) -> Map { - let mut table = Map::from_iter([ + Map::from_iter([ (IMPLEMENTATION_KEY.to_string(), json!(implementation)), ( ENDPOINT_KEY.to_string(), json!("https://bid.example/openrtb2/auction"), ), - ]); - if implementation == "auction.aps" { - table.insert( - ENDPOINT_KEY.to_string(), - json!("https://aps.example/e/pb/bid"), - ); - table.insert("account_id".to_string(), json!("example-account")); - } - table + ]) } /// A `[demand]` table selecting every name given, in order. @@ -913,7 +904,6 @@ mod tests { "fictional_exchange", "auction-protocol.openrtb", "auction.prebid-server", - "auction.aps", ] { assert!( message.contains(expected), @@ -1113,7 +1103,6 @@ mod tests { let plan = AuctionPlan::compile(config(vec![ ("openrtb_one", table("auction-protocol.openrtb")), ("pbs_one", table("auction.prebid-server")), - ("aps_one", table("auction.aps")), ("pbs_override", override_table), ])) .expect("should resolve timeouts"); @@ -1124,7 +1113,6 @@ mod tests { .collect::>(); assert_eq!(timeouts["openrtb_one"], 1500); assert_eq!(timeouts["pbs_one"], 1000); - assert_eq!(timeouts["aps_one"], 800); assert_eq!(timeouts["pbs_override"], 321); } @@ -1132,19 +1120,18 @@ mod tests { fn the_plan_reports_which_implementations_it_selected() { let plan = AuctionPlan::compile(config(vec![("pbs", table("auction.prebid-server"))])) .expect("should compile without Settings or browser integration state"); - assert!(!plan.has_implementation("auction.aps")); + assert!(!plan.has_implementation("auction-protocol.openrtb")); assert!(plan.has_implementation("auction.prebid-server")); - let plan = AuctionPlan::compile(config(vec![("aps_instance", table("auction.aps"))])) - .expect("should compile APS plan"); + let plan = AuctionPlan::compile(one_source()).expect("should compile a plain source"); assert!( - plan.has_implementation("auction.aps"), - "a validated plan should expose its APS renderer capability" + plan.has_implementation("auction-protocol.openrtb"), + "a validated plan should say which implementation it selected" ); assert!(!plan.has_implementation("auction.prebid-server")); assert!( - plan.providers()[0].demand.as_any().is::(), - "the APS source should compile its own settings" + plan.providers()[0].demand.as_any().is::(), + "the source should compile its own settings" ); } @@ -1191,15 +1178,6 @@ mod tests { "should reject {endpoint}" ); } - let mut aps = table("auction.aps"); - aps.insert( - ENDPOINT_KEY.to_string(), - json!("https://aps.example/e/dtb/bid"), - ); - assert!( - AuctionPlan::compile(config(vec![("aps", aps)])).is_err(), - "should refuse the legacy APS path" - ); } #[test] @@ -1279,13 +1257,6 @@ mod tests { plan.providers()[0].endpoint.as_str(), "https://bid.example/" ); - - let plan = AuctionPlan::compile(config(vec![("aps", table("auction.aps"))])) - .expect("should compile APS endpoint"); - assert_eq!( - plan.providers()[0].endpoint.as_str(), - "https://aps.example/e/pb/bid" - ); } #[test] @@ -1405,20 +1376,6 @@ mod tests { format!("{error:?}").contains("browser_only"), "should name the setting: {error:?}" ); - - let mut aps = table("auction.aps"); - aps.insert("inventory_domain".to_string(), json!("publisher.example")); - assert!( - AuctionPlan::compile(config(vec![("aps", aps)])).is_err(), - "should refuse an APS inventory domain without its page origin" - ); - - let mut without_account = table("auction.aps"); - without_account.remove("account_id"); - assert!( - AuctionPlan::compile(config(vec![("aps", without_account)])).is_err(), - "should refuse an APS source with no account" - ); } #[test] diff --git a/crates/trusted-server-core/src/auction/provider.rs b/crates/trusted-server-core/src/auction/provider.rs index 97b80668ab..d727629862 100644 --- a/crates/trusted-server-core/src/auction/provider.rs +++ b/crates/trusted-server-core/src/auction/provider.rs @@ -215,7 +215,7 @@ impl GenericOpenRtbProvider { self.plan.timeout_ms } - #[cfg(test)] + #[cfg(any(test, feature = "test-utils"))] pub(crate) fn parse_state_for_test(&self, input: ProviderAuctionInput) -> ProviderParseState { Box::new(GenericOpenRtbParseState { provider_id: self.provider_name().to_string(), diff --git a/crates/trusted-server-core/src/auction/routing.rs b/crates/trusted-server-core/src/auction/routing.rs index ac8c049b60..6d9784c6de 100644 --- a/crates/trusted-server-core/src/auction/routing.rs +++ b/crates/trusted-server-core/src/auction/routing.rs @@ -600,9 +600,12 @@ mod tests { table } - fn routing_plan_config(aps_routing: RoutingMode) -> crate::auction::plan::AuctionPlanConfig { + fn routing_plan_config(open_routing: RoutingMode) -> crate::auction::plan::AuctionPlanConfig { let mut config = plan_config(vec![ - ("aps_primary", provider("auction.aps", aps_routing)), + ( + "open_primary", + provider("auction-protocol.openrtb", open_routing), + ), ( "pbs_a", provider("auction.prebid-server", RoutingMode::Explicit), @@ -708,7 +711,7 @@ mod tests { } #[test] - fn pbs_admission_respects_intent_without_changing_aps_eligibility() { + fn pbs_admission_respects_intent_without_changing_all_eligible_routing() { for (params, pbs_ids) in [ (json!({"bidderParams":{}, "storedRequest":false}), vec![]), ( @@ -740,7 +743,7 @@ mod tests { &plan(), None, ); - let mut expected = vec!["aps_primary"]; + let mut expected = vec!["open_primary"]; expected.extend(pbs_ids); assert_eq!( routed @@ -776,7 +779,7 @@ mod tests { route_auction(request(vec![slot(bidders)]), &inbound(), &plan(), None); assert_eq!(routed.diagnostics().malformed_envelope_count(), 1); assert_eq!(routed.inputs().len(), if direct { 2 } else { 1 }); - assert_eq!(routed.inputs()[0].provider_id().as_str(), "aps_primary"); + assert_eq!(routed.inputs()[0].provider_id().as_str(), "open_primary"); if direct { let slot = &input(&routed, "pbs_a").slots()[0]; assert!(!slot.allows_stored_fallback()); @@ -796,11 +799,11 @@ mod tests { for (params, expected) in [ ( json!({"bidderParams": {}, "storedRequest": true}), - vec!["aps_primary", "pbs_a", "pbs_b"], + vec!["open_primary", "pbs_a", "pbs_b"], ), ( json!({"bidderParams": {"alpha": {"placement": 1}}, "storedRequest": false}), - vec!["aps_primary", "pbs_a"], + vec!["open_primary", "pbs_a"], ), ] { let routed = route_auction( @@ -848,8 +851,8 @@ mod tests { .collect::>(); assert_eq!( ids, - vec!["aps_primary", "pbs_a", "pbs_b"], - "{name} should fan out to both PBS providers while APS remains all-eligible" + vec!["open_primary", "pbs_a", "pbs_b"], + "{name} should fan out to both PBS providers while the all-eligible source keeps every slot" ); assert!( input(&routed, "pbs_a").slots()[0].allows_stored_fallback_without_candidates(), @@ -940,7 +943,7 @@ mod tests { routed .inputs() .iter() - .any(|provider| provider.provider_id().as_str() == "aps_primary"), + .any(|provider| provider.provider_id().as_str() == "open_primary"), "{name} should preserve independent all-eligible participation" ); } @@ -1011,11 +1014,11 @@ mod tests { assert_eq!( routed.inputs().len(), 1, - "only all-eligible APS should remain" + "only the all-eligible source should remain" ); assert_eq!( routed.inputs()[0].provider_id().as_str(), - "aps_primary", + "open_primary", "unknown demand should not cause PBS fallback" ); } @@ -1110,18 +1113,22 @@ mod tests { .collect::>(); assert_eq!( ids, - vec!["aps_primary", "pbs_a", "pbs_b", "openrtb_direct"], + vec!["open_primary", "pbs_a", "pbs_b", "openrtb_direct"], "inputs should follow deterministic provider-ID order" ); - let aps = input(&routed, "aps_primary") + let open = input(&routed, "open_primary") .slots() .first() .expect("should have slot"); assert!( - aps.bidder_params().is_empty(), - "APS must receive no foreign params" + open.bidder_params().is_empty(), + "a source that takes no bidder params must receive none" + ); + assert_eq!( + open.zone(), + None, + "a source that takes no bidder params must receive no zone" ); - assert_eq!(aps.zone(), None, "APS must receive no Prebid zone"); let pbs_a = &input(&routed, "pbs_a").slots()[0]; assert_eq!(pbs_a.bidder_params().len(), 1); assert!( @@ -1184,7 +1191,7 @@ mod tests { assert_eq!( routed.inputs().len(), 3, - "APS and two PBS providers should receive the eligible slot" + "the all-eligible source and two PBS providers should receive the eligible slot" ); for provider in routed.inputs() { assert_eq!(provider.slots().len(), 1); @@ -1211,7 +1218,7 @@ mod tests { .iter() .map(ProviderId::as_str) .collect::>(), - vec!["aps_primary", "pbs_a", "pbs_b", "openrtb_direct"], + vec!["open_primary", "pbs_a", "pbs_b", "openrtb_direct"], "no-banner auction should retain every provider's deterministic skip outcome" ); } @@ -1239,9 +1246,9 @@ mod tests { } #[test] - fn trusted_routes_admit_explicit_aps_and_standard_and_ignore_unknown_provider() { + fn trusted_routes_admit_explicit_sources_and_ignore_unknown_provider() { let trusted_routes = TrustedProviderRoutes::new(vec![vec![ - ProviderId::from_str("aps_primary").expect("should parse provider"), + ProviderId::from_str("open_primary").expect("should parse provider"), ProviderId::from_str("openrtb_direct").expect("should parse provider"), ProviderId::from_str("unknown_provider").expect("should parse provider"), ]]); @@ -1261,7 +1268,7 @@ mod tests { .iter() .map(|input| input.provider_id().as_str()) .collect::>(), - vec!["aps_primary", "openrtb_direct"], + vec!["open_primary", "openrtb_direct"], "only known server-owned provider routes should admit explicit providers" ); assert_eq!( @@ -1341,7 +1348,7 @@ mod tests { assert_eq!(routed.dnt(), Some(true)); for provider in routed.inputs() { let expected_timeout = match provider.provider_id().as_str() { - "aps_primary" => 800, + "open_primary" => 900, id if id.starts_with("pbs_") => 1000, _ => 900, }; diff --git a/crates/trusted-server-core/src/auction/test_support.rs b/crates/trusted-server-core/src/auction/test_support.rs index a9fa74f7ea..092ad592ba 100644 --- a/crates/trusted-server-core/src/auction/test_support.rs +++ b/crates/trusted-server-core/src/auction/test_support.rs @@ -1,3 +1,6 @@ +//! Helpers for tests of the auction, in core and in an implementation's own +//! crate. + use std::collections::HashMap; use std::sync::LazyLock; @@ -17,7 +20,8 @@ use crate::settings::Settings; static TEST_SERVICES: LazyLock = LazyLock::new(noop_services); -pub(crate) fn create_test_auction_context<'a>( +/// An auction context over services that do nothing. +pub fn create_test_auction_context<'a>( settings: &'a Settings, request: &'a Request, timeout_ms: u32, @@ -33,13 +37,13 @@ pub(crate) fn create_test_auction_context<'a>( } } -/// Build canonical request facts shared by the PBS and APS Stage 1 wire goldens. +/// Build canonical request facts shared by the implementations' wire goldens. /// -/// The supported and unsupported formats deliberately exercise each profile's -/// existing filtering and field-ownership policy. `trustedServer` bidder -/// parameters are included to pin that PBS consumes them while APS ignores -/// them. -pub(crate) fn canonical_parity_auction_request() -> AuctionRequest { +/// The supported and unsupported formats deliberately exercise each +/// implementation's filtering and field-ownership policy. `trustedServer` +/// bidder parameters are included to pin which implementation consumes them +/// and which ignores them. +pub fn canonical_parity_auction_request() -> AuctionRequest { AuctionRequest { id: "fictional-auction".to_string(), slots: vec![AdSlot { @@ -120,24 +124,172 @@ pub(crate) fn canonical_parity_auction_request() -> AuctionRequest { } } +/// The signing input the request goldens were captured with. +pub fn golden_signing_params() -> crate::request_signing::SigningParams { + crate::request_signing::SigningParams { + request_id: "fictional-auction".to_string(), + request_host: "publisher.example".to_string(), + request_scheme: "https".to_string(), + timestamp: 1_706_900_000, + } +} + +/// The one-source plan the request goldens were captured with: a source +/// called `fictional_provider`, running `implementation` with `settings`. +/// +/// The source takes every eligible slot when `all_eligible` is set. Otherwise +/// the bidder `exampleBidder` is routed to it, which is how a source that +/// takes only routed bidders is reached. +pub fn golden_plan_config( + implementation: &str, + settings: serde_json::Value, + all_eligible: bool, + extra: &[crate::integrations::IntegrationBuilder], +) -> crate::auction::plan::AuctionPlanConfig { + let mut table = demand_table(implementation, "https://exchange.example.test/openrtb"); + table.insert("timeout_ms".to_string(), json!(321)); + if all_eligible { + table.insert("routing".to_string(), json!("all_eligible")); + } + if let serde_json::Value::Object(settings) = settings { + table.extend(settings); + } + let mut config = plan_config_with(vec![("fictional_provider", table)], extra); + config.timeout_ms = 321; + if !all_eligible { + config.bidders.insert( + "exampleBidder".parse().expect("should parse bidder ID"), + crate::auction::plan::BidderRouteConfig { + module: "fictional_provider" + .parse() + .expect("should parse provider ID"), + }, + ); + } + config +} + +/// The inbound request the request goldens were captured with. +pub fn golden_inbound_request() -> Request { + Request::builder() + .uri("https://publisher.example/auction") + .header( + http::header::REFERER, + "https://referrer.example/story?fictional=1", + ) + .header(http::header::ACCEPT_LANGUAGE, "en-US,en;q=0.9") + .header("dnt", "1") + .body(EdgeBody::empty()) + .expect("should build inbound request") +} + +/// A signer over one fixed key, so a signed golden is the same on every run. +pub fn deterministic_signer() -> crate::request_signing::RequestSigner { + use base64::Engine as _; + + use crate::platform::test_support::{ + HashMapConfigStore, HashMapSecretStore, NoopHttpClient, + build_services_with_config_secret_and_http_client, + }; + + let config_data = HashMap::from([("current-kid".to_string(), "fictional-kid".to_string())]); + let secret_data = HashMap::from([( + "fictional-kid".to_string(), + base64::engine::general_purpose::STANDARD + .encode([7_u8; 32]) + .into_bytes(), + )]); + let services = build_services_with_config_secret_and_http_client( + HashMapConfigStore::new(config_data), + HashMapSecretStore::new(secret_data), + std::sync::Arc::new(NoopHttpClient), + ); + crate::request_signing::RequestSigner::from_services(&services) + .expect("should load deterministic signer") +} + +/// Builds the request the driver would send to the first demand source of +/// `plan` once `request` is routed to it, or `None` when the source keeps no +/// impression. +/// +/// A test in an implementation's own crate checks what its field policy and +/// its extensions do to a request through this. The signature, when `signer` +/// is given, is made over [`golden_signing_params`]. +/// +/// # Errors +/// +/// Returns the error the request builder returns. +pub fn build_for_first_source( + plan: &crate::auction::AuctionPlan, + request: AuctionRequest, + inbound: &Request, + transport_timeout_ms: u32, + signer: Option<&crate::request_signing::RequestSigner>, +) -> Result< + Option, + error_stack::Report, +> { + use crate::auction::openrtb::{OpenRtbBuildOutcome, RequestFinalization, build_request}; + + let routed = crate::auction::routing::route_auction(request, inbound, plan, None); + let outcome = build_request( + &routed.inputs()[0], + &routed, + &plan.providers()[0], + transport_timeout_ms, + &RequestFinalization { + signer, + signing_params: golden_signing_params(), + }, + )?; + Ok(match outcome { + OpenRtbBuildOutcome::Ready(request) => Some(request), + OpenRtbBuildOutcome::NoImpressions => None, + }) +} + +/// Reads `response` as the first demand source of `plan` would, once +/// `request` is routed to it. +/// +/// A test in an implementation's own crate checks how its implementation +/// answers through this, with no upstream. +/// +/// # Errors +/// +/// Returns the error the source's own reading returns. +pub async fn parse_as_first_source( + plan: &crate::auction::AuctionPlan, + request: AuctionRequest, + response: crate::platform::PlatformResponse, + response_time_ms: u64, +) -> Result< + crate::auction::types::AuctionResponse, + error_stack::Report, +> { + let inbound = Request::new(EdgeBody::empty()); + let routed = crate::auction::routing::route_auction(request, &inbound, plan, None); + let provider = + crate::auction::provider::GenericOpenRtbProvider::new(plan.providers()[0].clone()); + let state = provider.parse_state_for_test(routed.inputs()[0].clone()); + provider + .parse_response_with_state(response, response_time_ms, Some(state.as_ref())) + .await +} + /// One `[demand.]` table naming an implementation, for tests that need a /// compiled plan. -pub(crate) fn demand_table( +pub fn demand_table( implementation: &str, endpoint: &str, ) -> serde_json::Map { - let mut table = serde_json::Map::from_iter([ + serde_json::Map::from_iter([ ("implementation".to_string(), json!(implementation)), ("endpoint".to_string(), json!(endpoint)), - ]); - if implementation == "auction.aps" { - table.insert("account_id".to_string(), json!("example-account")); - } - table + ]) } /// An `[demand]` table selecting every name given, in the order given. -pub(crate) fn demand_selection( +pub fn demand_selection( tables: Vec<(&str, serde_json::Map)>, ) -> crate::provider_table::ProviderList { let selected = tables @@ -153,10 +305,21 @@ pub(crate) fn demand_selection( /// A plan configuration selecting the named demand tables, with every /// implementation the built-in builders register. -pub(crate) fn plan_config( +pub fn plan_config( + tables: Vec<(&str, serde_json::Map)>, +) -> crate::auction::plan::AuctionPlanConfig { + plan_config_with(tables, &[]) +} + +/// A plan configuration selecting the named demand tables, with every +/// implementation the built-in builders register followed by the ones +/// `extra` registers, which is how a crate's own tests reach its +/// implementation. +pub fn plan_config_with( tables: Vec<(&str, serde_json::Map)>, + extra: &[crate::integrations::IntegrationBuilder], ) -> crate::auction::plan::AuctionPlanConfig { - let builders = crate::integrations::all_builders(&[]).collect::>(); + let builders = crate::integrations::all_builders(extra).collect::>(); crate::auction::plan::AuctionPlanConfig { timeout_ms: 1000, demand: demand_selection(tables), @@ -174,7 +337,7 @@ pub(crate) fn plan_config( /// A `[demand]` selection of ordinary `OpenRTB` sources under the names given, /// each taking every eligible slot. -pub(crate) fn demand_named(names: &[&str]) -> crate::provider_table::ProviderList { +pub fn demand_named(names: &[&str]) -> crate::provider_table::ProviderList { demand_selection( names .iter() @@ -195,6 +358,7 @@ pub(crate) fn demand_named(names: &[&str]) -> crate::provider_table::ProviderLis /// /// Production compiles its sources from the plan instead, so this exists only /// so the parity tests can drive the pre-plan orchestrator. +#[cfg(test)] pub(crate) fn legacy_auction_config(settings: &Settings) -> crate::auction::AuctionConfig { let mut config = settings.auction.clone(); config.provider_names = settings @@ -213,6 +377,7 @@ pub(crate) fn legacy_auction_config(settings: &Settings) -> crate::auction::Auct /// `timeout_ms` from its table and refuses any other setting, as an /// implementation of a vendor's does. Its provider answers every request /// with no bid and calls nothing. +#[cfg(test)] pub(crate) mod adserver_fixture { use std::sync::Arc; diff --git a/crates/trusted-server-core/src/auction/types.rs b/crates/trusted-server-core/src/auction/types.rs index 74e1f72e38..db7818b4d8 100644 --- a/crates/trusted-server-core/src/auction/types.rs +++ b/crates/trusted-server-core/src/auction/types.rs @@ -559,7 +559,6 @@ impl AuctionResponse { #[cfg(test)] mod tests { use super::*; - use crate::integrations::aps::{APS_RENDERER_TYPE, ApsRendererV1, ApsTagType}; use serde_json::json; fn make_bid(bidder: &str) -> Bid { @@ -750,181 +749,48 @@ mod tests { ); } - #[test] - fn aps_renderer_serializes_to_versioned_camel_case_contract() { - let renderer = BidRenderer::from_typed( - APS_RENDERER_TYPE, - &ApsRendererV1 { - version: 1, - account_id: "example-account-id".to_string(), - bid_id: "fictional-bid-id".to_string(), - creative_id: Some("fictional-creative-id".to_string()), - tag_type: ApsTagType::Iframe, - creative_url: "https://creative.example/render".to_string(), - aax_response: "base64-data".to_string(), - width: 300, - height: 250, - }, - ) - .expect("should build APS renderer descriptor"); - - let serialized = serde_json::to_value(&renderer).expect("should serialize renderer"); - - assert_eq!( - serialized, - json!({ - "type": "aps", - "version": 1, - "accountId": "example-account-id", - "bidId": "fictional-bid-id", - "creativeId": "fictional-creative-id", - "tagType": "iframe", - "creativeUrl": "https://creative.example/render", - "aaxResponse": "base64-data", - "width": 300, - "height": 250 - }), - "should match renderer wire contract" - ); + /// A provider's own payload type, as an implementation in a crate of its + /// own defines one. + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] + #[serde(rename_all = "camelCase")] + struct ExampleRenderer { + version: u8, + bid_id: String, + creative_url: String, } - #[test] - fn aps_renderer_omits_absent_creative_id() { - let renderer = BidRenderer::from_typed( - APS_RENDERER_TYPE, - &ApsRendererV1 { - version: 1, - account_id: "example-account-id".to_string(), - bid_id: "fictional-bid-id".to_string(), - creative_id: None, - tag_type: ApsTagType::Iframe, - creative_url: "https://creative.example/render".to_string(), - aax_response: "base64-data".to_string(), - width: 300, - height: 250, - }, - ) - .expect("should build APS renderer descriptor"); - - let serialized = serde_json::to_value(&renderer).expect("should serialize renderer"); - - assert!( - serialized.get("creativeId").is_none(), - "should omit absent creative ID" - ); - } + const EXAMPLE_RENDERER_TYPE: &str = "example"; - /// Rewrites every object in `value` with its keys in sorted order, so - /// serializing the result gives one fixed key order. - /// - /// `serde_json::Map` is a `BTreeMap`, which serializes keys in sorted - /// order, only while the crate's `preserve_order` feature is off. With the - /// feature on it is an `IndexMap` and the order follows insertion instead. - /// Nothing in this crate asks for the feature, but Cargo unifies features - /// across everything built for one target, and `trusted-server-cli` pulls - /// it in through `edgezero-cli` and then `handlebars`. A maintainer - /// running `cargo test --workspace --target ` therefore builds this - /// crate with `preserve_order` on, and a test that pinned insertion order - /// would fail there for no reason. Sorting both sides removes the - /// dependence on which map `serde_json` was built with. - fn with_sorted_keys(value: &serde_json::Value) -> serde_json::Value { - match value { - serde_json::Value::Object(map) => { - let mut keys = map.keys().collect::>(); - keys.sort_unstable(); - let mut sorted = serde_json::Map::with_capacity(keys.len()); - for key in keys { - let child = map.get(key).expect("should find a key the map just listed"); - sorted.insert(key.clone(), with_sorted_keys(child)); - } - serde_json::Value::Object(sorted) - } - serde_json::Value::Array(items) => { - serde_json::Value::Array(items.iter().map(with_sorted_keys).collect()) - } - scalar => scalar.clone(), + fn example_descriptor() -> ExampleRenderer { + ExampleRenderer { + version: 1, + bid_id: "fictional-bid-id".to_string(), + creative_url: "https://creative.example/render".to_string(), } } #[test] - fn the_open_renderer_serializes_to_the_same_bytes_as_the_aps_variant_did() { - // Literal strings captured from the closed-enum form before this - // change, through the same `serde_json::to_value` path production - // uses: `BidExt::to_ext` for the OpenRTB response extension, and - // `build_bid_map` for `window.tsjs.bids`. - // - // Both sides go through `with_sorted_keys` first, because the key - // order `serde_json` emits is not ours to pin, and that function - // explains why. Sorting settles the order without weakening what is - // pinned, since two objects serialize to the same sorted bytes only - // when they carry exactly the same keys with exactly the same values. - let full = BidRenderer::from_typed( - APS_RENDERER_TYPE, - &ApsRendererV1 { - version: 1, - account_id: "example-account-id".to_string(), - bid_id: "fictional-bid-id".to_string(), - creative_id: Some("fictional-creative-id".to_string()), - tag_type: ApsTagType::Iframe, - creative_url: "https://creative.example/render".to_string(), - aax_response: "base64-data".to_string(), - width: 300, - height: 250, - }, - ) - .expect("should build APS renderer descriptor"); - let absent = BidRenderer::from_typed( - APS_RENDERER_TYPE, - &ApsRendererV1 { - version: 1, - account_id: "example-account-id".to_string(), - bid_id: "fictional-bid-id".to_string(), - creative_id: None, - tag_type: ApsTagType::Script, - creative_url: "https://creative.example/render".to_string(), - aax_response: "base64-data".to_string(), - width: 300, - height: 250, - }, - ) - .expect("should build APS renderer descriptor"); - - let full_bytes = serde_json::to_string(&with_sorted_keys( - &serde_json::to_value(&full).expect("should convert renderer to a JSON value"), - )) - .expect("should serialize renderer"); - let absent_bytes = serde_json::to_string(&with_sorted_keys( - &serde_json::to_value(&absent).expect("should convert renderer to a JSON value"), - )) - .expect("should serialize renderer"); + fn renderer_serializes_flat_under_its_type_tag() { + let renderer = BidRenderer::from_typed(EXAMPLE_RENDERER_TYPE, &example_descriptor()) + .expect("should build renderer descriptor"); assert_eq!( - full_bytes, - "{\"aaxResponse\":\"base64-data\",\"accountId\":\"example-account-id\",\"bidId\":\"fictional-bid-id\",\"creativeId\":\"fictional-creative-id\",\"creativeUrl\":\"https://creative.example/render\",\"height\":250,\"tagType\":\"iframe\",\"type\":\"aps\",\"version\":1,\"width\":300}", - "should serialize to the bytes the closed enum produced" - ); - assert_eq!( - absent_bytes, - "{\"aaxResponse\":\"base64-data\",\"accountId\":\"example-account-id\",\"bidId\":\"fictional-bid-id\",\"creativeUrl\":\"https://creative.example/render\",\"height\":250,\"tagType\":\"script\",\"type\":\"aps\",\"version\":1,\"width\":300}", - "should serialize to the bytes the closed enum produced with no creative ID" + serde_json::to_value(&renderer).expect("should serialize renderer"), + json!({ + "type": "example", + "version": 1, + "bidId": "fictional-bid-id", + "creativeUrl": "https://creative.example/render" + }), + "should carry the tag beside the payload's own keys" ); } #[test] fn renderer_round_trips_through_its_wire_form() { - let descriptor = ApsRendererV1 { - version: 1, - account_id: "example-account-id".to_string(), - bid_id: "fictional-bid-id".to_string(), - creative_id: Some("fictional-creative-id".to_string()), - tag_type: ApsTagType::Iframe, - creative_url: "https://creative.example/render".to_string(), - aax_response: "base64-data".to_string(), - width: 300, - height: 250, - }; - let renderer = BidRenderer::from_typed(APS_RENDERER_TYPE, &descriptor) - .expect("should build APS renderer descriptor"); + let descriptor = example_descriptor(); + let renderer = BidRenderer::from_typed(EXAMPLE_RENDERER_TYPE, &descriptor) + .expect("should build renderer descriptor"); let serialized = serde_json::to_string(&renderer).expect("should serialize renderer"); let restored: BidRenderer = @@ -932,13 +798,13 @@ mod tests { assert_eq!( restored.renderer_type(), - APS_RENDERER_TYPE, + EXAMPLE_RENDERER_TYPE, "should round-trip the renderer type tag" ); assert_eq!( restored - .payload_as::(APS_RENDERER_TYPE) - .expect("should deserialize the APS payload"), + .payload_as::(EXAMPLE_RENDERER_TYPE) + .expect("should deserialize the payload"), descriptor, "should round-trip the provider payload" ); @@ -946,51 +812,41 @@ mod tests { #[test] fn renderer_payload_is_hidden_from_a_different_type_tag() { - let renderer = BidRenderer::new(APS_RENDERER_TYPE, json!({ "version": 1 })) + let renderer = BidRenderer::new(EXAMPLE_RENDERER_TYPE, json!({ "version": 1 })) .expect("should build renderer descriptor"); assert!( - renderer.payload_as::("example").is_none(), + renderer.payload_as::("other").is_none(), "should refuse a payload requested under a different tag" ); } #[test] fn renderer_payload_field_borrows_the_same_value_the_whole_descriptor_carries() { - let descriptor = ApsRendererV1 { - version: 1, - account_id: "example-account-id".to_string(), - bid_id: "fictional-bid-id".to_string(), - creative_id: Some("fictional-creative-id".to_string()), - tag_type: ApsTagType::Iframe, - creative_url: "https://creative.example/render".to_string(), - aax_response: "base64-data".to_string(), - width: 300, - height: 250, - }; - let renderer = BidRenderer::from_typed(APS_RENDERER_TYPE, &descriptor) - .expect("should build APS renderer descriptor"); + let descriptor = example_descriptor(); + let renderer = BidRenderer::from_typed(EXAMPLE_RENDERER_TYPE, &descriptor) + .expect("should build renderer descriptor"); assert_eq!( renderer - .payload_field(APS_RENDERER_TYPE, "bidId") + .payload_field(EXAMPLE_RENDERER_TYPE, "bidId") .and_then(serde_json::Value::as_str), Some(descriptor.bid_id.as_str()), "should read the same bid id the whole descriptor carries" ); assert_eq!( renderer - .payload_field(APS_RENDERER_TYPE, "bidId") + .payload_field(EXAMPLE_RENDERER_TYPE, "bidId") .and_then(serde_json::Value::as_str), renderer - .payload_as::(APS_RENDERER_TYPE) + .payload_as::(EXAMPLE_RENDERER_TYPE) .as_ref() .map(|full| full.bid_id.as_str()), "should agree with the field read through the whole descriptor" ); assert!( renderer - .payload_field(APS_RENDERER_TYPE, "notAKey") + .payload_field(EXAMPLE_RENDERER_TYPE, "notAKey") .is_none(), "should return nothing for a key the payload does not carry" ); @@ -998,19 +854,59 @@ mod tests { #[test] fn renderer_payload_field_is_hidden_from_a_different_type_tag() { - let renderer = BidRenderer::new(APS_RENDERER_TYPE, json!({ "bidId": "fictional-bid-id" })) - .expect("should build renderer descriptor"); + let renderer = BidRenderer::new( + EXAMPLE_RENDERER_TYPE, + json!({ "bidId": "fictional-bid-id" }), + ) + .expect("should build renderer descriptor"); assert!( - renderer.payload_field("example", "bidId").is_none(), + renderer.payload_field("other", "bidId").is_none(), "should refuse a field requested under a different tag" ); } + #[test] + fn renderer_names_the_bid_it_picks_only_when_its_builder_said_which_key_holds_it() { + let unstated = BidRenderer::from_typed(EXAMPLE_RENDERER_TYPE, &example_descriptor()) + .expect("should build renderer descriptor"); + assert_eq!( + unstated.bid_id(), + None, + "should name no bid for a descriptor whose builder made no statement" + ); + + let stated = unstated.clone().picking_bid_by("bidId"); + assert_eq!( + stated.bid_id(), + Some("fictional-bid-id"), + "should read the key the builder stated" + ); + assert_eq!( + stated.clone().bid_id(), + Some("fictional-bid-id"), + "should keep the statement when the descriptor is cloned" + ); + assert_eq!( + serde_json::to_value(&stated).expect("should serialize renderer"), + serde_json::to_value(&unstated).expect("should serialize renderer"), + "should send the same bytes with or without the statement" + ); + + let absent = BidRenderer::new(EXAMPLE_RENDERER_TYPE, json!({ "version": 1 })) + .expect("should build renderer descriptor") + .picking_bid_by("bidId"); + assert_eq!( + absent.bid_id(), + None, + "should name no bid when the payload does not carry the stated key" + ); + } + #[test] fn renderer_rejects_a_payload_that_is_not_an_object() { assert!( - BidRenderer::new(APS_RENDERER_TYPE, json!("not-an-object")).is_err(), + BidRenderer::new(EXAMPLE_RENDERER_TYPE, json!("not-an-object")).is_err(), "should reject a payload that is not a JSON object" ); } @@ -1018,7 +914,11 @@ mod tests { #[test] fn renderer_rejects_a_payload_carrying_its_own_type_key() { assert!( - BidRenderer::new(APS_RENDERER_TYPE, json!({ "type": "other", "version": 1 })).is_err(), + BidRenderer::new( + EXAMPLE_RENDERER_TYPE, + json!({ "type": "other", "version": 1 }) + ) + .is_err(), "should reject a payload that would collide with the type tag" ); } diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index 7cc28c45e1..b75fbe75d2 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -750,27 +750,6 @@ formats = [{ width = 300, height = 250 }] } } - fn insert_aps_provider(settings: &mut Settings, account_id: &str) { - let table = serde_json::Map::from_iter([ - ( - "implementation".to_string(), - serde_json::json!("auction.aps"), - ), - ( - "endpoint".to_string(), - serde_json::json!("https://aps.example.com/e/pb/bid"), - ), - ("routing".to_string(), serde_json::json!("all_eligible")), - ("account_id".to_string(), serde_json::json!(account_id)), - ]); - settings.demand = crate::provider_table::ProviderList::new( - vec!["aps_main".to_string()], - std::collections::BTreeMap::from([("aps_main".to_string(), table)]), - ); - } - - /// The `[tinybird]` block is top-level and validated at parse time, so - /// uncommenting it with the documented `api_host` must parse cleanly. #[test] fn documented_tinybird_block_validates_when_uncommented() { let toml = uncomment_block(&template_with_resolved_required_secrets(), "[tinybird]"); @@ -1352,31 +1331,6 @@ password = "production-admin-password-32-bytes" ); } - #[test] - fn deploy_validation_rejects_blank_aps_account_id() { - for (label, account_id) in [("empty", ""), ("whitespace-only", " ")] { - let mut settings = valid_settings(); - insert_aps_provider(&mut settings, account_id); - - let err = validate_settings_for_deploy(&settings) - .expect_err("should reject blank APS account_id"); - - assert!( - format!("{err:?}").contains("account_id"), - "should mention the APS profile account_id for {label}: {err:?}" - ); - } - } - - #[test] - fn deploy_validation_normalizes_padded_aps_account_id() { - let mut settings = valid_settings(); - insert_aps_provider(&mut settings, " example-account "); - - validate_settings_for_deploy(&settings) - .expect("should accept a padded APS profile account_id after trimming it"); - } - #[test] fn deploy_validation_rejects_padded_request_signing_store_ids() { let mut settings = valid_settings(); @@ -1433,42 +1387,6 @@ password = "production-admin-password-32-bytes" .expect("deploy validation should leave unknown ids to the registry"); } - #[test] - fn deploy_validation_rejects_an_aps_demand_setting_it_does_not_know() { - let mut settings = valid_settings(); - let table = serde_json::Map::from_iter([ - ( - "implementation".to_string(), - serde_json::json!("auction.aps"), - ), - ( - "endpoint".to_string(), - serde_json::json!("https://aps.example.com/e/pb/bid"), - ), - ( - "account_id".to_string(), - serde_json::json!("example-account"), - ), - ("enabled".to_string(), serde_json::json!(false)), - ]); - settings.demand = crate::provider_table::ProviderList::new( - vec!["aps_main".to_string()], - std::collections::BTreeMap::from([("aps_main".to_string(), table)]), - ); - - let error = validate_settings_for_deploy(&settings) - .expect_err("should reject a setting the APS implementation does not know"); - let rendered = format!("{error:?}"); - assert!( - rendered.contains("aps_main"), - "should identify the demand source: {rendered}" - ); - assert!( - rendered.contains("enabled"), - "should identify the setting it does not know: {rendered}" - ); - } - #[test] fn deploy_validation_rejects_external_prebid_bundle_without_proxy_allowed_domains() { let mut settings = valid_settings(); diff --git a/crates/trusted-server-core/src/integrations/mod.rs b/crates/trusted-server-core/src/integrations/mod.rs index c91506bc74..7ac5ddf2cb 100644 --- a/crates/trusted-server-core/src/integrations/mod.rs +++ b/crates/trusted-server-core/src/integrations/mod.rs @@ -14,7 +14,6 @@ use crate::error::TrustedServerError; use crate::platform::{DEFAULT_FIRST_BYTE_TIMEOUT, PlatformBackendSpec, RuntimeServices}; use crate::settings::Settings; -pub mod aps; pub mod js_asset_proxy; pub mod openrtb; pub mod prebid; @@ -725,9 +724,6 @@ const BUILT_IN_BUILDERS: &[IntegrationBuilder] = &[ IntegrationBuilder::implementations(openrtb::MODULE, CORE_SOURCE).with_demand(&openrtb::DEMAND), IntegrationBuilder::implementations(prebid_server::MODULE, CORE_SOURCE) .with_demand(&prebid_server::DEMAND), - IntegrationBuilder::implementations(aps::APS_INTEGRATION_ID, CORE_SOURCE) - .with_demand(&aps::DEMAND) - .with_plan_registration(aps::register_for_plan), // A stand-in ad server, which core's own tests select where they need // one. #[cfg(test)] diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index d236e6c96b..43c5b47d59 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -4300,9 +4300,9 @@ mod tests { ); } - /// Prebid and APS are not builders, because the auction plan registers - /// them, yet an outside builder claiming either id is still refused, naming - /// both sources, whether or not the plan-backed integration runs. + /// Prebid registers from the auction plan, and an outside builder claiming + /// its id is refused all the same, naming both sources, whether or not the + /// plan-backed integration runs. #[test] fn with_registrations_rejects_an_outside_builder_claiming_a_plan_backed_id() { fn register_nothing( @@ -4312,29 +4312,25 @@ mod tests { } let settings = crate::test_support::tests::create_test_settings(); - for id in [ - crate::integrations::prebid::PREBID_INTEGRATION_ID, - crate::integrations::aps::APS_INTEGRATION_ID, - ] { - let extra = [crate::integrations::IntegrationBuilder::new( - id, - "seam-probe", - register_nothing, - validate_nothing, - )]; + let id = crate::integrations::prebid::PREBID_INTEGRATION_ID; + let extra = [crate::integrations::IntegrationBuilder::new( + id, + "seam-probe", + register_nothing, + validate_nothing, + )]; - let Err(error) = IntegrationRegistry::with_registrations(&settings, &extra) else { - panic!("should refuse an outside builder claiming `{id}`"); - }; + let Err(error) = IntegrationRegistry::with_registrations(&settings, &extra) else { + panic!("should refuse an outside builder claiming `{id}`"); + }; - let message = error.to_string(); - assert!( - message.contains(id) - && message.contains("trusted-server-core") - && message.contains("seam-probe"), - "error should name `{id}` and both sources: {message}" - ); - } + let message = error.to_string(); + assert!( + message.contains(id) + && message.contains("trusted-server-core") + && message.contains("seam-probe"), + "error should name `{id}` and both sources: {message}" + ); } fn enable_prebid(settings: &mut Settings) { @@ -4464,8 +4460,7 @@ mod tests { fn js_part_is_none_for_an_integration_registered_without_js() { // The carried lookup would answer `Some` on its own, so this proves the // without-JS check runs first. No built-in integration can stand in, because - // the only `without_js` built-in with a Rust registration, `aps`, has - // no compile-time module either. + // none registers without a browser script. let settings = settings_naming("probe"); let extra = [crate::integrations::IntegrationBuilder::new( "probe", @@ -5396,7 +5391,6 @@ mod tests { fn an_auction_implementation_named_in_a_section_is_refused() { for (section, name) in [ ("auction", "prebid-server"), - ("auction", "aps"), ("auction", "ad-server.fixture"), ("auction-protocol", "openrtb"), ] { diff --git a/crates/trusted-server-core/src/platform/test_support.rs b/crates/trusted-server-core/src/platform/test_support.rs index 10f51daade..768d3d5235 100644 --- a/crates/trusted-server-core/src/platform/test_support.rs +++ b/crates/trusted-server-core/src/platform/test_support.rs @@ -195,6 +195,82 @@ impl PlatformBackend for StubBackend { } } +// --------------------------------------------------------------------------- +// NamingBackend +// --------------------------------------------------------------------------- + +/// Test stub for [`PlatformBackend`] that names a backend the way the given +/// policy does, so two demand sources get two names, and records what it was +/// asked. +pub struct NamingBackend { + pub policy: super::BackendNamingPolicy, + /// How many names were predicted. + pub predicted: std::sync::atomic::AtomicUsize, + /// How many backends were registered or refused. + pub ensured: std::sync::atomic::AtomicUsize, + /// The specification of every backend registered, in order. + pub specs: Mutex>, + /// The discriminators whose registration fails. + pub fail_ensure_for: Mutex>, +} + +impl NamingBackend { + pub fn new(policy: super::BackendNamingPolicy) -> Self { + Self { + policy, + predicted: std::sync::atomic::AtomicUsize::new(0), + ensured: std::sync::atomic::AtomicUsize::new(0), + specs: Mutex::new(Vec::new()), + fail_ensure_for: Mutex::new(std::collections::HashSet::new()), + } + } + + /// Makes registration fail for the source `provider_id`. + pub fn fail_ensure_for(&self, provider_id: &str) { + self.fail_ensure_for + .lock() + .expect("should lock failing provider IDs") + .insert(provider_id.to_string()); + } + + fn name(&self, spec: &PlatformBackendSpec) -> Result> { + self.policy + .predict(spec) + .map(|prediction| prediction.name) + .change_context(PlatformError::Backend) + } +} + +impl PlatformBackend for NamingBackend { + fn naming_policy(&self) -> super::BackendNamingPolicy { + self.policy + } + + fn predict_name(&self, spec: &PlatformBackendSpec) -> Result> { + self.predicted + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + self.name(spec) + } + + fn ensure(&self, spec: &PlatformBackendSpec) -> Result> { + self.ensured + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + if spec.discriminator.as_deref().is_some_and(|provider_id| { + self.fail_ensure_for + .lock() + .expect("should lock failing provider IDs") + .contains(provider_id) + }) { + return Err(Report::new(PlatformError::Backend)); + } + self.specs + .lock() + .expect("should lock planned backend specs") + .push(spec.clone()); + self.name(spec) + } +} + // --------------------------------------------------------------------------- // StubHttpClient // --------------------------------------------------------------------------- diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index e7c4d66fd4..c77a350c03 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -3612,8 +3612,8 @@ pub(crate) fn prepend_auction_debug_comment( // untyped provider diagnostics cannot cross that boundary and one // large creative cannot dominate the payload. Bid-level fields // (`Bid.metadata`, `nurl`, `burl`) are NOT yet allowlisted; they pass - // through today because the only writer (`integrations/aps.rs`) emits - // opaque targeting keys. Tightening this to a fail-closed bid allowlist + // through today because the only writer, a demand implementation, + // emits opaque targeting keys. Tightening this to a fail-closed bid allowlist // is tracked in #925. // 2. `render_dump` below neutralises HTML comment terminators and caps the // total serialized size. @@ -22692,7 +22692,6 @@ mod tests { CreativeOpportunitiesConfig, CreativeOpportunityFormat, CreativeOpportunitySlot, }; use crate::http_util::RequestInfo; - use crate::integrations::aps::{APS_RENDERER_TYPE, ApsRendererV1, ApsTagType}; use crate::price_bucket::PriceGranularity; use crate::settings::Settings; use std::collections::HashMap; @@ -23857,45 +23856,40 @@ mod tests { } #[test] - fn bid_map_exposes_aps_renderer_and_selected_bid_id() { + fn bid_map_exposes_a_typed_renderer_and_the_bid_it_picks() { // Sanitization is opt-in, so enable it: the script-only creative // below is what drives this bid onto the renderer path. Left at the // default it would survive processing as an ordinary creative. let mut settings = test_settings(); settings.auction.sanitize_creatives = true; - let mut bid = make_bid("atf_sidebar_ad", 1.50, "aps", "fallback-ad", "", ""); + let mut bid = make_bid("atf_sidebar_ad", 1.50, "example", "fallback-ad", "", ""); bid.bid_id = Some("selected-bid".to_string()); bid.creative = Some("".to_string()); bid.nurl = None; bid.burl = None; bid.renderer = Some( - BidRenderer::from_typed( - APS_RENDERER_TYPE, - &ApsRendererV1 { - version: 1, - account_id: "example-account".to_string(), - bid_id: "selected-bid".to_string(), - creative_id: None, - tag_type: ApsTagType::Iframe, - creative_url: "https://creative.example/render".to_string(), - aax_response: "fictional-base64".to_string(), - width: 300, - height: 250, - }, + BidRenderer::new( + "example", + serde_json::json!({ + "version": 1, + "bidId": "selected-bid", + "creativeUrl": "https://creative.example/render", + "envelope": "fictional-base64", + }), ) - .expect("should build APS renderer descriptor") - .picking_bid_by(crate::integrations::aps::APS_RENDERER_BID_ID_KEY), + .expect("should build the example renderer descriptor") + .picking_bid_by("bidId"), ); let winning_bids = HashMap::from([("atf_sidebar_ad".to_string(), bid)]); let map = build_bid_map(&winning_bids, PriceGranularity::Dense, &settings, "", false); let obj = map["atf_sidebar_ad"] .as_object() - .expect("should include APS bid"); + .expect("should include the renderer bid"); - assert_eq!(obj["hb_bidder"], "aps"); + assert_eq!(obj["hb_bidder"], "example"); assert_eq!(obj["hb_adid"], "selected-bid"); - assert_eq!(obj["renderer"]["type"], "aps"); + assert_eq!(obj["renderer"]["type"], "example"); assert_eq!(obj["renderer"]["bidId"], "selected-bid"); assert!(obj.get("adm").is_none()); @@ -23912,34 +23906,29 @@ mod tests { // copy it. let mut settings = test_settings(); settings.auction.sanitize_creatives = true; - let mut bid = make_bid("atf_sidebar_ad", 1.50, "aps", "ad-id-value", "", ""); + let mut bid = make_bid("atf_sidebar_ad", 1.50, "example", "ad-id-value", "", ""); bid.bid_id = Some("openrtb-bid-id".to_string()); bid.cache_id = None; bid.creative = Some("".to_string()); bid.renderer = Some( - BidRenderer::from_typed( - APS_RENDERER_TYPE, - &ApsRendererV1 { - version: 1, - account_id: "example-account".to_string(), - bid_id: "renderer-bid-id".to_string(), - creative_id: None, - tag_type: ApsTagType::Iframe, - creative_url: "https://creative.example/render".to_string(), - aax_response: "A".repeat(200 * 1024), - width: 300, - height: 250, - }, + BidRenderer::new( + "example", + serde_json::json!({ + "version": 1, + "bidId": "renderer-bid-id", + "creativeUrl": "https://creative.example/render", + "envelope": "A".repeat(200 * 1024), + }), ) - .expect("should build APS renderer descriptor") - .picking_bid_by(crate::integrations::aps::APS_RENDERER_BID_ID_KEY), + .expect("should build the example renderer descriptor") + .picking_bid_by("bidId"), ); let winning_bids = HashMap::from([("atf_sidebar_ad".to_string(), bid)]); let map = build_bid_map(&winning_bids, PriceGranularity::Dense, &settings, "", false); let obj = map["atf_sidebar_ad"] .as_object() - .expect("should include APS bid"); + .expect("should include the renderer bid"); assert_eq!( obj["hb_adid"], "renderer-bid-id", diff --git a/crates/trusted-server-modules/Cargo.toml b/crates/trusted-server-modules/Cargo.toml index 2da46012ec..565630dc04 100644 --- a/crates/trusted-server-modules/Cargo.toml +++ b/crates/trusted-server-modules/Cargo.toml @@ -16,6 +16,7 @@ workspace = true [dependencies] trusted-server-ad-server-mock = { workspace = true } trusted-server-ad-tag-google = { workspace = true } +trusted-server-auction-aps = { workspace = true } trusted-server-audience-permutive = { workspace = true } trusted-server-bot-protection-datadome = { workspace = true } trusted-server-cmp-didomi = { workspace = true } diff --git a/crates/trusted-server-modules/src/lib.rs b/crates/trusted-server-modules/src/lib.rs index 049d55b767..72c2fd740a 100644 --- a/crates/trusted-server-modules/src/lib.rs +++ b/crates/trusted-server-modules/src/lib.rs @@ -31,6 +31,7 @@ pub fn builders() -> Vec { trusted_server_bot_protection_datadome::builder(), trusted_server_ad_tag_google::builder(), trusted_server_ad_tag_google::diagnostics::builder(), + trusted_server_auction_aps::builder(), trusted_server_ad_server_mock::builder(), ] } @@ -95,6 +96,7 @@ mod tests { "bot-protection.datadome", "ad-tag.google", "ad-tag.google.diagnostics", + "auction.aps", "ad-server.mock", ], "should offer the stock modules in the order their hooks run" diff --git a/docs/guide/integrations/aps.md b/docs/guide/integrations/aps.md index b77ac3a963..db2b0273ce 100644 --- a/docs/guide/integrations/aps.md +++ b/docs/guide/integrations/aps.md @@ -380,4 +380,4 @@ cd crates/trusted-server-js/lib npx vitest run test/integrations/aps/render.test.ts test/core/auction.test.ts ``` -See `crates/trusted-server-core/src/integrations/aps.rs` for the request/parser implementation and `crates/trusted-server-js/lib/src/integrations/aps/render.ts` for the browser renderer contract. +See `crates/auction/aps/src/lib.rs` for the request/parser implementation and `crates/trusted-server-js/lib/src/integrations/aps/render.ts` for the browser renderer contract. From 27648c0b7da2449951b0ab7d33079611ecfad5af Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 8 Oct 2026 05:17:16 +0100 Subject: [PATCH 103/124] Prebid's file is divided between the page integration and the demand `integrations/prebid.rs` held three things: the page integration, the request, response and bidder parameter override code the Prebid Server demand runs, and the provider and combined configuration kept for the test build. The shipped code falls into two parts with nothing shared between them, so the file is divided along that line and nothing it does changes. The page integration stays in `prebid.rs`. The override rules and their engine, the transport headers, the response reader and the upstream error text join the demand in `prebid_server.rs`, with the test-only provider that the tests compare the demand with. Neither file refers to the other. The combined test configuration is divided the same way, so each file keeps the fields its own code reads. The page's is `LegacyPrebidConfig`, the browser settings and the bidder list a plan would supply. The demand's keeps the name `LegacyPrebidServerConfig`. Of the file's 178 tests, 57 stay with the page integration and 120 go with the demand. One is not carried: `register_rejects_invalid_bid_param_override_rule` checked that the page registration built from the combined configuration refused a bad override rule, and that registration no longer compiles override rules. The provider's own test of the same refusal stays. `validate_config_for_startup` was test-only and nothing called it, so it is gone. --- .../src/integrations/prebid.rs | 9836 +++-------------- .../src/integrations/prebid_server.rs | 6991 +++++++++++- 2 files changed, 8466 insertions(+), 8361 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index c4606092bb..507045ca89 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -1,8 +1,5 @@ -use std::collections::HashMap; use std::collections::HashSet; use std::sync::{Arc, LazyLock}; -#[cfg(test)] -use std::time::Duration; use async_trait::async_trait; use base64::{ @@ -21,46 +18,16 @@ use serde_json::Value as Json; use url::{Url, Url as ParsedUrl}; use validator::{Validate, ValidationError}; -use crate::auction::openrtb::{ - BidDimensionIndex, BidRejectionReason, ResponseAdmissionDiagnostics, build_bid_dimension_index, - parse_optional_bid_dimension, resolve_bid_dimensions, -}; -use crate::auction::orchestrator::ERROR_TYPE_HTTP_STATUS; use crate::auction::plan::AuctionPlan; -#[cfg(test)] -use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; -use crate::auction::routing::{ProviderAuctionInput, TransportHeaders}; -#[cfg(test)] -use crate::auction::types::{AuctionContext, AuctionRequest, MediaType}; -use crate::auction::types::{AuctionResponse, Bid as AuctionBid}; use crate::cache_policy::{CacheControlPolicy, EdgeCacheHeader}; -use crate::consent_config::ConsentForwardingMode; -use crate::cookies::{CONSENT_COOKIE_NAMES, strip_cookies}; use crate::error::TrustedServerError; -#[cfg(test)] -use crate::http_util::RequestInfo; -use crate::integrations::prebid_server::PrebidServerDemand; use crate::integrations::{ AttributeRewriteAction, IntegrationAttributeContext, IntegrationAttributeRewriter, IntegrationEndpoint, IntegrationHeadInjector, IntegrationHtmlContext, IntegrationProxy, - IntegrationRegistration, UPSTREAM_RTB_MAX_RESPONSE_BYTES, collect_response_bounded, -}; -#[cfg(test)] -use crate::integrations::{ - ensure_integration_backend_with_timeout, predict_integration_backend_name, + IntegrationRegistration, }; -#[cfg(test)] -use crate::openrtb::{ - Banner, ConsentedProvidersSettings, Device, Format, Geo, Imp, ImpExt, ImpStoredRequest, - OpenRtbRequest, PrebidExt, PrebidImpExt, Publisher, Regs, RegsExt, RequestExt, Site, ToExt, - TrustedServerExt, User, UserExt, to_openrtb_i32, -}; -#[cfg(test)] -use crate::platform::PlatformHttpRequest; -use crate::platform::{PlatformResponse, RuntimeServices}; +use crate::platform::RuntimeServices; use crate::proxy::{ProxyRequestConfig, is_host_allowed, proxy_request}; -#[cfg(test)] -use crate::request_signing::{RequestSigner, SIGNING_VERSION, SigningParams}; use crate::settings::{IntegrationConfig, Settings}; pub(crate) const PREBID_INTEGRATION_ID: &str = "prebid"; @@ -119,153 +86,6 @@ const PREBID_BUNDLE_ERROR_CACHE_CONTROL: &str = "no-store"; const PREBID_BUNDLE_ERROR_CONTENT_TYPE: &str = "text/plain; charset=utf-8"; const PREBID_BUNDLE_NOSNIFF_HEADER: &str = "x-content-type-options"; const PREBID_BUNDLE_NOSNIFF_VALUE: &str = "nosniff"; -#[cfg(test)] -const TRUSTED_SERVER_BIDDER: &str = "trustedServer"; -#[cfg(test)] -const BIDDER_PARAMS_KEY: &str = "bidderParams"; -#[cfg(test)] -const ZONE_KEY: &str = "zone"; - -/// Default currency for `OpenRTB` bid floors and responses. -const DEFAULT_CURRENCY: &str = "USD"; - -const PREBID_PUBLIC_ERROR_MESSAGE_CHARS: usize = 500; -const PREBID_ERROR_BODY_PREVIEW_CHARS: usize = 1000; -const PREBID_ERROR_BODY_PREVIEW_BYTES: usize = PREBID_ERROR_BODY_PREVIEW_CHARS * 4; -const PREBID_ERROR_JSON_MAX_DEPTH: usize = 6; -const PREBID_ERROR_JSON_KEYS: [&str; 6] = - ["message", "error", "errors", "detail", "title", "reason"]; - -#[derive(Debug, Eq, PartialEq)] -struct BoundedPrebidErrorText { - text: String, - truncated: bool, -} - -fn bounded_prebid_error_text(value: &str, max_chars: usize) -> Option { - let mut text = String::new(); - let mut char_count = 0; - let mut pending_space = false; - let mut truncated = false; - - for character in value.chars() { - if character.is_whitespace() || character.is_control() { - pending_space = !text.is_empty(); - continue; - } - - if pending_space { - if char_count == max_chars { - truncated = true; - break; - } - text.push(' '); - char_count += 1; - pending_space = false; - } - - if char_count == max_chars { - truncated = true; - break; - } - text.push(character); - char_count += 1; - } - - (!text.is_empty()).then_some(BoundedPrebidErrorText { text, truncated }) -} - -fn prebid_body_preview(body: &[u8]) -> Option { - let bounded_body = &body[..body.len().min(PREBID_ERROR_BODY_PREVIEW_BYTES)]; - let mut preview = bounded_prebid_error_text( - &String::from_utf8_lossy(bounded_body), - PREBID_ERROR_BODY_PREVIEW_CHARS, - )?; - preview.truncated |= body.len() > bounded_body.len(); - Some(preview) -} - -fn nested_prebid_json_error_message( - value: &Json, - depth: usize, - allow_direct_string: bool, -) -> Option<&str> { - if depth > PREBID_ERROR_JSON_MAX_DEPTH { - return None; - } - - match value { - Json::String(message) if allow_direct_string => { - (!message.trim().is_empty()).then_some(message.as_str()) - } - Json::Array(values) => values.iter().find_map(|value| { - nested_prebid_json_error_message(value, depth + 1, allow_direct_string) - }), - Json::Object(values) => PREBID_ERROR_JSON_KEYS - .iter() - .find_map(|key| { - values - .get(*key) - .and_then(|value| nested_prebid_json_error_message(value, depth + 1, true)) - }) - .or_else(|| { - values - .values() - .find_map(|value| nested_prebid_json_error_message(value, depth + 1, false)) - }), - _ => None, - } -} - -fn prebid_json_error_message(value: &Json) -> Option<&str> { - let Json::Object(values) = value else { - return None; - }; - - PREBID_ERROR_JSON_KEYS.iter().find_map(|key| { - values - .get(*key) - .and_then(|value| nested_prebid_json_error_message(value, 0, true)) - }) -} - -fn is_plain_text_content_type(content_type: Option<&str>) -> bool { - content_type.is_some_and(|value| { - value - .split(';') - .next() - .is_some_and(|mime| mime.trim().eq_ignore_ascii_case("text/plain")) - }) -} - -fn extract_prebid_error_message( - body: &[u8], - content_type: Option<&str>, -) -> Option { - let candidate = match serde_json::from_slice::(body) { - Ok(value) => prebid_json_error_message(&value)?.to_owned(), - Err(_) if is_plain_text_content_type(content_type) => { - std::str::from_utf8(body).ok()?.to_owned() - } - Err(_) => return None, - }; - - // Do not expose an HTML error page even if an intermediary labels it as text/plain. - if candidate.trim_start().starts_with('<') { - return None; - } - - bounded_prebid_error_text(&candidate, PREBID_PUBLIC_ERROR_MESSAGE_CHARS) -} - -/// CCPA/US-privacy string sent when the `Sec-GPC` header signals opt-out. -/// -/// Encodes: version `1`, notice given (`Y`), user opted out (`Y`), LSPA not -/// signed (`N`). The opt-out (position 2 = `Y`) matches GPC intent. Position 3 -/// (`N` = LSPA not applicable) is a conservative default that may not hold for -/// all publishers — consider making this configurable per-publisher in the future. -#[cfg(test)] -const GPC_US_PRIVACY: &str = "1YYN"; /// Rejects a Prebid User ID identifier that Prebid.js could not address. /// @@ -379,11 +199,11 @@ pub struct PrebidManagedUserIdConfig { pub storage: Option, } +/// The configuration the tests build an integration from without an auction +/// plan: the browser settings, and the bidder list a plan would supply. #[cfg(test)] #[derive(Debug, Clone, Deserialize, Serialize, Validate)] -pub struct LegacyPrebidServerConfig { - #[validate(url)] - pub server_url: String, +pub struct LegacyPrebidConfig { /// Prebid Server account ID, injected into the client-side bundle via /// `window.__tsjs_prebid.accountId` so publishers don't need to configure /// it in JavaScript. @@ -406,14 +226,6 @@ pub struct LegacyPrebidServerConfig { pub bidders: Vec, #[serde(default)] pub debug: bool, - /// Sets the `OpenRTB` `test: 1` flag on outgoing requests. When enabled, - /// bidders treat the auction as non-billable test traffic, which can - /// significantly reduce fill rates. Separate from `debug` so you can get - /// debug diagnostics without suppressing real demand. - #[serde(default)] - pub test_mode: bool, - #[serde(default)] - pub debug_query_params: Option, /// Patterns to match Prebid script URLs for serving empty JS. /// Supports suffix matching (e.g., "/prebid.min.js" matches any path ending with that) /// and wildcard patterns (e.g., "/static/prebid/*" matches paths under that prefix). @@ -456,80 +268,10 @@ pub struct LegacyPrebidServerConfig { #[serde(default, deserialize_with = "crate::settings::vec_from_seq_or_map")] #[validate(custom(function = "validate_excluded_gam_ad_unit_path_suffixes"))] pub excluded_gam_ad_unit_path_suffixes: Vec, - /// Compatibility sugar for per-bidder, per-zone param overrides. - /// - /// This preserves the natural `bidder -> zone -> params` config shape for - /// the existing zone-based use case, but it is normalized into the - /// canonical [`bid_param_override_rules`](Self::bid_param_override_rules) - /// engine before runtime use. - /// - /// Example in TOML: - /// ```toml - /// [auction.prebid.bid_param_zone_overrides.kargo] - /// header = {placementId = "_s2sHeaderId"} - /// in_content = {placementId = "_s2sContentId"} - /// fixed_bottom = {placementId = "_s2sBottomId"} - /// ``` - #[serde(default)] - pub bid_param_zone_overrides: HashMap>>, - /// Compatibility sugar for static per-bidder parameter overrides. - /// - /// These rules are normalized into the canonical - /// [`bid_param_override_rules`](Self::bid_param_override_rules) engine and - /// therefore share the same validation and precedence behavior as explicit - /// rules. - /// - /// Example in TOML: - /// ```toml - /// [auction.prebid.bid_param_overrides.bidder-name] - /// param1 = 12345 - /// param2 = "value" - /// ``` - #[serde(default)] - pub bid_param_overrides: HashMap>, - /// Canonical ordered bidder-param override rules. - /// - /// Each rule has structured `when` matchers and a non-empty `set` object - /// that is shallow-merged into the bidder params when every matcher - /// matches. Compatibility fields such as [`bid_param_overrides`](Self::bid_param_overrides) - /// and [`bid_param_zone_overrides`](Self::bid_param_zone_overrides) are - /// normalized into the same runtime rule engine before request handling. - /// - /// Example in TOML: - /// ```toml - /// [[auction.prebid.bid_param_override_rules]] - /// when.bidder = "kargo" - /// when.zone = "header" - /// set = { placementId = "_abc" } - /// ``` - #[serde(default)] - pub bid_param_override_rules: Vec, - /// How consent signals are forwarded to Prebid Server. - /// - /// - `openrtb_only` — consent in `OpenRTB` body only, consent cookies stripped - /// - `cookies_only` — consent cookies forwarded, body consent fields omitted - /// - `both` — consent in both cookies and body (default) - #[serde(default)] - pub consent_forwarding: ConsentForwardingMode, - /// Strip `nurl` and `burl` from PBS bids before they reach `window.tsjs.bids`. - /// - /// Set to `true` when the PBS deployment is configured to fire win/billing - /// notifications server-side (e.g. `ext.prebid.events.enabled`), so the - /// client does not double-fire them via `sendBeacon`. Default: `false`. - #[serde(default)] - pub suppress_nurl: bool, - /// Bidder seats whose `nurl` and `burl` should be stripped before they reach - /// `window.tsjs.bids`. - /// - /// Use this when only specific PBS seats fire win/billing notifications - /// internally. The global [`suppress_nurl`](Self::suppress_nurl) switch still - /// suppresses every bidder when set. - #[serde(default, deserialize_with = "crate::settings::vec_from_seq_or_map")] - pub suppress_nurl_bidders: Vec, } #[cfg(test)] -impl IntegrationConfig for LegacyPrebidServerConfig {} +impl IntegrationConfig for LegacyPrebidConfig {} /// CLI build inputs retained in app config but ignored safely by the runtime. #[derive(Debug, Clone, Default, Deserialize, Serialize)] @@ -624,8 +366,8 @@ impl Default for PrebidIntegrationConfig { impl IntegrationConfig for PrebidIntegrationConfig {} #[cfg(test)] -impl From<&LegacyPrebidServerConfig> for PrebidIntegrationConfig { - fn from(config: &LegacyPrebidServerConfig) -> Self { +impl From<&LegacyPrebidConfig> for PrebidIntegrationConfig { + fn from(config: &LegacyPrebidConfig) -> Self { Self { account_id: config.account_id.clone(), timeout_ms: config.timeout_ms, @@ -643,7 +385,7 @@ impl From<&LegacyPrebidServerConfig> for PrebidIntegrationConfig { } #[cfg(test)] -fn remove_aps_bidders(config: &mut LegacyPrebidServerConfig) { +fn remove_aps_bidders(config: &mut LegacyPrebidConfig) { for (field, bidders) in [ ("bidders", &mut config.bidders), ("client_side_bidders", &mut config.client_side_bidders), @@ -701,7 +443,7 @@ fn validate_excluded_gam_ad_unit_path_suffixes(values: &[String]) -> Result<(), } #[cfg(test)] -fn canonicalize_excluded_gam_ad_unit_path_suffixes(config: &mut LegacyPrebidServerConfig) { +fn canonicalize_excluded_gam_ad_unit_path_suffixes(config: &mut LegacyPrebidConfig) { let mut canonical = Vec::with_capacity(config.excluded_gam_ad_unit_path_suffixes.len()); for suffix in std::mem::take(&mut config.excluded_gam_ad_unit_path_suffixes) { if !canonical.contains(&suffix) { @@ -714,8 +456,8 @@ fn canonicalize_excluded_gam_ad_unit_path_suffixes(config: &mut LegacyPrebidServ #[cfg(test)] fn load_config( settings: &Settings, -) -> Result, Report> { - let Some(mut config) = settings.module_config::(MODULE)? else { +) -> Result, Report> { + let Some(mut config) = settings.module_config::(MODULE)? else { return Ok(None); }; canonicalize_excluded_gam_ad_unit_path_suffixes(&mut config); @@ -723,54 +465,6 @@ fn load_config( Ok(Some(config)) } -/// Validate the Prebid config the provider list names, using the same -/// startup-only checks as runtime registration. -/// -/// # Errors -/// -/// Returns a configuration error if the Prebid settings fail typed parsing, -/// schema validation, or bidder-param override compilation. -#[cfg(test)] -pub fn validate_config_for_startup( - settings: &Settings, -) -> Result, Report> { - let Some(config) = load_config(settings)? else { - return Ok(None); - }; - BidParamOverrideEngine::try_from_config(&config)?; - validate_external_bundle_config(&config, &settings.proxy.allowed_domains)?; - Ok(Some(config)) -} - -/// Canonical bidder-param override rule. -/// -/// A rule matches against the request-time facts in [`BidParamOverrideWhen`] -/// and shallow-merges [`set`](Self::set) into the bidder params when all -/// populated matchers are equal. -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct BidParamOverrideRule { - /// Structured exact-match conditions for this rule. - pub when: BidParamOverrideWhen, - /// Parameters shallow-merged into bidder params when the rule matches. - /// Top-level keys in this object are inserted or replaced; nested objects - /// are replaced wholesale rather than recursed into. - pub set: serde_json::Map, -} - -/// Structured exact-match conditions for a [`BidParamOverrideRule`]. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct BidParamOverrideWhen { - /// Bidder name matcher. - #[serde(default)] - pub bidder: Option, - /// Zone matcher from `mediaTypes.banner.name` propagated via - /// `trustedServer.zone`. - #[serde(default)] - pub zone: Option, -} - fn default_timeout_ms() -> u32 { 1000 } @@ -990,7 +684,7 @@ pub(crate) fn validate_browser_bidder_ownership( #[cfg(test)] fn validate_external_bundle_config( - config: &LegacyPrebidServerConfig, + config: &LegacyPrebidConfig, allowed_domains: &[String], ) -> Result<(), Report> { validate_external_bundle_url_allowed(config.external_bundle_url.as_deref(), allowed_domains) @@ -1000,26 +694,17 @@ pub struct PrebidIntegration { config: PrebidIntegrationConfig, planned_head_inserts: Option>, #[cfg(test)] - legacy_config: Option, - #[cfg(test)] - engine: Arc, + legacy_config: Option, } impl PrebidIntegration { #[cfg(test)] - fn try_new(config: LegacyPrebidServerConfig) -> Result, Report> { - let engine = Arc::new(BidParamOverrideEngine::try_from_config(&config)?); - Ok(Arc::new(Self { + fn new(config: LegacyPrebidConfig) -> Arc { + Arc::new(Self { config: PrebidIntegrationConfig::from(&config), planned_head_inserts: None, legacy_config: Some(config), - engine, - })) - } - - #[cfg(test)] - fn new(config: LegacyPrebidServerConfig) -> Arc { - Self::try_new(config).expect("should compile prebid bid param overrides") + }) } fn for_browser_plan(config: &PrebidIntegrationConfig, plan: &AuctionPlan) -> Arc { @@ -1028,24 +713,11 @@ impl PrebidIntegration { planned_head_inserts: None, #[cfg(test)] legacy_config: None, - #[cfg(test)] - engine: Arc::new(BidParamOverrideEngine::default()), }; integration.planned_head_inserts = Some(integration.head_inserts_for_plan(config, plan)); Arc::new(integration) } - #[cfg(test)] - fn auction_provider(&self) -> PrebidAuctionProvider { - PrebidAuctionProvider { - config: self - .legacy_config - .clone() - .expect("should retain legacy config for legacy provider tests"), - bid_param_override_engine: Arc::clone(&self.engine), - } - } - fn matches_script_url(&self, attr_value: &str) -> bool { let trimmed = attr_value.trim(); let without_query = trimmed.split(['?', '#']).next().unwrap_or(trimmed); @@ -1365,7 +1037,7 @@ fn build( } } - Ok(Some(PrebidIntegration::try_new(config)?)) + Ok(Some(PrebidIntegration::new(config))) } /// Register the Prebid integration when a section selects it. @@ -1625,8370 +1297,1950 @@ impl<'a> From<&'a PrebidIntegrationConfig> for InjectedPrebidClientConfig<'a> { } } -/// Returns `true` when `params` is not a usable PBS bidder-params object — an -/// empty object `{}` or any non-object value such as `null`. -/// -/// PBS rejects an imp whose `bidder.` carries such a value, and it cannot -/// tell a fabricated empty from an explicitly supplied one — they are identical -/// bytes on the wire. The merge uses this to stop an unusable value from -/// clobbering real params, and the final pass uses it to drop whatever remains. -#[cfg(test)] -fn is_unusable_bidder_params(params: &Json) -> bool { - // `None` covers non-object values (e.g. `null`); an empty map covers `{}`. - params.as_object().is_none_or(serde_json::Map::is_empty) -} - #[cfg(test)] -fn expand_trusted_server_bidders( - configured_bidders: &[String], - params: &Json, -) -> HashMap { - let per_bidder = params.get(BIDDER_PARAMS_KEY).and_then(Json::as_object); - - if configured_bidders.is_empty() { - return per_bidder - .map(|m| m.iter().map(|(k, v)| (k.clone(), v.clone())).collect()) - .unwrap_or_default(); - } - - configured_bidders - .iter() - .map(|bidder| { - let value = per_bidder - .and_then(|m| m.get(bidder).cloned()) - .unwrap_or_else(|| { - // No per-bidder map → use entire params as shared config - if per_bidder.is_some() { - Json::Object(Default::default()) - } else { - params.clone() - } - }); - (bidder.clone(), value) - }) - .collect() -} +mod tests { + use std::sync::Arc; -/// Shallow-merges `override_obj` into `params`. -/// -/// When `params` is a JSON object, each top-level key in `override_obj` is -/// inserted or replaced in `params`. Nested objects are replaced wholesale — -/// they are not recursed into. When `params` is not an object, it is left -/// untouched to preserve pre-existing behavior. -/// -/// Returns whether `params` was an object and the merge path ran. -fn merge_bidder_param_object( - params: &mut Json, - override_obj: &serde_json::Map, -) -> bool { - if let Json::Object(base) = params { - for (k, v) in override_obj { - base.insert(k.clone(), v.clone()); - } + use super::*; - true - } else { - false - } -} + use crate::auction::plan::{BidderId, BidderRouteConfig, ProviderId}; -// ============================================================================ -// Generic bid-parameter override engine -// ============================================================================ + use crate::html_processor::{HtmlProcessorConfig, create_html_processor}; + use crate::integrations::{ + AttributeRewriteAction, IntegrationDocumentState, IntegrationRegistry, + }; + use crate::platform::test_support::{StubHttpClient, build_services_with_http_client}; -#[cfg(test)] -fn warn_unconfigured_bidder(config: &LegacyPrebidServerConfig, bidder: &str, field: &str) { - if !config.bidders.iter().any(|b| b == bidder) { - if config.client_side_bidders.iter().any(|b| b == bidder) { - log::warn!( - "prebid: {field} entry targets client-side-only bidder \ - '{bidder}' — server-side override will never apply" - ); - } else { - log::warn!( - "prebid: {field} entry references unconfigured bidder \ - '{bidder}' — rule will never fire" - ); - } - } -} + use crate::settings::Settings; + use crate::streaming_processor::{Compression, PipelineConfig, StreamingPipeline}; + use crate::test_support::tests::create_test_settings; + use base64::engine::general_purpose::STANDARD as TEST_BASE64_STANDARD; -#[derive(Debug, Default, Clone)] -pub(crate) struct BidParamOverrideEngine { - rules: Vec, - // Maps bidder name to the indices (into `rules`) of rules that constrain on that bidder. - // Rules with no bidder constraint (zone-only or catch-all) are kept in `wildcard_indices`. - // Both slices are in declaration order; `apply` merges them without allocation. - bidder_index: HashMap>, - wildcard_indices: Vec, -} + use http::Method; + use serde_json::json; + use std::collections::BTreeMap; + use std::io::Cursor; + use std::str::FromStr as _; -#[derive(Debug, Clone, PartialEq, Eq)] -struct CompiledBidParamOverrideRule { - bidder: Option, - zone: Option, - set: serde_json::Map, -} + #[test] + fn external_bundle_sha256_validation_matches_hex_pattern() { + use validator::Validate as _; -#[derive(Debug, Copy, Clone)] -struct BidParamOverrideFacts<'a> { - bidder: &'a str, - zone: Option<&'a str>, -} + let config = |sha: &str| -> PrebidIntegrationConfig { + serde_json::from_value(serde_json::json!({ + "external_bundle_sha256": sha, + })) + .expect("should deserialize prebid config") + }; -impl BidParamOverrideEngine { - #[cfg(test)] - fn try_from_config( - config: &LegacyPrebidServerConfig, - ) -> Result> { - let mut rules = Vec::new(); - - let mut bidder_overrides = config.bid_param_overrides.iter().collect::>(); - bidder_overrides.sort_by(|(left, _), (right, _)| left.as_str().cmp(right.as_str())); - - for (bidder, set) in bidder_overrides { - warn_unconfigured_bidder(config, bidder, "bid_param_overrides"); - rules.push(CompiledBidParamOverrideRule::from_bidder_override( - bidder.as_str(), - set, - )?); - } + // Exactly 64 hex digits (either case) passes. + config(&"a".repeat(64)) + .validate() + .expect("64-char lowercase hex sha256 should pass"); + config("ABCDEF0123456789abcdef0123456789ABCDEF0123456789abcdef0123456789") + .validate() + .expect("mixed-case 64-char hex sha256 should pass"); - let mut zone_overrides = config.bid_param_zone_overrides.iter().collect::>(); - zone_overrides.sort_by(|(left, _), (right, _)| left.as_str().cmp(right.as_str())); - - for (bidder, zone_override_sets) in zone_overrides { - warn_unconfigured_bidder(config, bidder, "bid_param_zone_overrides"); - let mut sorted_zone_overrides = zone_override_sets.iter().collect::>(); - sorted_zone_overrides - .sort_by(|(left, _), (right, _)| left.as_str().cmp(right.as_str())); - - for (zone, set) in sorted_zone_overrides { - rules.push(CompiledBidParamOverrideRule::from_zone_override( - bidder.as_str(), - zone.as_str(), - set, - )?); - } + // Wrong length or non-hex characters are rejected. + for bad in [ + "a".repeat(63), + "a".repeat(65), + "g".repeat(64), + String::new(), + ] { + config(&bad) + .validate() + .expect_err(&format!("invalid sha256 {bad:?} should be rejected")); } + } - for rule in &config.bid_param_override_rules { - if let Some(bidder) = &rule.when.bidder { - warn_unconfigured_bidder(config, bidder, "bid_param_override_rules"); - } - rules.push(CompiledBidParamOverrideRule::try_from(rule)?); - } + fn make_settings() -> Settings { + create_test_settings() + } - let mut bidder_index: HashMap> = HashMap::new(); - let mut wildcard_indices: Vec = Vec::new(); - for (idx, rule) in rules.iter().enumerate() { - match &rule.bidder { - Some(bidder) => bidder_index.entry(bidder.clone()).or_default().push(idx), - None => wildcard_indices.push(idx), - } - } + #[test] + fn injected_prebid_config_escapes_every_less_than_sign() { + let config_json = serialize_injected_prebid_config(&json!({ + "accountId": "x