diff --git a/CHANGELOG.md b/CHANGELOG.md index 4cae2f48d5..5447933a50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- **Breaking:** auction telemetry is selected the way every other module is. `[tinybird]` is now `[analytics.tinybird]`, and `[analytics] module = "tinybird"` selects it, which replaces the `enabled` line. A configuration that still carries `[tinybird]` is refused with directions. A stored configuration written by an earlier version carries the table even where telemetry was off, so push the configuration again with this version. The secret reference is `analytics.tinybird.auction_token_secret`, the table refuses a key it does not know, and a section that selects nothing, or a module this build does not have, is refused. A row is sent only for a reader whose permissions include `analytics.ad_reporting.measure_ad_performance`, so none is sent for a reader the consent gate refuses unless they permit it. The Axum, Cloudflare and Spin adapters send no telemetry, and log a warning at startup when the section selects a module. - The DataDome module asks its Protection API about a publisher's pages beneath `/admin/` like any other page. It skipped every path beneath `/admin/` and `/_ts/admin/`, which left a publisher's own `/admin/` pages out of bot protection. - **Breaking:** `ts cache purge` asks the hosting platform to purge instead of calling the service. It takes `--service-id ` where it took `--service `, reads a Fastly API token from `FASTLY_API_TOKEN` where it read `TRUSTED_SERVER_ADMIN_PASSWORD`, and no longer takes `--username`. It runs `fastly service purge --key`, which needs Fastly CLI 14.0.0 or later, with the surrogate key it derives by the function the service tags its cached objects with, so `--all` and `--page ` purge exactly what they purged before. - **Breaking:** A module's change to a page runs only on the pages where a `[[fetch]]` or `[[serve]]` entry names its middleware. A deployment adds the entries when it upgrades. Until it does, a module changes no page, and startup logs a warning naming each middleware that no entry names. "Placing page changes" in the configuration guide lists the middleware each module supplies and the phase each runs in. To keep a page as it was, name the middleware of the modules a deployment selects in one `[[fetch]]` entry and one `[[serve]]` entry for `text/html`, in the order `trusted-server.example.toml` shows them, which is the order the modules' hooks ran in. diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 165f6250bd..5a7be228f8 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -1,5 +1,6 @@ use core::future::Future; use std::sync::Arc; +use std::sync::atomic::AtomicBool; use edgezero_core::app::Hooks; use edgezero_core::context::RequestContext; @@ -10,6 +11,7 @@ use edgezero_core::http::{ use edgezero_core::router::RouterService; use error_stack::Report; use trusted_server_core::auction::endpoints::handle_auction; +use trusted_server_core::auction::telemetry::warn_once_when_analytics_is_unsent; use trusted_server_core::auction::{ AuctionOrchestrator, build_orchestrator_with_plan, compile_auction_plan_with, }; @@ -78,6 +80,9 @@ pub struct AppState { services: Option, } +/// Set once startup has logged that this adapter sends no auction telemetry. +static UNSENT_ANALYTICS_LOGGED: AtomicBool = AtomicBool::new(false); + /// The permission signal modules this adapter links, in the order they run /// when configuration names none. Global Privacy Control is first because it /// is a browser setting with no interface of its own, and the three that @@ -213,6 +218,14 @@ fn build_state_with_registrations_and_services( &settings, &shipped_signal_modules(), )?; + // This adapter builds no telemetry sink, so a module `[analytics]` selects + // sends nothing unless the caller's services carry a sink. + warn_once_when_analytics_is_unsent( + &settings, + services.as_ref(), + "Axum", + &UNSENT_ANALYTICS_LOGGED, + ); let permission_policy = settings.required_permission_policy()?; Ok(Arc::new(AppState { @@ -929,6 +942,27 @@ mod tests { ); } + /// `[analytics]` selecting a module still loads on this adapter, which + /// sends no auction telemetry, and startup says so. + #[test] + fn startup_logs_that_this_adapter_sends_no_auction_telemetry() { + let settings = Settings::from_toml(&format!( + "{}\n{}", + trusted_server_core::test_support::tests::crate_test_settings_str(), + trusted_server_core::test_support::template::documented_block("[analytics]") + )) + .expect("should load settings selecting the documented analytics module") + // A loaded domain carries its policy. + .with_permission_policy(trusted_server_core::permissions::PermissionPolicy::sample()); + + build_state_with_settings(settings).expect("should accept a selected analytics module"); + + assert!( + UNSENT_ANALYTICS_LOGGED.load(std::sync::atomic::Ordering::Relaxed), + "startup should log that this adapter sends no auction telemetry" + ); + } + mod startup_policy { use std::path::Path; diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index 81fa864e87..b5681bffbc 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -1,6 +1,7 @@ use core::future::Future; use core::pin::Pin; use std::sync::Arc; +use std::sync::atomic::AtomicBool; use edgezero_core::app::Hooks; use edgezero_core::context::RequestContext; @@ -9,6 +10,7 @@ use edgezero_core::http::{HeaderValue, Method, Request, Response, header}; use edgezero_core::router::RouterService; use error_stack::Report; use trusted_server_core::auction::endpoints::handle_auction; +use trusted_server_core::auction::telemetry::warn_once_when_analytics_is_unsent; use trusted_server_core::auction::{ AuctionOrchestrator, build_orchestrator_with_plan, compile_auction_plan_with, }; @@ -257,6 +259,14 @@ fn build_state_with_registrations_and_services( &settings, &shipped_signal_modules(), )?; + // This adapter builds no telemetry sink, so a module `[analytics]` selects + // sends nothing unless the caller's services carry a sink. + warn_once_when_analytics_is_unsent( + &settings, + services.as_ref(), + "Cloudflare", + &UNSENT_ANALYTICS_LOGGED, + ); let permission_policy = settings.required_permission_policy()?; Ok(Arc::new(AppState { @@ -270,6 +280,11 @@ fn build_state_with_registrations_and_services( })) } +/// Set once startup has logged that this adapter sends no auction telemetry. +/// The state is built for every request, so the flag keeps the warning to +/// one for each instance. +static UNSENT_ANALYTICS_LOGGED: AtomicBool = AtomicBool::new(false); + /// The permission signal modules this adapter links, in the order they run /// when configuration names none. Global Privacy Control is first because it /// is a browser setting with no interface of its own, and the three that @@ -1131,4 +1146,25 @@ mod tests { } } } + + /// `[analytics]` selecting a module still loads on this adapter, which + /// sends no auction telemetry, and startup says so. + #[test] + fn startup_logs_that_this_adapter_sends_no_auction_telemetry() { + let settings = Settings::from_toml(&format!( + "{}\n{}", + trusted_server_core::test_support::tests::crate_test_settings_str(), + trusted_server_core::test_support::template::documented_block("[analytics]") + )) + .expect("should load settings selecting the documented analytics module") + // A loaded domain carries its policy. + .with_permission_policy(trusted_server_core::permissions::PermissionPolicy::sample()); + + build_state_with_settings(settings).expect("should accept a selected analytics module"); + + assert!( + UNSENT_ANALYTICS_LOGGED.load(std::sync::atomic::Ordering::Relaxed), + "startup should log that this adapter sends no auction telemetry" + ); + } } diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 98f8b48b50..382d0f1a13 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -1840,6 +1840,41 @@ mod tests { ); } + /// The sink startup builds from `[analytics]` is the one each request's + /// services carry, so a selected module sends from the request path and a + /// deployment that selects none sends nothing. + #[test] + fn per_request_services_carry_the_sink_analytics_selects() { + let base = trusted_server_core::test_support::tests::crate_test_settings_str(); + let analytics = + trusted_server_core::test_support::template::documented_block("[analytics]"); + let context = RequestContext::new( + empty_request(Method::GET, "/article"), + PathParams::default(), + ); + + for (document, sends) in [ + (base.clone(), false), + (format!("{base}\n{analytics}"), true), + ] { + let settings = Settings::from_toml(&document) + .expect("should load the settings") + // A loaded domain carries its policy. + .with_permission_policy( + trusted_server_core::permissions::PermissionPolicy::sample(), + ); + let state = build_state_from_settings(settings).expect("should build state"); + + let services = build_per_request_services(&state, &context); + + assert_eq!( + services.auction_telemetry_sink().is_enabled(), + sends, + "a request's sink should send exactly when [analytics] selects a module" + ); + } + } + /// Builds a router whose `AppState` uses a registry containing the given /// request filters (and no routes), so dispatch-level request-filter /// behavior can be exercised without a real integration. diff --git a/crates/trusted-server-adapter-fastly/src/tinybird.rs b/crates/trusted-server-adapter-fastly/src/tinybird.rs index db13584b53..bec4bad6fb 100644 --- a/crates/trusted-server-adapter-fastly/src/tinybird.rs +++ b/crates/trusted-server-adapter-fastly/src/tinybird.rs @@ -20,21 +20,18 @@ const TINYBIRD_FIRST_BYTE_TIMEOUT: Duration = Duration::from_secs(2); const TINYBIRD_BETWEEN_BYTES_TIMEOUT: Duration = Duration::from_secs(2); const TINYBIRD_MAX_ROWS_PER_AUCTION_BATCH: usize = 512; -/// Build the configured auction telemetry sink. +/// Build the auction telemetry sink, which sends only when `[analytics]` +/// selects the analytics module. #[must_use] pub(crate) fn auction_sink_from_settings(settings: &Settings) -> Arc { - if settings.tinybird.enabled { - Arc::new(FastlyTinybirdAuctionTelemetrySink::new( - settings.tinybird.clone(), - )) - } else { - Arc::new(NoopAuctionTelemetrySink) + match settings.analytics_module() { + Some(telemetry) => Arc::new(FastlyTinybirdAuctionTelemetrySink::new(telemetry.clone())), + None => Arc::new(NoopAuctionTelemetrySink), } } #[derive(Debug, Clone)] struct FastlyTinybirdAuctionTelemetrySink { - enabled: bool, target: TinybirdEventsTarget, } @@ -57,7 +54,7 @@ impl TinybirdEventsTarget { dataset: config.auction_dataset, append_token: config .auction_token_secret - .expect("should contain a resolved Tinybird auction token when enabled"), + .expect("should contain a resolved Tinybird auction token when selected"), uri, backend_spec, max_body_bytes: config.max_body_bytes, @@ -67,9 +64,7 @@ impl TinybirdEventsTarget { impl FastlyTinybirdAuctionTelemetrySink { fn new(config: TinybirdSettings) -> Self { - let enabled = config.enabled; Self { - enabled, target: TinybirdEventsTarget::from_config(config), } } @@ -149,16 +144,12 @@ impl FastlyTinybirdAuctionTelemetrySink { #[async_trait::async_trait(?Send)] impl AuctionTelemetrySink for FastlyTinybirdAuctionTelemetrySink { - fn is_enabled(&self) -> bool { - self.enabled - } - async fn emit_auction_events( &self, services: &RuntimeServices, batch: AuctionEventBatch, ) -> Result<(), Report> { - if !self.enabled || batch.is_empty() { + if batch.is_empty() { return Ok(()); } @@ -431,9 +422,8 @@ mod tests { .build() } - fn enabled_config() -> TinybirdSettings { + fn tinybird_settings() -> TinybirdSettings { TinybirdSettings { - enabled: true, api_host: "api.us-east.aws.tinybird.co".to_owned(), secret_store: None, auction_dataset: "auction_events_raw".to_owned(), @@ -480,7 +470,7 @@ mod tests { Arc::clone(&http_client), HashMap::new(), ); - let sink = FastlyTinybirdAuctionTelemetrySink::new(enabled_config()); + let sink = FastlyTinybirdAuctionTelemetrySink::new(tinybird_settings()); futures::executor::block_on( sink.emit_auction_events(&services, AuctionEventBatch::new(vec![test_row()])), @@ -528,7 +518,7 @@ mod tests { } #[test] - fn disabled_sink_does_not_dispatch() { + fn settings_selecting_no_analytics_module_build_a_sink_that_sends_nothing() { let backend = Arc::new(RecordingBackend::default()); let http_client = Arc::new(RecordingHttpClient::default()); let services = services( @@ -536,18 +526,17 @@ mod tests { Arc::clone(&http_client), HashMap::new(), ); - let mut config = enabled_config(); - config.enabled = false; - let sink = FastlyTinybirdAuctionTelemetrySink::new(config); + let sink = auction_sink_from_settings(&Settings::default()); + assert!(!sink.is_enabled(), "should build the no-op sink"); futures::executor::block_on( sink.emit_auction_events(&services, AuctionEventBatch::new(vec![test_row()])), ) - .expect("should ignore disabled sink"); + .expect("the no-op sink should accept the batch"); assert!( backend.specs.lock().expect("should lock specs").is_empty(), - "should not ensure backend when disabled" + "should not ensure a backend when nothing is selected" ); assert!( http_client @@ -555,10 +544,23 @@ mod tests { .lock() .expect("should lock recorded requests") .is_empty(), - "should not send when disabled" + "should not send when nothing is selected" ); } + #[test] + fn settings_selecting_the_analytics_module_build_a_sink_that_sends() { + let mut settings = Settings::default(); + settings.analytics = Some(trusted_server_core::settings::AnalyticsConfig { + module: Some("tinybird".to_owned()), + tinybird: Some(tinybird_settings()), + }); + + let sink = auction_sink_from_settings(&settings); + + assert!(sink.is_enabled(), "should build the sink that sends"); + } + #[test] fn empty_batch_does_not_dispatch() { let backend = Arc::new(RecordingBackend::default()); @@ -568,7 +570,7 @@ mod tests { Arc::clone(&http_client), HashMap::new(), ); - let sink = FastlyTinybirdAuctionTelemetrySink::new(enabled_config()); + let sink = FastlyTinybirdAuctionTelemetrySink::new(tinybird_settings()); futures::executor::block_on( sink.emit_auction_events(&services, AuctionEventBatch::new(Vec::new())), @@ -601,7 +603,7 @@ mod tests { b"append-token".to_vec(), )]), ); - let sink = FastlyTinybirdAuctionTelemetrySink::new(enabled_config()); + let sink = FastlyTinybirdAuctionTelemetrySink::new(tinybird_settings()); let rows = vec![test_row(); TINYBIRD_MAX_ROWS_PER_AUCTION_BATCH + 1]; let result = futures::executor::block_on( diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index d302976c85..2129b8e6cc 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -1,5 +1,6 @@ use std::net::{IpAddr, SocketAddr}; use std::sync::Arc; +use std::sync::atomic::AtomicBool; #[cfg(all(feature = "spin", target_arch = "wasm32"))] use edgezero_adapter_spin::config_store::SpinConfigStore; @@ -13,6 +14,7 @@ use edgezero_core::http::{HeaderValue, Method, Request, Response, StatusCode, he use edgezero_core::router::RouterService; use error_stack::Report; use trusted_server_core::auction::endpoints::handle_auction; +use trusted_server_core::auction::telemetry::warn_once_when_analytics_is_unsent; use trusted_server_core::auction::{ AuctionOrchestrator, build_orchestrator_with_plan, compile_auction_plan_with, }; @@ -245,6 +247,14 @@ fn build_state_with_registrations_and_services( &settings, &shipped_signal_modules(), )?; + // This adapter builds no telemetry sink, so a module `[analytics]` selects + // sends nothing unless the caller's services carry a sink. + warn_once_when_analytics_is_unsent( + &settings, + services.as_ref(), + "Spin", + &UNSENT_ANALYTICS_LOGGED, + ); let permission_policy = settings.required_permission_policy()?; Ok(Arc::new(AppState { @@ -258,6 +268,11 @@ fn build_state_with_registrations_and_services( })) } +/// Set once startup has logged that this adapter sends no auction telemetry. +/// The state is built for every request, so the flag keeps the warning to +/// one for each instance. +static UNSENT_ANALYTICS_LOGGED: AtomicBool = AtomicBool::new(false); + /// The permission signal modules this adapter links, in the order they run /// when configuration names none. Global Privacy Control is first because it /// is a browser setting with no interface of its own, and the three that @@ -1725,4 +1740,25 @@ mod tests { "startup-fallback health body should be `ok`" ); } + + /// `[analytics]` selecting a module still loads on this adapter, which + /// sends no auction telemetry, and startup says so. + #[test] + fn startup_logs_that_this_adapter_sends_no_auction_telemetry() { + let settings = Settings::from_toml(&format!( + "{}\n{}", + trusted_server_core::test_support::tests::crate_test_settings_str(), + trusted_server_core::test_support::template::documented_block("[analytics]") + )) + .expect("should load settings selecting the documented analytics module") + // A loaded domain carries its policy. + .with_permission_policy(trusted_server_core::permissions::PermissionPolicy::sample()); + + build_state_with_settings(settings).expect("should accept a selected analytics module"); + + assert!( + UNSENT_ANALYTICS_LOGGED.load(std::sync::atomic::Ordering::Relaxed), + "startup should log that this adapter sends no auction telemetry" + ); + } } diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index 7610fc248d..28429f0850 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -646,6 +646,7 @@ mod tests { use crate::ec::tests::{CANONICAL_COOKIE_VALUE, CANONICAL_KV_KEY, CanonicalizingModule}; use crate::error::IntoHttpResponse as _; use crate::openrtb::Uid; + use crate::permissions::{Permission, PermissionSet, PermissionState}; use crate::platform::test_support::{ NoopBackend, NoopConfigStore, NoopGeo, NoopHttpClient, NoopSecretStore, StubHttpClient, noop_services, @@ -1312,7 +1313,11 @@ mod tests { let ec_id = format!("{}.ABC123", "a".repeat(64)); // The default consent context keeps the jurisdiction unknown, so the // server-side auction gate fails closed; the EC gate is off to match. - let mut ec_context = make_ec_context(false, Some(&ec_id)); + // The reader permits ad performance measurement alone, so the skip is + // still recorded. + let mut ec_context = make_ec_context(false, Some(&ec_id)).with_permissions_for_test( + PermissionState::new(PermissionSet::none().with(Permission::MeasureAdPerformance)), + ); let body = json!({ "adUnits": [ @@ -1378,6 +1383,60 @@ mod tests { ); } + #[tokio::test] + async fn a_consent_denied_auction_sends_no_row_without_the_ad_measurement_permission() { + // The server-side auction gate refuses this reader, who has set no + // permission at all. Every row describes the reader, so none is sent. + let settings = create_test_settings(); + let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { + enabled: true, + provider_names: vec!["panic_provider".to_string()], + timeout_ms: 2000, + adserver_name: None, + ..Default::default() + }); + orchestrator.register_provider(Arc::new(PanicOnBidProvider)); + let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); + let services = services_with_telemetry(Arc::clone(&telemetry_sink)); + let mut ec_context = make_ec_context(false, None); + let body = json!({ + "adUnits": [{ + "code": "div-gpt-ad-1", + "mediaTypes": { "banner": { "sizes": [[300, 250]] } } + }] + }); + let req = Request::builder() + .method("POST") + .uri("https://test-publisher.com/auction") + .header(header::USER_AGENT, "FictionalBrowser/1.0 (FictionalOS)") + .body(EdgeBody::from( + serde_json::to_vec(&body).expect("should serialize body"), + )) + .expect("should build auction request"); + + handle_auction( + &settings, + &orchestrator, + None, + None, + &mut ec_context, + &services, + req, + ) + .await + .expect("gated auction should still return a valid response"); + + let batches = telemetry_sink + .batches + .lock() + .expect("should lock telemetry batches"); + assert!( + batches.is_empty(), + "should send no row about a reader who has not permitted ad measurement, got {:?}", + *batches + ); + } + /// Provider that records whether the auction request it received carried /// EIDs, then fails its launch so no real transport handle is needed. struct EidCapturingProvider { diff --git a/crates/trusted-server-core/src/auction/telemetry.rs b/crates/trusted-server-core/src/auction/telemetry.rs index 0beb1508e4..e97ccb687c 100644 --- a/crates/trusted-server-core/src/auction/telemetry.rs +++ b/crates/trusted-server-core/src/auction/telemetry.rs @@ -1,9 +1,11 @@ //! Auction telemetry row construction and sink abstraction. //! //! Core owns the auction observation model and pure row builder. Platform -//! adapters provide the concrete sink implementation. +//! adapters provide the concrete sink implementation. Rows are built only for +//! a reader who permits measuring ad performance. use std::collections::HashSet; +use std::sync::atomic::{AtomicBool, Ordering}; use chrono::Utc; use error_stack::Report; @@ -15,7 +17,9 @@ use crate::auction::orchestrator::OrchestrationResult; use crate::auction::types::{AuctionRequest, AuctionResponse, Bid, BidStatus, MediaType}; use crate::ec::EcContext; use crate::error::TrustedServerError; +use crate::permissions::Permission; use crate::platform::RuntimeServices; +use crate::settings::Settings; const MAX_PAGE_PATH_BYTES: usize = 256; const DYNAMIC_SEGMENT_REPLACEMENT: &str = ":id"; @@ -131,6 +135,9 @@ pub struct AuctionObservationContext { /// `Some(false)` — a dashboard that reads absence as "not shareable" will be wrong for /// every `/auction` row. pub origin_cache_shareable: Option, + /// Whether the reader permits measuring ad performance, which is what + /// every row does. See [`build_auction_events`]. + ad_measurement_permitted: bool, started_at: Instant, } @@ -198,6 +205,9 @@ impl AuctionObservationContext { consent_present: !consent.is_empty(), slot_count, origin_cache_shareable: None, + ad_measurement_permitted: ec_context + .permissions() + .is_set(Permission::MeasureAdPerformance), started_at: Instant::now(), } } @@ -237,6 +247,7 @@ impl AuctionObservationContext { consent_present: false, slot_count, origin_cache_shareable: None, + ad_measurement_permitted: true, started_at: Instant::now(), } } @@ -514,6 +525,32 @@ impl AuctionTelemetrySink for NoopAuctionTelemetrySink { } } +/// Logs once, at warning level, that `adapter` sends no auction telemetry +/// when `[analytics]` selects a module. +/// +/// An adapter that builds no sink of its own calls this as it builds its +/// state, so the selection is not ignored without a word. `supplied` is the +/// services a caller handed the adapter, whose sink its requests use, and +/// nothing is logged when that sink sends. `logged` is the adapter's own flag, +/// so the line is written once however often the state is built. +pub fn warn_once_when_analytics_is_unsent( + settings: &Settings, + supplied: Option<&RuntimeServices>, + adapter: &str, + logged: &AtomicBool, +) { + if settings.analytics_module().is_none() + || supplied.is_some_and(|services| services.auction_telemetry_sink().is_enabled()) + || logged.swap(true, Ordering::Relaxed) + { + return; + } + log::warn!( + "[analytics] selects a module, but the {adapter} adapter sends no auction \ + telemetry, so nothing is sent" + ); +} + /// Emit a telemetry batch without letting errors affect customer traffic. pub async fn emit_auction_events_best_effort(services: &RuntimeServices, batch: AuctionEventBatch) { emit_auction_events_best_effort_lazy(services, || batch).await; @@ -542,6 +579,10 @@ pub async fn emit_auction_events_best_effort_lazy( } /// Build auction telemetry rows for one terminal observation. +/// +/// Every row measures ad performance and describes the reader, so there is no +/// row at all for a reader whose permissions do not include +/// [`Permission::MeasureAdPerformance`], whatever the outcome. #[allow( clippy::needless_pass_by_value, reason = "call sites hand off terminal observations by value to keep ownership explicit" @@ -551,6 +592,9 @@ pub fn build_auction_events( observation: AuctionObservationContext, terminal: AuctionTerminalOutcome<'_>, ) -> AuctionEventBatch { + if !observation.ad_measurement_permitted { + return AuctionEventBatch::default(); + } let event_ts = current_event_timestamp(); let mut rows = Vec::new(); @@ -1166,6 +1210,172 @@ mod tests { ); } + #[test] + fn no_row_is_built_for_a_reader_who_has_not_permitted_ad_measurement() { + use crate::consent::types::ConsentContext; + use crate::permissions::{PermissionSet, PermissionState}; + + let refused = EcContext::new_for_test_gated(None, ConsentContext::default(), false); + let permitted = EcContext::new_for_test_gated(None, ConsentContext::default(), false) + .with_permissions_for_test(PermissionState::new( + PermissionSet::none().with(Permission::MeasureAdPerformance), + )); + let observation = |ec_context: &EcContext| { + AuctionObservationContext::from_parts( + AuctionSource::InitialNavigation, + "test-publisher.example", + "/article", + 1, + Some(TEST_USER_AGENT), + ec_context, + ) + }; + let request = test_request("request-id"); + let result = OrchestrationResult { + provider_responses: vec![AuctionResponse::no_bid("provider", 10)], + adserver_response: None, + winning_bids: HashMap::new(), + total_time_ms: 10, + metadata: HashMap::new(), + }; + let skipped = || AuctionTerminalOutcome::Skipped { + reason: "consent_denied", + elapsed_ms: 0, + }; + + assert!( + build_auction_events(observation(&refused), skipped()).is_empty(), + "a skipped auction should give no row without the permission" + ); + assert!( + build_auction_events( + observation(&refused), + AuctionTerminalOutcome::Completed { + request: &request, + result: &result, + delivered_winner_slots: None, + }, + ) + .is_empty(), + "a completed auction should give no row without the permission" + ); + assert_eq!( + build_auction_events(observation(&permitted), skipped()).row_count(), + 1, + "the permission alone should let the summary row through" + ); + } + + thread_local! { + /// The lines logged on this thread while a test is capturing them. + static CAPTURED_LOG: std::cell::RefCell>> = + const { std::cell::RefCell::new(None) }; + } + + /// A logger that keeps the lines logged on a thread that asked for them, + /// so a test reads its own lines and none of another test's. + struct ThreadLog; + + impl log::Log for ThreadLog { + fn enabled(&self, _metadata: &log::Metadata<'_>) -> bool { + true + } + + fn log(&self, record: &log::Record<'_>) { + CAPTURED_LOG.with(|captured| { + if let Some(lines) = captured.borrow_mut().as_mut() { + lines.push((record.level(), record.args().to_string())); + } + }); + } + + fn flush(&self) {} + } + + /// The lines at warning level or above that `run` logs on this thread. + fn warnings_logged_by(run: impl FnOnce()) -> Vec<(log::Level, String)> { + static LOGGER: ThreadLog = ThreadLog; + static INSTALL: std::sync::Once = std::sync::Once::new(); + INSTALL.call_once(|| { + log::set_logger(&LOGGER).expect("no other test should install a logger"); + log::set_max_level(log::LevelFilter::Warn); + }); + CAPTURED_LOG.with(|captured| *captured.borrow_mut() = Some(Vec::new())); + run(); + CAPTURED_LOG.with(|captured| captured.borrow_mut().take().unwrap_or_default()) + } + + /// Settings whose `[analytics]` selects the module the example documents. + fn settings_selecting_analytics() -> Settings { + Settings::from_toml(&format!( + "{}\n{}", + crate::test_support::tests::crate_test_settings_str(), + crate::test_support::template::documented_block("[analytics]") + )) + .expect("should load settings selecting the documented analytics module") + } + + /// A sink that sends, standing in for one a caller supplies. + struct SendingSink; + + #[async_trait::async_trait(?Send)] + impl AuctionTelemetrySink for SendingSink { + async fn emit_auction_events( + &self, + _services: &RuntimeServices, + _batch: AuctionEventBatch, + ) -> Result<(), Report> { + Ok(()) + } + } + + #[test] + fn an_adapter_that_sends_nothing_warns_once_when_analytics_selects_a_module() { + let settings = settings_selecting_analytics(); + let logged = AtomicBool::new(false); + + let lines = warnings_logged_by(|| { + warn_once_when_analytics_is_unsent(&settings, None, "Example", &logged); + warn_once_when_analytics_is_unsent(&settings, None, "Example", &logged); + }); + + assert_eq!(lines.len(), 1, "should log one line, got {lines:?}"); + assert_eq!(lines[0].0, log::Level::Warn, "should log a warning"); + assert!( + lines[0] + .1 + .contains("the Example adapter sends no auction telemetry"), + "should name the adapter, got {:?}", + lines[0].1 + ); + } + + #[test] + fn no_warning_when_analytics_selects_nothing_or_a_supplied_sink_sends() { + let selecting_nothing = crate::test_support::tests::create_test_settings(); + let selecting = settings_selecting_analytics(); + let sending = crate::platform::test_support::noop_services_with_telemetry_sink( + std::sync::Arc::new(SendingSink), + ); + + let lines = warnings_logged_by(|| { + warn_once_when_analytics_is_unsent( + &selecting_nothing, + None, + "Example", + &AtomicBool::new(false), + ); + warn_once_when_analytics_is_unsent( + &selecting, + Some(&sending), + "Example", + &AtomicBool::new(false), + ); + }); + + assert!(lines.is_empty(), "should log nothing, got {lines:?}"); + } + #[test] fn normalize_page_path_strips_query_and_redacts_dynamic_segments() { assert_eq!( diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index 32a2df53a7..6e939ac7db 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -272,7 +272,11 @@ impl edgezero_core::app_config::AppConfigMeta for TrustedServerAppConfig { false, ), field( - vec![optional_object("tinybird"), object("auction_token_secret")], + vec![ + optional_object("analytics"), + optional_object("tinybird"), + object("auction_token_secret"), + ], true, ), field( @@ -498,15 +502,6 @@ fn validate_secret_key_references( )?; } - if settings.tinybird.enabled { - let token = settings - .tinybird - .auction_token_secret - .as_ref() - .ok_or_else(|| missing_secret_key_reference("tinybird.auction_token_secret"))?; - validate_secret_key_reference("tinybird.auction_token_secret", token.expose())?; - } - // Each setting a selected module declares as naming a secret, where the // module's table puts it to use. for builder in crate::integrations::all_builders(extra_integrations) { @@ -584,7 +579,10 @@ mod tests { use crate::integrations::IntegrationRegistration; use crate::integrations::js_asset_proxy::JS_ASSET_PROXY_INTEGRATION_ID; use crate::redacted::Redacted; - use crate::settings::{ProxyAssetRoute, S3SigV4AuthConfig, TrustedClientIpConfig}; + use crate::settings::{ + AnalyticsConfig, ProxyAssetRoute, S3SigV4AuthConfig, TinybirdSettings, + TrustedClientIpConfig, + }; use crate::test_support::template::{template_with_resolved_required_secrets, uncomment_block}; use crate::test_support::tests::{ crate_test_settings_str, crate_test_settings_str_with_ec_section, select_hmac_module, @@ -730,13 +728,15 @@ formats = [{ width = 300, height = 250 }] } #[test] - fn documented_tinybird_block_validates_when_uncommented() { - let toml = uncomment_block(&template_with_resolved_required_secrets(), "[tinybird]"); + fn documented_analytics_block_validates_when_uncommented() { + let toml = uncomment_block(&template_with_resolved_required_secrets(), "[analytics]"); let settings = Settings::from_toml(&toml) - .expect("uncommented [tinybird] with documented api_host should parse and validate"); + .expect("uncommented [analytics] with documented api_host should parse and validate"); assert!( - settings.tinybird.enabled && !settings.tinybird.api_host.is_empty(), - "tinybird should be enabled with a non-empty api_host" + settings + .analytics_module() + .is_some_and(|module| !module.api_host.is_empty()), + "the selected analytics module should have a non-empty api_host" ); } @@ -1010,7 +1010,7 @@ formats = [{ width = 300, height = 250 }] ("ec.partners[*].api_token".to_owned(), true), ("ec.partners[*].ts_pull_token".to_owned(), true), ("trusted_client_ip.shared_secret".to_owned(), false), - ("tinybird.auction_token_secret".to_owned(), true), + ("analytics.tinybird.auction_token_secret".to_owned(), true), ("proxy.asset_routes[*].auth.access_key_id".to_owned(), true), ( "proxy.asset_routes[*].auth.secret_access_key".to_owned(), @@ -1059,10 +1059,21 @@ formats = [{ width = 300, height = 250 }] assert_eq!(serialized["secret_access_key"], "secret_access_key"); } + /// An `[analytics]` section selecting the analytics module with `settings`. + fn analytics_selecting(settings: TinybirdSettings) -> AnalyticsConfig { + AnalyticsConfig { + module: Some("analytics.tinybird".to_string()), + tinybird: Some(settings), + } + } + #[test] fn legacy_static_secret_store_selectors_are_accepted_but_not_serialized() { let mut settings = valid_settings(); - settings.tinybird.secret_store = Some("legacy-tinybird-store".to_string()); + settings.analytics = Some(analytics_selecting(TinybirdSettings { + secret_store: Some("legacy-tinybird-store".to_string()), + ..TinybirdSettings::default() + })); let mut route = ProxyAssetRoute::new( "/assets/", "https://examplebucket.s3.us-east-1.amazonaws.com", @@ -1093,8 +1104,10 @@ formats = [{ width = 300, height = 250 }] #[test] fn settings_debug_redacts_resolved_static_credentials() { let mut settings = valid_settings(); - settings.tinybird.auction_token_secret = - Some(Redacted::new("resolved-tinybird-secret".to_string())); + settings.analytics = Some(analytics_selecting(TinybirdSettings { + auction_token_secret: Some(Redacted::new("resolved-tinybird-secret".to_string())), + ..TinybirdSettings::default() + })); settings .insert_module_config( "testing", @@ -1235,6 +1248,50 @@ gam_network_id = "99999" ); } + /// The analytics module's token is a secret reference like every other, + /// so an empty or absent one is refused for deploy, by its path. + #[test] + fn app_config_new_rejects_an_analytics_secret_key_reference_that_is_empty_or_absent() { + for (case, reference) in [ + ("an empty reference", Some(Redacted::new(String::new()))), + ( + "a whitespace reference", + Some(Redacted::new(" ".to_owned())), + ), + ("no reference", None), + ] { + let mut settings = valid_settings(); + settings.analytics = Some(analytics_selecting(TinybirdSettings { + api_host: "api.tinybird.co".to_owned(), + auction_token_secret: reference, + ..TinybirdSettings::default() + })); + + let err = TrustedServerAppConfig::new(settings) + .err() + .unwrap_or_else(|| panic!("{case}: should refuse the telemetry secret reference")); + + assert!( + err.to_string() + .contains("analytics.tinybird.auction_token_secret"), + "{case}: the error should identify the telemetry secret reference: {err:?}" + ); + } + } + + #[test] + fn app_config_new_accepts_an_analytics_secret_key_reference() { + let mut settings = valid_settings(); + settings.analytics = Some(analytics_selecting(TinybirdSettings { + api_host: "api.tinybird.co".to_owned(), + auction_token_secret: Some(Redacted::new("tinybird_auction_token".to_owned())), + ..TinybirdSettings::default() + })); + + TrustedServerAppConfig::new(settings) + .expect("should validate the token's key name without treating it as the value"); + } + #[test] fn app_config_new_rejects_a_resolve_allowed_origin_that_is_not_bare() { let mut settings = valid_settings(); diff --git a/crates/trusted-server-core/src/config_payload.rs b/crates/trusted-server-core/src/config_payload.rs index f43c3de7a3..46dad7d959 100644 --- a/crates/trusted-server-core/src/config_payload.rs +++ b/crates/trusted-server-core/src/config_payload.rs @@ -198,16 +198,14 @@ fn log_permission_baseline(policy: &PermissionPolicy) { } fn remove_inactive_secret_references(data: &mut serde_json::Value) { - if data - .pointer("/tinybird/enabled") - .and_then(serde_json::Value::as_bool) - != Some(true) - && let Some(tinybird) = data - .get_mut("tinybird") + // References of a table `[analytics]` does not select are cleared before resolution. + if !crate::settings::document_selects_analytics_module(data) + && let Some(table) = data + .pointer_mut("/analytics/tinybird") .and_then(serde_json::Value::as_object_mut) { - tinybird.remove("auction_token_secret"); - tinybird.remove("access_token_secret"); + table.remove("auction_token_secret"); + table.remove("access_token_secret"); } if let Some(partners) = data @@ -237,7 +235,8 @@ mod tests { use crate::platform::{PlatformError, StoreId}; use crate::redacted::Redacted; use crate::settings::{ - AssetOriginAuth, EcPartner, ProxyAssetRoute, S3SigV4AuthConfig, TrustedClientIpConfig, + AnalyticsConfig, AssetOriginAuth, EcPartner, ProxyAssetRoute, S3SigV4AuthConfig, + TinybirdSettings, TrustedClientIpConfig, }; use crate::test_support::tests::{ crate_test_settings_str, hmac_passphrase, sample_permissions, select_hmac_module, @@ -613,10 +612,14 @@ mod tests { #[test] fn resolves_all_static_credentials_from_the_mapped_default_store() { let mut original = test_settings(); - original.tinybird.enabled = true; - original.tinybird.api_host = "api.example.com".to_string(); - original.tinybird.auction_token_secret = - Some(Redacted::new("tinybird-token-key".to_string())); + original.analytics = Some(AnalyticsConfig { + module: Some("analytics.tinybird".to_string()), + tinybird: Some(TinybirdSettings { + api_host: "api.example.com".to_string(), + auction_token_secret: Some(Redacted::new("tinybird-token-key".to_string())), + ..TinybirdSettings::default() + }), + }); let mut route = ProxyAssetRoute::new( "/assets/", "https://examplebucket.s3.us-east-1.amazonaws.com", @@ -645,9 +648,8 @@ mod tests { assert_eq!( reconstructed - .tinybird - .auction_token_secret - .as_ref() + .analytics_module() + .and_then(|module| module.auction_token_secret.as_ref()) .map(Redacted::expose) .map(String::as_str), Some("resolved-tinybird-token") @@ -988,8 +990,6 @@ mod tests { #[test] fn inactive_optional_features_do_not_resolve_stale_secret_references() { let mut original = test_settings(); - original.tinybird.auction_token_secret = - Some(Redacted::new("unused-tinybird-key".to_string())); original .ec .partners @@ -1003,10 +1003,40 @@ mod tests { ) .expect("should skip inactive optional feature references"); - assert!(reconstructed.tinybird.auction_token_secret.is_none()); assert!(reconstructed.ec.partners[0].ts_pull_token.is_none()); } + #[test] + fn an_unselected_analytics_module_table_is_refused_without_resolving_its_secret() { + let mut original = test_settings(); + original.analytics = Some(AnalyticsConfig { + module: None, + tinybird: Some(TinybirdSettings { + api_host: "api.example.com".to_string(), + auction_token_secret: Some(Redacted::new("unused-telemetry-key".to_string())), + ..TinybirdSettings::default() + }), + }); + + let error = settings_from_config_blob( + &envelope_json(&original), + &UnifiedSecretStore, + &StoreName::from("ts_secrets"), + PolicyOrigin::Blob, + ) + .expect_err("should refuse a table nothing selects"); + let rendered = format!("{error:?}"); + + assert!( + rendered.contains("[analytics] selects no module"), + "should name the section and why: {rendered}" + ); + assert!( + !rendered.contains("unused-telemetry-key") && !rendered.contains("secret reference"), + "should not have tried to resolve the stale secret reference: {rendered}" + ); + } + /// A table for a module its section does not select is refused, and its /// secret references are dropped before resolution, so the operator reads /// the table's own fault rather than a secret-store failure that follows diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index 34be0def15..d54a72e973 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -1302,15 +1302,20 @@ impl EcContext { } } -/// The permissions a test context stands in for: storage and personalised-ad -/// selection when the gate is open, none when it is closed. +/// The permissions a test context stands in for: storage, personalised-ad +/// selection and ad performance measurement when the gate is open, none when +/// it is closed. #[cfg(test)] fn permissions_for_test(ec_allowed: bool) -> PermissionState { if ec_allowed { PermissionState::new( - [Permission::StoreOnDevice, Permission::SelectPersonalisedAds] - .into_iter() - .collect(), + [ + Permission::StoreOnDevice, + Permission::SelectPersonalisedAds, + Permission::MeasureAdPerformance, + ] + .into_iter() + .collect(), ) } else { PermissionState::default() diff --git a/crates/trusted-server-core/src/inspect/config/tests.rs b/crates/trusted-server-core/src/inspect/config/tests.rs index 1d00380b85..f92ab52a7d 100644 --- a/crates/trusted-server-core/src/inspect/config/tests.rs +++ b/crates/trusted-server-core/src/inspect/config/tests.rs @@ -238,10 +238,12 @@ fn documents_holding_every_declared_secret() -> Vec<(&'static str, Value)> { "session_token": "route-session", }, }]); - main["tinybird"] = json!({ - "enabled": true, - "api_host": "api.tinybird.example.com", - "auction_token_secret": "telemetry", + main["analytics"] = json!({ + "module": "tinybird", + "tinybird": { + "api_host": "api.tinybird.example.com", + "auction_token_secret": "telemetry", + }, }); let mut host_signals = document(""); diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 0809c70782..6763568530 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -27036,7 +27036,9 @@ mod tests { /// The same capturing setup also covers how both paths key the identity /// graph. A module whose canonical form differs from the cookie value has /// its row read under the canonical key, so the partner ID stored there - /// reaches the dispatched auction request. + /// reaches the dispatched auction request. It also covers that neither + /// path sends a telemetry row about a reader the consent gate refuses and + /// who has not permitted ad performance measurement. mod navigation_publisher_domain_tests { use super::*; use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; @@ -27195,6 +27197,26 @@ mod tests { EcContext::new_for_test(None, consent) } + /// [`EcContext`] in an unknown jurisdiction with no consent record, + /// which the server-side auction gate refuses, for a reader who has + /// set no permission, ad performance measurement included. + fn consent_denying_ec_context() -> EcContext { + EcContext::new_for_test_gated(None, crate::consent::ConsentContext::default(), false) + } + + /// Asserts the sink received no telemetry row at all. + fn assert_no_telemetry(telemetry_sink: &RecordingTelemetrySink) { + let batches = telemetry_sink + .batches + .lock() + .expect("should lock telemetry batches"); + assert!( + batches.is_empty(), + "should send no row about a reader who has not permitted ad measurement, got {:?}", + *batches + ); + } + fn services_with( http_client: Arc, telemetry_sink: Arc, @@ -27368,6 +27390,94 @@ mod tests { assert_configured_domain(&captured, &telemetry_sink); } + #[tokio::test] + async fn navigation_sends_no_row_when_consent_denies_without_ad_measurement() { + let settings = settings_with_capturing_provider(); + let captured = Arc::new(Mutex::new(None)); + let orchestrator = orchestrator_capturing_request(&settings, &captured); + let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"ok".to_vec()); + let services = services_with( + Arc::clone(&stub) as Arc, + Arc::clone(&telemetry_sink), + ); + let mut ec_context = consent_denying_ec_context(); + let req = HttpRequest::builder() + .method(Method::GET) + .uri(format!("https://{EDGE_HOST}/2024/01/my-article/")) + .header(header::HOST, EDGE_HOST) + .header(header::USER_AGENT, "FictionalBrowser/1.0 (FictionalOS)") + .header("sec-fetch-dest", "document") + .body(EdgeBody::empty()) + .expect("should build test request"); + + let registry = test_registry(&settings); + let _ = handle_publisher_request( + AppContext { + settings: &settings, + integration_registry: ®istry, + }, + &services, + None, + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &article_slot(), + registry: None, + }, + req, + EdgeCacheHeader::SMaxageFallback, + ) + .await + .expect("should proxy publisher request"); + + assert_no_telemetry(&telemetry_sink); + } + + #[tokio::test] + async fn page_bids_sends_no_row_when_consent_denies_without_ad_measurement() { + let settings = settings_with_capturing_provider(); + let captured = Arc::new(Mutex::new(None)); + let orchestrator = orchestrator_capturing_request(&settings, &captured); + let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); + let services = services_with( + Arc::new(crate::platform::test_support::NoopHttpClient), + Arc::clone(&telemetry_sink), + ); + let mut ec_context = consent_denying_ec_context(); + let mut req = HttpRequest::builder() + .method(Method::GET) + .uri(format!( + "https://{EDGE_HOST}/_ts/page-bids?path=/2024/01/my-article/" + )) + .header(header::HOST, EDGE_HOST) + .header(header::USER_AGENT, "FictionalBrowser/1.0 (FictionalOS)") + .body(EdgeBody::empty()) + .expect("should build test request"); + req.headers_mut().insert( + header::HeaderName::from_static("sec-fetch-site"), + HeaderValue::from_static("same-origin"), + ); + + let _ = handle_page_bids( + &settings, + &services, + None, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &article_slot(), + registry: None, + }, + &mut ec_context, + req, + ) + .await + .expect("should return ok response"); + + assert_no_telemetry(&telemetry_sink); + } + #[tokio::test] async fn initial_navigation_auctions_only_renderable_slots() { let settings = settings_with_capturing_provider(); diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 2a0b8bd008..df4af8a75e 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -2724,12 +2724,15 @@ impl Proxy { } } -/// Direct Tinybird Events API telemetry configuration. +/// The analytics module's name in full, which is also the table holding its +/// settings and which every message about them names. +const ANALYTICS_MODULE: &str = "analytics.tinybird"; + +/// Direct Tinybird Events API telemetry configuration, the settings of the +/// module `[analytics]` selects. #[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct TinybirdSettings { - /// Master enablement for auction telemetry ingestion. - #[serde(default)] - pub enabled: bool, /// Regional Tinybird API host, without scheme or path. #[serde(default)] pub api_host: String, @@ -2777,7 +2780,6 @@ fn default_tinybird_max_body_bytes() -> usize { impl Default for TinybirdSettings { fn default() -> Self { Self { - enabled: false, api_host: String::new(), secret_store: None, auction_dataset: default_tinybird_auction_dataset(), @@ -2796,7 +2798,7 @@ impl TinybirdSettings { self.api_host = self.api_host.trim().to_ascii_lowercase(); if self.secret_store.take().is_some() { log::warn!( - "tinybird.secret_store is deprecated and ignored; static credentials resolve through the default app-config secret store" + "{ANALYTICS_MODULE}.secret_store is deprecated and ignored; static credentials resolve through the default app-config secret store" ); } self.auction_dataset = self.auction_dataset.trim().to_owned(); @@ -2812,32 +2814,39 @@ impl TinybirdSettings { self.normalize(); if !(0.0..=1.0).contains(&self.access_sample_rate) { return Err(Report::new(TrustedServerError::Configuration { - message: "tinybird.access_sample_rate must be between 0.0 and 1.0".to_owned(), + message: format!( + "{ANALYTICS_MODULE}.access_sample_rate must be between 0.0 and 1.0" + ), })); } if self.max_body_bytes < 1024 { return Err(Report::new(TrustedServerError::Configuration { - message: "tinybird.max_body_bytes must be at least 1024".to_owned(), + message: format!("{ANALYTICS_MODULE}.max_body_bytes must be at least 1024"), })); } if self.access_enabled { return Err(Report::new(TrustedServerError::Configuration { - message: "tinybird.access_enabled is reserved for future access-log telemetry; no emitter is currently wired".to_owned(), + message: format!( + "{ANALYTICS_MODULE}.access_enabled is reserved for future access-log telemetry; no emitter is currently wired" + ), })); } - if !self.enabled { - return Ok(()); - } validate_tinybird_api_host(&self.api_host)?; - validate_tinybird_dataset(&self.auction_dataset, "tinybird.auction_dataset")?; + validate_tinybird_dataset( + &self.auction_dataset, + &format!("{ANALYTICS_MODULE}.auction_dataset"), + )?; let token = self.auction_token_secret.as_ref().ok_or_else(|| { Report::new(TrustedServerError::Configuration { - message: - "tinybird.auction_token_secret is required when Tinybird telemetry is enabled" - .to_owned(), + message: format!( + "{ANALYTICS_MODULE}.auction_token_secret is required when [analytics] selects the module" + ), }) })?; - validate_tinybird_secret(token.expose(), "tinybird.auction_token_secret") + validate_tinybird_secret( + token.expose(), + &format!("{ANALYTICS_MODULE}.auction_token_secret"), + ) } } @@ -2850,13 +2859,14 @@ fn validate_tinybird_api_host(host: &str) -> Result<(), Report Result<(), Report, + /// The analytics module's settings. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tinybird: Option, +} + +/// Whether `written`, selected in `[analytics]`, names the analytics module, +/// in full or by its name within the section. +fn names_analytics_module(written: &str) -> bool { + written == ANALYTICS_MODULE + || written == crate::module_name::short_form("analytics", ANALYTICS_MODULE) +} + +/// Whether the document `data` selects the analytics module in `[analytics]`, +/// read before the settings are, to tell which secret references are live. +pub(crate) fn document_selects_analytics_module(data: &JsonValue) -> bool { + data.pointer("/analytics/module") + .and_then(JsonValue::as_str) + .is_some_and(names_analytics_module) +} + +impl AnalyticsConfig { + /// Whether `module` selects the analytics module. + fn selects_analytics_module(&self) -> bool { + self.module.as_deref().is_some_and(names_analytics_module) + } + + /// The analytics module's settings, when `module` selects it. + #[must_use] + pub fn analytics_module(&self) -> Option<&TinybirdSettings> { + if self.selects_analytics_module() { + self.tinybird.as_ref() + } else { + None + } + } + + fn normalize(&mut self) { + if let Some(settings) = &mut self.tinybird { + settings.normalize(); + } + } + + /// Refuses a section that selects nothing or a module this build does + /// not have, and checks the selected module's settings. + fn prepare_runtime(&mut self) -> Result<(), Report> { + let short = crate::module_name::short_form("analytics", ANALYTICS_MODULE); + match self.module.as_deref() { + None => { + return Err(Report::new(TrustedServerError::Configuration { + message: "[analytics] selects no module. A module type's section names \ + what runs with `module` or `modules`, so select one or remove \ + the section" + .to_owned(), + })); + } + Some(written) if !self.selects_analytics_module() => { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "[analytics] selects `{written}`, which no module this deployment \ + supplies. The analytics modules it supplies are [{short}]" + ), + })); + } + Some(_) => {} + } + match &mut self.tinybird { + Some(settings) => settings.prepare_runtime(), + // Checking the defaults reports the first setting the module + // needs and the table does not have. + None => TinybirdSettings::default().prepare_runtime(), + } + } +} + /// Cache behavior configuration. #[derive(Debug, Default, Clone, Deserialize, Serialize)] #[serde(deny_unknown_fields)] @@ -3601,6 +3692,13 @@ refused_table! { `--secret-store-id`" } +refused_table! { + /// The `[tinybird]` table, now `[analytics.tinybird]`. + MovedTinybirdTable => "Configuration table `[tinybird]` is now `[analytics.tinybird]`. \ + Select it with `[analytics] module = \"tinybird\"`, which replaces its `enabled` \ + line, and move its other settings there unchanged" +} + refused_table! { /// The `[permission_signal]` table, renamed `[permission-signal]`. RenamedPermissionSignalTable => "Configuration table `[permission_signal]` is now \ @@ -3995,8 +4093,19 @@ pub struct Settings { pub creative_opportunities: Option, #[serde(default)] pub image_optimizer: ImageOptimizerSettings, - #[serde(default)] - pub tinybird: TinybirdSettings, + /// Analytics. `[analytics] module` selects the module that runs, with its + /// settings in the table at its name. Omitted from a serialized + /// configuration when absent, for the reason given on `trusted_client_ip`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub analytics: Option, + /// The `[tinybird]` table, kept so a configuration carrying it is told + /// where it moved. + #[serde(rename = "tinybird", default, skip_serializing)] + #[allow( + dead_code, + reason = "the field exists so that reading the moved table fails with directions" + )] + moved_tinybird: MovedTinybirdTable, #[serde(default)] pub debug: DebugConfig, #[serde(default, skip_serializing_if = "is_default_device_config")] @@ -4188,6 +4297,14 @@ impl Settings { Self::finalize_deserialized(settings, "Build-time configuration") } + /// The analytics module's settings, when `[analytics]` selects it. + #[must_use] + pub fn analytics_module(&self) -> Option<&TinybirdSettings> { + self.analytics + .as_ref() + .and_then(AnalyticsConfig::analytics_module) + } + /// Normalize settings after deserialization. /// /// # Errors @@ -4198,7 +4315,9 @@ impl Settings { self.proxy.normalize(); self.image_optimizer.normalize()?; self.debug.auction_html_comment_options.normalize(); - self.tinybird.normalize(); + if let Some(analytics) = &mut self.analytics { + analytics.normalize(); + } self.consent.validate(); Ok(()) } @@ -4245,7 +4364,9 @@ impl Settings { self.image_optimizer.prepare_runtime()?; self.cache.prepare_runtime()?; self.proxy.prepare_runtime()?; - self.tinybird.prepare_runtime()?; + if let Some(analytics) = &mut self.analytics { + analytics.prepare_runtime()?; + } self.debug .auction_html_comment_options .validate_metadata_keys()?; @@ -5346,18 +5467,22 @@ module = \"none\"", })); settings.proxy.asset_routes = vec![asset_route]; - settings.tinybird = TinybirdSettings { - auction_token_secret: Some(Redacted::new(CANARY_TINYBIRD_AUCTION_TOKEN.to_string())), - access_token_secret: Some(Redacted::new(CANARY_TINYBIRD_ACCESS_TOKEN.to_string())), - enabled: false, - api_host: String::new(), - secret_store: None, - auction_dataset: String::new(), - access_enabled: false, - access_dataset: String::new(), - access_sample_rate: 0.0f64, - max_body_bytes: 0, - }; + settings.analytics = Some(AnalyticsConfig { + module: Some(ANALYTICS_MODULE.to_string()), + tinybird: Some(TinybirdSettings { + auction_token_secret: Some(Redacted::new( + CANARY_TINYBIRD_AUCTION_TOKEN.to_string(), + )), + access_token_secret: Some(Redacted::new(CANARY_TINYBIRD_ACCESS_TOKEN.to_string())), + api_host: String::new(), + secret_store: None, + auction_dataset: String::new(), + access_enabled: false, + access_dataset: String::new(), + access_sample_rate: 0.0f64, + max_body_bytes: 0, + }), + }); // A module's table is opaque JSON, and the section's hand-written // `Debug` impl is the only thing keeping a secret a module's table @@ -5408,10 +5533,13 @@ module = \"none\"", CANARY_S3_SESSION_TOKEN, ), ( - "tinybird.auction_token_secret", + "analytics.tinybird.auction_token_secret", CANARY_TINYBIRD_AUCTION_TOKEN, ), - ("tinybird.access_token_secret", CANARY_TINYBIRD_ACCESS_TOKEN), + ( + "analytics.tinybird.access_token_secret", + CANARY_TINYBIRD_ACCESS_TOKEN, + ), ("testing.example.key_name", CANARY_MODULE_KEY), ]; @@ -6166,61 +6294,235 @@ module = \"none\"", ); } + /// `[analytics]` selecting the analytics module, with `table` as the + /// lines of its settings table. + fn settings_str_with_analytics_module(table: &str) -> String { + format!( + "{}\n[analytics]\nmodule = \"tinybird\"\n\n[analytics.tinybird]\n{table}\n", + crate_test_settings_str() + ) + } + + /// The lines of a telemetry table that passes every check. + const ANALYTICS_MODULE_LINES: &str = + "api_host = \"api.us-east.aws.tinybird.co\"\nauction_token_secret = \"test-auction-token\""; + #[test] - fn tinybird_defaults_to_disabled_placeholders() { + fn settings_without_an_analytics_section_select_no_module() { let settings = Settings::from_toml(&crate_test_settings_str()) - .expect("should parse settings without tinybird block"); + .expect("should parse settings without an analytics section"); + assert!(settings.analytics.is_none(), "should hold no section"); assert!( - !settings.tinybird.enabled, - "Tinybird should default disabled" + settings.analytics_module().is_none(), + "should select no module" + ); + let value = serde_json::to_value(&settings).expect("should serialize the settings"); + assert!( + value.get("analytics").is_none() && value.get("tinybird").is_none(), + "should write neither the section nor the table it replaced: {value}" ); - assert_eq!(settings.tinybird.secret_store, None); - assert_eq!(settings.tinybird.auction_dataset, "auction_events_raw"); - assert!(settings.tinybird.auction_token_secret.is_none()); } #[test] - fn tinybird_enabled_requires_host_dataset_and_token() { + fn the_analytics_module_is_selected_by_its_short_or_full_name() { + for name in ["tinybird", "analytics.tinybird"] { + let toml = format!( + "{}\n[analytics]\nmodule = \"{name}\"\n\n[analytics.tinybird]\n{ANALYTICS_MODULE_LINES}\n", + crate_test_settings_str() + ); + + let settings = Settings::from_toml(&toml).expect("should accept the selection"); + + let telemetry = settings + .analytics_module() + .expect("should select the analytics module"); + assert_eq!(telemetry.api_host, "api.us-east.aws.tinybird.co"); + assert_eq!(telemetry.auction_dataset, "auction_events_raw"); + } + } + + #[test] + fn a_selected_analytics_module_survives_the_stored_form() { + let settings = + Settings::from_toml(&settings_str_with_analytics_module(ANALYTICS_MODULE_LINES)) + .expect("should accept the selection"); + + let stored = serde_json::to_value(&settings).expect("should serialize the settings"); + + assert_eq!(stored["analytics"]["module"], "tinybird"); + assert_eq!( + stored["analytics"]["tinybird"]["api_host"], + "api.us-east.aws.tinybird.co" + ); + assert!( + stored.get("tinybird").is_none(), + "should not write the table that moved: {stored}" + ); + } + + #[test] + fn the_analytics_module_needs_a_regional_host() { + let toml = + settings_str_with_analytics_module("api_host = \"https://api.example.com/path\""); + + let err = Settings::from_toml(&toml).expect_err("should reject invalid api host"); + + assert!( + format!("{err:?}").contains("analytics.tinybird.api_host"), + "should name the setting by its table: {err:?}" + ); + } + + #[test] + fn the_analytics_module_needs_its_token() { + let toml = settings_str_with_analytics_module("api_host = \"api.us-east.aws.tinybird.co\""); + + let err = Settings::from_toml(&toml).expect_err("should require the token"); + + assert!( + format!("{err:?}").contains( + "analytics.tinybird.auction_token_secret is required when [analytics] selects \ + the module" + ), + "should say what is missing and why: {err:?}" + ); + } + + #[test] + fn a_selected_analytics_module_with_no_table_is_told_what_it_needs() { let toml = format!( - "{}\n[tinybird]\nenabled = true\napi_host = \"https://api.example.com/path\"\n", + "{}\n[analytics]\nmodule = \"tinybird\"\n", crate_test_settings_str() ); - let err = Settings::from_toml(&toml).expect_err("should reject invalid api host"); + let err = Settings::from_toml(&toml).expect_err("should need the table's settings"); + + assert!( + format!("{err:?}").contains("analytics.tinybird.api_host"), + "should name the first setting the module needs: {err:?}" + ); + } + + #[test] + fn access_telemetry_is_rejected_until_an_emitter_is_wired() { + let toml = settings_str_with_analytics_module(&format!( + "{ANALYTICS_MODULE_LINES}\naccess_enabled = true" + )); + + let err = Settings::from_toml(&toml) + .expect_err("should reject access telemetry before an emitter exists"); + assert!( - format!("{err:?}").contains("tinybird.api_host"), - "should report tinybird.api_host validation error: {err:?}" + format!("{err:?}").contains("analytics.tinybird.access_enabled"), + "should name the unsupported setting: {err:?}" ); } #[test] - fn tinybird_accepts_region_host_without_scheme() { + fn an_analytics_section_selecting_nothing_is_refused() { let toml = format!( - "{}\n[tinybird]\nenabled = true\napi_host = \"api.us-east.aws.tinybird.co\"\nauction_token_secret = \"test-auction-token\"\n", + "{}\n[analytics.tinybird]\n{ANALYTICS_MODULE_LINES}\n", crate_test_settings_str() ); - let settings = Settings::from_toml(&toml).expect("should accept Tinybird region host"); - assert!(settings.tinybird.enabled); - assert_eq!(settings.tinybird.api_host, "api.us-east.aws.tinybird.co"); + let err = Settings::from_toml(&toml).expect_err("nothing would read the table"); + + assert!( + format!("{err:?}").contains("[analytics] selects no module"), + "should say the section selects nothing: {err:?}" + ); } #[test] - fn tinybird_access_enabled_is_rejected_until_emitter_is_wired() { + fn an_analytics_module_this_build_does_not_have_is_refused() { let toml = format!( - "{}\n[tinybird]\naccess_enabled = true\n", + "{}\n[analytics]\nmodule = \"another\"\n", crate_test_settings_str() ); - let err = Settings::from_toml(&toml) - .expect_err("should reject access telemetry before emitter exists"); + let err = Settings::from_toml(&toml).expect_err("no such module"); + let message = format!("{err:?}"); + + assert!( + message.contains( + "[analytics] selects `another`, which no module this deployment supplies" + ), + "should name what was written: {message}" + ); + assert!( + message.contains("The analytics modules it supplies are [tinybird]"), + "should list what can be selected: {message}" + ); + } + + /// The token is a secret reference, and an empty or absent one is refused + /// as the settings load, before any deploy step sees the document. + #[test] + fn an_analytics_token_reference_that_is_empty_or_absent_is_refused_when_the_settings_load() { + for (case, table) in [ + ( + "an empty token", + "api_host = \"api.us-east.aws.tinybird.co\"\nauction_token_secret = \"\"", + ), + ("no token", "api_host = \"api.us-east.aws.tinybird.co\""), + ] { + let err = Settings::from_toml(&settings_str_with_analytics_module(table)) + .err() + .unwrap_or_else(|| panic!("{case}: the settings should not load")); + + assert!( + format!("{err:?}").contains("auction_token_secret"), + "{case}: should name the token: {err:?}" + ); + } + } + + #[test] + fn the_analytics_module_table_refuses_a_key_it_does_not_know() { + let toml = settings_str_with_analytics_module(&format!( + "{ANALYTICS_MODULE_LINES}\nenabled = true" + )); + + let err = Settings::from_toml(&toml).expect_err("the enabled line is gone"); + + assert!( + format!("{err:?}").contains("enabled"), + "should name the key it does not know: {err:?}" + ); + } + + #[test] + fn a_table_the_section_does_not_select_gives_no_analytics_module() { + let analytics = AnalyticsConfig { + module: None, + tinybird: Some(TinybirdSettings::default()), + }; + assert!( - format!("{err:?}").contains("tinybird.access_enabled"), - "should report unsupported tinybird.access_enabled setting: {err:?}" + analytics.analytics_module().is_none(), + "a table nothing selects should give no settings" ); } + #[test] + fn the_tinybird_table_is_refused_with_directions() { + for table in [ + "enabled = true\napi_host = \"api.us-east.aws.tinybird.co\"", + "enabled = false", + ] { + let toml = format!("{}\n[tinybird]\n{table}\n", crate_test_settings_str()); + + let err = Settings::from_toml(&toml).expect_err("the table has moved"); + + assert!( + format!("{err:?}") + .contains("Configuration table `[tinybird]` is now `[analytics.tinybird]`"), + "should say where the table went: {err:?}" + ); + } + } + #[test] fn settings_rejects_removed_consent_store_toml() { let toml = format!( diff --git a/crates/trusted-server-core/src/test_support.rs b/crates/trusted-server-core/src/test_support.rs index fe0d304883..4a0514bfca 100644 --- a/crates/trusted-server-core/src/test_support.rs +++ b/crates/trusted-server-core/src/test_support.rs @@ -275,6 +275,31 @@ pub mod template { out.join("\n") } + + /// Only the block of [`EXAMPLE_TEMPLATE`] that begins at the line + /// `# {header}`, uncommented, for a test that adds a documented section to + /// settings of its own. The block ends at the first line that is not a + /// comment. + /// + /// # Panics + /// + /// Panics if the template documents no such block. + #[must_use] + pub fn documented_block(header: &str) -> String { + let header_line = format!("# {header}"); + let block: Vec<&str> = EXAMPLE_TEMPLATE + .lines() + .skip_while(|line| *line != header_line) + .take_while(|line| line.trim_start().starts_with('#')) + .map(|line| { + line.strip_prefix("# ") + .or_else(|| line.strip_prefix('#')) + .unwrap_or(line) + }) + .collect(); + assert!(!block.is_empty(), "the template should document {header}"); + block.join("\n") + } } /// Shared Next.js + auction origin fixture. diff --git a/docs/guide/axum-dev.md b/docs/guide/axum-dev.md index 225d227e15..4e785f977b 100644 --- a/docs/guide/axum-dev.md +++ b/docs/guide/axum-dev.md @@ -72,5 +72,8 @@ Axum sanitizes forwarded client-IP headers but does not resolve `[trusted_client_ip]`; see the [adapter matrix](./api-reference#adapter-and-startup-support). +Axum sends no [auction telemetry](./telemetry#adapter-support), and warns at +startup when `[analytics]` selects a module. + For an edge deployment, use the [Fastly](./fastly), [Cloudflare](./cloudflare), or [Spin](./spin) guide. diff --git a/docs/guide/cloudflare.md b/docs/guide/cloudflare.md index b4ecdd4360..2113f256b4 100644 --- a/docs/guide/cloudflare.md +++ b/docs/guide/cloudflare.md @@ -74,5 +74,8 @@ Request-time store registries are unwired, so EC KV lookup is not available. Forwarded client-IP headers are sanitized, but `[trusted_client_ip]` is not resolved. See the [API reference](./api-reference) for route-level behavior. +The Worker sends no [auction telemetry](./telemetry#adapter-support), and warns +at startup when `[analytics]` selects a module. + Compare the [Fastly](./fastly), [Spin](./spin), and [Axum](./axum-dev) adapter journeys. diff --git a/docs/guide/configuration-rules.md b/docs/guide/configuration-rules.md index fd4ac994f5..b36dc2778d 100644 --- a/docs/guide/configuration-rules.md +++ b/docs/guide/configuration-rules.md @@ -10,7 +10,8 @@ request, and why the file is built this way. Everything Trusted Server can switch on is a module, selected the same way: the Edge Cookie identity, the location and device lookups, the permission signals, the demand sources in an auction, the ad server that picks the -winner, the rules in `robots.txt`, and the page integrations. +winner, the rules in `robots.txt`, where auction telemetry is sent, and the +page integrations. ```toml [] @@ -22,8 +23,8 @@ setting = "value" 1. **The type is the job.** Each type is one top-level table, named for what its modules do: `ec`, `geo`, `device`, `permission-signal`, `demand`, - `ad-server` and `robots-txt`, and the section of each module type, such as - `cmp` or `tag`. + `ad-server`, `robots-txt` and `analytics`, and the section of each module + type, such as `cmp` or `tag`. 2. **The selector chooses what runs.** A type that runs one module takes `module`, a string. A type that runs several takes `modules`, a list. 3. **`[.]` holds the settings.** A name with nothing to set @@ -53,6 +54,7 @@ setting = "value" | `demand` | several | `modules`, a list | `auction-protocol.openrtb`, `auction.prebid-server`, `auction.aps` | | `ad-server` | one | `module`, a string | `mock` | | `robots-txt` | several, in order | `modules`, a list | `refuse_all`, `allow_all`, or a module that supplies robots.txt rules | +| `analytics` | one | `module`, a string | `tinybird` | | `cmp` | one | `module`, a string | `didomi`, `sourcepoint`, `osano` | | `tag` | several | `modules`, a list | `google-tag-manager` | | `ad-tag` | several | `modules`, a list | `google`, `google.diagnostics` | @@ -168,7 +170,7 @@ They keep their own keys and have no selector. | `[creative_opportunities]` | server-rendered ad slots | | `[proxy]`, `[cache]`, `[rewrite]` | first-party proxying, caching and URL rewriting | | `[request_signing]`, `[trusted_client_ip]` | signing and client addresses | -| `[debug]`, `[tinybird]` | diagnostics and telemetry | +| `[debug]` | diagnostics | ## Why the file works this way @@ -296,6 +298,7 @@ endpoint = "https://house.example.com/openrtb2/auction" | `profile = "standard"` | `implementation = "auction-protocol.openrtb"` | | `profile = "prebid-server"` and `profile = "aps"` | `implementation = "auction.prebid-server"` and `implementation = "auction.aps"` | | `[auction] mediator = "adserver_mock"` and `[integrations.adserver_mock]` | `[ad-server] module = "mock"` and `[ad-server.mock]` | +| `[tinybird]` with `enabled = true` | `[analytics] module = "tinybird"` and `[analytics.tinybird]` | | `[integrations.aps] rendering_mode` | `rendering_mode` in the `[demand.]` table of the `auction.aps` implementation | | `[debug.auction_html_comment_options] include_mediator_response` | `include_adserver_response` | diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 26d266df19..60621a579d 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -83,7 +83,7 @@ publisher, trusted-client-IP, EC, Tinybird, DataDome, and S3 fields: - `ec.host_signals.passphrase`, when `[ec] module = "host_signals"` - `ec.partners[*].api_token`, when inbound identify or batch sync is used - `ec.partners[*].ts_pull_token`, when pull sync is enabled -- `tinybird.auction_token_secret`, when Tinybird auction telemetry is enabled +- `analytics.tinybird.auction_token_secret`, when `[analytics] module = "tinybird"` - `bot-protection.datadome.server_side_key_secret_name`, when protection is enabled - `bot-protection.datadome.protection_test_bypass.credential_secret_name`, when the bypass is enabled - `proxy.asset_routes[*].auth.access_key_id`, `secret_access_key`, and optional `session_token` @@ -97,9 +97,9 @@ Two accepted secret-shaped fields are deliberately different: - `trusted_client_ip.shared_secret` is an inline value in the app-config blob; it is redacted by debug formatting but is not resolved from a secret store. -- `tinybird.access_token_secret` is deprecated input. It is accepted for - migration, then discarded and omitted from serialized config; use - `tinybird.auction_token_secret` as the store key name instead. +- `analytics.tinybird.access_token_secret` is deprecated input. It is accepted + for migration, then discarded and omitted from serialized config; use + `analytics.tinybird.auction_token_secret` as the store key name instead. The four deprecated `secret_store` selectors under Tinybird, DataDome, its protection-test bypass, and S3 route authentication are also accepted and @@ -180,8 +180,10 @@ Tinybird uses the same typed secret-reference path as the other static credentials. Do not configure a feature-specific store: ```toml -[tinybird] -enabled = true +[analytics] +module = "tinybird" + +[analytics.tinybird] api_host = "api.example.com" auction_dataset = "auction_events_raw" auction_token_secret = "tinybird_auction_append_token" @@ -189,9 +191,9 @@ auction_token_secret = "tinybird_auction_append_token" Store the APPEND token value under `tinybird_auction_append_token` in the physical store mapped from `trusted_server_secrets`. The token is resolved once -at startup. Disabled Tinybird telemetry does not require or resolve the token. -The legacy `tinybird.secret_store` field is accepted for one migration release, -but it is ignored and omitted from newly pushed config. +at startup. Settings that do not select the module neither require nor resolve +the token. The legacy `analytics.tinybird.secret_store` field is accepted for +one migration release, but it is ignored and omitted from newly pushed config. ### Generate Secure Secrets @@ -219,7 +221,7 @@ fail and the service will return its startup-error response. ## Key Sections -10 of these sections select what runs, with `module` where one runs and +11 of these sections select what runs, with `module` where one runs and `modules` where several run, and each gives every selected name its own `[.]` settings table, as [Configuration Rules](/guide/configuration-rules) describes. @@ -227,6 +229,7 @@ fail and the service will return its startup-error response. | Section | Selects | Purpose | | ----------------------------------------------------------------------------------------------------------------- | --------------------- | --------------------------------------------------------------------------------------------- | | `[ad-server]` | one module | The ad server that picks the winner | +| `[analytics]` | one module | The module that receives auction telemetry | | `[auction]` | several modules | Auction orchestration, bidder routes, and the modules the auction runs, `prebid` among them | | `[cache]` | nothing | Static and rehosted asset cache policy | | `[consent]` | nothing | Consent interpretation, forwarding, and conflict resolution | @@ -249,7 +252,6 @@ fail and the service will return its startup-error response. | `[robots-txt]` | several modules | What `/robots.txt` answers, and `X-Robots-Tag` on its responses when every crawler is refused | | `[[serve]]` | nothing | Which page changes run on each reader's copy of a page, and in what order | | `[tester_cookie]` | nothing | Optional tester-cookie endpoints | -| `[tinybird]` | nothing | Direct Tinybird auction telemetry | | `[trusted_client_ip]` | nothing | Authenticated front-door client-IP forwarding | | `[]`, such as `[cmp]`, `[tag]`, `[ad-tag]`, `[bot-protection]`, `[identity]`, `[audience]` or `[framework]` | one module or several | The section of a module type, selecting the modules of that type that run | @@ -1566,30 +1568,48 @@ when_missing = "smart" See [Asset Routes](/guide/asset-routes) for request flow, S3 auth details, and Image Optimizer behavior. -## Tinybird Configuration +## Analytics Configuration + +`[analytics]` selects the module that receives auction telemetry, the way every +other section selects its module. `module = "tinybird"` sends auction events +directly to the Tinybird Events API, with its settings in +`[analytics.tinybird]`. Without the section nothing is sent. Access-log +emission is not wired. + +### `[analytics]` + +| Field | Type | Default | Contract | +| -------- | ------ | ------- | ------------------------------------------------------------------------------------------------ | +| `module` | String | none | Required when the section is present. `tinybird`, or the same name in full, `analytics.tinybird` | + +**Refused when the settings load**: a section that selects no module, a module +this build does not have, and a key `[analytics.tinybird]` does not know. -`[tinybird]` sends auction events directly to the Tinybird Events API. The -emitter is active only when `enabled = true`; access-log emission is not wired. +A configuration that still carries `[tinybird]` is refused with directions. +Move its settings to `[analytics.tinybird]` unchanged, and select the module +with `[analytics] module = "tinybird"`, which replaces the `enabled` line. A +stored configuration written by an earlier version carries the `[tinybird]` +table even where telemetry was off, so push the configuration again with this +version. -### `[tinybird]` +### `[analytics.tinybird]` -| Field | Type | Default | Contract | -| ---------------------- | -------------- | ---------------------- | ------------------------------------------------------------------ | -| `enabled` | Boolean | `false` | Enable auction telemetry | -| `api_host` | String | `""` | Required when enabled; regional host without scheme, port, or path | -| `auction_dataset` | String | `"auction_events_raw"` | 1–128 ASCII letters, digits, or `_` | -| `auction_token_secret` | String or null | `null` | Store key required when enabled; resolved value must be nonempty | -| `access_enabled` | Boolean | `false` | Reserved; `true` fails startup because no emitter is wired | -| `access_dataset` | String | `"access_logs_raw"` | Reserved access-log dataset name | -| `access_sample_rate` | Number | `0.0` | `0.0..=1.0`; reserved while access emission is disabled | -| `max_body_bytes` | Integer | `1048576` | At least `1024` bytes | +| Field | Type | Default | Contract | +| ---------------------- | -------------- | ---------------------- | ---------------------------------------------------------- | +| `api_host` | String | `""` | Required; regional host without scheme, port, or path | +| `auction_dataset` | String | `"auction_events_raw"` | 1–128 ASCII letters, digits, or `_` | +| `auction_token_secret` | String or null | `null` | Store key, required; resolved value must be nonempty | +| `access_enabled` | Boolean | `false` | Reserved; `true` fails startup because no emitter is wired | +| `access_dataset` | String | `"access_logs_raw"` | Reserved access-log dataset name | +| `access_sample_rate` | Number | `0.0` | `0.0..=1.0`; reserved while access emission is disabled | +| `max_body_bytes` | Integer | `1048576` | At least `1024` bytes | `secret_store` and `access_token_secret` are deprecated compatibility inputs. Both are removed during normalization; neither reaches runtime or serialized output. New configurations use only `auction_token_secret`, whose value is resolved through `trusted_server_secrets`. -The complete enabled example appears in +The complete example appears in [Tinybird auction telemetry](#tinybird-auction-telemetry). ## Cache Configuration diff --git a/docs/guide/spin.md b/docs/guide/spin.md index 075fc2357f..90c4eb769e 100644 --- a/docs/guide/spin.md +++ b/docs/guide/spin.md @@ -80,6 +80,9 @@ unavailable. Forwarded client-IP headers are sanitized, and Spin derives its runtime authority, scheme, and client-address headers in the innermost normalization layer. +Spin sends no [auction telemetry](./telemetry#adapter-support), and warns at +startup when `[analytics]` selects a module. + Spin is not installed in the integration CI environment. The recurring manual smoke therefore needs a named owner, tested commit, exact tool versions, and an expiry; those details are recorded with each run in the documentation-refresh diff --git a/docs/guide/telemetry.md b/docs/guide/telemetry.md index a4ad08b93a..cb55984879 100644 --- a/docs/guide/telemetry.md +++ b/docs/guide/telemetry.md @@ -2,7 +2,17 @@ Trusted Server can emit bounded auction observations directly to Tinybird. Emission is currently wired only by the Fastly adapter. Other adapters use the -core no-op sink even if they can parse the shared settings. +core no-op sink even if they can parse the shared settings, and log a warning +at startup when `[analytics]` selects a module. + +## Adapter support + +| Adapter | Auction telemetry | Access telemetry | +| ------------ | ---------------------------- | ---------------- | +| `axum` | not sent, warning at startup | refused | +| `cloudflare` | not sent, warning at startup | refused | +| `fastly` | sent | refused | +| `spin` | not sent, warning at startup | refused | ## Data path @@ -16,8 +26,8 @@ request to the configured auction datasource. Emission is best effort. Failure to construct or start telemetry is logged and must not change the customer response. The sink drops the in-flight handle after dispatch; it does not wait for Tinybird's response. Each batch is limited -to 512 rows and `tinybird.max_body_bytes` bytes. The Fastly backend uses -two-second first-byte and between-byte timeouts. +to 512 rows and `analytics.tinybird.max_body_bytes` bytes. The Fastly backend +uses two-second first-byte and between-byte timeouts. ## Privacy boundary @@ -27,6 +37,11 @@ provider status and timing, slot dimensions, media type, seat, price, currency, win state, and selected creative identifiers. Tests reject the request's derived EC ID from serialized NDJSON. +Every row also carries the reader's complete User-Agent, country, region and +page path, so rows are built only when the reader's permissions include +`analytics.ad_reporting.measure_ad_performance` (TCF Purpose 7). A reader +without it gets no row, whatever the outcome, a skip for consent included. + `browser_family` is a coarse User-Agent classification. Recognized values are `chrome`, `safari`, `firefox`, `edge`, and `opera`; unrecognized clients produce no value. Edge, Opera, and iOS-specific tokens are matched @@ -35,17 +50,18 @@ misclassify them. ## Configuration -Set `tinybird.enabled = true`, provide the regional API host, select the -auction datasource, and set `auction_token_secret` to the key holding that +Select the module with `[analytics] module = "tinybird"`. In +`[analytics.tinybird]` provide the regional API host, select the auction +datasource, and set `auction_token_secret` to the key holding that datasource's APPEND token in the default app-config secret store. The default datasource is `auction_events_raw`; the default body limit is 1 MiB and the minimum accepted limit is 1 KiB. -`tinybird.access_enabled = true` is rejected because no access-log emitter is -wired. `access_dataset` and `access_sample_rate` are reserved. +`analytics.tinybird.access_enabled = true` is rejected because no access-log +emitter is wired. `access_dataset` and `access_sample_rate` are reserved. `access_token_secret` is deprecated input and is normalized away; it cannot enable access telemetry. The exact settings and secret-handling dispositions are in -[Configuration](/guide/configuration#tinybird). The tracked Tinybird project is -described in [`tinybird/README.md`](https://github.com/IABTechLab/trusted-server/blob/main/tinybird/README.md). +[Configuration](/guide/configuration#analytics-configuration). The tracked +Tinybird project is described in [`tinybird/README.md`](https://github.com/IABTechLab/trusted-server/blob/main/tinybird/README.md). diff --git a/tinybird/README.md b/tinybird/README.md index c0a0e976ce..57571e3172 100644 --- a/tinybird/README.md +++ b/tinybird/README.md @@ -19,8 +19,9 @@ directory, run `tb test`, inspect `tb deploy --dry-run`, and deploy with Create an APPEND token scoped to `auction_events_raw`, store its value in the platform secret store, and put only its key name in -`tinybird.auction_token_secret`. The runtime sends directly to the regional -Events API host configured by `tinybird.api_host`. +`analytics.tinybird.auction_token_secret`. The runtime sends directly to the +regional Events API host configured by `analytics.tinybird.api_host`, when +`[analytics] module = "tinybird"` selects the module. For runtime behavior, limits, privacy properties, and the Fastly-only emission boundary, see [Auction Telemetry](../docs/guide/telemetry.md). For exact @@ -61,7 +62,9 @@ Publisher summary rows cover requests with matching renderable slots while creat opportunities are enabled. They include completed auctions **and skipped candidates**: bots, prefetches, consent-denied readers, and requests with auctions disabled can emit a `Skipped` summary when slots match. A page with no matching slot, a non-GET request, or -traffic with creative opportunities disabled produces no publisher summary. +traffic with creative opportunities disabled produces no publisher summary, and neither +does a reader whose permissions do not include measuring ad performance, whatever the +outcome. The query below therefore measures shareability among matching-slot candidates, including skipped summaries. It is neither an ad-serving-pageview rate nor a site-wide rate. The diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 55b3d96090..607ecbd17a 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -28,9 +28,10 @@ # setting = "value" # # The types are ec, geo, device, permission-signal, demand, ad-server, -# robots-txt and integration. A settings table its type's selector does not -# name refuses startup, so there is no stale block to misread. The complete -# rules, and what is checked when, are in docs/guide/configuration-rules.md. +# robots-txt, analytics and integration. A settings table its type's selector +# does not name refuses startup, so there is no stale block to misread. The +# complete rules, and what is checked when, are in +# docs/guide/configuration-rules.md. # # `TRUSTED_SERVER__` env vars can override values when the `ts` CLI builds and # validates this config for push (e.g. TRUSTED_SERVER__PUBLISHER__DOMAIN=...; @@ -648,11 +649,13 @@ auction_timeout_ms = 500 # "/news/x" -> /123456789/example/news # "/reviews/y" -> /123456789/example/reviews -# Direct Tinybird auction telemetry (off by default). When enabled, `api_host` -# is required and must be a bare host (no scheme or path). -# [tinybird] -# enabled = true -# api_host = "api.us-east.tinybird.example" # required when enabled; host only +# Direct Tinybird auction telemetry, sent only when [analytics] selects it. +# `api_host` is required and must be a bare host (no scheme or path). +# [analytics] +# module = "tinybird" +# +# [analytics.tinybird] +# api_host = "api.us-east.tinybird.example" # required; host only # auction_dataset = "auction_events" # Events API datasource name # auction_token_secret = "tinybird_auction_append_token" # Key in trusted_server_secrets