diff --git a/.cargo/config.toml b/.cargo/config.toml index b88693a6c2..a82cddef61 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -26,10 +26,10 @@ test_details = "test --target aarch64-apple-darwin" # native crate needs no change here. Axum (native), Cloudflare # (wasm32-unknown-unknown), Spin, the CLI (native), and integration-tests # (native) are simply not listed. -build-fastly = "build -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-iabtl -p trusted-server-headers-geo -p trusted-server-asset-cache-51degrees -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-iabtl -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" -check-fastly = "check -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-iabtl -p trusted-server-headers-geo -p trusted-server-asset-cache-51degrees -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-iabtl -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" -clippy-fastly = "clippy -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-iabtl -p trusted-server-headers-geo -p trusted-server-asset-cache-51degrees -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-iabtl -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --all-targets --all-features --target wasm32-wasip1 -- -D warnings" -test-fastly = "test -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-iabtl -p trusted-server-headers-geo -p trusted-server-asset-cache-51degrees -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-iabtl -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" +build-fastly = "build -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-iabtl -p trusted-server-headers-geo -p trusted-server-asset-cache-51degrees -p trusted-server-analytics-tinybird -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-iabtl -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" +check-fastly = "check -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-iabtl -p trusted-server-headers-geo -p trusted-server-asset-cache-51degrees -p trusted-server-analytics-tinybird -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-iabtl -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" +clippy-fastly = "clippy -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-iabtl -p trusted-server-headers-geo -p trusted-server-asset-cache-51degrees -p trusted-server-analytics-tinybird -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-iabtl -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --all-targets --all-features --target wasm32-wasip1 -- -D warnings" +test-fastly = "test -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-iabtl -p trusted-server-headers-geo -p trusted-server-asset-cache-51degrees -p trusted-server-analytics-tinybird -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-iabtl -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" # Feature-on counterpart of `test-fastly`. `test-fastly` does NOT pass # --all-features, so without this the reusable-sandbox code is linted by # `clippy-fastly` but its tests never execute. @@ -40,8 +40,8 @@ test-fastly-reuse = "test -p trusted-server-adapter-fastly --target wasm32-wasip # run with the Axum adapter in CI. build-axum = "build -p trusted-server-adapter-axum" check-axum = "check -p trusted-server-adapter-axum" -clippy-axum = "clippy -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-iabtl -p trusted-server-headers-geo -p trusted-server-loading-cache-51degrees -p trusted-server-asset-cache-51degrees -p trusted-server-shared-51degrees -p trusted-server-middleware-51degrees -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-iabtl -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb --all-targets --all-features -- -D warnings" -test-axum = "test -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-iabtl -p trusted-server-headers-geo -p trusted-server-loading-cache-51degrees -p trusted-server-asset-cache-51degrees -p trusted-server-shared-51degrees -p trusted-server-middleware-51degrees -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-iabtl -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb" +clippy-axum = "clippy -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-iabtl -p trusted-server-headers-geo -p trusted-server-analytics-tinybird -p trusted-server-loading-cache-51degrees -p trusted-server-asset-cache-51degrees -p trusted-server-shared-51degrees -p trusted-server-middleware-51degrees -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-iabtl -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb --all-targets --all-features -- -D warnings" +test-axum = "test -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-iabtl -p trusted-server-headers-geo -p trusted-server-analytics-tinybird -p trusted-server-loading-cache-51degrees -p trusted-server-asset-cache-51degrees -p trusted-server-shared-51degrees -p trusted-server-middleware-51degrees -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-iabtl -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb" # --- Cloudflare adapter (native host + wasm32-unknown-unknown) --- # Build/check target the WASM runtime (requires the `cloudflare` feature); diff --git a/CHANGELOG.md b/CHANGELOG.md index 39d721872c..c41cd4df78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Core names no platform's cache header. It removes from a response it makes private only `Surrogate-Control`, `CDN-Cache-Control` and a platform header it wrote its own policy in, so on every adapter `fastly-surrogate-control` and `cloudflare-cdn-cache-control` stay where an origin or an operator wrote them, until a fetch middleware removes them, which none in the stack does yet. The Fastly and Cloudflare adapters declare their own platform's header, which keeps an origin response with a policy in it out of the stored copies on that adapter alone, and each adapter writes its policy in the header it wrote before. **For crates outside core**, `EdgeCacheHeader::CloudflareCdnCacheControl` is `EdgeCacheHeader::Platform(name)`, `EDGE_CACHE_HEADER_NAMES` lists only the two standard headers, the two platform header constants leave core, an adapter declares its platform's headers with `platform::declare_platform_cache_headers`, and a policy written with `CachePolicy::apply_to_response` is removed by `cache_policy::remove_owned_cache_headers` when the response is made private. +- Auction telemetry is an analytics module from a crate of its own. `[analytics]` is read as every module type's section is, and the Tinybird module, its settings and its checks live in `crates/analytics/tinybird`, which each adapter and the `ts` tool list among the modules they link, where the Fastly adapter alone built it. The Fastly adapter alone hands the module's sink to the auction, so the Axum, Cloudflare and Spin adapters link the module and still send nothing. The settings a deployment writes do not change. A module named in full in its own type's section, as `analytics.tinybird` in `[analytics]`, reads its table without the type folder, at `[analytics.tinybird]`, when the settings load, at deploy and when a stored configuration loads. A table missing its host or token, a key the table does not know and a reserved setting turned on are refused when the deployment starts and by `ts config validate`, naming the table, where they were refused as the settings loaded. A `[tinybird]` table at the top level is refused as the section of a module type nothing supplies. **For crates outside core**, a registration declares the sink it supplies with `IntegrationRegistrationBuilder::with_analytics_module`, and `IntegrationRegistry::analytics_module` gives the one `[analytics] module` selected. `Settings::analytics_module`, `AnalyticsConfig` and `TinybirdSettings` leave core. - **Breaking for crates outside core:** core's auction names no protocol. A demand source implements `CompiledDemand`, which builds its own request from the routed input (`build_request`, answering the body, the headers, the slot ids and any state kept for the answer) and reads its own answer, and core sends what it built. OpenRTB 2.6 is the protocol module `crates/auction-protocol/openrtb`, where the request builder, the response reader, the wire types with their extension objects, and `OpenRtbDemand`, the hook trait a demand implementation that speaks OpenRTB fills in, now live, run through core's seam by `OpenRtbDriver`. `auction::openrtb` and core's `openrtb` module are gone, and core's own manifest keeps the generated types only for the response `POST /auction` returns. The dimension admission helpers are `auction::admission`, `apply_notification_policy` and `ignored_bidder_params_count` are `auction::provider`, and `Eid` and `Uid` are `ec::eids`. A deployment's settings do not change. - **Breaking:** The ad server this repository ships is `ad-server.iabtl`, from `crates/ad-server/iabtl` as the package `trusted-server-ad-server-iabtl`, where it was `ad-server.mock` from `crates/ad-server/mock`. A deployment that selected it writes `[ad-server] module = "iabtl"` and `[ad-server.iabtl]` where it wrote `module = "mock"` and `[ad-server.mock]`, and an `implementation = "ad-server.mock"` line names `ad-server.iabtl`. The seam probe, the test fixture that proves the integration seam from outside core, is `crates/testing/iabtl` as the package `trusted-server-testing-iabtl`, selected as `testing.iabtl`, where it was `crates/testing/seam-probe` selected as `testing.seam-probe`, with its shared table `[shared.iabtl]`, its loading cache `loading-cache.iabtl` and its declared cache `iabtl-answers` named the same way. The module that adds sellers to `ads.txt` is `ads-txt.iabtl`, from `crates/ads-txt/iabtl` as the package `trusted-server-ads-txt-iabtl`, where it was `ads-txt.sellers` from `crates/ads-txt/sellers`. A deployment that selected it writes `[ads-txt] modules = ["iabtl"]` and `[ads-txt.iabtl]` where it wrote `modules = ["sellers"]` and `[ads-txt.sellers]`, and the `[[fetch]]` entry on `/ads.txt` names the middleware `ads-txt.iabtl` where it named `ads-txt.sellers`. Each module describes the middleware it supplies on its builder with `IntegrationBuilder::with_middleware`, as `SuppliedMiddleware` values naming the middleware, the phase an entry names it in and the media type the entry covers, and `trusted_server_core::integrations::supplied_middleware(builders)` lists what a build supplies in the order the registry registers them, core's robots.txt and service version middleware included. The registry refuses a registration whose middleware contradicts its builder's description, in a phase it does not run in or for a media type it does not work on. **Breaking for crates outside core:** `crates/trusted-server-modules` is gone. Each adapter lists the module builders it links in its own `app.rs`, as `shipped_module_builders()`, with `module_builders_with(extra)` putting a deployment's builders after them, and the `ts` tool lists the builders it links and writes its entries and a module's section from what they describe. A deployment that took its list from `trusted_server_modules::builders()` or `builders_with(...)` names the crates it links instead, or takes an adapter's list. - **Breaking for crates outside core:** the consent decoders live in the crates named for their standards. The TCF string decoder is in `trusted-server-permission-signal-tcf`, the GPP string decoder in `trusted-server-permission-signal-gpp`, and the US Privacy string decoder in `trusted-server-permission-signal-us-privacy`, and each module reads its own string from the request through `PermissionSignalModule::read`, once, and reports a `SignalRecord`. `ConsentContext` is the records the configured modules reported and the request's jurisdiction, read by scheme name, with nothing of any standard in it, and a reader that needs a string in its standard's own form asks that standard's crate (`tc_string`, `applies`, `gpp_string`, `section_ids`, `us_privacy_string`, `signaled`). A record also carries the permissions its signal consents to, and the server-side auction gate keeps its rule by reading them, so in a GDPR or unknown jurisdiction, or where a record says its regulation applies, the auction needs consent to device storage (TCF Purpose 1) from a record that applies. Without `tcf` in the modules no TC String is read, and one that cannot be read, standalone or in a GPP string, is no consent to any purpose TCF maps. `[consent]` keeps its keys, with `mode` parsed by core and the rest read by the modules, and `signals..authoritative` in `permissions.yaml` is read for any scheme. The identity graph record writes every vouched signal under its scheme, in the shape it wrote before for the two it stored. `CONSENT_COOKIE_NAMES`, `forward_cookie_header`, the consent cookie and header constants, `ConsentForwardingMode` (now the Prebid Server crate's), the `GdprConsent` error and the never-filled Additional Consent field and `additional_consent` policy flag are removed. diff --git a/Cargo.lock b/Cargo.lock index e6dadef1af..009ce86797 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6644,6 +6644,7 @@ dependencies = [ "trusted-server-ad-server-iabtl", "trusted-server-ad-tag-google", "trusted-server-ads-txt-iabtl", + "trusted-server-analytics-tinybird", "trusted-server-auction-aps", "trusted-server-auction-prebid", "trusted-server-auction-prebid-server", @@ -6689,6 +6690,7 @@ dependencies = [ "trusted-server-ad-server-iabtl", "trusted-server-ad-tag-google", "trusted-server-ads-txt-iabtl", + "trusted-server-analytics-tinybird", "trusted-server-auction-aps", "trusted-server-auction-prebid", "trusted-server-auction-prebid-server", @@ -6740,6 +6742,7 @@ dependencies = [ "trusted-server-ad-server-iabtl", "trusted-server-ad-tag-google", "trusted-server-ads-txt-iabtl", + "trusted-server-analytics-tinybird", "trusted-server-auction-aps", "trusted-server-auction-prebid", "trusted-server-auction-prebid-server", @@ -6766,7 +6769,6 @@ dependencies = [ "trusted-server-tag-google-tag-manager", "trusted-server-testing-iabtl", "trusted-server-testing-testlight", - "urlencoding", ] [[package]] @@ -6791,6 +6793,7 @@ dependencies = [ "trusted-server-ad-server-iabtl", "trusted-server-ad-tag-google", "trusted-server-ads-txt-iabtl", + "trusted-server-analytics-tinybird", "trusted-server-auction-aps", "trusted-server-auction-prebid", "trusted-server-auction-prebid-server", @@ -6828,6 +6831,22 @@ dependencies = [ "validator", ] +[[package]] +name = "trusted-server-analytics-tinybird" +version = "0.1.0" +dependencies = [ + "async-trait", + "edgezero-core", + "error-stack", + "futures", + "log", + "serde", + "serde_json", + "trusted-server-core", + "urlencoding", + "validator", +] + [[package]] name = "trusted-server-asset-cache-51degrees" version = "0.1.0" @@ -7026,6 +7045,7 @@ dependencies = [ "trusted-server-ad-server-iabtl", "trusted-server-ad-tag-google", "trusted-server-ads-txt-iabtl", + "trusted-server-analytics-tinybird", "trusted-server-auction-aps", "trusted-server-auction-prebid", "trusted-server-auction-prebid-server", diff --git a/Cargo.toml b/Cargo.toml index 9d81ade1fb..9eeff2c681 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ members = [ "crates/cmp/didomi", "crates/cmp/inmobi", "crates/ads-txt/iabtl", + "crates/analytics/tinybird", "crates/headers/geo", "crates/loading-cache/51degrees", "crates/shared/51degrees", @@ -172,6 +173,7 @@ trusted-server-cmp-sourcepoint = { path = "crates/cmp/sourcepoint" } trusted-server-cmp-didomi = { path = "crates/cmp/didomi" } trusted-server-cmp-inmobi = { path = "crates/cmp/inmobi" } trusted-server-ads-txt-iabtl = { path = "crates/ads-txt/iabtl" } +trusted-server-analytics-tinybird = { path = "crates/analytics/tinybird" } trusted-server-headers-geo = { path = "crates/headers/geo" } trusted-server-loading-cache-51degrees = { path = "crates/loading-cache/51degrees" } trusted-server-shared-51degrees = { path = "crates/shared/51degrees" } diff --git a/crates/ad-tag/google/src/diagnostics.rs b/crates/ad-tag/google/src/diagnostics.rs index cc9443b4e9..bc8f53171c 100644 --- a/crates/ad-tag/google/src/diagnostics.rs +++ b/crates/ad-tag/google/src/diagnostics.rs @@ -676,8 +676,7 @@ mod tests { .header(header::ETAG, "\"origin\"") .header(header::LAST_MODIFIED, "Wed, 12 Aug 2026 00:00:00 GMT") .header("surrogate-control", "max-age=60") - .header("fastly-surrogate-control", "max-age=60") - .header("cloudflare-cdn-cache-control", "public, max-age=60") + .header("cdn-cache-control", "public, max-age=60") .body(EdgeBody::empty()) .expect("should build response"); @@ -690,12 +689,7 @@ mod tests { ); assert_eq!(response.headers()[header::SET_COOKIE], SET_CONSOLE_COOKIE); assert!(!response.headers().contains_key("surrogate-control")); - assert!(!response.headers().contains_key("fastly-surrogate-control")); - assert!( - !response - .headers() - .contains_key("cloudflare-cdn-cache-control") - ); + assert!(!response.headers().contains_key("cdn-cache-control")); assert!( !response.headers().contains_key(header::ETAG), "should drop the origin validator with the shared-cache policy" diff --git a/crates/analytics/README.md b/crates/analytics/README.md new file mode 100644 index 0000000000..170a381515 --- /dev/null +++ b/crates/analytics/README.md @@ -0,0 +1,17 @@ +# Analytics modules + +Analytics module crates live here, one per vendor, each supplying an +`AuctionTelemetrySink` from `trusted-server-core` under the name +`[analytics] module` selects it by: + +- `crates/analytics/tinybird` (`trusted-server-analytics-tinybird`) sends each + auction's telemetry rows to the Tinybird Events API as one request. It is + selected by `[analytics] module = "tinybird"` and reads + `[analytics.tinybird]`. +- Vendor analytics modules (for example `crates/analytics/`) slot in + alongside it, one per vendor. + +Core builds the rows, being the auction's summary, each provider call and +each bid, and asks the selected sink to send them once the auction has +settled. With no `[analytics]` section nothing is sent, through the no-op +sink core supplies itself. diff --git a/crates/analytics/tinybird/Cargo.toml b/crates/analytics/tinybird/Cargo.toml new file mode 100644 index 0000000000..f6f88059c0 --- /dev/null +++ b/crates/analytics/tinybird/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "trusted-server-analytics-tinybird" +description = "Tinybird as an analytics module: each auction's telemetry rows sent to the Events API as one request, selected by `[analytics]`." +authors = { workspace = true } +edition = { workspace = true } +license = { workspace = true } +publish = { workspace = true } +version = { workspace = true } + +[lib] +doctest = false + +[lints] +workspace = true + +[dependencies] +async-trait = { workspace = true } +edgezero-core = { workspace = true } +error-stack = { workspace = true } +log = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +trusted-server-core = { workspace = true } +urlencoding = { workspace = true } +validator = { workspace = true } + +[dev-dependencies] +futures = { workspace = true } +trusted-server-core = { workspace = true, features = ["test-utils"] } + +# The visible owner of this vendor crate, the way Prebid.js requires a named +# maintainer of every adapter. Tinybird has not yet adopted this crate, so the +# Trusted Server maintainers own it until a vendor owner steps up. +[package.metadata.maintainers] +owner = "Trusted Server maintainers" +status = "seeking vendor owner" diff --git a/crates/analytics/tinybird/README.md b/crates/analytics/tinybird/README.md new file mode 100644 index 0000000000..d3d91f98cf --- /dev/null +++ b/crates/analytics/tinybird/README.md @@ -0,0 +1,31 @@ +# trusted-server-analytics-tinybird + +Tinybird as an analytics module. It sends each auction's telemetry rows to +the Tinybird Events API as one NDJSON request, started and not waited for, +so a slow or failing Events API never delays a reader's page. + +Select it with `[analytics] module = "tinybird"` and give it the regional +API host, the datasource and the name of the secret holding the +datasource's APPEND token in `[analytics.tinybird]`: + +```toml +[analytics] +module = "tinybird" + +[analytics.tinybird] +api_host = "api.us-east.aws.tinybird.co" +auction_dataset = "auction_events_raw" +auction_token_secret = "tinybird_auction_append_token" +``` + +Each adapter, and the `ts` tool, lists the module's builder among the +modules it links. The settings, what each one accepts and what is refused are +in the configuration guide under "Analytics Configuration", and the data path +in the telemetry guide. + +Build and test from the repository root: + +```bash +cargo test -p trusted-server-analytics-tinybird +cargo clippy -p trusted-server-analytics-tinybird --all-targets -- -D warnings +``` diff --git a/crates/analytics/tinybird/src/lib.rs b/crates/analytics/tinybird/src/lib.rs new file mode 100644 index 0000000000..3c11292ca0 --- /dev/null +++ b/crates/analytics/tinybird/src/lib.rs @@ -0,0 +1,1229 @@ +//! Tinybird as an analytics module: each auction's telemetry rows sent to the +//! Events API as one request. +//! +//! `[analytics] module = "tinybird"` selects the module and +//! `[analytics.tinybird]` holds its settings. The module supplies the auction +//! telemetry sink core hands each settled auction's rows to. The sink +//! serializes a batch as NDJSON, starts one Events API request for it and +//! drops the pending request without waiting, so the Events API never delays +//! a reader's page. + +use std::sync::Arc; +use std::time::Duration; + +use edgezero_core::body::Body; +use edgezero_core::http::{HeaderValue, Method, header, request_builder}; +use error_stack::{Report, ResultExt as _}; +use serde::{Deserialize, Serialize}; +use validator::Validate; + +use trusted_server_core::auction::telemetry::{AuctionEventBatch, AuctionTelemetrySink}; +use trusted_server_core::error::TrustedServerError; +use trusted_server_core::host_header::validate_host_header_override_value; +use trusted_server_core::integrations::{ + IntegrationBuilder, IntegrationRegistration, ModuleSecretSetting, +}; +use trusted_server_core::platform::{PlatformBackendSpec, PlatformHttpRequest, RuntimeServices}; +use trusted_server_core::redacted::Redacted; +use trusted_server_core::settings::{IntegrationConfig, Settings}; + +/// The id the module's registration carries. +const TINYBIRD_INTEGRATION_ID: &str = "tinybird"; + +/// The name `[analytics]` selects the module by, from this crate's folder. +/// Its settings are the table `[analytics.tinybird]`. +pub const MODULE: &str = "analytics.tinybird"; + +const EVENTS_PATH: &str = "/v0/events"; +const NDJSON_CONTENT_TYPE: &str = "application/x-ndjson"; +const FIRST_BYTE_TIMEOUT: Duration = Duration::from_secs(2); +const BETWEEN_BYTES_TIMEOUT: Duration = Duration::from_secs(2); +const MAX_ROWS_PER_AUCTION_BATCH: usize = 512; + +/// The settings of `[analytics.tinybird]`. +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct TinybirdSettings { + /// Regional Tinybird API host, without scheme or path. + #[serde(default)] + pub api_host: String, + /// Deprecated feature-specific store selector accepted for migration only. + #[serde(default, skip_serializing)] + pub secret_store: Option, + /// Auction Events API datasource name. + #[serde(default = "default_auction_dataset")] + pub auction_dataset: String, + /// The name of the secret holding the auction datasource's APPEND token, + /// which holds the token itself once the settings have loaded. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub auction_token_secret: Option>, + /// Reserved for future access-log telemetry. + /// + /// `true` is refused until an access-log emitter exists, so an operator + /// cannot enable a setting that silently emits nothing. + #[serde(default)] + pub access_enabled: bool, + /// Future access-log Events API datasource name. + #[serde(default = "default_access_dataset")] + pub access_dataset: String, + /// Deprecated placeholder for the unwired access-log APPEND token. + #[serde(default, skip_serializing)] + pub access_token_secret: Option>, + /// Future fraction of requests to emit for optional access telemetry. + #[serde(default)] + pub access_sample_rate: f64, + /// Defensive maximum NDJSON body size for one Events API request. + #[serde(default = "default_max_body_bytes")] + pub max_body_bytes: usize, +} + +fn default_auction_dataset() -> String { + "auction_events_raw".to_owned() +} + +fn default_access_dataset() -> String { + "access_logs_raw".to_owned() +} + +fn default_max_body_bytes() -> usize { + 1024 * 1024 +} + +impl Default for TinybirdSettings { + fn default() -> Self { + Self { + api_host: String::new(), + secret_store: None, + auction_dataset: default_auction_dataset(), + auction_token_secret: None, + access_enabled: false, + access_dataset: default_access_dataset(), + access_token_secret: None, + access_sample_rate: 0.0, + max_body_bytes: default_max_body_bytes(), + } + } +} + +// The checks need the table trimmed first, which a borrowed `validate` cannot +// do, so they run in `checked` and this reads the table as written. +impl Validate for TinybirdSettings { + fn validate(&self) -> Result<(), validator::ValidationErrors> { + Ok(()) + } +} + +impl IntegrationConfig for TinybirdSettings {} + +impl TinybirdSettings { + /// The table trimmed, with the host lower cased and the deprecated + /// fields dropped, then checked. + /// + /// # Errors + /// + /// Naming the setting that is missing or malformed. + fn checked(mut self) -> Result> { + self.api_host = self.api_host.trim().to_ascii_lowercase(); + if self.secret_store.take().is_some() { + log::warn!( + "{MODULE}.secret_store is deprecated and ignored; the token resolves through \ + the default app-config secret store" + ); + } + self.auction_dataset = self.auction_dataset.trim().to_owned(); + self.auction_token_secret = self.auction_token_secret.take().and_then(|value| { + let value = value.expose().trim().to_owned(); + (!value.is_empty()).then(|| Redacted::new(value)) + }); + self.access_dataset = self.access_dataset.trim().to_owned(); + self.access_token_secret = None; + self.check()?; + Ok(self) + } + + fn check(&self) -> Result<(), Report> { + if !(0.0..=1.0).contains(&self.access_sample_rate) { + return Err(configuration(format!( + "{MODULE}.access_sample_rate must be between 0.0 and 1.0" + ))); + } + if self.max_body_bytes < 1024 { + return Err(configuration(format!( + "{MODULE}.max_body_bytes must be at least 1024" + ))); + } + if self.access_enabled { + return Err(configuration(format!( + "{MODULE}.access_enabled is reserved for future access-log telemetry; no \ + emitter is currently wired" + ))); + } + validate_api_host(&self.api_host)?; + validate_dataset(&self.auction_dataset, &format!("{MODULE}.auction_dataset"))?; + let token = self.auction_token_secret.as_ref().ok_or_else(|| { + configuration(format!( + "{MODULE}.auction_token_secret is required when [analytics] selects the module" + )) + })?; + if token.expose().chars().any(char::is_control) { + return Err(configuration(format!( + "{MODULE}.auction_token_secret must be non-empty after secret resolution" + ))); + } + Ok(()) + } +} + +fn configuration(message: String) -> Report { + Report::new(TrustedServerError::Configuration { message }) +} + +fn validate_api_host(host: &str) -> Result<(), Report> { + if host.is_empty() + || host.contains('/') + || host.contains(':') + || host.chars().any(char::is_control) + || host.starts_with("http://") + || host.starts_with("https://") + { + return Err(configuration(format!( + "{MODULE}.api_host must be a regional host without scheme, port, or path" + ))); + } + validate_host_header_override_value(host) + .map_err(|reason| configuration(format!("{MODULE}.api_host {reason}"))) +} + +fn validate_dataset(value: &str, setting: &str) -> Result<(), Report> { + if value.is_empty() + || value.len() > 128 + || !value + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || ch == '_') + { + return Err(configuration(format!( + "{setting} must be a non-empty datasource identifier" + ))); + } + Ok(()) +} + +/// The token is in use whenever the section selects the module. +fn always_in_use(_table: &serde_json::Map) -> bool { + true +} + +/// The one setting of the table that names a secret. +const SECRET_SETTINGS: &[ModuleSecretSetting] = &[ModuleSecretSetting { + path: &["auction_token_secret"], + in_use: always_in_use, +}]; + +/// Validates the table and reports whether `[analytics]` selects the module. +/// +/// # Errors +/// +/// Returns an error when the table cannot be read, carries a key the module +/// does not know, or fails a check. +pub(crate) fn validate(settings: &Settings) -> Result> { + let Some(config) = settings.module_config::(MODULE)? else { + return Ok(false); + }; + config.checked()?; + Ok(true) +} + +/// Registers the module's sink when `[analytics]` selects the module. +/// +/// # Errors +/// +/// Returns an error when the table cannot be read, carries a key the module +/// does not know, or fails a check. +pub fn register( + settings: &Settings, +) -> Result, Report> { + let Some(config) = settings.module_config::(MODULE)? else { + return Ok(None); + }; + let sink = TinybirdAuctionTelemetrySink::new(config.checked()?)?; + Ok(Some( + IntegrationRegistration::builder(TINYBIRD_INTEGRATION_ID) + // No browser module. The sink is the whole module. + .without_js() + .with_analytics_module(MODULE, Arc::new(sink)) + .build(), + )) +} + +/// The builder a deployment installs to offer this module. +#[must_use] +pub fn builder() -> IntegrationBuilder { + IntegrationBuilder::new( + TINYBIRD_INTEGRATION_ID, + env!("CARGO_PKG_NAME"), + register, + validate, + ) + .with_module_name(MODULE) + .with_secret_settings(SECRET_SETTINGS) +} + +/// The sink that sends each batch to the Events API. +#[derive(Debug, Clone)] +pub struct TinybirdAuctionTelemetrySink { + target: EventsTarget, +} + +#[derive(Debug, Clone)] +struct EventsTarget { + api_host: String, + dataset: String, + append_token: Redacted, + uri: String, + backend_spec: PlatformBackendSpec, + max_body_bytes: usize, +} + +impl TinybirdAuctionTelemetrySink { + /// The sink for a checked table. + /// + /// # Errors + /// + /// When the table carries no token, which [`TinybirdSettings::checked`] + /// refuses before this is reached. + fn new(config: TinybirdSettings) -> Result> { + let append_token = config + .auction_token_secret + .ok_or_else(|| configuration(format!("{MODULE}.auction_token_secret is required")))?; + Ok(Self { + target: EventsTarget { + uri: events_uri(&config.api_host, &config.auction_dataset), + backend_spec: backend_spec(&config.api_host), + api_host: config.api_host, + dataset: config.auction_dataset, + append_token, + max_body_bytes: config.max_body_bytes, + }, + }) + } + + fn validate_batch(batch: &AuctionEventBatch) -> Result<(), Report> { + if batch.row_count() > MAX_ROWS_PER_AUCTION_BATCH { + return Err(Report::new(TrustedServerError::Proxy { + message: format!( + "auction telemetry batch has {} rows, exceeding {} row limit", + batch.row_count(), + MAX_ROWS_PER_AUCTION_BATCH + ), + })); + } + Ok(()) + } + + fn serialize_batch( + &self, + batch: &AuctionEventBatch, + ) -> Result> { + batch.to_ndjson(self.target.max_body_bytes) + } + + fn ensure_backend( + &self, + services: &RuntimeServices, + ) -> Result> { + services + .backend() + .ensure(&self.target.backend_spec) + .change_context(TrustedServerError::Proxy { + message: "Tinybird backend registration failed".to_owned(), + }) + } + + fn authorization_header(token: &str) -> Result> { + HeaderValue::from_str(&format!("Bearer {token}")).change_context( + TrustedServerError::InvalidHeaderValue { + message: "invalid Tinybird authorization header".to_owned(), + }, + ) + } + + fn build_events_request( + &self, + body: String, + auth_header: HeaderValue, + ) -> Result> { + request_builder() + .method(Method::POST) + .uri(self.target.uri.as_str()) + .header(header::AUTHORIZATION, auth_header) + .header(header::CONTENT_TYPE, NDJSON_CONTENT_TYPE) + .body(Body::from(body)) + .change_context(TrustedServerError::Proxy { + message: "failed to build Tinybird Events API request".to_owned(), + }) + } + + async fn send_fire_and_forget( + services: &RuntimeServices, + request: edgezero_core::http::Request, + backend_name: String, + ) -> Result<(), Report> { + let pending = services + .http_client() + .send_async(PlatformHttpRequest::new(request, backend_name)) + .await + .change_context(TrustedServerError::Proxy { + message: "failed to start Tinybird Events API request".to_owned(), + })?; + drop(pending); + Ok(()) + } +} + +#[async_trait::async_trait(?Send)] +impl AuctionTelemetrySink for TinybirdAuctionTelemetrySink { + async fn emit_auction_events( + &self, + services: &RuntimeServices, + batch: AuctionEventBatch, + ) -> Result<(), Report> { + if batch.is_empty() { + return Ok(()); + } + + Self::validate_batch(&batch)?; + let body = self.serialize_batch(&batch)?; + let body_len = body.len(); + let auth_header = Self::authorization_header(self.target.append_token.expose())?; + let backend_name = self.ensure_backend(services)?; + let request = self.build_events_request(body, auth_header)?; + + log::info!( + "sending auction telemetry to Tinybird dataset={} rows={} bytes={} host={} backend={}", + self.target.dataset, + batch.row_count(), + body_len, + self.target.api_host, + backend_name + ); + + Self::send_fire_and_forget(services, request, backend_name).await + } +} + +fn backend_spec(api_host: &str) -> PlatformBackendSpec { + PlatformBackendSpec { + scheme: "https".to_owned(), + host: api_host.to_owned(), + port: None, + host_header_override: None, + certificate_check: true, + first_byte_timeout: FIRST_BYTE_TIMEOUT, + between_bytes_timeout: BETWEEN_BYTES_TIMEOUT, + discriminator: None, + } +} + +fn events_uri(api_host: &str, dataset: &str) -> String { + format!( + "https://{api_host}{EVENTS_PATH}?name={}", + urlencoding::encode(dataset) + ) +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::sync::{Arc, Mutex}; + + use edgezero_core::blob_envelope::BlobEnvelope; + use error_stack::Report; + use trusted_server_core::auction::telemetry::{AuctionEventBatch, AuctionEventRow}; + use trusted_server_core::config::validate_settings_for_deploy_with; + use trusted_server_core::config_payload::settings_from_config_blob_with; + use trusted_server_core::integrations::IntegrationRegistry; + use trusted_server_core::platform::{ + BackendNamingPolicy, ClientInfo, PlatformBackend, PlatformConfigStore, PlatformError, + PlatformGeo, PlatformHttpClient, PlatformPendingRequest, PlatformResponse, + PlatformSecretStore, PlatformSelectResult, RuntimeServices, StoreId, StoreName, + }; + use trusted_server_core::test_support::template::{ + template_with_resolved_required_secrets, uncomment_block, + }; + use trusted_server_core::test_support::tests::crate_test_settings_str; + + use super::*; + + const TEST_USER_AGENT: &str = + "FictionalBrowser/123.4 (FictionalOS 10.2; FictionalDevice) ExampleRenderer/567.8"; + + /// The lines of a table that passes every check. + const TABLE: &str = + "api_host = \"api.us-east.aws.tinybird.co\"\nauction_token_secret = \"test-auction-token\""; + + /// A document selecting the module, with `table` as its settings. + fn settings_str_selecting(table: &str) -> String { + format!( + "{}\n[analytics]\nmodule = \"tinybird\"\n\n[analytics.tinybird]\n{table}\n", + crate_test_settings_str() + ) + } + + fn settings_selecting(table: &str) -> Settings { + Settings::from_toml(&settings_str_selecting(table)).expect("the document should parse") + } + + /// The error `validate` reports for a document selecting the module with + /// `table` as its settings. + fn refusal_of(table: &str) -> String { + let err = validate(&settings_selecting(table)).expect_err("the table should be refused"); + format!("{err:?}") + } + + struct NoopConfigStore; + + impl PlatformConfigStore for NoopConfigStore { + fn get( + &self, + _store_name: &StoreName, + _key: &str, + ) -> Result> { + Err(Report::new(PlatformError::Unsupported)) + } + + fn put( + &self, + _store_id: &StoreId, + _key: &str, + _value: &str, + ) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } + + fn delete(&self, _store_id: &StoreId, _key: &str) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } + } + + struct MapSecretStore(HashMap>); + + impl PlatformSecretStore for MapSecretStore { + fn get_bytes( + &self, + _store_name: &StoreName, + key: &str, + ) -> Result, Report> { + self.0 + .get(key) + .cloned() + .ok_or_else(|| Report::new(PlatformError::SecretStore)) + } + + fn create( + &self, + _store_id: &StoreId, + _name: &str, + _value: &str, + ) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } + + fn delete(&self, _store_id: &StoreId, _name: &str) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } + } + + #[derive(Default)] + struct RecordingBackend { + specs: Mutex>, + } + + impl PlatformBackend for RecordingBackend { + fn naming_policy(&self) -> BackendNamingPolicy { + BackendNamingPolicy::Fastly + } + + fn predict_name( + &self, + _spec: &PlatformBackendSpec, + ) -> Result> { + Ok("tinybird-backend".to_owned()) + } + + fn ensure(&self, spec: &PlatformBackendSpec) -> Result> { + self.specs + .lock() + .expect("should lock backend specs") + .push(spec.clone()); + Ok("tinybird-backend".to_owned()) + } + } + + #[derive(Debug)] + struct RecordedRequest { + backend_name: String, + method: String, + uri: String, + headers: Vec<(String, String)>, + body: Vec, + } + + #[derive(Default)] + struct RecordingHttpClient { + requests: Mutex>, + select_calls: Mutex, + } + + #[async_trait::async_trait(?Send)] + impl PlatformHttpClient for RecordingHttpClient { + async fn send( + &self, + _request: PlatformHttpRequest, + ) -> Result> { + Err(Report::new(PlatformError::Unsupported)) + } + + async fn send_async( + &self, + request: PlatformHttpRequest, + ) -> Result> { + let backend_name = request.backend_name; + let (parts, body) = request.request.into_parts(); + let headers = parts + .headers + .iter() + .filter_map(|(name, value)| { + value + .to_str() + .ok() + .map(|value| (name.as_str().to_owned(), value.to_owned())) + }) + .collect(); + let recorded = RecordedRequest { + backend_name, + method: parts.method.to_string(), + uri: parts.uri.to_string(), + headers, + body: body.into_bytes().unwrap_or_default().to_vec(), + }; + self.requests + .lock() + .expect("should lock recorded requests") + .push(recorded); + Ok(PlatformPendingRequest::new(()).with_backend_name("tinybird-backend")) + } + + async fn select( + &self, + _pending_requests: Vec, + ) -> Result> { + *self.select_calls.lock().expect("should lock select calls") += 1; + Err(Report::new(PlatformError::Unsupported)) + } + } + + struct NoopGeo; + + #[async_trait::async_trait(?Send)] + impl PlatformGeo for NoopGeo { + async fn lookup( + &self, + _client_ip: Option, + _services: &RuntimeServices, + ) -> Result, Report> { + Ok(None) + } + } + + fn test_row() -> AuctionEventRow { + AuctionEventRow { + event_ts: "2026-06-23 00:00:00.000".to_owned(), + event_kind: "summary".to_owned(), + auction_id: "550e8400-e29b-41d4-a716-446655440000".to_owned(), + auction_source: "auction_api".to_owned(), + publisher_domain: "test-publisher.example".to_owned(), + page_path: "/".to_owned(), + country: "US".to_owned(), + region: None, + is_mobile: 0, + is_known_browser: 1, + user_agent: Some(TEST_USER_AGENT.to_owned()), + gdpr_applies: 0, + consent_present: 0, + terminal_status: Some("completed".to_owned()), + terminal_reason: None, + slot_count: Some(1), + total_time_ms: Some(1), + winning_bid_count: Some(0), + provider: None, + provider_role: None, + status: None, + provider_response_time_ms: None, + provider_bid_count: None, + slot_id: None, + slot_w: None, + slot_h: None, + media_type: None, + seat: None, + price_cpm: None, + currency: None, + is_win: None, + ad_domain: None, + ad_id: None, + origin_cache_shareable: None, + } + } + + fn services( + backend: Arc, + http_client: Arc, + secrets: HashMap>, + ) -> RuntimeServices { + RuntimeServices::builder() + .config_store(Arc::new(NoopConfigStore)) + .secret_store(Arc::new(MapSecretStore(secrets))) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(backend) + .http_client(http_client) + .geo(Arc::new( + trusted_server_core::platform::PlatformGeoModule::new("noop", Arc::new(NoopGeo)), + )) + .client_info(ClientInfo::default()) + .permission_policy(trusted_server_core::permissions::PermissionPolicy::sample()) + .build() + } + + fn tinybird_settings() -> TinybirdSettings { + TinybirdSettings { + api_host: "api.us-east.aws.tinybird.co".to_owned(), + secret_store: None, + auction_dataset: "auction_events_raw".to_owned(), + auction_token_secret: Some(Redacted::new("append-token".to_owned())), + access_enabled: false, + access_dataset: "access_logs_raw".to_owned(), + access_token_secret: None, + access_sample_rate: 0.0, + max_body_bytes: 1024 * 1024, + } + } + + fn sink() -> TinybirdAuctionTelemetrySink { + TinybirdAuctionTelemetrySink::new(tinybird_settings()).expect("the table carries a token") + } + + #[test] + fn events_uri_targets_dataset_on_region_host() { + assert_eq!( + events_uri("api.us-east.aws.tinybird.co", "auction_events_raw"), + "https://api.us-east.aws.tinybird.co/v0/events?name=auction_events_raw" + ); + } + + #[test] + fn events_uri_urlencodes_dataset_name() { + assert_eq!( + events_uri("api.us-east.aws.tinybird.co", "auction events/raw"), + "https://api.us-east.aws.tinybird.co/v0/events?name=auction%20events%2Fraw" + ); + } + + #[test] + fn backend_spec_uses_matching_tls_host() { + let spec = backend_spec("api.us-east.aws.tinybird.co"); + assert_eq!(spec.scheme, "https"); + assert_eq!(spec.host, "api.us-east.aws.tinybird.co"); + assert_eq!(spec.host_header_override, None); + assert!(spec.certificate_check, "should verify Tinybird TLS cert"); + } + + #[test] + fn sink_posts_ndjson_with_resolved_token_and_does_not_wait() { + let backend = Arc::new(RecordingBackend::default()); + let http_client = Arc::new(RecordingHttpClient::default()); + let services = services( + Arc::clone(&backend), + Arc::clone(&http_client), + HashMap::new(), + ); + + futures::executor::block_on( + sink().emit_auction_events(&services, AuctionEventBatch::new(vec![test_row()])), + ) + .expect("should start Tinybird request"); + + let specs = backend.specs.lock().expect("should lock specs"); + assert_eq!(specs.len(), 1, "should ensure one backend"); + assert_eq!(specs[0].host, "api.us-east.aws.tinybird.co"); + drop(specs); + + let requests = http_client + .requests + .lock() + .expect("should lock recorded requests"); + assert_eq!(requests.len(), 1, "should start one async POST"); + assert_eq!(requests[0].backend_name, "tinybird-backend"); + assert_eq!(requests[0].method, Method::POST.to_string()); + assert_eq!( + requests[0].uri, + "https://api.us-east.aws.tinybird.co/v0/events?name=auction_events_raw" + ); + assert_eq!( + header_value(&requests[0].headers, header::CONTENT_TYPE.as_str()), + Some(NDJSON_CONTENT_TYPE) + ); + assert_eq!( + header_value(&requests[0].headers, header::AUTHORIZATION.as_str()), + Some("Bearer append-token") + ); + let body = std::str::from_utf8(&requests[0].body).expect("should record utf8 ndjson body"); + assert!(body.ends_with('\n'), "should send newline-delimited JSON"); + assert!( + body.contains(TEST_USER_AGENT), + "should send the complete user agent to Tinybird" + ); + assert_eq!( + *http_client + .select_calls + .lock() + .expect("should lock select calls"), + 0, + "should not wait for the Tinybird response" + ); + } + + #[test] + fn empty_batch_does_not_dispatch() { + let backend = Arc::new(RecordingBackend::default()); + let http_client = Arc::new(RecordingHttpClient::default()); + let services = services( + Arc::clone(&backend), + Arc::clone(&http_client), + HashMap::new(), + ); + + futures::executor::block_on( + sink().emit_auction_events(&services, AuctionEventBatch::new(Vec::new())), + ) + .expect("should ignore empty batch"); + + assert!( + backend.specs.lock().expect("should lock specs").is_empty(), + "should not ensure backend for empty batches" + ); + assert!( + http_client + .requests + .lock() + .expect("should lock recorded requests") + .is_empty(), + "should not send empty batches" + ); + } + + #[test] + fn sink_drops_row_count_oversize_before_sending() { + let backend = Arc::new(RecordingBackend::default()); + let http_client = Arc::new(RecordingHttpClient::default()); + let services = services( + backend, + Arc::clone(&http_client), + HashMap::from([( + "tinybird_auction_append_token".to_owned(), + b"append-token".to_vec(), + )]), + ); + let rows = vec![test_row(); MAX_ROWS_PER_AUCTION_BATCH + 1]; + + let result = futures::executor::block_on( + sink().emit_auction_events(&services, AuctionEventBatch::new(rows)), + ); + + assert!( + result.is_err(), + "best-effort caller will suppress this error" + ); + assert!( + http_client + .requests + .lock() + .expect("should lock recorded requests") + .is_empty(), + "should not send oversized row batches" + ); + } + + #[test] + fn a_table_without_a_token_gives_no_sink() { + let err = TinybirdAuctionTelemetrySink::new(TinybirdSettings::default()) + .expect_err("no token, no sink"); + + assert!(format!("{err:?}").contains("auction_token_secret is required")); + } + + #[test] + fn settings_without_an_analytics_section_register_nothing() { + let settings = Settings::from_toml(&crate_test_settings_str()) + .expect("should parse settings without an analytics section"); + + assert!( + register(&settings).expect("nothing to refuse").is_none(), + "should register nothing" + ); + assert!( + !validate(&settings).expect("nothing to refuse"), + "should report the module unselected" + ); + let value = serde_json::to_value(&settings).expect("should serialize"); + assert!( + value.get("analytics").is_none() && value.get("tinybird").is_none(), + "should carry no analytics key: {value}" + ); + } + + #[test] + fn the_module_is_selected_by_its_short_or_full_name() { + for name in ["tinybird", "analytics.tinybird"] { + let toml = format!( + "{}\n[analytics]\nmodule = \"{name}\"\n\n[analytics.tinybird]\n{TABLE}\n", + crate_test_settings_str() + ); + let settings = + Settings::from_toml(&toml).unwrap_or_else(|err| panic!("{name}: {err:?}")); + + let registration = register(&settings) + .unwrap_or_else(|err| panic!("{name}: {err:?}")) + .unwrap_or_else(|| panic!("{name}: should register the sink")); + + let (offered, sink) = registration + .analytics_module + .expect("the registration should carry the sink"); + assert_eq!(offered, MODULE); + assert!(sink.is_enabled(), "{name}: the sink should send"); + } + } + + #[test] + fn a_selected_module_survives_the_stored_form() { + let settings = settings_selecting(TABLE); + + let stored = serde_json::to_value(&settings).expect("should serialize"); + + assert_eq!(stored["analytics"]["module"], "tinybird"); + assert_eq!( + stored["analytics"]["tinybird"]["api_host"], + "api.us-east.aws.tinybird.co" + ); + assert!( + stored.get("tinybird").is_none(), + "should hold no table at the old name" + ); + } + + #[test] + fn the_module_needs_a_regional_host() { + for table in [ + "api_host = \"https://api.example.com/path\"\nauction_token_secret = \"token\"", + "auction_token_secret = \"token\"", + ] { + assert!( + refusal_of(table).contains("analytics.tinybird.api_host"), + "should name the host setting: {}", + refusal_of(table) + ); + } + } + + #[test] + fn the_module_needs_its_token() { + for table in [ + "api_host = \"api.us-east.aws.tinybird.co\"", + "api_host = \"api.us-east.aws.tinybird.co\"\nauction_token_secret = \"\"", + "api_host = \"api.us-east.aws.tinybird.co\"\nauction_token_secret = \" \"", + ] { + assert!( + refusal_of(table).contains( + "analytics.tinybird.auction_token_secret is required when [analytics] \ + selects the module" + ), + "should say the token is required: {}", + refusal_of(table) + ); + } + } + + #[test] + fn a_selected_module_with_no_table_is_told_what_it_needs() { + let toml = format!( + "{}\n[analytics]\nmodule = \"tinybird\"\n", + crate_test_settings_str() + ); + let settings = Settings::from_toml(&toml).expect("a selection without a table parses"); + + let err = validate(&settings).expect_err("an empty table lacks the host"); + + assert!( + format!("{err:?}").contains("analytics.tinybird.api_host"), + "should name the first setting the module needs: {err:?}" + ); + } + + #[test] + fn access_telemetry_cannot_be_enabled() { + assert!( + refusal_of(&format!("{TABLE}\naccess_enabled = true")) + .contains("analytics.tinybird.access_enabled"), + "should refuse the reserved setting" + ); + } + + #[test] + fn the_sample_rate_and_body_limit_are_bounded() { + assert!( + refusal_of(&format!("{TABLE}\naccess_sample_rate = 1.5")) + .contains("analytics.tinybird.access_sample_rate"), + ); + assert!( + refusal_of(&format!("{TABLE}\nmax_body_bytes = 512")) + .contains("analytics.tinybird.max_body_bytes"), + ); + } + + #[test] + fn the_table_refuses_a_key_it_does_not_know() { + let refusal = refusal_of(&format!("{TABLE}\nenabled = true")); + + assert!( + refusal.contains("[analytics.tinybird]") && refusal.contains("enabled"), + "should name the table and the key: {refusal}" + ); + } + + #[test] + fn the_deprecated_fields_are_accepted_and_dropped() { + let settings = settings_selecting(&format!( + "{TABLE}\nsecret_store = \"legacy-store\"\naccess_token_secret = \"legacy-token\"" + )); + + let config = settings + .module_config::(MODULE) + .expect("should read the table") + .expect("should select the module") + .checked() + .expect("the deprecated fields are accepted"); + + assert!(config.secret_store.is_none() && config.access_token_secret.is_none()); + let stored = serde_json::to_string(&config).expect("should serialize"); + assert!( + !stored.contains("legacy-store") && !stored.contains("legacy-token"), + "should serialize neither deprecated field: {stored}" + ); + } + + #[test] + fn a_table_the_section_does_not_select_is_refused_when_the_settings_load() { + let toml = format!( + "{}\n[analytics.tinybird]\n{TABLE}\n", + crate_test_settings_str() + ); + + let err = Settings::from_toml(&toml).expect_err("a table nothing selects is refused"); + + assert!( + format!("{err:?}").contains("[analytics] selects no module"), + "should name the section and why: {err:?}" + ); + } + + #[test] + fn the_old_top_level_table_is_refused() { + for table in [ + "enabled = true\napi_host = \"api.us-east.aws.tinybird.co\"", + "enabled = false", + ] { + let toml = format!("{}\n[tinybird]\n{table}\n", crate_test_settings_str()); + + let err = Settings::from_toml(&toml).expect_err("the table is not read"); + + assert!( + format!("{err:?}").contains("[tinybird]"), + "should name the table: {err:?}" + ); + } + } + + #[test] + fn a_name_the_build_does_not_have_is_refused_where_the_registry_is_built() { + let toml = format!( + "{}\n[analytics]\nmodule = \"another\"\n", + crate_test_settings_str() + ); + let settings = Settings::from_toml(&toml).expect("the selection parses"); + + let err = IntegrationRegistry::with_registrations(&settings, &[builder()]) + .err() + .expect("a name nothing supplies is refused"); + + let message = format!("{err:?}"); + assert!( + message.contains("[analytics] selects `another`, which no module this deployment") + && message.contains("The analytics modules it supplies are [tinybird]"), + "should list what the section could select: {message}" + ); + } + + #[test] + fn the_registry_resolves_the_selected_sink() { + let registry = + IntegrationRegistry::with_registrations(&settings_selecting(TABLE), &[builder()]) + .expect("should build the registry"); + + let sink = registry + .analytics_module() + .expect("should resolve the selected sink"); + + assert!(sink.is_enabled(), "the module's sink sends"); + } + + #[test] + fn a_token_reference_that_is_empty_or_absent_is_refused_at_deploy() { + for (case, table) in [ + ( + "an empty reference", + "api_host = \"api.tinybird.co\"\nauction_token_secret = \"\"", + ), + ( + "a whitespace reference", + "api_host = \"api.tinybird.co\"\nauction_token_secret = \" \t \"", + ), + ("no reference", "api_host = \"api.tinybird.co\""), + ] { + let settings = settings_selecting(table); + + let err = validate_settings_for_deploy_with(&settings, &[builder()]).expect_err(case); + + assert!( + format!("{err:?}").contains("analytics.tinybird.auction_token_secret"), + "{case}: the error should identify the token reference: {err:?}" + ); + } + } + + #[test] + fn a_token_key_name_is_accepted_at_deploy() { + let settings = settings_selecting( + "api_host = \"api.tinybird.co\"\nauction_token_secret = \"tinybird_auction_token\"", + ); + + validate_settings_for_deploy_with(&settings, &[builder()]) + .expect("should validate the token's key name without treating it as the value"); + } + + /// A document selecting the module by its full name, with `table` as its + /// settings at `[analytics.tinybird]`. + fn settings_selecting_the_full_name(table: &str) -> Settings { + Settings::from_toml(&format!( + "{}\n[analytics]\nmodule = \"{MODULE}\"\n\n[analytics.tinybird]\n{table}\n", + crate_test_settings_str() + )) + .expect("the document should parse") + } + + #[test] + fn the_full_name_s_token_is_checked_at_deploy_in_the_table_the_module_reads() { + validate_settings_for_deploy_with(&settings_selecting_the_full_name(TABLE), &[builder()]) + .expect("should find the token's key name in [analytics.tinybird]"); + + let err = validate_settings_for_deploy_with( + &settings_selecting_the_full_name("api_host = \"api.tinybird.co\""), + &[builder()], + ) + .expect_err("a missing token reference is refused"); + + assert!( + format!("{err:?}").contains("`analytics.tinybird.auction_token_secret`"), + "should name the token in the table the module reads: {err:?}" + ); + } + + /// Answers the keys the test document names, and the token's key with + /// the token. + struct StoredSecrets; + + impl PlatformSecretStore for StoredSecrets { + fn get_bytes( + &self, + _store_name: &StoreName, + key: &str, + ) -> Result, Report> { + let value = match key { + "unit-test-proxy-secret" => "unit-test-proxy-secret-32-bytes-ok", + "test-auction-token" => "resolved-auction-token", + _ => key, + }; + Ok(value.as_bytes().to_vec()) + } + + fn create( + &self, + _store_id: &StoreId, + _name: &str, + _value: &str, + ) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } + + fn delete(&self, _store_id: &StoreId, _name: &str) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } + } + + #[test] + fn the_full_name_loads_from_the_stored_form_with_its_token() { + let mut settings = settings_selecting_the_full_name(TABLE); + // The blob a push writes names the policy and carries its file. + settings.permissions = trusted_server_core::test_support::tests::sample_permissions(); + let stored = serde_json::to_value(settings).expect("should serialize the settings"); + let envelope = BlobEnvelope::new(stored, "2026-01-01T00:00:00Z".to_owned()); + + let loaded = settings_from_config_blob_with( + &serde_json::to_string(&envelope).expect("should serialize the envelope"), + &StoredSecrets, + &StoreName::from("trusted_server_secrets"), + trusted_server_core::platform::PolicyOrigin::Blob, + &[builder()], + ) + .expect("should load the stored configuration"); + + let config = loaded + .module_config::(MODULE) + .expect("should read the table") + .expect("should select the module"); + assert_eq!( + config + .auction_token_secret + .as_ref() + .map(|token| token.expose().as_str()), + Some("resolved-auction-token"), + "should keep the token's key name in the table the module reads and look it up" + ); + } + + #[test] + fn the_documented_block_validates_when_uncommented() { + let toml = uncomment_block(&template_with_resolved_required_secrets(), "[analytics]"); + let settings = Settings::from_toml(&toml) + .expect("uncommented [analytics] with documented api_host should parse"); + + assert!( + validate(&settings).expect("the documented block should validate"), + "the documented block should select the module" + ); + } + + fn header_value<'a>(headers: &'a [(String, String)], name: &str) -> Option<&'a str> { + headers + .iter() + .find(|(header_name, _)| header_name.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.as_str()) + } +} diff --git a/crates/trusted-server-adapter-axum/Cargo.toml b/crates/trusted-server-adapter-axum/Cargo.toml index 5d65e36095..f1349d438d 100644 --- a/crates/trusted-server-adapter-axum/Cargo.toml +++ b/crates/trusted-server-adapter-axum/Cargo.toml @@ -32,6 +32,7 @@ trusted-server-ad-server-iabtl = { workspace = true } trusted-server-ad-tag-google = { workspace = true } trusted-server-ads-txt-iabtl = { workspace = true } trusted-server-middleware-51degrees = { workspace = true } +trusted-server-analytics-tinybird = { workspace = true } trusted-server-auction-aps = { workspace = true } trusted-server-auction-prebid = { workspace = true } trusted-server-auction-prebid-server = { workspace = true } diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index d7af2dc56c..cea59103b6 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -138,6 +138,8 @@ pub fn shipped_module_builders() -> Vec { trusted_server_headers_geo::builder(), // Run by `[shared.51degrees]` rather than selected by a section. trusted_server_middleware_51degrees::builder(), + // The analytics module `[analytics]` selects, which changes no page. + trusted_server_analytics_tinybird::builder(), // Implementations `[demand]` and `[ad-server]` can name, which no // section selects. trusted_server_auction_protocol_openrtb::builder(), @@ -192,6 +194,9 @@ fn build_state_with_settings( build_state_with_registrations(settings, &[]) } +/// The header core writes a shared cache policy in on this adapter. +const EDGE_CACHE_HEADER: EdgeCacheHeader = EdgeCacheHeader::SMaxageFallback; + /// Build the application state from explicit settings, composing the modules /// this adapter links with the externally supplied builders in /// `integrations`. @@ -476,7 +481,7 @@ async fn dispatch_fallback( let method = req.method().clone(); if method == Method::GET && path.starts_with("/static/tsjs=") { - return handle_tsjs_dynamic(&req, &state.registry, EdgeCacheHeader::SMaxageFallback); + return handle_tsjs_dynamic(&req, &state.registry, EDGE_CACHE_HEADER); } if state.registry.has_route(&method, &path) { @@ -520,7 +525,7 @@ async fn dispatch_fallback( &mut ec_context, auction, req, - EdgeCacheHeader::SMaxageFallback, + EDGE_CACHE_HEADER, ) .await?; // Async finalize so the dispatched auction is collected and its bids are @@ -1129,6 +1134,65 @@ mod tests { ); } + /// A response core makes private carries the private policy, with no + /// `s-maxage`, and keeps a platform's header the origin sent, which is a + /// fetch middleware's to remove. The adapter declares no platform's header. + #[test] + fn a_private_response_follows_its_policy_and_keeps_the_origins_header() { + let settings = Settings::from_toml( + r#" + [source] + module = "origin" + + [source.origin] + url = "https://origin.publisher.example" + + [publisher] + domain = "www.publisher.example" + cookie_domain = ".publisher.example" + proxy_secret = "fictional-proxy-secret-32-bytes-ok" + + [geo] + assume_single_jurisdiction = true + "#, + ) + .expect("should parse the settings") + // A loaded domain carries its policy. + .with_permission_policy(trusted_server_core::permissions::PermissionPolicy::sample()); + build_state_with_settings(settings).expect("should build the state"); + assert!( + trusted_server_core::platform::platform_cache_headers().is_empty(), + "should declare no platform's header" + ); + let mut response = edgezero_core::http::response_builder() + .header("example-platform-cache-control", "max-age=600") + .body(edgezero_core::body::Body::empty()) + .expect("should build the response"); + trusted_server_core::cache_policy::CachePolicy::public_immutable( + std::time::Duration::from_secs(600), + ) + .apply_to_response(&mut response, EDGE_CACHE_HEADER); + + trusted_server_core::response_privacy::enforce_private_no_store(&mut response); + + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("no-store, private"), + "should carry the private policy" + ); + assert_eq!( + response + .headers() + .get("example-platform-cache-control") + .and_then(|value| value.to_str().ok()), + Some("max-age=600"), + "should keep the header the origin sent" + ); + } + /// The modules this adapter links are offered in the order their hooks /// run. A module is named by what its section selects, and an /// implementation by what `[demand]` or `[ad-server]` names. @@ -1165,6 +1229,7 @@ mod tests { "ads-txt.iabtl", "headers.geo", "51degrees", + "analytics.tinybird", "auction-protocol.openrtb", "auction.prebid-server", "auction.aps", diff --git a/crates/trusted-server-adapter-cloudflare/Cargo.toml b/crates/trusted-server-adapter-cloudflare/Cargo.toml index bbbda065d1..7be645761e 100644 --- a/crates/trusted-server-adapter-cloudflare/Cargo.toml +++ b/crates/trusted-server-adapter-cloudflare/Cargo.toml @@ -33,6 +33,7 @@ trusted-server-ad-server-iabtl = { workspace = true } trusted-server-ad-tag-google = { workspace = true } trusted-server-ads-txt-iabtl = { workspace = true } trusted-server-middleware-51degrees = { workspace = true } +trusted-server-analytics-tinybird = { workspace = true } trusted-server-auction-aps = { workspace = true } trusted-server-auction-prebid = { workspace = true } trusted-server-auction-prebid-server = { workspace = true } diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index 7ee42969ff..9f27551663 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -220,11 +220,20 @@ fn build_state_with_services( build_state_with_registrations_and_services(settings, &[], services) } +/// The header Workers reads a shared cache policy from, which core writes +/// the policy in. Declaring it keeps an origin response with a policy in it +/// out of the stored copies. +pub const CDN_CACHE_CONTROL_HEADER: &str = "cloudflare-cdn-cache-control"; + +/// The header core writes a shared cache policy in on this adapter. +const EDGE_CACHE_HEADER: EdgeCacheHeader = EdgeCacheHeader::Platform(CDN_CACHE_CONTROL_HEADER); + fn build_state_with_registrations_and_services( settings: Settings, integrations: &[IntegrationBuilder], services: Option, ) -> Result, Report> { + trusted_server_core::platform::declare_platform_cache_headers(&[CDN_CACHE_CONTROL_HEADER]); // The modules this adapter links come first and the deployment's own // follow, which is the order their hooks run in. let integrations = module_builders_with(integrations); @@ -343,6 +352,8 @@ pub fn shipped_module_builders() -> Vec { trusted_server_headers_geo::builder(), // Run by `[shared.51degrees]` rather than selected by a section. trusted_server_middleware_51degrees::builder(), + // The analytics module `[analytics]` selects, which changes no page. + trusted_server_analytics_tinybird::builder(), // Implementations `[demand]` and `[ad-server]` can name, which no // section selects. trusted_server_auction_protocol_openrtb::builder(), @@ -745,11 +756,7 @@ fn build_router(state: &Arc) -> RouterService { let allow_tsjs = method == Method::GET; let result = if allow_tsjs && path.starts_with("/static/tsjs=") { - handle_tsjs_dynamic( - &req, - &state.registry, - EdgeCacheHeader::CloudflareCdnCacheControl, - ) + handle_tsjs_dynamic(&req, &state.registry, EDGE_CACHE_HEADER) } else if state.registry.has_route(&method, &path) { // A module route is handed what was resolved for its request, // the permissions among them, as the publisher path is. @@ -799,7 +806,7 @@ fn build_router(state: &Arc) -> RouterService { &mut ec_context, auction, req, - EdgeCacheHeader::CloudflareCdnCacheControl, + EDGE_CACHE_HEADER, ) .await { @@ -1335,6 +1342,74 @@ mod tests { ); } + /// A response core makes private loses the policy core wrote in the + /// Workers header and carries the private policy, and keeps that header + /// where the origin sent it, which is a fetch middleware's to remove. The + /// adapter declares its own platform's header and no other. + #[test] + fn a_private_response_follows_its_policy_and_keeps_the_origins_header() { + let settings = Settings::from_toml( + r#" + [source] + module = "origin" + + [source.origin] + url = "https://origin.publisher.example" + + [publisher] + domain = "www.publisher.example" + cookie_domain = ".publisher.example" + proxy_secret = "fictional-proxy-secret-32-bytes-ok" + + [geo] + assume_single_jurisdiction = true + "#, + ) + .expect("should parse the settings") + // A loaded domain carries its policy. + .with_permission_policy(trusted_server_core::permissions::PermissionPolicy::sample()); + build_state_with_settings(settings).expect("should build the state"); + assert_eq!( + trusted_server_core::platform::platform_cache_headers(), + vec![CDN_CACHE_CONTROL_HEADER], + "should declare the Workers header and no other platform's" + ); + let mut written = edgezero_core::http::response_builder() + .body(edgezero_core::body::Body::empty()) + .expect("should build the response"); + trusted_server_core::cache_policy::CachePolicy::public_immutable( + std::time::Duration::from_secs(600), + ) + .apply_to_response(&mut written, EDGE_CACHE_HEADER); + assert!(written.headers().contains_key(CDN_CACHE_CONTROL_HEADER)); + trusted_server_core::response_privacy::enforce_private_no_store(&mut written); + assert_eq!( + written + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("no-store, private"), + "should carry the private policy" + ); + assert!( + !written.headers().contains_key(CDN_CACHE_CONTROL_HEADER), + "should remove the policy core wrote" + ); + + let mut sent = edgezero_core::http::response_builder() + .header(CDN_CACHE_CONTROL_HEADER, "max-age=600") + .body(edgezero_core::body::Body::empty()) + .expect("should build the response"); + trusted_server_core::response_privacy::enforce_private_no_store(&mut sent); + assert_eq!( + sent.headers() + .get(CDN_CACHE_CONTROL_HEADER) + .and_then(|value| value.to_str().ok()), + Some("max-age=600"), + "should keep the header the origin sent" + ); + } + /// The modules this adapter links are offered in the order their hooks /// run. A module is named by what its section selects, and an /// implementation by what `[demand]` or `[ad-server]` names. @@ -1371,6 +1446,7 @@ mod tests { "ads-txt.iabtl", "headers.geo", "51degrees", + "analytics.tinybird", "auction-protocol.openrtb", "auction.prebid-server", "auction.aps", diff --git a/crates/trusted-server-adapter-fastly/Cargo.toml b/crates/trusted-server-adapter-fastly/Cargo.toml index 5c869e96ec..a2d33fdc2d 100644 --- a/crates/trusted-server-adapter-fastly/Cargo.toml +++ b/crates/trusted-server-adapter-fastly/Cargo.toml @@ -37,6 +37,7 @@ trusted-server-ad-server-iabtl = { workspace = true } trusted-server-ad-tag-google = { workspace = true } trusted-server-ads-txt-iabtl = { workspace = true } trusted-server-middleware-51degrees = { workspace = true } +trusted-server-analytics-tinybird = { workspace = true } trusted-server-auction-aps = { workspace = true } trusted-server-auction-prebid = { workspace = true } trusted-server-auction-prebid-server = { workspace = true } @@ -61,7 +62,6 @@ trusted-server-tag-google-tag-manager = { workspace = true } trusted-server-testing-testlight = { workspace = true } trusted-server-device-fastly = { workspace = true } trusted-server-geo-fastly = { workspace = true } -urlencoding = { workspace = true } [dev-dependencies] bytes = { workspace = true } diff --git a/crates/trusted-server-adapter-fastly/README.md b/crates/trusted-server-adapter-fastly/README.md index 24558148ae..d1915d8666 100644 --- a/crates/trusted-server-adapter-fastly/README.md +++ b/crates/trusted-server-adapter-fastly/README.md @@ -5,7 +5,7 @@ Production Fastly Compute adapter for Trusted Server, targeting This crate owns the Fastly entry point, dynamic backend construction, edge cache behavior, config and secret-store access, EC KV operations, rate limits, -streaming template assembly, and Fastly-only Tinybird auction emission. Its +and streaming template assembly. Its `/health` response is served before application construction, so liveness does not prove that configuration loaded. Platform-neutral routing, settings, auctions, integrations, and rewrites remain in `trusted-server-core`. diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 711fcf8569..a25ac4965a 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -301,6 +301,14 @@ fn registered_integrations() -> &'static [IntegrationBuilder] { REGISTERED_INTEGRATIONS.get().map_or(&[], Vec::as_slice) } +/// The second header Compute reads a shared cache policy from, beside +/// `Surrogate-Control`. Core never writes it, and declaring it keeps an origin +/// response with a policy in it out of the stored copies. +pub const FASTLY_SURROGATE_CONTROL_HEADER: &str = "fastly-surrogate-control"; + +/// The header core writes a shared cache policy in on this adapter. +pub(crate) const EDGE_CACHE_HEADER: EdgeCacheHeader = EdgeCacheHeader::SurrogateControl; + /// Build the application state from explicit settings, composing the modules /// this adapter links with the externally supplied builders in /// `integrations`. @@ -321,6 +329,9 @@ pub(crate) fn build_state_with_registrations( settings: Settings, integrations: &[IntegrationBuilder], ) -> Result, Report> { + trusted_server_core::platform::declare_platform_cache_headers(&[ + FASTLY_SURROGATE_CONTROL_HEADER, + ]); warn_if_certificate_check_disabled(&settings); // The modules this adapter links come first and the deployment's own @@ -364,7 +375,11 @@ pub(crate) fn build_state_with_registrations( )?; let permission_policy = settings.required_permission_policy()?; - let auction_telemetry_sink = crate::tinybird::auction_sink_from_settings(&settings); + // The sink `[analytics] module` selected, or core's no-op sink. + let auction_telemetry_sink = registry.analytics_module().unwrap_or_else(|| { + Arc::new(trusted_server_core::auction::NoopAuctionTelemetrySink) + as Arc + }); let default_kv_store = Arc::new(UnavailableKvStore) as Arc; let loading_caches = trusted_server_core::integrations::loading_caches_from(&settings, integrations)?; @@ -451,6 +466,8 @@ pub fn shipped_module_builders() -> Vec { trusted_server_headers_geo::builder(), // Run by `[shared.51degrees]` rather than selected by a section. trusted_server_middleware_51degrees::builder(), + // The analytics module `[analytics]` selects, which changes no page. + trusted_server_analytics_tinybird::builder(), // Implementations `[demand]` and `[ad-server]` can name, which no // section selects. trusted_server_auction_protocol_openrtb::builder(), @@ -1160,7 +1177,7 @@ async fn dispatch_fallback( }; let result = if uses_dynamic_tsjs_fallback(&method, &path) { - handle_tsjs_dynamic(&req, &state.registry, EdgeCacheHeader::SurrogateControl) + handle_tsjs_dynamic(&req, &state.registry, EDGE_CACHE_HEADER) } else if state.registry.has_route(&method, &path) { // Integration-proxy responses are not bounded by // publisher.max_buffered_body_bytes. Publisher fallback below uses the @@ -1236,7 +1253,7 @@ async fn dispatch_fallback( &mut ec.ec_context, auction, req, - EdgeCacheHeader::SurrogateControl, + EDGE_CACHE_HEADER, ) .await { @@ -1910,6 +1927,51 @@ mod tests { assert_eq!(services.public_prefix().as_str(), "ts"); } + /// A response core makes private carries the private policy in the + /// headers core owns and keeps the platform header the origin sent, which + /// is a fetch middleware's to remove. The adapter declares its own + /// platform's header and no other. + #[test] + fn a_private_response_follows_its_policy_and_keeps_the_origins_header() { + build_state_from_settings(test_settings()).expect("should build test state"); + assert_eq!( + trusted_server_core::platform::platform_cache_headers(), + vec![super::FASTLY_SURROGATE_CONTROL_HEADER], + "should declare Compute's own header and no other platform's" + ); + let mut response = edgezero_core::http::response_builder() + .header(super::FASTLY_SURROGATE_CONTROL_HEADER, "max-age=600") + .body(Body::empty()) + .expect("should build the response"); + trusted_server_core::cache_policy::CachePolicy::public_immutable( + std::time::Duration::from_secs(600), + ) + .apply_to_response(&mut response, super::EDGE_CACHE_HEADER); + + trusted_server_core::response_privacy::enforce_private_no_store(&mut response); + + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("no-store, private"), + "should carry the private policy" + ); + assert!( + !response.headers().contains_key("surrogate-control"), + "should remove the policy core wrote" + ); + assert_eq!( + response + .headers() + .get(super::FASTLY_SURROGATE_CONTROL_HEADER) + .and_then(|value| value.to_str().ok()), + Some("max-age=600"), + "should keep the header the origin sent" + ); + } + #[test] fn only_the_blob_at_the_logical_store_id_carries_its_policy_file() { assert!(blob_carries_its_policy(DEFAULT_CONFIG_STORE_ID)); @@ -3533,6 +3595,7 @@ mod tests { "ads-txt.iabtl", "headers.geo", "51degrees", + "analytics.tinybird", "auction-protocol.openrtb", "auction.prebid-server", "auction.aps", diff --git a/crates/trusted-server-adapter-fastly/src/lib.rs b/crates/trusted-server-adapter-fastly/src/lib.rs index bff902f173..96e7a003dd 100644 --- a/crates/trusted-server-adapter-fastly/src/lib.rs +++ b/crates/trusted-server-adapter-fastly/src/lib.rs @@ -22,7 +22,7 @@ use fastly::http::Method as FastlyMethod; use fastly::{Request as FastlyRequest, Response as FastlyResponse}; use trusted_server_core::addressing::{ByHostAddressing, RemovedPathPrefix}; -use trusted_server_core::cache_policy::{EdgeCacheHeader, cache_control_headers_have_directive}; +use trusted_server_core::cache_policy::cache_control_headers_have_directive; use trusted_server_core::ec::device::{DeviceModule, DeviceSignals, build_device_module}; use trusted_server_core::ec::finalize::ec_finalize_response; use trusted_server_core::ec::kv::KvIdentityGraph; @@ -53,7 +53,6 @@ mod platform; mod rate_limiter; mod sandbox; mod template_cache; -mod tinybird; mod value_cache; use crate::app::{ @@ -858,7 +857,7 @@ fn edgezero_main(mut req: FastlyRequest, sandbox: &mut Sandbox, ordinal: u64, re } if let Some(policy) = asset_cache_policy { - policy.apply_after_route_finalization(&mut response, EdgeCacheHeader::SurrogateControl); + policy.apply_after_route_finalization(&mut response, crate::app::EDGE_CACHE_HEADER); } if let Some(mut ec_state) = ec_state { diff --git a/crates/trusted-server-adapter-fastly/src/tinybird.rs b/crates/trusted-server-adapter-fastly/src/tinybird.rs deleted file mode 100644 index 5f3f51a613..0000000000 --- a/crates/trusted-server-adapter-fastly/src/tinybird.rs +++ /dev/null @@ -1,635 +0,0 @@ -//! Tinybird direct-ingest telemetry sink for the Fastly adapter. - -use std::sync::Arc; -use std::time::Duration; - -use edgezero_core::body::Body; -use edgezero_core::http::{HeaderValue, Method, header, request_builder}; -use error_stack::{Report, ResultExt as _}; -use trusted_server_core::auction::telemetry::{ - AuctionEventBatch, AuctionTelemetrySink, NoopAuctionTelemetrySink, -}; -use trusted_server_core::error::TrustedServerError; -use trusted_server_core::platform::{PlatformBackendSpec, PlatformHttpRequest, RuntimeServices}; -use trusted_server_core::redacted::Redacted; -use trusted_server_core::settings::{Settings, TinybirdSettings}; - -const TINYBIRD_EVENTS_PATH: &str = "/v0/events"; -const TINYBIRD_NDJSON_CONTENT_TYPE: &str = "application/x-ndjson"; -const TINYBIRD_FIRST_BYTE_TIMEOUT: Duration = Duration::from_secs(2); -const TINYBIRD_BETWEEN_BYTES_TIMEOUT: Duration = Duration::from_secs(2); -const TINYBIRD_MAX_ROWS_PER_AUCTION_BATCH: usize = 512; - -/// Build the auction telemetry sink, which sends only when `[analytics]` -/// selects the analytics module. -#[must_use] -pub(crate) fn auction_sink_from_settings(settings: &Settings) -> Arc { - match settings.analytics_module() { - Some(telemetry) => Arc::new(FastlyTinybirdAuctionTelemetrySink::new(telemetry.clone())), - None => Arc::new(NoopAuctionTelemetrySink), - } -} - -#[derive(Debug, Clone)] -struct FastlyTinybirdAuctionTelemetrySink { - target: TinybirdEventsTarget, -} - -#[derive(Debug, Clone)] -struct TinybirdEventsTarget { - api_host: String, - dataset: String, - append_token: Redacted, - uri: String, - backend_spec: PlatformBackendSpec, - max_body_bytes: usize, -} - -impl TinybirdEventsTarget { - fn from_config(config: TinybirdSettings) -> Self { - let uri = tinybird_events_uri(&config.api_host, &config.auction_dataset); - let backend_spec = tinybird_backend_spec(&config.api_host); - Self { - api_host: config.api_host, - dataset: config.auction_dataset, - append_token: config - .auction_token_secret - .expect("should contain a resolved Tinybird auction token when selected"), - uri, - backend_spec, - max_body_bytes: config.max_body_bytes, - } - } -} - -impl FastlyTinybirdAuctionTelemetrySink { - fn new(config: TinybirdSettings) -> Self { - Self { - target: TinybirdEventsTarget::from_config(config), - } - } - - fn validate_batch(batch: &AuctionEventBatch) -> Result<(), Report> { - if batch.row_count() > TINYBIRD_MAX_ROWS_PER_AUCTION_BATCH { - return Err(Report::new(TrustedServerError::Proxy { - message: format!( - "auction telemetry batch has {} rows, exceeding {} row limit", - batch.row_count(), - TINYBIRD_MAX_ROWS_PER_AUCTION_BATCH - ), - })); - } - Ok(()) - } - - fn serialize_batch( - &self, - batch: &AuctionEventBatch, - ) -> Result> { - batch.to_ndjson(self.target.max_body_bytes) - } - - fn ensure_backend( - &self, - services: &RuntimeServices, - ) -> Result> { - services - .backend() - .ensure(&self.target.backend_spec) - .change_context(TrustedServerError::Proxy { - message: "Tinybird backend registration failed".to_owned(), - }) - } - - fn authorization_header(token: &str) -> Result> { - HeaderValue::from_str(&format!("Bearer {token}")).change_context( - TrustedServerError::InvalidHeaderValue { - message: "invalid Tinybird authorization header".to_owned(), - }, - ) - } - - fn build_events_request( - &self, - body: String, - auth_header: HeaderValue, - ) -> Result> { - request_builder() - .method(Method::POST) - .uri(self.target.uri.as_str()) - .header(header::AUTHORIZATION, auth_header) - .header(header::CONTENT_TYPE, TINYBIRD_NDJSON_CONTENT_TYPE) - .body(Body::from(body)) - .change_context(TrustedServerError::Proxy { - message: "failed to build Tinybird Events API request".to_owned(), - }) - } - - async fn send_fire_and_forget( - services: &RuntimeServices, - request: edgezero_core::http::Request, - backend_name: String, - ) -> Result<(), Report> { - let pending = services - .http_client() - .send_async(PlatformHttpRequest::new(request, backend_name)) - .await - .change_context(TrustedServerError::Proxy { - message: "failed to start Tinybird Events API request".to_owned(), - })?; - drop(pending); - Ok(()) - } -} - -#[async_trait::async_trait(?Send)] -impl AuctionTelemetrySink for FastlyTinybirdAuctionTelemetrySink { - async fn emit_auction_events( - &self, - services: &RuntimeServices, - batch: AuctionEventBatch, - ) -> Result<(), Report> { - if batch.is_empty() { - return Ok(()); - } - - Self::validate_batch(&batch)?; - let body = self.serialize_batch(&batch)?; - let body_len = body.len(); - let auth_header = Self::authorization_header(self.target.append_token.expose())?; - let backend_name = self.ensure_backend(services)?; - let request = self.build_events_request(body, auth_header)?; - - log::info!( - "sending auction telemetry to Tinybird dataset={} rows={} bytes={} host={} backend={}", - self.target.dataset, - batch.row_count(), - body_len, - self.target.api_host, - backend_name - ); - - Self::send_fire_and_forget(services, request, backend_name).await - } -} - -fn tinybird_backend_spec(api_host: &str) -> PlatformBackendSpec { - PlatformBackendSpec { - scheme: "https".to_owned(), - host: api_host.to_owned(), - port: None, - host_header_override: None, - certificate_check: true, - first_byte_timeout: TINYBIRD_FIRST_BYTE_TIMEOUT, - between_bytes_timeout: TINYBIRD_BETWEEN_BYTES_TIMEOUT, - discriminator: None, - } -} - -fn tinybird_events_uri(api_host: &str, dataset: &str) -> String { - format!( - "https://{api_host}{TINYBIRD_EVENTS_PATH}?name={}", - urlencoding::encode(dataset) - ) -} - -#[cfg(test)] -mod tests { - use std::collections::HashMap; - use std::sync::{Arc, Mutex}; - - use error_stack::Report; - use trusted_server_core::auction::telemetry::{AuctionEventBatch, AuctionEventRow}; - use trusted_server_core::platform::{ - ClientInfo, PlatformBackend, PlatformConfigStore, PlatformError, PlatformGeo, - PlatformHttpClient, PlatformPendingRequest, PlatformResponse, PlatformSecretStore, - PlatformSelectResult, RuntimeServices, StoreId, StoreName, - }; - - use super::*; - - const TEST_USER_AGENT: &str = - "FictionalBrowser/123.4 (FictionalOS 10.2; FictionalDevice) ExampleRenderer/567.8"; - - struct NoopConfigStore; - - impl PlatformConfigStore for NoopConfigStore { - fn get( - &self, - _store_name: &StoreName, - _key: &str, - ) -> Result> { - Err(Report::new(PlatformError::Unsupported)) - } - - fn put( - &self, - _store_id: &StoreId, - _key: &str, - _value: &str, - ) -> Result<(), Report> { - Err(Report::new(PlatformError::Unsupported)) - } - - fn delete(&self, _store_id: &StoreId, _key: &str) -> Result<(), Report> { - Err(Report::new(PlatformError::Unsupported)) - } - } - - struct MapSecretStore(HashMap>); - - impl PlatformSecretStore for MapSecretStore { - fn get_bytes( - &self, - _store_name: &StoreName, - key: &str, - ) -> Result, Report> { - self.0 - .get(key) - .cloned() - .ok_or_else(|| Report::new(PlatformError::SecretStore)) - } - - fn create( - &self, - _store_id: &StoreId, - _name: &str, - _value: &str, - ) -> Result<(), Report> { - Err(Report::new(PlatformError::Unsupported)) - } - - fn delete(&self, _store_id: &StoreId, _name: &str) -> Result<(), Report> { - Err(Report::new(PlatformError::Unsupported)) - } - } - - #[derive(Default)] - struct RecordingBackend { - specs: Mutex>, - } - - impl PlatformBackend for RecordingBackend { - fn naming_policy(&self) -> trusted_server_core::platform::BackendNamingPolicy { - trusted_server_core::platform::BackendNamingPolicy::Fastly - } - - fn predict_name( - &self, - _spec: &PlatformBackendSpec, - ) -> Result> { - Ok("tinybird-backend".to_owned()) - } - - fn ensure(&self, spec: &PlatformBackendSpec) -> Result> { - self.specs - .lock() - .expect("should lock backend specs") - .push(spec.clone()); - Ok("tinybird-backend".to_owned()) - } - } - - #[derive(Debug)] - struct RecordedRequest { - backend_name: String, - method: String, - uri: String, - headers: Vec<(String, String)>, - body: Vec, - } - - #[derive(Default)] - struct RecordingHttpClient { - requests: Mutex>, - select_calls: Mutex, - } - - #[async_trait::async_trait(?Send)] - impl PlatformHttpClient for RecordingHttpClient { - async fn send( - &self, - _request: PlatformHttpRequest, - ) -> Result> { - Err(Report::new(PlatformError::Unsupported)) - } - - async fn send_async( - &self, - request: PlatformHttpRequest, - ) -> Result> { - let backend_name = request.backend_name; - let (parts, body) = request.request.into_parts(); - let headers = parts - .headers - .iter() - .filter_map(|(name, value)| { - value - .to_str() - .ok() - .map(|value| (name.as_str().to_owned(), value.to_owned())) - }) - .collect(); - let recorded = RecordedRequest { - backend_name, - method: parts.method.to_string(), - uri: parts.uri.to_string(), - headers, - body: body.into_bytes().unwrap_or_default().to_vec(), - }; - self.requests - .lock() - .expect("should lock recorded requests") - .push(recorded); - Ok(PlatformPendingRequest::new(()).with_backend_name("tinybird-backend")) - } - - async fn select( - &self, - _pending_requests: Vec, - ) -> Result> { - *self.select_calls.lock().expect("should lock select calls") += 1; - Err(Report::new(PlatformError::Unsupported)) - } - } - - struct NoopGeo; - - #[async_trait::async_trait(?Send)] - impl PlatformGeo for NoopGeo { - async fn lookup( - &self, - _client_ip: Option, - _services: &trusted_server_core::platform::RuntimeServices, - ) -> Result, Report> { - Ok(None) - } - } - - fn test_row() -> AuctionEventRow { - AuctionEventRow { - event_ts: "2026-06-23 00:00:00.000".to_owned(), - event_kind: "summary".to_owned(), - auction_id: "550e8400-e29b-41d4-a716-446655440000".to_owned(), - auction_source: "auction_api".to_owned(), - publisher_domain: "test-publisher.example".to_owned(), - page_path: "/".to_owned(), - country: "US".to_owned(), - region: None, - is_mobile: 0, - is_known_browser: 1, - user_agent: Some(TEST_USER_AGENT.to_owned()), - gdpr_applies: 0, - consent_present: 0, - terminal_status: Some("completed".to_owned()), - terminal_reason: None, - slot_count: Some(1), - total_time_ms: Some(1), - winning_bid_count: Some(0), - provider: None, - provider_role: None, - status: None, - provider_response_time_ms: None, - provider_bid_count: None, - slot_id: None, - slot_w: None, - slot_h: None, - media_type: None, - seat: None, - price_cpm: None, - currency: None, - is_win: None, - ad_domain: None, - ad_id: None, - origin_cache_shareable: None, - } - } - - fn services( - backend: Arc, - http_client: Arc, - secrets: HashMap>, - ) -> RuntimeServices { - RuntimeServices::builder() - .config_store(Arc::new(NoopConfigStore)) - .secret_store(Arc::new(MapSecretStore(secrets))) - .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) - .backend(backend) - .http_client(http_client) - .geo(Arc::new( - trusted_server_core::platform::PlatformGeoModule::new("noop", Arc::new(NoopGeo)), - )) - .client_info(ClientInfo::default()) - .permission_policy(trusted_server_core::permissions::PermissionPolicy::sample()) - .build() - } - - fn tinybird_settings() -> TinybirdSettings { - TinybirdSettings { - api_host: "api.us-east.aws.tinybird.co".to_owned(), - secret_store: None, - auction_dataset: "auction_events_raw".to_owned(), - auction_token_secret: Some(Redacted::new("append-token".to_owned())), - access_enabled: false, - access_dataset: "access_logs_raw".to_owned(), - access_token_secret: None, - access_sample_rate: 0.0, - max_body_bytes: 1024 * 1024, - } - } - - #[test] - fn events_uri_targets_dataset_on_region_host() { - assert_eq!( - tinybird_events_uri("api.us-east.aws.tinybird.co", "auction_events_raw"), - "https://api.us-east.aws.tinybird.co/v0/events?name=auction_events_raw" - ); - } - - #[test] - fn events_uri_urlencodes_dataset_name() { - assert_eq!( - tinybird_events_uri("api.us-east.aws.tinybird.co", "auction events/raw"), - "https://api.us-east.aws.tinybird.co/v0/events?name=auction%20events%2Fraw" - ); - } - - #[test] - fn backend_spec_uses_matching_tls_host() { - let spec = tinybird_backend_spec("api.us-east.aws.tinybird.co"); - assert_eq!(spec.scheme, "https"); - assert_eq!(spec.host, "api.us-east.aws.tinybird.co"); - assert_eq!(spec.host_header_override, None); - assert!(spec.certificate_check, "should verify Tinybird TLS cert"); - } - - #[test] - fn sink_posts_ndjson_with_resolved_token_and_does_not_wait() { - let backend = Arc::new(RecordingBackend::default()); - let http_client = Arc::new(RecordingHttpClient::default()); - let services = services( - Arc::clone(&backend), - Arc::clone(&http_client), - HashMap::new(), - ); - let sink = FastlyTinybirdAuctionTelemetrySink::new(tinybird_settings()); - - futures::executor::block_on( - sink.emit_auction_events(&services, AuctionEventBatch::new(vec![test_row()])), - ) - .expect("should start Tinybird request"); - - let specs = backend.specs.lock().expect("should lock specs"); - assert_eq!(specs.len(), 1, "should ensure one backend"); - assert_eq!(specs[0].host, "api.us-east.aws.tinybird.co"); - drop(specs); - - let requests = http_client - .requests - .lock() - .expect("should lock recorded requests"); - assert_eq!(requests.len(), 1, "should start one async POST"); - assert_eq!(requests[0].backend_name, "tinybird-backend"); - assert_eq!(requests[0].method, Method::POST.to_string()); - assert_eq!( - requests[0].uri, - "https://api.us-east.aws.tinybird.co/v0/events?name=auction_events_raw" - ); - assert_eq!( - header_value(&requests[0].headers, header::CONTENT_TYPE.as_str()), - Some(TINYBIRD_NDJSON_CONTENT_TYPE) - ); - assert_eq!( - header_value(&requests[0].headers, header::AUTHORIZATION.as_str()), - Some("Bearer append-token") - ); - let body = std::str::from_utf8(&requests[0].body).expect("should record utf8 ndjson body"); - assert!(body.ends_with('\n'), "should send newline-delimited JSON"); - assert!( - body.contains(TEST_USER_AGENT), - "should send the complete user agent to Tinybird" - ); - assert_eq!( - *http_client - .select_calls - .lock() - .expect("should lock select calls"), - 0, - "should not wait for the Tinybird response" - ); - } - - #[test] - fn settings_selecting_no_analytics_module_build_a_sink_that_sends_nothing() { - let backend = Arc::new(RecordingBackend::default()); - let http_client = Arc::new(RecordingHttpClient::default()); - let services = services( - Arc::clone(&backend), - Arc::clone(&http_client), - HashMap::new(), - ); - let sink = auction_sink_from_settings(&Settings::default()); - assert!(!sink.is_enabled(), "should build the no-op sink"); - - futures::executor::block_on( - sink.emit_auction_events(&services, AuctionEventBatch::new(vec![test_row()])), - ) - .expect("the no-op sink should accept the batch"); - - assert!( - backend.specs.lock().expect("should lock specs").is_empty(), - "should not ensure a backend when nothing is selected" - ); - assert!( - http_client - .requests - .lock() - .expect("should lock recorded requests") - .is_empty(), - "should not send when nothing is selected" - ); - } - - #[test] - fn settings_selecting_the_analytics_module_build_a_sink_that_sends() { - let mut settings = Settings::default(); - settings.analytics = Some(trusted_server_core::settings::AnalyticsConfig { - module: Some("tinybird".to_owned()), - tinybird: Some(tinybird_settings()), - }); - - let sink = auction_sink_from_settings(&settings); - - assert!(sink.is_enabled(), "should build the sink that sends"); - } - - #[test] - fn empty_batch_does_not_dispatch() { - let backend = Arc::new(RecordingBackend::default()); - let http_client = Arc::new(RecordingHttpClient::default()); - let services = services( - Arc::clone(&backend), - Arc::clone(&http_client), - HashMap::new(), - ); - let sink = FastlyTinybirdAuctionTelemetrySink::new(tinybird_settings()); - - futures::executor::block_on( - sink.emit_auction_events(&services, AuctionEventBatch::new(Vec::new())), - ) - .expect("should ignore empty batch"); - - assert!( - backend.specs.lock().expect("should lock specs").is_empty(), - "should not ensure backend for empty batches" - ); - assert!( - http_client - .requests - .lock() - .expect("should lock recorded requests") - .is_empty(), - "should not send empty batches" - ); - } - - #[test] - fn sink_drops_row_count_oversize_before_sending() { - let backend = Arc::new(RecordingBackend::default()); - let http_client = Arc::new(RecordingHttpClient::default()); - let services = services( - backend, - Arc::clone(&http_client), - HashMap::from([( - "tinybird_auction_append_token".to_owned(), - b"append-token".to_vec(), - )]), - ); - let sink = FastlyTinybirdAuctionTelemetrySink::new(tinybird_settings()); - let rows = vec![test_row(); TINYBIRD_MAX_ROWS_PER_AUCTION_BATCH + 1]; - - let result = futures::executor::block_on( - sink.emit_auction_events(&services, AuctionEventBatch::new(rows)), - ); - - assert!( - result.is_err(), - "best-effort caller will suppress this error" - ); - assert!( - http_client - .requests - .lock() - .expect("should lock recorded requests") - .is_empty(), - "should not send oversized row batches" - ); - } - - fn header_value<'a>(headers: &'a [(String, String)], name: &str) -> Option<&'a str> { - headers - .iter() - .find(|(header_name, _)| header_name.eq_ignore_ascii_case(name)) - .map(|(_, value)| value.as_str()) - } -} diff --git a/crates/trusted-server-adapter-spin/Cargo.toml b/crates/trusted-server-adapter-spin/Cargo.toml index b50dcbdc6b..71b3c8b27c 100644 --- a/crates/trusted-server-adapter-spin/Cargo.toml +++ b/crates/trusted-server-adapter-spin/Cargo.toml @@ -35,6 +35,7 @@ trusted-server-ad-server-iabtl = { workspace = true } trusted-server-ad-tag-google = { workspace = true } trusted-server-ads-txt-iabtl = { workspace = true } trusted-server-middleware-51degrees = { workspace = true } +trusted-server-analytics-tinybird = { workspace = true } trusted-server-auction-aps = { workspace = true } trusted-server-auction-prebid = { workspace = true } trusted-server-auction-prebid-server = { workspace = true } diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index 57148f8f92..798d2e8bc3 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -176,6 +176,9 @@ fn build_state_with_settings( build_state_with_registrations(settings, &[]) } +/// The header core writes a shared cache policy in on this adapter. +const EDGE_CACHE_HEADER: EdgeCacheHeader = EdgeCacheHeader::SMaxageFallback; + /// Build the application state from explicit settings, composing the modules /// this adapter links with the externally supplied builders in /// `integrations`. @@ -355,6 +358,8 @@ pub fn shipped_module_builders() -> Vec { trusted_server_headers_geo::builder(), // Run by `[shared.51degrees]` rather than selected by a section. trusted_server_middleware_51degrees::builder(), + // The analytics module `[analytics]` selects, which changes no page. + trusted_server_analytics_tinybird::builder(), // Implementations `[demand]` and `[ad-server]` can name, which no // section selects. trusted_server_auction_protocol_openrtb::builder(), @@ -1128,7 +1133,7 @@ fn build_router(state: &Arc) -> RouterService { // Dynamic tsjs serving is GET-only; other methods fall through to the // integration/publisher fallback. let result = if method == Method::GET && path.starts_with("/static/tsjs=") { - handle_tsjs_dynamic(&req, &state.registry, EdgeCacheHeader::SMaxageFallback) + handle_tsjs_dynamic(&req, &state.registry, EDGE_CACHE_HEADER) } else if state.registry.has_route(&method, &path) { // A module route is handed what was resolved for its request, // the permissions among them, as the publisher path is. @@ -1178,7 +1183,7 @@ fn build_router(state: &Arc) -> RouterService { &mut ec_context, auction, req, - EdgeCacheHeader::SMaxageFallback, + EDGE_CACHE_HEADER, ) .await { @@ -1940,6 +1945,65 @@ mod tests { ); } + /// A response core makes private carries the private policy, with no + /// `s-maxage`, and keeps a platform's header the origin sent, which is a + /// fetch middleware's to remove. The adapter declares no platform's header. + #[test] + fn a_private_response_follows_its_policy_and_keeps_the_origins_header() { + let settings = Settings::from_toml( + r#" + [source] + module = "origin" + + [source.origin] + url = "https://origin.publisher.example" + + [publisher] + domain = "www.publisher.example" + cookie_domain = ".publisher.example" + proxy_secret = "fictional-proxy-secret-32-bytes-ok" + + [geo] + assume_single_jurisdiction = true + "#, + ) + .expect("should parse the settings") + // A loaded domain carries its policy. + .with_permission_policy(trusted_server_core::permissions::PermissionPolicy::sample()); + build_state_with_settings(settings).expect("should build the state"); + assert!( + trusted_server_core::platform::platform_cache_headers().is_empty(), + "should declare no platform's header" + ); + let mut response = edgezero_core::http::response_builder() + .header("example-platform-cache-control", "max-age=600") + .body(edgezero_core::body::Body::empty()) + .expect("should build the response"); + trusted_server_core::cache_policy::CachePolicy::public_immutable( + std::time::Duration::from_secs(600), + ) + .apply_to_response(&mut response, EDGE_CACHE_HEADER); + + trusted_server_core::response_privacy::enforce_private_no_store(&mut response); + + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("no-store, private"), + "should carry the private policy" + ); + assert_eq!( + response + .headers() + .get("example-platform-cache-control") + .and_then(|value| value.to_str().ok()), + Some("max-age=600"), + "should keep the header the origin sent" + ); + } + /// The modules this adapter links are offered in the order their hooks /// run. A module is named by what its section selects, and an /// implementation by what `[demand]` or `[ad-server]` names. @@ -1976,6 +2040,7 @@ mod tests { "ads-txt.iabtl", "headers.geo", "51degrees", + "analytics.tinybird", "auction-protocol.openrtb", "auction.prebid-server", "auction.aps", diff --git a/crates/trusted-server-cli/Cargo.toml b/crates/trusted-server-cli/Cargo.toml index 6ade072a65..900f08eb01 100644 --- a/crates/trusted-server-cli/Cargo.toml +++ b/crates/trusted-server-cli/Cargo.toml @@ -62,6 +62,7 @@ trusted-server-ad-server-iabtl = { workspace = true } trusted-server-ad-tag-google = { workspace = true } trusted-server-ads-txt-iabtl = { workspace = true } trusted-server-middleware-51degrees = { workspace = true } +trusted-server-analytics-tinybird = { workspace = true } trusted-server-auction-aps = { workspace = true } trusted-server-auction-prebid = { workspace = true } trusted-server-auction-prebid-server = { workspace = true } diff --git a/crates/trusted-server-cli/src/app_config.rs b/crates/trusted-server-cli/src/app_config.rs index 17a36d28d0..d3aefbf361 100644 --- a/crates/trusted-server-cli/src/app_config.rs +++ b/crates/trusted-server-cli/src/app_config.rs @@ -57,6 +57,8 @@ pub fn linked_builders() -> Vec { trusted_server_headers_geo::builder(), // Run by `[shared.51degrees]` rather than selected by a section. trusted_server_middleware_51degrees::builder(), + // The analytics module `[analytics]` selects, which changes no page. + trusted_server_analytics_tinybird::builder(), // Implementations `[demand]` and `[ad-server]` can name, which no // section selects. trusted_server_auction_protocol_openrtb::builder(), @@ -281,6 +283,7 @@ mod tests { "ads-txt.iabtl", "headers.geo", "51degrees", + "analytics.tinybird", "auction-protocol.openrtb", "auction.prebid-server", "auction.aps", diff --git a/crates/trusted-server-core/src/auction/telemetry.rs b/crates/trusted-server-core/src/auction/telemetry.rs index 725e814313..17517f3d43 100644 --- a/crates/trusted-server-core/src/auction/telemetry.rs +++ b/crates/trusted-server-core/src/auction/telemetry.rs @@ -540,7 +540,11 @@ pub fn warn_once_when_analytics_is_unsent( adapter: &str, logged: &AtomicBool, ) { - if settings.analytics_module().is_none() + let selects_a_module = settings + .sections + .section("analytics") + .is_some_and(|section| !section.selected().is_empty()); + if !selects_a_module || supplied.is_some_and(|services| services.auction_telemetry_sink().is_enabled()) || logged.swap(true, Ordering::Relaxed) { diff --git a/crates/trusted-server-core/src/cache_policy.rs b/crates/trusted-server-core/src/cache_policy.rs index 4d920f54a6..c77fb2f2d9 100644 --- a/crates/trusted-server-core/src/cache_policy.rs +++ b/crates/trusted-server-core/src/cache_policy.rs @@ -1,58 +1,57 @@ //! Structured cache-policy rendering helpers. //! //! Cache policy is expressed once as typed data and then rendered into the -//! runtime-specific headers used by each edge platform. The helpers in this -//! module only write cache-control headers; response privacy hardening still -//! runs later so personalized or cookie-bearing responses cannot be made -//! shared-cacheable by accident. +//! header the running platform reads a shared cache policy from. The helpers +//! in this module only write cache-control headers; response privacy +//! hardening still runs later so personalized or cookie-bearing responses +//! cannot be made shared-cacheable by accident. +//! +//! Core owns `Cache-Control`, the two standard shared cache headers and, on +//! one response, a platform header it wrote a policy in. Of the headers that +//! carry a shared cache policy, those are the only ones it writes or removes. use std::time::Duration; use http::header::{self, HeaderName}; use http::{HeaderMap, HeaderValue}; -/// String name Fastly uses for shared-cache control. +/// String name of the shared-cache control header from the Edge Architecture +/// Specification, which core reads for every platform. pub const HEADER_SURROGATE_CONTROL_NAME: &str = "surrogate-control"; -/// String name Fastly may use for shared-cache control in some configurations. -pub const HEADER_FASTLY_SURROGATE_CONTROL_NAME: &str = "fastly-surrogate-control"; -/// String name for the standards-track CDN-only shared-cache control header. +/// String name of the CDN-only shared-cache control header from RFC 9213, +/// which core reads for every platform. pub const HEADER_CDN_CACHE_CONTROL_NAME: &str = "cdn-cache-control"; -/// String name for Cloudflare-specific CDN-only shared-cache control. -pub const HEADER_CLOUDFLARE_CDN_CACHE_CONTROL_NAME: &str = "cloudflare-cdn-cache-control"; - -/// Runtime edge-cache header names owned by this crate. -pub const EDGE_CACHE_HEADER_NAMES: &[&str] = &[ - HEADER_SURROGATE_CONTROL_NAME, - HEADER_FASTLY_SURROGATE_CONTROL_NAME, - HEADER_CDN_CACHE_CONTROL_NAME, - HEADER_CLOUDFLARE_CDN_CACHE_CONTROL_NAME, -]; - -/// Header name Fastly uses for shared-cache control. + +/// Header name of the shared-cache control header from the Edge Architecture +/// Specification. pub const HEADER_SURROGATE_CONTROL: HeaderName = HeaderName::from_static(HEADER_SURROGATE_CONTROL_NAME); -/// Header name Fastly may use for shared-cache control in some configurations. -pub const HEADER_FASTLY_SURROGATE_CONTROL: HeaderName = - HeaderName::from_static(HEADER_FASTLY_SURROGATE_CONTROL_NAME); -/// Standards-track header name for CDN-only shared-cache control. +/// Header name of the CDN-only shared-cache control header from RFC 9213. pub const HEADER_CDN_CACHE_CONTROL: HeaderName = HeaderName::from_static(HEADER_CDN_CACHE_CONTROL_NAME); -/// Cloudflare-specific header name for CDN-only shared-cache control. -pub const HEADER_CLOUDFLARE_CDN_CACHE_CONTROL: HeaderName = - HeaderName::from_static(HEADER_CLOUDFLARE_CDN_CACHE_CONTROL_NAME); + +/// The shared cache headers core reads and may write on every platform. +pub const EDGE_CACHE_HEADER_NAMES: &[&str] = + &[HEADER_SURROGATE_CONTROL_NAME, HEADER_CDN_CACHE_CONTROL_NAME]; + +/// The platform header core writes a shared cache policy in on one response, +/// kept in the response's extensions. Core owns it on that response as it +/// owns the standard headers, so making the response private removes it. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub struct WrittenPlatformCacheHeader(pub &'static str); /// Cache-control value used when a response must not be stored. pub const NO_STORE_PRIVATE_CACHE_CONTROL: &str = "no-store, private"; -/// Shared-cache header family emitted for the current runtime. +/// The header the running platform reads a shared cache policy from. #[derive(Debug, Clone, Copy, Eq, PartialEq)] pub enum EdgeCacheHeader { - /// Emit Fastly's `Surrogate-Control` header. + /// Emit the `Surrogate-Control` header. SurrogateControl, - /// Emit the standards-track `CDN-Cache-Control` header. + /// Emit the `CDN-Cache-Control` header. CdnCacheControl, - /// Emit Cloudflare's `Cloudflare-CDN-Cache-Control` header. - CloudflareCdnCacheControl, + /// Emit the platform's own header, named in lower case by its adapter. + Platform(&'static str), /// Put `s-maxage` into `Cache-Control` instead of emitting a separate edge header. SMaxageFallback, /// Do not emit edge-cache directives. @@ -82,7 +81,13 @@ impl EdgeCacheHeader { match self { Self::SurrogateControl => Some(HEADER_SURROGATE_CONTROL), Self::CdnCacheControl => Some(HEADER_CDN_CACHE_CONTROL), - Self::CloudflareCdnCacheControl => Some(HEADER_CLOUDFLARE_CDN_CACHE_CONTROL), + Self::Platform(name) => { + let parsed = HeaderName::from_bytes(name.as_bytes()).ok(); + if parsed.is_none() { + log::warn!("the platform's cache header `{name}` is not a header name"); + } + parsed + } Self::SMaxageFallback | Self::None => None, } } @@ -209,7 +214,10 @@ impl CachePolicy { Some(directives.join(", ")) } - /// Apply the policy to response headers for the selected runtime edge header. + /// Apply the policy to response headers for the selected runtime edge header, + /// replacing what the standard shared cache headers and the selected one + /// held. On a response, [`Self::apply_to_response`] also records a + /// platform header the policy is written in. /// /// # Panics /// @@ -225,14 +233,36 @@ impl CachePolicy { ); remove_edge_cache_headers(headers); - if let Some(header_name) = edge_header.header_name() - && let Some(value) = self.edge_header_value(edge_header) - { - headers.insert( - header_name, - HeaderValue::from_str(&value) - .expect("should render a valid edge cache-control header"), - ); + if let Some(header_name) = edge_header.header_name() { + headers.remove(&header_name); + if let Some(value) = self.edge_header_value(edge_header) { + headers.insert( + header_name, + HeaderValue::from_str(&value) + .expect("should render a valid edge cache-control header"), + ); + } + } + } + + /// Apply the policy to `response` as [`Self::apply_to_headers`] does, and + /// record a platform header it is written in as the response's + /// [`WrittenPlatformCacheHeader`], so that making the response private + /// removes the policy core wrote there. + /// + /// # Panics + /// + /// Panics as [`Self::apply_to_headers`] does. + pub fn apply_to_response( + self, + response: &mut http::Response, + edge_header: EdgeCacheHeader, + ) { + self.apply_to_headers(response.headers_mut(), edge_header); + if let EdgeCacheHeader::Platform(name) = edge_header { + response + .extensions_mut() + .insert(WrittenPlatformCacheHeader(name)); } } } @@ -242,7 +272,8 @@ impl CachePolicy { pub enum CacheControlPolicy { /// Apply a regular TTL-based cache policy. Store(CachePolicy), - /// Apply `Cache-Control: no-store, private` and strip shared-cache headers. + /// Apply `Cache-Control: no-store, private` and remove the standard shared + /// cache headers. NoStorePrivate, } @@ -268,14 +299,28 @@ impl From for CacheControlPolicy { } } -/// Remove every runtime-specific shared-cache header owned by this crate. +/// Remove the standard shared cache headers, [`EDGE_CACHE_HEADER_NAMES`]. pub fn remove_edge_cache_headers(headers: &mut HeaderMap) { for name in EDGE_CACHE_HEADER_NAMES { headers.remove(*name); } } -/// Return true when `name` is an edge-cache header owned by this crate. +/// Remove the headers carrying a shared cache policy that core owns on +/// `response`: the standard ones and its [`WrittenPlatformCacheHeader`]. +pub fn remove_owned_cache_headers(response: &mut http::Response) { + remove_edge_cache_headers(response.headers_mut()); + if let Some(WrittenPlatformCacheHeader(name)) = response + .extensions() + .get::() + .copied() + { + response.headers_mut().remove(name); + } +} + +/// Return true when `name` is one of the standard shared cache headers, +/// [`EDGE_CACHE_HEADER_NAMES`]. #[must_use] pub fn is_edge_cache_header_name(name: &str) -> bool { EDGE_CACHE_HEADER_NAMES @@ -283,6 +328,17 @@ pub fn is_edge_cache_header_name(name: &str) -> bool { .any(|candidate| name.eq_ignore_ascii_case(candidate)) } +/// Return true when `name` is a header carrying a shared cache policy that +/// core owns on `response`, see [`remove_owned_cache_headers`]. +#[must_use] +pub fn is_owned_cache_header_name(response: &http::Response, name: &str) -> bool { + is_edge_cache_header_name(name) + || response + .extensions() + .get::() + .is_some_and(|WrittenPlatformCacheHeader(written)| name.eq_ignore_ascii_case(written)) +} + /// Return true when a `Cache-Control` field value contains `directive`. /// /// Matching is directive-name exact and case-insensitive. Pseudo-directives such @@ -340,7 +396,8 @@ pub fn cache_control_headers_are_private_or_no_store(headers: &HeaderMap) -> boo || cache_control_headers_have_directive(headers, "no-store") } -/// Apply `Cache-Control: no-store, private` and strip all shared-cache headers. +/// Apply `Cache-Control: no-store, private` and remove the standard shared +/// cache headers. /// /// # Panics /// @@ -359,7 +416,7 @@ mod tests { use super::*; #[test] - fn public_immutable_renders_browser_and_fastly_headers() { + fn public_immutable_renders_browser_and_surrogate_control_headers() { let policy = CachePolicy::public_immutable(Duration::from_secs(31_536_000)); let mut headers = HeaderMap::new(); @@ -377,7 +434,7 @@ mod tests { .get(HEADER_SURROGATE_CONTROL) .and_then(|v| v.to_str().ok()), Some("max-age=31536000"), - "should render Fastly edge TTL" + "should render the edge TTL in Surrogate-Control" ); } @@ -424,7 +481,7 @@ mod tests { } #[test] - fn cloudflare_specific_header_renders_cdn_only_policy() { + fn a_platform_header_renders_cdn_only_policy() { let policy = CachePolicy::public_short_with_stale( Duration::from_secs(300), Duration::from_secs(60), @@ -432,7 +489,10 @@ mod tests { ); let mut headers = HeaderMap::new(); - policy.apply_to_headers(&mut headers, EdgeCacheHeader::CloudflareCdnCacheControl); + policy.apply_to_headers( + &mut headers, + EdgeCacheHeader::Platform("example-cdn-cache-control"), + ); assert_eq!( headers @@ -443,10 +503,10 @@ mod tests { ); assert_eq!( headers - .get(HEADER_CLOUDFLARE_CDN_CACHE_CONTROL) + .get("example-cdn-cache-control") .and_then(|v| v.to_str().ok()), Some("max-age=300, stale-while-revalidate=60, stale-if-error=86400"), - "should render Cloudflare-specific CDN cache policy" + "should render the policy in the platform's own header" ); assert!( headers.get(HEADER_CDN_CACHE_CONTROL).is_none(), @@ -455,19 +515,104 @@ mod tests { } #[test] - fn private_policy_removes_stale_edge_headers() { - let policy = CachePolicy::private_revalidate(); + fn a_declared_platform_header_stays_on_a_private_response() { + crate::platform::declare_platform_cache_headers(&["example-platform-cache-control"]); let mut headers = HeaderMap::new(); headers.insert( HEADER_SURROGATE_CONTROL, HeaderValue::from_static("max-age=60"), ); headers.insert( - HEADER_CDN_CACHE_CONTROL, + "example-platform-cache-control", HeaderValue::from_static("max-age=60"), ); + + assert!(!is_edge_cache_header_name("Example-Platform-Cache-Control")); + CacheControlPolicy::NoStorePrivate.apply_to_headers(&mut headers, EdgeCacheHeader::None); + + assert!( + headers.get(HEADER_SURROGATE_CONTROL).is_none(), + "should remove the standard header" + ); + assert_eq!( + headers + .get("example-platform-cache-control") + .and_then(|v| v.to_str().ok()), + Some("max-age=60"), + "should leave a platform header core did not write" + ); + } + + #[test] + fn a_private_policy_empties_the_platform_header_it_is_written_in() { + let mut headers = HeaderMap::new(); headers.insert( - HEADER_CLOUDFLARE_CDN_CACHE_CONTROL, + "example-written-cache-control", + HeaderValue::from_static("max-age=600"), + ); + + CachePolicy::private_revalidate().apply_to_headers( + &mut headers, + EdgeCacheHeader::Platform("example-written-cache-control"), + ); + + assert!( + headers.get("example-written-cache-control").is_none(), + "a private policy should leave no shared policy in the header it is written in" + ); + } + + #[test] + fn a_policy_written_in_a_platform_header_is_removed_with_the_owned_headers() { + let mut response = http::Response::builder() + .header("example-origin-cache-control", "max-age=600") + .body(()) + .expect("should build the response"); + + CachePolicy::public_immutable(Duration::from_secs(600)).apply_to_response( + &mut response, + EdgeCacheHeader::Platform("example-written-cache-control"), + ); + assert!( + response + .headers() + .contains_key("example-written-cache-control") + ); + assert!(is_owned_cache_header_name( + &response, + "Example-Written-Cache-Control" + )); + assert!(!is_owned_cache_header_name( + &response, + "example-origin-cache-control" + )); + + remove_owned_cache_headers(&mut response); + + assert!( + !response + .headers() + .contains_key("example-written-cache-control"), + "should remove the policy core wrote in the platform header" + ); + assert!( + response + .headers() + .contains_key("example-origin-cache-control"), + "should leave a platform header core did not write" + ); + } + + #[test] + fn private_policy_removes_stale_edge_headers() { + let policy = CachePolicy::private_revalidate(); + let mut headers = HeaderMap::new(); + headers.insert( + HEADER_SURROGATE_CONTROL, + HeaderValue::from_static("max-age=60"), + ); + headers.insert( + HEADER_CDN_CACHE_CONTROL, HeaderValue::from_static("max-age=60"), ); @@ -482,16 +627,12 @@ mod tests { ); assert!( headers.get(HEADER_SURROGATE_CONTROL).is_none(), - "should remove Fastly shared-cache headers for private responses" + "should remove Surrogate-Control for private responses" ); assert!( headers.get(HEADER_CDN_CACHE_CONTROL).is_none(), "should remove generic CDN cache headers for private responses" ); - assert!( - headers.get(HEADER_CLOUDFLARE_CDN_CACHE_CONTROL).is_none(), - "should remove Cloudflare cache headers for private responses" - ); } #[test] @@ -501,18 +642,10 @@ mod tests { HEADER_SURROGATE_CONTROL, HeaderValue::from_static("max-age=60"), ); - headers.insert( - HEADER_FASTLY_SURROGATE_CONTROL, - HeaderValue::from_static("max-age=60"), - ); headers.insert( HEADER_CDN_CACHE_CONTROL, HeaderValue::from_static("max-age=60"), ); - headers.insert( - HEADER_CLOUDFLARE_CDN_CACHE_CONTROL, - HeaderValue::from_static("max-age=60"), - ); CacheControlPolicy::NoStorePrivate.apply_to_headers(&mut headers, EdgeCacheHeader::None); @@ -525,9 +658,7 @@ mod tests { ); assert!( headers.get(HEADER_SURROGATE_CONTROL).is_none() - && headers.get(HEADER_FASTLY_SURROGATE_CONTROL).is_none() - && headers.get(HEADER_CDN_CACHE_CONTROL).is_none() - && headers.get(HEADER_CLOUDFLARE_CDN_CACHE_CONTROL).is_none(), + && headers.get(HEADER_CDN_CACHE_CONTROL).is_none(), "should remove all shared-cache headers" ); } diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index 337ec3b781..98354b0e22 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -271,14 +271,6 @@ impl edgezero_core::app_config::AppConfigMeta for TrustedServerAppConfig { ], false, ), - field( - vec![ - optional_object("analytics"), - optional_object("tinybird"), - object("auction_token_secret"), - ], - true, - ), ]; // An asset route's source and the publisher's take the same signing // table, so each holds the same three key references. @@ -531,12 +523,16 @@ fn validate_secret_key_references( table.clone(), ) } else { - let Some((section, written)) = builder + let Some((section, selected)) = builder .module_name() .and_then(|name| settings.module_selection(name)) else { continue; }; + // The table is keyed by the name without its type folder, however + // the section's selector wrote it, as `Settings::module_config` + // reads it. + let written = crate::module_name::short_form(section, selected); ( format!("{section}.{written}"), settings.section_table(section, written), @@ -619,10 +615,7 @@ mod tests { use crate::integrations::IntegrationRegistration; use crate::integrations::js_asset_proxy::JS_ASSET_PROXY_INTEGRATION_ID; use crate::redacted::Redacted; - use crate::settings::{ - AnalyticsConfig, ProxyAssetRoute, S3SigV4AuthConfig, TinybirdSettings, - TrustedClientIpConfig, - }; + use crate::settings::{ProxyAssetRoute, S3SigV4AuthConfig, TrustedClientIpConfig}; use crate::test_support::template::{template_with_resolved_required_secrets, uncomment_block}; use crate::test_support::tests::{ crate_test_settings_str, crate_test_settings_str_with_ec_section, edit_origin, @@ -768,19 +761,6 @@ formats = [{ width = 300, height = 250 }] } } - #[test] - fn documented_analytics_block_validates_when_uncommented() { - let toml = uncomment_block(&template_with_resolved_required_secrets(), "[analytics]"); - let settings = Settings::from_toml(&toml) - .expect("uncommented [analytics] with documented api_host should parse and validate"); - assert!( - settings - .analytics_module() - .is_some_and(|module| !module.api_host.is_empty()), - "the selected analytics module should have a non-empty api_host" - ); - } - #[test] fn wrapper_serializes_as_settings_shape() { let settings = valid_settings(); @@ -1051,7 +1031,6 @@ formats = [{ width = 300, height = 250 }] ("ec.partners[*].api_token".to_owned(), true), ("ec.partners[*].ts_pull_token".to_owned(), true), ("trusted_client_ip.shared_secret".to_owned(), false), - ("analytics.tinybird.auction_token_secret".to_owned(), true), ( "proxy.asset_routes[*].source.origin.auth.access_key_id".to_owned(), true @@ -1109,21 +1088,9 @@ formats = [{ width = 300, height = 250 }] assert_eq!(serialized["secret_access_key"], "secret_access_key"); } - /// An `[analytics]` section selecting the analytics module with `settings`. - fn analytics_selecting(settings: TinybirdSettings) -> AnalyticsConfig { - AnalyticsConfig { - module: Some("analytics.tinybird".to_string()), - tinybird: Some(settings), - } - } - #[test] fn legacy_static_secret_store_selectors_are_accepted_but_not_serialized() { let mut settings = valid_settings(); - settings.analytics = Some(analytics_selecting(TinybirdSettings { - secret_store: Some("legacy-tinybird-store".to_string()), - ..TinybirdSettings::default() - })); let mut route = ProxyAssetRoute::new( "/assets/", "https://examplebucket.s3.us-east-1.amazonaws.com", @@ -1146,21 +1113,15 @@ formats = [{ width = 300, height = 250 }] .expect("should normalize settings without key collisions"); let serialized = serde_json::to_string(&settings).expect("should serialize settings"); - for legacy_store in ["legacy-tinybird-store", "legacy-s3-store"] { - assert!( - !serialized.contains(legacy_store), - "serialized config should omit deprecated selector {legacy_store}" - ); - } + assert!( + !serialized.contains("legacy-s3-store"), + "serialized config should omit the deprecated selector" + ); } #[test] fn settings_debug_redacts_resolved_static_credentials() { let mut settings = valid_settings(); - settings.analytics = Some(analytics_selecting(TinybirdSettings { - auction_token_secret: Some(Redacted::new("resolved-tinybird-secret".to_string())), - ..TinybirdSettings::default() - })); settings .insert_module_config( "testing", @@ -1173,7 +1134,6 @@ formats = [{ width = 300, height = 250 }] let debug = format!("{settings:?}"); - assert!(!debug.contains("resolved-tinybird-secret")); assert!(!debug.contains("resolved-module-secret")); assert!(debug.contains("example")); } @@ -1301,50 +1261,6 @@ gam_network_id = "99999" ); } - /// The analytics module's token is a secret reference like every other, - /// so an empty or absent one is refused for deploy, by its path. - #[test] - fn app_config_new_rejects_an_analytics_secret_key_reference_that_is_empty_or_absent() { - for (case, reference) in [ - ("an empty reference", Some(Redacted::new(String::new()))), - ( - "a whitespace reference", - Some(Redacted::new(" ".to_owned())), - ), - ("no reference", None), - ] { - let mut settings = valid_settings(); - settings.analytics = Some(analytics_selecting(TinybirdSettings { - api_host: "api.tinybird.co".to_owned(), - auction_token_secret: reference, - ..TinybirdSettings::default() - })); - - let err = TrustedServerAppConfig::new(settings) - .err() - .unwrap_or_else(|| panic!("{case}: should refuse the telemetry secret reference")); - - assert!( - err.to_string() - .contains("analytics.tinybird.auction_token_secret"), - "{case}: the error should identify the telemetry secret reference: {err:?}" - ); - } - } - - #[test] - fn app_config_new_accepts_an_analytics_secret_key_reference() { - let mut settings = valid_settings(); - settings.analytics = Some(analytics_selecting(TinybirdSettings { - api_host: "api.tinybird.co".to_owned(), - auction_token_secret: Some(Redacted::new("tinybird_auction_token".to_owned())), - ..TinybirdSettings::default() - })); - - TrustedServerAppConfig::new(settings) - .expect("should validate the token's key name without treating it as the value"); - } - #[test] fn app_config_new_rejects_a_resolve_allowed_origin_that_is_not_bare() { let mut settings = valid_settings(); @@ -1947,6 +1863,50 @@ passphrase = "production-secret-key-32-bytes-min" } } + /// A module selected by its full name in its own type's section reads its + /// table at the name without the type folder, so its secret settings are + /// checked there. + #[test] + fn deploy_validation_checks_the_secret_of_a_module_selected_by_its_full_name() { + for (table, expected) in [ + ( + serde_json::json!({ "lock": true }), + Some("`testing.probe.spare.key_name`"), + ), + ( + serde_json::json!({ "lock": true, "spare": { "key_name": "module_key" } }), + None, + ), + ] { + let mut value = + serde_json::to_value(valid_settings()).expect("should serialize settings"); + value + .as_object_mut() + .expect("settings should serialize as an object") + .insert( + "testing".to_owned(), + serde_json::json!({ "module": "testing.probe", "probe": table }), + ); + let settings = + Settings::from_json_value(value).expect("should read the module's table"); + + let result = validate_settings_for_deploy_with(&settings, &[module_with_a_secret()]); + + match expected { + Some(path) => { + let error = result.expect_err("should refuse a setting that names no key"); + assert!( + format!("{error:?}").contains(path), + "should name the setting in the table the module reads: {error:?}" + ); + } + None => result.unwrap_or_else(|error| { + panic!("should accept {table}: {error:?}"); + }), + } + } + } + const PLAN_RULE_MESSAGE: &str = "probe module refuses an enabled auction"; fn refuse_an_enabled_auction( diff --git a/crates/trusted-server-core/src/config_payload.rs b/crates/trusted-server-core/src/config_payload.rs index b62e4951c3..d9d1450240 100644 --- a/crates/trusted-server-core/src/config_payload.rs +++ b/crates/trusted-server-core/src/config_payload.rs @@ -198,16 +198,6 @@ fn log_permission_baseline(policy: &PermissionPolicy) { } fn remove_inactive_secret_references(data: &mut serde_json::Value) { - // References of a table `[analytics]` does not select are cleared before resolution. - if !crate::settings::document_selects_analytics_module(data) - && let Some(table) = data - .pointer_mut("/analytics/tinybird") - .and_then(serde_json::Value::as_object_mut) - { - table.remove("auction_token_secret"); - table.remove("access_token_secret"); - } - if let Some(partners) = data .pointer_mut("/ec/partners") .and_then(serde_json::Value::as_array_mut) @@ -235,8 +225,7 @@ mod tests { use crate::platform::{PlatformError, StoreId}; use crate::redacted::Redacted; use crate::settings::{ - AnalyticsConfig, AssetOriginAuth, EcPartner, ProxyAssetRoute, S3SigV4AuthConfig, - TinybirdSettings, TrustedClientIpConfig, + AssetOriginAuth, EcPartner, ProxyAssetRoute, S3SigV4AuthConfig, TrustedClientIpConfig, }; use crate::test_support::tests::{ crate_test_settings_str, edit_origin, hmac_passphrase, sample_permissions, @@ -483,6 +472,30 @@ mod tests { ); } + #[test] + fn the_load_looks_up_the_secret_of_a_module_selected_by_its_full_name() { + let mut value = serde_json::to_value(test_settings()).expect("should serialize settings"); + value + .as_object_mut() + .expect("settings should serialize as an object") + .insert( + "testing".to_owned(), + serde_json::json!({ + "module": "testing.probe", + "probe": { "lock": true, "key_name": "module-key" }, + }), + ); + let settings = Settings::from_json_value(value).expect("should read the module's table"); + + let loaded = load_with_module(&settings); + + assert_eq!( + loaded.section_table("testing", "probe").get("key_name"), + Some(&serde_json::json!("resolved-module-secret")), + "should keep the setting in the table the module reads and look it up" + ); + } + #[test] fn a_table_s_key_name_is_left_as_written_without_its_module_s_builder() { let mut settings = test_settings(); @@ -612,14 +625,6 @@ mod tests { #[test] fn resolves_all_static_credentials_from_the_mapped_default_store() { let mut original = test_settings(); - original.analytics = Some(AnalyticsConfig { - module: Some("analytics.tinybird".to_string()), - tinybird: Some(TinybirdSettings { - api_host: "api.example.com".to_string(), - auction_token_secret: Some(Redacted::new("tinybird-token-key".to_string())), - ..TinybirdSettings::default() - }), - }); let mut route = ProxyAssetRoute::new( "/assets/", "https://examplebucket.s3.us-east-1.amazonaws.com", @@ -649,14 +654,6 @@ mod tests { ) .expect("should resolve every static credential from the mapped store"); - assert_eq!( - reconstructed - .analytics_module() - .and_then(|module| module.auction_token_secret.as_ref()) - .map(Redacted::expose) - .map(String::as_str), - Some("resolved-tinybird-token") - ); let auth = reconstructed.proxy.asset_routes[0] .origin_source() .and_then(|origin| origin.auth) @@ -1056,41 +1053,6 @@ mod tests { assert!(reconstructed.ec.partners[0].ts_pull_token.is_none()); } - #[test] - fn an_unselected_analytics_module_table_is_refused_without_resolving_its_secret() { - let mut original = test_settings(); - original.analytics = Some(AnalyticsConfig { - module: None, - tinybird: Some(TinybirdSettings { - api_host: "api.example.com".to_string(), - auction_token_secret: Some(Redacted::new("unused-telemetry-key".to_string())), - ..TinybirdSettings::default() - }), - }); - - let error = settings_from_config_blob( - &envelope_json(&original), - &UnifiedSecretStore, - &StoreName::from("ts_secrets"), - PolicyOrigin::Blob, - ) - .expect_err("should refuse a table nothing selects"); - let rendered = format!("{error:?}"); - - assert!( - rendered.contains("[analytics] selects no module"), - "should name the section and why: {rendered}" - ); - assert!( - !rendered.contains("unused-telemetry-key") && !rendered.contains("secret reference"), - "should not have tried to resolve the stale secret reference: {rendered}" - ); - } - - /// A table for a module its section does not select is refused, and its - /// secret references are dropped before resolution, so the operator reads - /// the table's own fault rather than a secret-store failure that follows - /// from it. #[test] fn an_unselected_module_s_table_is_refused_without_resolving_its_secrets() { let original = test_settings(); diff --git a/crates/trusted-server-core/src/http_util.rs b/crates/trusted-server-core/src/http_util.rs index 62bb825c11..a5706f8981 100644 --- a/crates/trusted-server-core/src/http_util.rs +++ b/crates/trusted-server-core/src/http_util.rs @@ -368,7 +368,7 @@ pub fn serve_static_with_etag( .header(header::VARY, "Accept-Encoding") .body(EdgeBody::empty()) .expect("should build 304 static response"); - short_policy.apply_to_headers(response.headers_mut(), edge_header); + short_policy.apply_to_response(&mut response, edge_header); return response; } @@ -379,7 +379,7 @@ pub fn serve_static_with_etag( .header(header::VARY, "Accept-Encoding") .body(EdgeBody::from(body.as_bytes())) .expect("should build static response"); - short_policy.apply_to_headers(response.headers_mut(), edge_header); + short_policy.apply_to_response(&mut response, edge_header); response } diff --git a/crates/trusted-server-core/src/inspect/config/tests.rs b/crates/trusted-server-core/src/inspect/config/tests.rs index e9d6ad5639..a66f1ec850 100644 --- a/crates/trusted-server-core/src/inspect/config/tests.rs +++ b/crates/trusted-server-core/src/inspect/config/tests.rs @@ -250,14 +250,6 @@ fn documents_holding_every_declared_secret() -> Vec<(&'static str, Value)> { }, }, }]); - main["analytics"] = json!({ - "module": "tinybird", - "tinybird": { - "api_host": "api.tinybird.example.com", - "auction_token_secret": "telemetry", - }, - }); - let mut host_signals = document(""); let ec = host_signals["ec"].as_object_mut().expect("an ec table"); ec.remove("hmac"); diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index d8b07f13fd..e521d3fa46 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -11,6 +11,7 @@ use sha2::{Digest as _, Sha256}; use crate::asset_cache::AssetCacheModule; use crate::auction::AuctionPlan; +use crate::auction::telemetry::AuctionTelemetrySink; use crate::constants::HEADER_X_TS_EC; use crate::ec::EcContext; use crate::ec::device::DeviceModule; @@ -694,6 +695,12 @@ pub struct IntegrationRegistration { /// Declaring one does not make it serve anything, because a source is only /// built where `[source]` or an asset route's `source` table names it. pub source_module: Option<(&'static str, Arc)>, + /// Analytics module this registration supplies, with the name + /// `[analytics] module` selects it by. + /// + /// Declaring one does not make it send, because core hands an auction's + /// rows only to the sink `[analytics] module` names. + pub analytics_module: Option<(&'static str, Arc)>, /// Identifier module this registration supplies, see /// [`crate::ec::identifiers`]. /// @@ -738,6 +745,7 @@ impl IntegrationRegistrationBuilder { prefix_module: None, robots_txt_contributor: None, source_module: None, + analytics_module: None, identifier_module: None, asset_cache_module: None, }, @@ -846,6 +854,22 @@ impl IntegrationRegistrationBuilder { self } + /// Declare the analytics module this registration supplies, under its + /// `name`: the sink core hands each settled auction's rows to. + /// + /// It only receives rows when `[analytics] module` names it, and a + /// module the selector does not choose is logged as a warning when the + /// registry is built, the same as geo. + #[must_use] + pub fn with_analytics_module( + mut self, + name: &'static str, + sink: Arc, + ) -> Self { + self.registration.analytics_module = Some((name, sink)); + self + } + /// Declare the identifier module this registration supplies, which core /// asks for the identifiers it learned on every request the registration /// runs for, see [`crate::ec::identifiers`]. @@ -1009,6 +1033,12 @@ struct IntegrationRegistryInner { prefix_module: SelectedPrefixModule, // The source modules the running registrations supply, each with its name. source_modules: Vec<(&'static str, Arc)>, + // The analytics modules the running registrations supply, each with its + // name, in registration order. `[analytics] module` picks at most one. + analytics_modules: Vec<(&'static str, Arc)>, + // The sink `[analytics] module` resolved to, or `None` when no section + // selects one, so core's no-op sink stands. + analytics_module: Option>, // The publisher's source and each asset route's, in route order, built // from the settings. `None` for a registry built without settings. sources: Option, @@ -1063,6 +1093,8 @@ impl Default for IntegrationRegistryInner { prefix_modules: Vec::new(), prefix_module: SelectedPrefixModule::default(), source_modules: Vec::new(), + analytics_modules: Vec::new(), + analytics_module: None, sources: None, identifier_modules: Arc::default(), asset_cache_module: None, @@ -1091,6 +1123,7 @@ const DEVICE_MODULE_FASTLY: &str = "fastly"; /// written in its section may leave off. const EDGECOOKIE_TYPE: &str = crate::ec::module::MODULE_TYPE; const DEVICE_TYPE: &str = "device"; +const ANALYTICS_TYPE: &str = "analytics"; const GEO_TYPE: &str = "geo"; const PREFIX_TYPE: &str = crate::prefix::MODULE_TYPE; const ROBOTS_TXT_TYPE: &str = crate::robots_txt::MODULE_TYPE; @@ -1484,6 +1517,53 @@ fn resolve_geo_module( Ok(resolved) } +/// Resolves the analytics module `[analytics] module` names, when a +/// registered module supplies it. +/// +/// Core has no analytics module of its own, so with no `[analytics]` section +/// nothing is resolved and the no-op sink stands. +/// +/// # Errors +/// +/// Returns [`TrustedServerError::Configuration`] when the section selects +/// more than one name, or a name no registered module answers to. +fn resolve_analytics_module( + settings: &Settings, + inner: &IntegrationRegistryInner, +) -> Result>, Report> { + let selected = settings + .sections + .section(ANALYTICS_TYPE) + .map(crate::provider_table::SectionModules::selected) + .unwrap_or_default(); + let resolved = match selected { + [] => None, + [written] => Some(named_module( + ANALYTICS_TYPE, + ANALYTICS_TYPE, + written, + &inner.analytics_modules, + inner, + )?), + _ => { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "[{ANALYTICS_TYPE}] selects {} modules, and an auction's rows go to \ + one. Select one with `module`", + selected.len() + ), + })); + } + }; + warn_unselected( + ANALYTICS_TYPE, + ANALYTICS_TYPE, + resolved.as_ref(), + &inner.analytics_modules, + ); + Ok(resolved) +} + /// Summary of registered integration capabilities. #[derive(Debug, Clone)] pub struct IntegrationMetadata { @@ -2330,6 +2410,10 @@ impl IntegrationRegistry { )?; inner.robots_txt_contributors.push((name, contributor)); } + if let Some((name, sink)) = registration.analytics_module { + claim_module_name(&mut claimed, "analytics", name, registration.integration_id)?; + inner.analytics_modules.push((name, sink)); + } if let Some((name, module)) = registration.asset_cache_module { if asset_cache_supplier.map(|(id, _, _)| id) != Some(registration.integration_id) { return Err(refuse_unselected_asset_cache( @@ -2407,6 +2491,7 @@ impl IntegrationRegistry { inner.ec_module = resolve_ec_module(settings, &inner); inner.device_module = resolve_device_module(settings, &inner)?; inner.prefix_module = resolve_prefix_module(settings, &inner)?; + inner.analytics_module = resolve_analytics_module(settings, &inner)?; inner.sources = Some(build_sources(settings, &inner.source_modules)?); inner.identifier_modules = Arc::from(identifier_modules); inner.geo_module = geo_module; @@ -2467,6 +2552,13 @@ impl IntegrationRegistry { self.inner.device_module.clone() } + /// The analytics module `[analytics] module` selected, or `None` when no + /// section selects one and core's no-op sink stands. + #[must_use] + pub fn analytics_module(&self) -> Option> { + self.inner.analytics_module.clone() + } + /// The asset cache module of the builder `[asset-cache]` selects, or /// `None` when it selects none. #[must_use] @@ -3092,6 +3184,8 @@ impl IntegrationRegistry { prefix_modules: Vec::new(), prefix_module: SelectedPrefixModule::default(), source_modules: Vec::new(), + analytics_modules: Vec::new(), + analytics_module: None, sources: None, identifier_modules: Arc::default(), asset_cache_module: None, @@ -3178,6 +3272,8 @@ impl IntegrationRegistry { prefix_modules: Vec::new(), prefix_module: SelectedPrefixModule::default(), source_modules: Vec::new(), + analytics_modules: Vec::new(), + analytics_module: None, sources: None, identifier_modules: Arc::default(), asset_cache_module: None, @@ -7225,6 +7321,76 @@ mod tests { ); } + #[test] + fn analytics_selector_rejects_a_module_that_declares_no_analytics_module() { + let mut settings = crate::test_support::tests::create_test_settings(); + settings.select_module("analytics", "testing.probe"); + let extra = [crate::integrations::IntegrationBuilder::new( + "probe", + "seam-probe", + probe_registration, + validate_nothing, + ) + .with_module_name("testing.probe")]; + + let error = IntegrationRegistry::with_registrations(&settings, &extra) + .err() + .expect("should reject a module that declares no analytics module"); + + let message = error.to_string(); + assert!( + message.contains("`[analytics] module` names `testing.probe`") + && message.contains("It runs no analytics module") + && message.contains("is selected and supplies no analytics module"), + "error should name the module and say it supplies no analytics module: {message}" + ); + } + + #[test] + fn analytics_selector_takes_one_module() { + let mut settings = crate::test_support::tests::create_test_settings(); + settings.select_module("analytics", "testing.probe"); + settings.select_module("analytics", "testing.probe-unnamed"); + let extra = [ + crate::integrations::IntegrationBuilder::new( + "probe", + "seam-probe", + probe_registration, + validate_nothing, + ) + .with_module_name("testing.probe"), + crate::integrations::IntegrationBuilder::new( + "probe-unnamed", + "seam-probe", + never_registering_builder, + validate_nothing, + ) + .with_module_name("testing.probe-unnamed"), + ]; + + let error = IntegrationRegistry::with_registrations(&settings, &extra) + .err() + .expect("should reject a section selecting two analytics modules"); + + assert!( + error.to_string().contains("[analytics] selects 2 modules"), + "error should say one module is selected: {error}" + ); + } + + #[test] + fn no_analytics_section_resolves_no_sink() { + let settings = crate::test_support::tests::create_test_settings(); + + let registry = IntegrationRegistry::with_registrations(&settings, &[]) + .expect("should build registry without an analytics section"); + + assert!( + registry.analytics_module().is_none(), + "nothing selected should leave the no-op sink standing" + ); + } + #[test] fn geo_selector_rejects_a_module_no_section_selects() { // The module is registered, but no section selects it, so its geo diff --git a/crates/trusted-server-core/src/module_secrets.rs b/crates/trusted-server-core/src/module_secrets.rs index 24e9653fc6..5215d68c1c 100644 --- a/crates/trusted-server-core/src/module_secrets.rs +++ b/crates/trusted-server-core/src/module_secrets.rs @@ -91,8 +91,10 @@ fn remove_at(table: &mut Map, path: &[&str]) { /// configuration whose secrets have not been looked up yet. /// /// A setting is in use when its section selects the module and the module's -/// rule for the setting says its table uses it. A table the section does not -/// select is refused once the settings are deserialized, and clearing its +/// rule for the setting says its table uses it. The table is the one at the +/// name without the section's type folder, however the selection wrote the +/// name, because that is the table the module reads. A table the section does +/// not select is refused once the settings are deserialized, and clearing its /// references first means that refusal is what an operator sees, rather than /// a secret lookup failing. pub(crate) fn clear_unused(data: &mut Value, builders: &[IntegrationBuilder]) { @@ -110,11 +112,12 @@ pub(crate) fn clear_unused(data: &mut Value, builders: &[IntegrationBuilder]) { Some(vendor) => Some(vendor), None => selected_as(data, section, &forms), }; + let read = selected.map(|written| short_form(section, written)); for form in &forms { let Some(table) = table_mut(data, section, form) else { continue; }; - let runs = selected == Some(*form); + let runs = read == Some(*form); let unused = secrets .iter() .filter(|secret| !(runs && (secret.in_use)(table))) @@ -292,7 +295,7 @@ mod tests { } #[test] - fn a_module_selected_by_its_full_name_keeps_the_settings_in_the_table_at_that_name() { + fn a_module_selected_by_its_full_name_keeps_the_settings_in_the_table_it_reads() { let mut data = json!({ "testing": { "module": "testing.probe", @@ -304,18 +307,19 @@ mod tests { clear_unused(&mut data, &[builder()]); assert_eq!( - data.pointer("/testing/testing/probe/key_name"), + data.pointer("/testing/probe/key_name"), Some(&json!("main-key")), - "should keep the settings of the table the selection names" + "should keep the settings of the table at the name without its type folder" ); assert_eq!( - data.pointer("/testing/testing/probe/spare/key_name"), + data.pointer("/testing/probe/spare/key_name"), Some(&json!("spare-key")) ); assert_eq!( - data.pointer("/testing/probe/key_name"), + data.pointer("/testing/testing/probe/key_name"), None, - "should clear the settings of a table under the name not selected" + "should clear the settings of the table at the full name's parts, which the \ + module never reads" ); } diff --git a/crates/trusted-server-core/src/platform/cache_headers.rs b/crates/trusted-server-core/src/platform/cache_headers.rs new file mode 100644 index 0000000000..48e9069664 --- /dev/null +++ b/crates/trusted-server-core/src/platform/cache_headers.rs @@ -0,0 +1,53 @@ +//! The response headers a hosting platform reads a shared cache policy from. +//! +//! Core reads `Surrogate-Control` and `CDN-Cache-Control` for every platform. +//! A platform's own header is declared by its adapter, once, before it +//! serves, so that core does not keep an origin response whose policy is in +//! a header it does not read. The declaration is read for nothing else. + +use std::sync::{Mutex, PoisonError}; + +static DECLARED: Mutex> = Mutex::new(Vec::new()); + +/// Declares the headers the hosting platform reads a shared cache policy +/// from, each in lower case, beyond the two core reads for every platform. +/// +/// An origin response carrying a policy in a declared header is not kept for +/// other readers, because core does not read that policy. A name declared +/// twice is kept once. +pub fn declare_platform_cache_headers(names: &[&'static str]) { + let mut declared = DECLARED.lock().unwrap_or_else(PoisonError::into_inner); + for name in names { + if !declared.contains(name) { + declared.push(name); + } + } +} + +/// The headers the running platform declared, in declaration order. +#[must_use] +pub fn platform_cache_headers() -> Vec<&'static str> { + DECLARED + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_declared_header_is_listed_once() { + declare_platform_cache_headers(&["example-twice-cache-control"]); + declare_platform_cache_headers(&["example-twice-cache-control"]); + + assert_eq!( + platform_cache_headers() + .iter() + .filter(|name| **name == "example-twice-cache-control") + .count(), + 1 + ); + } +} diff --git a/crates/trusted-server-core/src/platform/mod.rs b/crates/trusted-server-core/src/platform/mod.rs index 5a93aaa952..af160d2bbe 100644 --- a/crates/trusted-server-core/src/platform/mod.rs +++ b/crates/trusted-server-core/src/platform/mod.rs @@ -38,6 +38,7 @@ use std::time::Duration; mod backend_naming; +mod cache_headers; mod error; mod geo; mod http; @@ -66,6 +67,7 @@ pub use backend_naming::{ AuctionTargetCapabilities, AuctionTargetDescriptor, AuctionTargetId, BackendNamingError, BackendNamingPolicy, PredictedBackend, }; +pub use cache_headers::{declare_platform_cache_headers, platform_cache_headers}; pub use edgezero_core::key_value_store::{KvError, KvHandle, KvStore as PlatformKvStore}; pub use error::PlatformError; pub use geo::{DisabledGeo, PlatformGeoModule}; diff --git a/crates/trusted-server-core/src/provider_table.rs b/crates/trusted-server-core/src/provider_table.rs index b73e9a39a8..a59cacf8eb 100644 --- a/crates/trusted-server-core/src/provider_table.rs +++ b/crates/trusted-server-core/src/provider_table.rs @@ -537,16 +537,23 @@ fn check_tables( tables: &Map, prefix: &str, ) -> Result<(), String> { + // A module may be written with its type folder or without it, as + // `analytics.tinybird` or `tinybird` in `[analytics]`, and its table is + // keyed by the name without it. + let written: Vec<&str> = selected + .iter() + .map(|name| crate::module_name::short_form(section, name)) + .collect(); for (name, value) in tables { let path = if prefix.is_empty() { name.clone() } else { format!("{prefix}.{name}") }; - if selected.contains(&path) { + if written.contains(&path.as_str()) { continue; } - let leads_to_one = selected + let leads_to_one = written .iter() .any(|written| written.starts_with(&format!("{path}."))); match value { diff --git a/crates/trusted-server-core/src/proxy.rs b/crates/trusted-server-core/src/proxy.rs index 082d81d3cb..7a4c8e480e 100644 --- a/crates/trusted-server-core/src/proxy.rs +++ b/crates/trusted-server-core/src/proxy.rs @@ -15,7 +15,7 @@ use crate::asset_cache::{AssetAnswer, AssetCacheModule, AssetFile, AssetVia, Ori use crate::cache_policy::{ CachePolicy, EdgeCacheHeader, NO_STORE_PRIVATE_CACHE_CONTROL, apply_no_store_private_to_headers, cache_control_headers_are_private_or_no_store, - remove_edge_cache_headers, + remove_owned_cache_headers, }; use crate::constants::{ HEADER_ACCEPT, HEADER_ACCEPT_ENCODING, HEADER_ACCEPT_LANGUAGE, HEADER_REFERER, @@ -150,9 +150,9 @@ impl AssetProxyCachePolicy { Self::NoStorePrivate => apply_no_store_cache_control(response), Self::Normalized(policy) => { if cache_control_headers_are_private_or_no_store(response.headers()) { - remove_edge_cache_headers(response.headers_mut()); + remove_owned_cache_headers(response); } else { - policy.apply_to_headers(response.headers_mut(), edge_header); + policy.apply_to_response(response, edge_header); } } } @@ -6019,7 +6019,7 @@ mod tests { .get("surrogate-control") .and_then(|value| value.to_str().ok()), Some("max-age=31536000"), - "Fastly finalization should render Surrogate-Control" + "finalization should render Surrogate-Control" ); }); } @@ -6031,7 +6031,6 @@ mod tests { .header(header::CACHE_CONTROL, cache_control) .header("surrogate-control", "max-age=31536000") .header("cdn-cache-control", "max-age=31536000") - .header("cloudflare-cdn-cache-control", "max-age=31536000") .body(EdgeBody::empty()) .expect("should build asset response"); @@ -6049,13 +6048,9 @@ mod tests { "final privacy directive should veto normalized cache policy" ); assert!( - [ - "surrogate-control", - "cdn-cache-control", - "cloudflare-cdn-cache-control", - ] - .iter() - .all(|name| !response.headers().contains_key(*name)), + ["surrogate-control", "cdn-cache-control"] + .iter() + .all(|name| !response.headers().contains_key(*name)), "final privacy directive should remove every edge-cache header" ); } diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 949f2c43a3..2358035d5e 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -602,7 +602,7 @@ fn serve_tsjs_static( ); if request_version_hash(req).is_some_and(|hash| hash == expected_hash) { CachePolicy::public_immutable(Duration::from_secs(31_536_000)) - .apply_to_headers(response.headers_mut(), edge_header); + .apply_to_response(&mut response, edge_header); } response .headers_mut() @@ -2247,7 +2247,7 @@ fn apply_publisher_asset_cache_policy( } if let Some(policy) = settings.asset_cache_policy_for_path(path)? { - policy.apply_to_headers(response.headers_mut(), edge_header); + policy.apply_to_response(response, edge_header); } Ok(()) @@ -7617,10 +7617,10 @@ fn parse_delta_seconds(value: &str) -> Result { .map_err(|_| KeepRefusal::MalformedPolicy) } -/// Parse the Fastly-specific freshness policy used by template cache's hosting platform. +/// Parse the `Surrogate-Control` freshness policy the template cache's hosting +/// platform reads. /// -/// Fastly documents `max-age`, `stale-while-revalidate`, and `stale-if-error` for -/// `Surrogate-Control`. Template cache uses only `max-age` as fresh lifetime; the stale windows +/// The header carries `max-age`, `stale-while-revalidate`, and `stale-if-error`. Template cache uses only `max-age` as fresh lifetime; the stale windows /// are validated so malformed policy cannot hide beside a valid max age, but Core /// Cache assembly does not serve stale templates under either extension. /// @@ -7762,8 +7762,9 @@ fn origin_shared_ttl_at( }) .transpose()?, }; - // Fastly gives Surrogate-Control precedence for its edge cache. Standard - // restrictive directives were still parsed above and remain hard refusals. + // A platform reading Surrogate-Control gives it precedence for its edge + // cache. Standard restrictive directives were still parsed above and + // remain hard refusals. let freshness = surrogate_control_freshness(headers)? .or(standard_freshness) .ok_or(KeepRefusal::NoLifetime)?; @@ -7865,13 +7866,16 @@ pub fn origin_response_may_be_kept( if headers.contains_key(header::SET_COOKIE) { return Err(KeepRefusal::SetsCookie); } - // Core Cache has no HTTP semantics. Fastly's documented Surrogate-Control subset - // is parsed by `origin_shared_ttl`; every other vendor-specific policy remains a - // bypass rather than guessing that unrelated CDNs share its grammar or precedence. + // Core Cache has no HTTP semantics. The Surrogate-Control subset + // `origin_shared_ttl` parses is read; a policy in any other shared cache + // header, the ones the platform's adapter declared included, is a bypass + // rather than a guess at its grammar or precedence. if crate::cache_policy::EDGE_CACHE_HEADER_NAMES .iter() - .filter(|name| **name != "surrogate-control") - .any(|name| headers.contains_key(*name)) + .copied() + .filter(|name| *name != crate::cache_policy::HEADER_SURROGATE_CONTROL_NAME) + .chain(crate::platform::platform_cache_headers()) + .any(|name| headers.contains_key(name)) { return Err(KeepRefusal::NotShareable); } @@ -18470,7 +18474,12 @@ mod tests { #[test] fn cdn_specific_cache_policy_cannot_be_overridden_by_public_cache_control() { - for name in crate::cache_policy::EDGE_CACHE_HEADER_NAMES { + crate::platform::declare_platform_cache_headers(&["example-platform-cache-control"]); + for name in crate::cache_policy::EDGE_CACHE_HEADER_NAMES + .iter() + .copied() + .chain(["example-platform-cache-control"]) + { let mut split = shareable(); split.insert( header::HeaderName::from_static(name), @@ -18495,9 +18504,12 @@ mod tests { #[test] fn unsupported_vendor_freshness_does_not_authorize_template_cache() { + crate::platform::declare_platform_cache_headers(&["example-platform-cache-control"]); for name in crate::cache_policy::EDGE_CACHE_HEADER_NAMES .iter() - .filter(|name| **name != "surrogate-control") + .copied() + .filter(|name| *name != "surrogate-control") + .chain(["example-platform-cache-control"]) { let mut split = shareable(); split.insert( @@ -18516,7 +18528,7 @@ mod tests { ¬hing_covered(), ), Some(TemplateCacheBypassReason::OriginNotShareable), - "the Fastly exception must not authorize the vendor policy {name}" + "the Surrogate-Control exception must not authorize the policy in {name}" ); } } @@ -19467,7 +19479,7 @@ mod tests { .get("surrogate-control") .and_then(|value| value.to_str().ok()), Some("max-age=31536000"), - "matched publisher asset should receive Fastly edge policy" + "matched publisher asset should receive the edge policy" ); } @@ -20117,9 +20129,7 @@ mod tests { ("etag", ORIGIN_ETAG), ("last-modified", ORIGIN_LAST_MODIFIED), ("surrogate-control", "max-age=300"), - ("fastly-surrogate-control", "max-age=300"), ("cdn-cache-control", "max-age=300"), - ("cloudflare-cdn-cache-control", "max-age=300"), ], ); } @@ -20455,9 +20465,7 @@ mod tests { header::ETAG, header::LAST_MODIFIED, header::HeaderName::from_static("surrogate-control"), - header::HeaderName::from_static("fastly-surrogate-control"), header::HeaderName::from_static("cdn-cache-control"), - header::HeaderName::from_static("cloudflare-cdn-cache-control"), ] { assert!( !response_head.headers.contains_key(&header_name), @@ -20562,18 +20570,10 @@ mod tests { header::HeaderName::from_static("surrogate-control"), "max-age=300", ), - ( - header::HeaderName::from_static("fastly-surrogate-control"), - "max-age=300", - ), ( header::HeaderName::from_static("cdn-cache-control"), "max-age=300", ), - ( - header::HeaderName::from_static("cloudflare-cdn-cache-control"), - "max-age=300", - ), ] { assert_eq!( response_head @@ -20812,18 +20812,10 @@ mod tests { header::HeaderName::from_static("surrogate-control"), "max-age=300", ), - ( - header::HeaderName::from_static("fastly-surrogate-control"), - "max-age=300", - ), ( header::HeaderName::from_static("cdn-cache-control"), "max-age=300", ), - ( - header::HeaderName::from_static("cloudflare-cdn-cache-control"), - "max-age=300", - ), ] { assert_eq!( response_head @@ -20931,18 +20923,10 @@ mod tests { header::HeaderName::from_static("surrogate-control"), "max-age=300", ), - ( - header::HeaderName::from_static("fastly-surrogate-control"), - "max-age=300", - ), ( header::HeaderName::from_static("cdn-cache-control"), "max-age=300", ), - ( - header::HeaderName::from_static("cloudflare-cdn-cache-control"), - "max-age=300", - ), ] { assert_eq!( response_head @@ -21169,7 +21153,6 @@ mod tests { ("etag", ORIGIN_ETAG), ("last-modified", ORIGIN_LAST_MODIFIED), ("surrogate-control", "max-age=300"), - ("fastly-surrogate-control", "max-age=300"), ]; if let Some(content_type) = content_type { origin_headers.push(("content-type", content_type)); @@ -21224,7 +21207,6 @@ mod tests { header::ETAG, header::LAST_MODIFIED, header::HeaderName::from_static("surrogate-control"), - header::HeaderName::from_static("fastly-surrogate-control"), ] { assert!( !response.headers().contains_key(&header_name), @@ -21319,7 +21301,6 @@ mod tests { ("etag", ORIGIN_ETAG), ("last-modified", ORIGIN_LAST_MODIFIED), ("surrogate-control", "max-age=300"), - ("fastly-surrogate-control", "max-age=300"), ], ); let services = build_services_with_http_client( @@ -21359,10 +21340,6 @@ mod tests { header::HeaderName::from_static("surrogate-control"), "max-age=300", ), - ( - header::HeaderName::from_static("fastly-surrogate-control"), - "max-age=300", - ), ] { assert_eq!( response @@ -21858,8 +21835,6 @@ mod tests { .status(StatusCode::OK) .header(header::CACHE_CONTROL, "public, max-age=600") .header("surrogate-control", "max-age=600") - .header("fastly-surrogate-control", "max-age=600") - .header("cloudflare-cdn-cache-control", "max-age=600") .header("cdn-cache-control", "max-age=600") .header(header::ETAG, "\"origin-tag\"") .header(header::LAST_MODIFIED, "Wed, 21 Oct 2015 07:28:00 GMT") @@ -21885,17 +21860,6 @@ mod tests { response.headers().get("surrogate-control").is_none(), "personalized HTML should not retain Surrogate-Control" ); - assert!( - response.headers().get("fastly-surrogate-control").is_none(), - "personalized HTML should not retain Fastly-Surrogate-Control" - ); - assert!( - response - .headers() - .get("cloudflare-cdn-cache-control") - .is_none(), - "personalized HTML should not retain Cloudflare-CDN-Cache-Control" - ); assert!( response.headers().get("cdn-cache-control").is_none(), "personalized HTML should not retain CDN-Cache-Control" @@ -21974,8 +21938,6 @@ mod tests { .status(StatusCode::OK) .header(header::CACHE_CONTROL, "public, max-age=600") .header("surrogate-control", "max-age=600") - .header("fastly-surrogate-control", "max-age=600") - .header("cloudflare-cdn-cache-control", "max-age=600") .header("cdn-cache-control", "max-age=600") .header(header::ETAG, "\"origin-tag\"") .header(header::LAST_MODIFIED, "Wed, 21 Oct 2015 07:28:00 GMT") @@ -21997,12 +21959,7 @@ mod tests { Some("no-store, private"), "a personalized response should be private and non-storable" ); - for header_name in [ - "surrogate-control", - "fastly-surrogate-control", - "cloudflare-cdn-cache-control", - "cdn-cache-control", - ] { + for header_name in ["surrogate-control", "cdn-cache-control"] { assert!( response.headers().get(header_name).is_none(), "a personalized response should not retain {header_name}" @@ -23266,12 +23223,12 @@ mod tests { .get("surrogate-control") .and_then(|value| value.to_str().ok()), Some("max-age=31536000"), - "matching content-versioned bundle should set the Fastly edge TTL" + "matching content-versioned bundle should set the edge TTL" ); } #[test] - fn tsjs_dynamic_uses_cloudflare_edge_header_when_selected() { + fn tsjs_dynamic_uses_the_platform_edge_header_when_selected() { let settings = create_test_settings(); let registry = IntegrationRegistry::new(&settings).expect("should create integration registry"); @@ -23285,21 +23242,21 @@ mod tests { let response = handle_tsjs_dynamic( &request, ®istry, - EdgeCacheHeader::CloudflareCdnCacheControl, + EdgeCacheHeader::Platform("example-cdn-cache-control"), ) .expect("should handle tsjs request"); assert_eq!( response .headers() - .get("cloudflare-cdn-cache-control") + .get("example-cdn-cache-control") .and_then(|value| value.to_str().ok()), Some("max-age=31536000"), - "Cloudflare requests should use its edge cache header" + "the platform's own edge cache header should carry the policy" ); assert!( response.headers().get("surrogate-control").is_none(), - "Cloudflare requests should not emit Fastly's edge cache header" + "no other edge cache header should be written" ); } @@ -25238,7 +25195,7 @@ mod tests { .get("surrogate-control") .and_then(|value| value.to_str().ok()), Some("max-age=300, stale-while-revalidate=60, stale-if-error=86400"), - "mismatched hash should retain the short Fastly edge TTL" + "mismatched hash should retain the short edge TTL" ); } diff --git a/crates/trusted-server-core/src/response_privacy.rs b/crates/trusted-server-core/src/response_privacy.rs index fc7105b2ef..1c1456eb11 100644 --- a/crates/trusted-server-core/src/response_privacy.rs +++ b/crates/trusted-server-core/src/response_privacy.rs @@ -5,15 +5,15 @@ //! never reach a shared cache. Each adapter's `apply_finalize_headers` applies //! operator-configured `settings.response_headers`, so the cookie-privacy //! downgrade and the uncacheable-operator-header guard have to live in one place -//! and run byte-identically on Fastly, Cloudflare, Axum, and Spin — a shared -//! cache such as Cloudflare would otherwise serve an operator/origin -//! `Cache-Control: public` on a cookie-bearing response as-is. +//! and run byte-identically on every adapter, because a shared cache would +//! otherwise serve an operator/origin `Cache-Control: public` on a +//! cookie-bearing response as-is. use edgezero_core::http::{HeaderMap, HeaderName, HeaderValue, Response, header}; use crate::cache_policy::{ CacheControlPolicy, EdgeCacheHeader, cache_control_headers_are_private_or_no_store, - is_edge_cache_header_name, remove_edge_cache_headers, + is_owned_cache_header_name, remove_owned_cache_headers, }; use crate::settings::Settings; @@ -78,12 +78,13 @@ pub fn apply_inactive_ad_stack_browser_cache_policy(response: &mut Response) { /// Reassert the terminal privacy invariant for a synthesized per-reader response. /// /// Call this after every configurable response mutation. It deliberately overwrites -/// `Cache-Control` and strips validators, expiry metadata, and runtime edge-cache -/// directives so a later integration cannot turn an assembled document into something a -/// shared cache would store. +/// `Cache-Control` and strips validators, expiry metadata, and the shared cache +/// headers core owns so a later integration cannot turn an assembled document into +/// something a shared cache would store. pub fn enforce_private_no_store(response: &mut Response) { CacheControlPolicy::NoStorePrivate .apply_to_headers(response.headers_mut(), EdgeCacheHeader::None); + remove_owned_cache_headers(response); for name in [ header::ETAG.as_str(), header::LAST_MODIFIED.as_str(), @@ -112,14 +113,15 @@ pub(crate) fn enforce_synthesized_html_cache_privacy(response: &mut Response) { enforce_terminal_private_cache_privacy(response); } -/// Removes runtime edge-cache headers from a response finalized as uncacheable. +/// Removes the shared cache headers core owns from a response finalized as +/// uncacheable. /// /// Call this after any late response-header mutations so a final `private` or /// `no-store` directive cannot coexist with an independently authoritative edge /// cache header. pub fn enforce_uncacheable_cache_privacy(response: &mut Response) { if cache_control_is_private_or_no_store(response) { - remove_edge_cache_headers(response.headers_mut()); + remove_owned_cache_headers(response); } } @@ -130,18 +132,18 @@ pub fn enforce_uncacheable_cache_privacy(response: &mut Response) { /// `Set-Cookie` to others. /// /// Origin `private`/`no-store` policies remain unchanged, but the generated -/// inactive-stack browser policy is downgraded to `private, max-age=0`. CDN -/// cache headers are always stripped so a cookie response cannot retain shared -/// cacheability. +/// inactive-stack browser policy is downgraded to `private, max-age=0`. The +/// shared cache headers core owns are always stripped so a cookie response +/// cannot retain shared cacheability. pub fn enforce_set_cookie_cache_privacy(response: &mut Response) { if !response.headers().contains_key(header::SET_COOKIE) { return; } - // Edge-cache headers must come off every cookie-bearing response, even one - // already carrying a stricter `no-store`/`private` directive — they are - // independent of Cache-Control and would otherwise let a shared cache store - // and replay one visitor's Set-Cookie. - remove_edge_cache_headers(response.headers_mut()); + // The shared cache headers core owns must come off every cookie-bearing + // response, even one already carrying a stricter `no-store`/`private` + // directive, because they are independent of Cache-Control and would + // otherwise let a shared cache store and replay one visitor's Set-Cookie. + remove_owned_cache_headers(response); // Cookie privacy takes precedence over the generated inactive-stack browser // policy, while unrelated origin private/no-store policies remain unchanged. let already_forbids_shared_storage = cache_control_forbids_shared_storage(response.headers()); @@ -168,11 +170,11 @@ pub struct RequiredHeaders(pub HeaderMap); /// hardening. /// /// First downgrades cookie-bearing responses via -/// [`enforce_set_cookie_cache_privacy`], then applies operator headers — but on +/// [`enforce_set_cookie_cache_privacy`], then applies operator headers, but on /// an uncacheable (`private`/`no-store`) response the cache-controlling headers -/// (`Cache-Control` and runtime edge-cache headers) are skipped so operators -/// cannot re-enable shared caching for per-user payloads. After the operator -/// headers are applied the cookie-privacy downgrade runs once more, so a +/// (`Cache-Control` and the shared cache headers core owns) are skipped so +/// operators cannot re-enable shared caching for per-user payloads. After the +/// operator headers are applied the cookie-privacy downgrade runs once more, so a /// configured `Set-Cookie` combined with public edge-cache headers cannot /// produce a shared-cacheable cookie-bearing response. /// @@ -187,7 +189,7 @@ pub fn apply_response_headers_with_cache_privacy(settings: &Settings, response: for (key, value) in &settings.response_headers { if response_is_uncacheable && (key.eq_ignore_ascii_case(header::CACHE_CONTROL.as_str()) - || is_edge_cache_header_name(key)) + || is_owned_cache_header_name(response, key)) { continue; } @@ -229,7 +231,7 @@ mod tests { use edgezero_core::http::response_builder; - use crate::cache_policy::EDGE_CACHE_HEADER_NAMES; + use crate::cache_policy::{CachePolicy, EDGE_CACHE_HEADER_NAMES}; fn settings_with_response_headers(headers: &[(&str, &str)]) -> Settings { let mut s = Settings::from_toml( @@ -270,9 +272,7 @@ mod tests { .header(header::ETAG, "\"origin\"") .header(header::LAST_MODIFIED, "Wed, 21 Oct 2015 07:28:00 GMT") .header("surrogate-control", "max-age=600") - .header("fastly-surrogate-control", "max-age=600") .header("cdn-cache-control", "max-age=600") - .header("cloudflare-cdn-cache-control", "max-age=600") .body(edgezero_core::body::Body::empty()) .expect("should build response"); @@ -385,7 +385,6 @@ mod tests { .header(header::SET_COOKIE, "id=abc") .header("surrogate-control", "max-age=600") .header("cdn-cache-control", "max-age=600") - .header("cloudflare-cdn-cache-control", "max-age=600") .body(edgezero_core::body::Body::empty()) .expect("should build response"); @@ -401,10 +400,7 @@ mod tests { ); assert!( !response.headers().contains_key("surrogate-control") - && !response.headers().contains_key("cdn-cache-control") - && !response - .headers() - .contains_key("cloudflare-cdn-cache-control"), + && !response.headers().contains_key("cdn-cache-control"), "edge cache headers must be stripped on cookie responses" ); } @@ -418,7 +414,6 @@ mod tests { ("cache-control", "public, max-age=600"), ("surrogate-control", "max-age=600"), ("cdn-cache-control", "max-age=600"), - ("cloudflare-cdn-cache-control", "max-age=600"), ]); let mut response = response_builder() .body(edgezero_core::body::Body::empty()) @@ -436,10 +431,7 @@ mod tests { ); assert!( !response.headers().contains_key("surrogate-control") - && !response.headers().contains_key("cdn-cache-control") - && !response - .headers() - .contains_key("cloudflare-cdn-cache-control"), + && !response.headers().contains_key("cdn-cache-control"), "edge cache headers must be stripped when operator headers add Set-Cookie" ); assert!( @@ -511,9 +503,7 @@ mod tests { let settings = settings_with_response_headers(&[ ("cache-control", "public, max-age=600"), ("surrogate-control", "max-age=600"), - ("fastly-surrogate-control", "max-age=600"), ("cdn-cache-control", "public, max-age=600"), - ("cloudflare-cdn-cache-control", "public, max-age=600"), ]); let mut response = response_builder() .header(header::CACHE_CONTROL, "private, no-store") @@ -537,15 +527,11 @@ mod tests { #[test] fn strips_edge_headers_from_uncacheable_cookieless_response() { - let settings = settings_with_response_headers(&[ - ("cdn-cache-control", "max-age=600"), - ("cloudflare-cdn-cache-control", "max-age=600"), - ]); + let settings = settings_with_response_headers(&[("cdn-cache-control", "max-age=600")]); let mut response = response_builder() .header(header::CACHE_CONTROL, "private, max-age=0") .header("surrogate-control", "max-age=600") .header("cdn-cache-control", "max-age=600") - .header("cloudflare-cdn-cache-control", "max-age=600") .body(edgezero_core::body::Body::empty()) .expect("should build response"); @@ -553,10 +539,7 @@ mod tests { assert!( !response.headers().contains_key("surrogate-control") - && !response.headers().contains_key("cdn-cache-control") - && !response - .headers() - .contains_key("cloudflare-cdn-cache-control"), + && !response.headers().contains_key("cdn-cache-control"), "uncacheable responses must not retain or receive edge-cache headers" ); } @@ -567,7 +550,6 @@ mod tests { .header(header::CACHE_CONTROL, "no-store") .header("surrogate-control", "max-age=600") .header("cdn-cache-control", "max-age=600") - .header("cloudflare-cdn-cache-control", "max-age=600") .body(edgezero_core::body::Body::empty()) .expect("should build response"); @@ -589,6 +571,61 @@ mod tests { ); } + #[test] + fn a_platform_header_core_did_not_write_is_left_on_a_cookie_response() { + crate::platform::declare_platform_cache_headers(&[ + "example-cdn-cache-control", + "example-operator-cache-control", + ]); + let settings = + settings_with_response_headers(&[("example-operator-cache-control", "max-age=600")]); + let mut response = response_builder() + .header(header::CACHE_CONTROL, "private, max-age=0") + .header(header::SET_COOKIE, "id=abc") + .header("example-cdn-cache-control", "max-age=600") + .body(edgezero_core::body::Body::empty()) + .expect("should build response"); + + apply_response_headers_with_cache_privacy(&settings, &mut response); + + for name in [ + "example-cdn-cache-control", + "example-operator-cache-control", + ] { + assert_eq!( + response + .headers() + .get(name) + .and_then(|value| value.to_str().ok()), + Some("max-age=600"), + "a platform header core did not write should be left as written: {name}" + ); + } + } + + #[test] + fn a_policy_core_wrote_in_a_platform_header_leaves_a_cookie_response() { + let settings = + settings_with_response_headers(&[("example-written-cache-control", "max-age=600")]); + let mut response = response_builder() + .header(header::SET_COOKIE, "id=abc") + .body(edgezero_core::body::Body::empty()) + .expect("should build response"); + CachePolicy::public_immutable(std::time::Duration::from_secs(600)).apply_to_response( + &mut response, + EdgeCacheHeader::Platform("example-written-cache-control"), + ); + + apply_response_headers_with_cache_privacy(&settings, &mut response); + + assert!( + !response + .headers() + .contains_key("example-written-cache-control"), + "the policy core wrote should come off a cookie response and stay off" + ); + } + #[test] fn terminal_private_stamp_removes_every_cache_and_validator_header() { let mut response = response_builder() diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 7c69d9f3bc..a7e08422f9 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -2987,257 +2987,6 @@ impl Proxy { } } -/// The analytics module's name in full, which is also the table holding its -/// settings and which every message about them names. -const ANALYTICS_MODULE: &str = "analytics.tinybird"; - -/// Direct Tinybird Events API telemetry configuration, the settings of the -/// module `[analytics]` selects. -#[derive(Debug, Clone, Deserialize, Serialize)] -#[serde(deny_unknown_fields)] -pub struct TinybirdSettings { - /// Regional Tinybird API host, without scheme or path. - #[serde(default)] - pub api_host: String, - /// Deprecated feature-specific store selector accepted for migration only. - #[serde(default, skip_serializing)] - pub secret_store: Option, - /// Auction Events API datasource name. - #[serde(default = "default_tinybird_auction_dataset")] - pub auction_dataset: String, - /// Secret reference containing the auction datasource APPEND token. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub auction_token_secret: Option>, - /// Reserved for future access-log telemetry. - /// - /// `true` is rejected until an access-log emitter is wired, so operators - /// cannot enable a setting that silently emits nothing. - #[serde(default)] - pub access_enabled: bool, - /// Future access-log Events API datasource name. - #[serde(default = "default_tinybird_access_dataset")] - pub access_dataset: String, - /// Deprecated placeholder for the unwired access-log APPEND token. - #[serde(default, skip_serializing)] - pub access_token_secret: Option>, - /// Future fraction of requests to emit for optional access telemetry. - #[serde(default)] - pub access_sample_rate: f64, - /// Defensive maximum NDJSON body size for one Events API request. - #[serde(default = "default_tinybird_max_body_bytes")] - pub max_body_bytes: usize, -} - -fn default_tinybird_auction_dataset() -> String { - "auction_events_raw".to_owned() -} - -fn default_tinybird_access_dataset() -> String { - "access_logs_raw".to_owned() -} - -fn default_tinybird_max_body_bytes() -> usize { - 1024 * 1024 -} - -impl Default for TinybirdSettings { - fn default() -> Self { - Self { - api_host: String::new(), - secret_store: None, - auction_dataset: default_tinybird_auction_dataset(), - auction_token_secret: None, - access_enabled: false, - access_dataset: default_tinybird_access_dataset(), - access_token_secret: None, - access_sample_rate: 0.0, - max_body_bytes: default_tinybird_max_body_bytes(), - } - } -} - -impl TinybirdSettings { - fn normalize(&mut self) { - self.api_host = self.api_host.trim().to_ascii_lowercase(); - if self.secret_store.take().is_some() { - log::warn!( - "{ANALYTICS_MODULE}.secret_store is deprecated and ignored; static credentials resolve through the default app-config secret store" - ); - } - self.auction_dataset = self.auction_dataset.trim().to_owned(); - self.auction_token_secret = self.auction_token_secret.take().and_then(|value| { - let value = value.expose().trim().to_owned(); - (!value.is_empty()).then(|| Redacted::new(value)) - }); - self.access_dataset = self.access_dataset.trim().to_owned(); - self.access_token_secret = None; - } - - fn prepare_runtime(&mut self) -> Result<(), Report> { - self.normalize(); - if !(0.0..=1.0).contains(&self.access_sample_rate) { - return Err(Report::new(TrustedServerError::Configuration { - message: format!( - "{ANALYTICS_MODULE}.access_sample_rate must be between 0.0 and 1.0" - ), - })); - } - if self.max_body_bytes < 1024 { - return Err(Report::new(TrustedServerError::Configuration { - message: format!("{ANALYTICS_MODULE}.max_body_bytes must be at least 1024"), - })); - } - if self.access_enabled { - return Err(Report::new(TrustedServerError::Configuration { - message: format!( - "{ANALYTICS_MODULE}.access_enabled is reserved for future access-log telemetry; no emitter is currently wired" - ), - })); - } - validate_tinybird_api_host(&self.api_host)?; - validate_tinybird_dataset( - &self.auction_dataset, - &format!("{ANALYTICS_MODULE}.auction_dataset"), - )?; - let token = self.auction_token_secret.as_ref().ok_or_else(|| { - Report::new(TrustedServerError::Configuration { - message: format!( - "{ANALYTICS_MODULE}.auction_token_secret is required when [analytics] selects the module" - ), - }) - })?; - validate_tinybird_secret( - token.expose(), - &format!("{ANALYTICS_MODULE}.auction_token_secret"), - ) - } -} - -fn validate_tinybird_api_host(host: &str) -> Result<(), Report> { - if host.is_empty() - || host.contains('/') - || host.contains(':') - || host.chars().any(char::is_control) - || host.starts_with("http://") - || host.starts_with("https://") - { - return Err(Report::new(TrustedServerError::Configuration { - message: format!( - "{ANALYTICS_MODULE}.api_host must be a regional host without scheme, port, or path" - ), - })); - } - validate_host_header_override_value(host).map_err(|reason| { - Report::new(TrustedServerError::Configuration { - message: format!("{ANALYTICS_MODULE}.api_host {reason}"), - }) - }) -} - -fn validate_tinybird_dataset(value: &str, setting: &str) -> Result<(), Report> { - if value.is_empty() - || value.len() > 128 - || !value - .chars() - .all(|ch| ch.is_ascii_alphanumeric() || ch == '_') - { - return Err(Report::new(TrustedServerError::Configuration { - message: format!("{setting} must be a non-empty datasource identifier"), - })); - } - Ok(()) -} - -fn validate_tinybird_secret(value: &str, setting: &str) -> Result<(), Report> { - if value.is_empty() || value.chars().any(char::is_control) { - return Err(Report::new(TrustedServerError::Configuration { - message: format!("{setting} must be non-empty after secret resolution"), - })); - } - Ok(()) -} - -/// The `[analytics]` section, selecting the module that sends auction telemetry. -#[derive(Debug, Clone, Default, Deserialize, Serialize)] -#[serde(deny_unknown_fields)] -pub struct AnalyticsConfig { - /// The module that runs. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub module: Option, - /// The analytics module's settings. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub tinybird: Option, -} - -/// Whether `written`, selected in `[analytics]`, names the analytics module, -/// in full or by its name within the section. -fn names_analytics_module(written: &str) -> bool { - written == ANALYTICS_MODULE - || written == crate::module_name::short_form("analytics", ANALYTICS_MODULE) -} - -/// Whether the document `data` selects the analytics module in `[analytics]`, -/// read before the settings are, to tell which secret references are live. -pub(crate) fn document_selects_analytics_module(data: &JsonValue) -> bool { - data.pointer("/analytics/module") - .and_then(JsonValue::as_str) - .is_some_and(names_analytics_module) -} - -impl AnalyticsConfig { - /// Whether `module` selects the analytics module. - fn selects_analytics_module(&self) -> bool { - self.module.as_deref().is_some_and(names_analytics_module) - } - - /// The analytics module's settings, when `module` selects it. - #[must_use] - pub fn analytics_module(&self) -> Option<&TinybirdSettings> { - if self.selects_analytics_module() { - self.tinybird.as_ref() - } else { - None - } - } - - fn normalize(&mut self) { - if let Some(settings) = &mut self.tinybird { - settings.normalize(); - } - } - - /// Refuses a section that selects nothing or a module this build does - /// not have, and checks the selected module's settings. - fn prepare_runtime(&mut self) -> Result<(), Report> { - let short = crate::module_name::short_form("analytics", ANALYTICS_MODULE); - match self.module.as_deref() { - None => { - return Err(Report::new(TrustedServerError::Configuration { - message: "[analytics] selects no module. A module type's section names \ - what runs with `module` or `modules`, so select one or remove \ - the section" - .to_owned(), - })); - } - Some(written) if !self.selects_analytics_module() => { - return Err(Report::new(TrustedServerError::Configuration { - message: format!( - "[analytics] selects `{written}`, which no module this deployment \ - supplies. The analytics modules it supplies are [{short}]" - ), - })); - } - Some(_) => {} - } - match &mut self.tinybird { - Some(settings) => settings.prepare_runtime(), - // Checking the defaults reports the first setting the module - // needs and the table does not have. - None => TinybirdSettings::default().prepare_runtime(), - } - } -} - /// Cache behavior configuration. #[derive(Debug, Default, Clone, Deserialize, Serialize)] pub struct CacheSettings { @@ -4119,13 +3868,6 @@ refused_table! { `--secret-store-id`" } -refused_table! { - /// The `[tinybird]` table, now `[analytics.tinybird]`. - MovedTinybirdTable => "Configuration table `[tinybird]` is now `[analytics.tinybird]`. \ - Select it with `[analytics] module = \"tinybird\"`, which replaces its `enabled` \ - line, and move its other settings there unchanged" -} - refused_table! { /// The `[permission_signal]` table, renamed `[permission-signal]`. RenamedPermissionSignalTable => "Configuration table `[permission_signal]` is now \ @@ -4534,19 +4276,6 @@ pub struct Settings { pub creative_opportunities: Option, #[serde(default)] pub image_optimizer: ImageOptimizerSettings, - /// Analytics. `[analytics] module` selects the module that runs, with its - /// settings in the table at its name. Omitted from a serialized - /// configuration when absent, for the reason given on `trusted_client_ip`. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub analytics: Option, - /// The `[tinybird]` table, kept so a configuration carrying it is told - /// where it moved. - #[serde(rename = "tinybird", default, skip_serializing)] - #[allow( - dead_code, - reason = "the field exists so that reading the moved table fails with directions" - )] - moved_tinybird: MovedTinybirdTable, #[serde(default)] pub debug: DebugConfig, #[serde(default, skip_serializing_if = "is_default_device_config")] @@ -4742,14 +4471,6 @@ impl Settings { Self::finalize_deserialized(settings, "Build-time configuration") } - /// The analytics module's settings, when `[analytics]` selects it. - #[must_use] - pub fn analytics_module(&self) -> Option<&TinybirdSettings> { - self.analytics - .as_ref() - .and_then(AnalyticsConfig::analytics_module) - } - /// Normalize settings after deserialization. /// /// # Errors @@ -4760,9 +4481,6 @@ impl Settings { self.proxy.normalize(); self.image_optimizer.normalize()?; self.debug.auction_html_comment_options.normalize(); - if let Some(analytics) = &mut self.analytics { - analytics.normalize(); - } Ok(()) } @@ -4826,9 +4544,6 @@ impl Settings { self.cache.prepare_runtime()?; crate::source::check_selection(&self.source, crate::source::SOURCE_SECTION)?; self.proxy.prepare_runtime()?; - if let Some(analytics) = &mut self.analytics { - analytics.prepare_runtime()?; - } self.debug .auction_html_comment_options .validate_metadata_keys()?; @@ -5124,9 +4839,12 @@ impl Settings { where T: IntegrationConfig, { - let Some((section, written)) = self.module_selection(name) else { + let Some((section, selected)) = self.module_selection(name) else { return Ok(None); }; + // The table is keyed by the name without its type folder, however the + // section's selector wrote it. + let written = crate::module_name::short_form(section, selected); let table = self .module_sections() .find(|(candidate, _)| *candidate == section) @@ -5880,8 +5598,6 @@ mod tests { #[serde(default)] image_optimizer: serde::de::IgnoredAny, #[serde(default)] - tinybird: serde::de::IgnoredAny, - #[serde(default)] debug: serde::de::IgnoredAny, } @@ -6014,9 +5730,8 @@ module = \"none\"", // hand-written `Debug` impl prints only integration IDs, never values. // // Do not use `..Struct::default()` anywhere in this function. A default - // spread would let a new secret field be added to `EcPartner`, - // `TinybirdSettings`, or any other struct built here without forcing - // anyone to consider it. The compile break is the prompt; the canary + // spread would let a new secret field be added to `EcPartner` or any + // other struct built here without forcing anyone to consider it. The compile break is the prompt; the canary // list below is still maintained by hand. List every field explicitly. #[test] fn settings_debug_output_redacts_every_secret_field() { @@ -6029,8 +5744,6 @@ module = \"none\"", const CANARY_S3_ACCESS_KEY_ID: &str = "CANARY-S3-ACCESS-KEY-ID-0123456789"; const CANARY_S3_SECRET_ACCESS_KEY: &str = "CANARY-S3-SECRET-ACCESS-KEY-0123456789"; const CANARY_S3_SESSION_TOKEN: &str = "CANARY-S3-SESSION-TOKEN-0123456789"; - const CANARY_TINYBIRD_AUCTION_TOKEN: &str = "CANARY-TINYBIRD-AUCTION-TOKEN-0123456789"; - const CANARY_TINYBIRD_ACCESS_TOKEN: &str = "CANARY-TINYBIRD-ACCESS-TOKEN-0123456789"; const CANARY_MODULE_KEY: &str = "CANARY-MODULE-TABLE-KEY-0123456789"; let mut settings = create_test_settings(); @@ -6073,23 +5786,6 @@ module = \"none\"", }); settings.proxy.asset_routes = vec![asset_route]; - settings.analytics = Some(AnalyticsConfig { - module: Some(ANALYTICS_MODULE.to_string()), - tinybird: Some(TinybirdSettings { - auction_token_secret: Some(Redacted::new( - CANARY_TINYBIRD_AUCTION_TOKEN.to_string(), - )), - access_token_secret: Some(Redacted::new(CANARY_TINYBIRD_ACCESS_TOKEN.to_string())), - api_host: String::new(), - secret_store: None, - auction_dataset: String::new(), - access_enabled: false, - access_dataset: String::new(), - access_sample_rate: 0.0f64, - max_body_bytes: 0, - }), - }); - // A module's table is opaque JSON, and the section's hand-written // `Debug` impl is the only thing keeping a secret a module's table // holds out of this output, so pin it here. @@ -6138,14 +5834,6 @@ module = \"none\"", "proxy.asset_routes[].auth.session_token", CANARY_S3_SESSION_TOKEN, ), - ( - "analytics.tinybird.auction_token_secret", - CANARY_TINYBIRD_AUCTION_TOKEN, - ), - ( - "analytics.tinybird.access_token_secret", - CANARY_TINYBIRD_ACCESS_TOKEN, - ), ("testing.example.key_name", CANARY_MODULE_KEY), ]; @@ -6900,235 +6588,6 @@ module = \"none\"", ); } - /// `[analytics]` selecting the analytics module, with `table` as the - /// lines of its settings table. - fn settings_str_with_analytics_module(table: &str) -> String { - format!( - "{}\n[analytics]\nmodule = \"tinybird\"\n\n[analytics.tinybird]\n{table}\n", - crate_test_settings_str() - ) - } - - /// The lines of a telemetry table that passes every check. - const ANALYTICS_MODULE_LINES: &str = - "api_host = \"api.us-east.aws.tinybird.co\"\nauction_token_secret = \"test-auction-token\""; - - #[test] - fn settings_without_an_analytics_section_select_no_module() { - let settings = Settings::from_toml(&crate_test_settings_str()) - .expect("should parse settings without an analytics section"); - - assert!(settings.analytics.is_none(), "should hold no section"); - assert!( - settings.analytics_module().is_none(), - "should select no module" - ); - let value = serde_json::to_value(&settings).expect("should serialize the settings"); - assert!( - value.get("analytics").is_none() && value.get("tinybird").is_none(), - "should write neither the section nor the table it replaced: {value}" - ); - } - - #[test] - fn the_analytics_module_is_selected_by_its_short_or_full_name() { - for name in ["tinybird", "analytics.tinybird"] { - let toml = format!( - "{}\n[analytics]\nmodule = \"{name}\"\n\n[analytics.tinybird]\n{ANALYTICS_MODULE_LINES}\n", - crate_test_settings_str() - ); - - let settings = Settings::from_toml(&toml).expect("should accept the selection"); - - let telemetry = settings - .analytics_module() - .expect("should select the analytics module"); - assert_eq!(telemetry.api_host, "api.us-east.aws.tinybird.co"); - assert_eq!(telemetry.auction_dataset, "auction_events_raw"); - } - } - - #[test] - fn a_selected_analytics_module_survives_the_stored_form() { - let settings = - Settings::from_toml(&settings_str_with_analytics_module(ANALYTICS_MODULE_LINES)) - .expect("should accept the selection"); - - let stored = serde_json::to_value(&settings).expect("should serialize the settings"); - - assert_eq!(stored["analytics"]["module"], "tinybird"); - assert_eq!( - stored["analytics"]["tinybird"]["api_host"], - "api.us-east.aws.tinybird.co" - ); - assert!( - stored.get("tinybird").is_none(), - "should not write the table that moved: {stored}" - ); - } - - #[test] - fn the_analytics_module_needs_a_regional_host() { - let toml = - settings_str_with_analytics_module("api_host = \"https://api.example.com/path\""); - - let err = Settings::from_toml(&toml).expect_err("should reject invalid api host"); - - assert!( - format!("{err:?}").contains("analytics.tinybird.api_host"), - "should name the setting by its table: {err:?}" - ); - } - - #[test] - fn the_analytics_module_needs_its_token() { - let toml = settings_str_with_analytics_module("api_host = \"api.us-east.aws.tinybird.co\""); - - let err = Settings::from_toml(&toml).expect_err("should require the token"); - - assert!( - format!("{err:?}").contains( - "analytics.tinybird.auction_token_secret is required when [analytics] selects \ - the module" - ), - "should say what is missing and why: {err:?}" - ); - } - - #[test] - fn a_selected_analytics_module_with_no_table_is_told_what_it_needs() { - let toml = format!( - "{}\n[analytics]\nmodule = \"tinybird\"\n", - crate_test_settings_str() - ); - - let err = Settings::from_toml(&toml).expect_err("should need the table's settings"); - - assert!( - format!("{err:?}").contains("analytics.tinybird.api_host"), - "should name the first setting the module needs: {err:?}" - ); - } - - #[test] - fn access_telemetry_is_rejected_until_an_emitter_is_wired() { - let toml = settings_str_with_analytics_module(&format!( - "{ANALYTICS_MODULE_LINES}\naccess_enabled = true" - )); - - let err = Settings::from_toml(&toml) - .expect_err("should reject access telemetry before an emitter exists"); - - assert!( - format!("{err:?}").contains("analytics.tinybird.access_enabled"), - "should name the unsupported setting: {err:?}" - ); - } - - #[test] - fn an_analytics_section_selecting_nothing_is_refused() { - let toml = format!( - "{}\n[analytics.tinybird]\n{ANALYTICS_MODULE_LINES}\n", - crate_test_settings_str() - ); - - let err = Settings::from_toml(&toml).expect_err("nothing would read the table"); - - assert!( - format!("{err:?}").contains("[analytics] selects no module"), - "should say the section selects nothing: {err:?}" - ); - } - - #[test] - fn an_analytics_module_this_build_does_not_have_is_refused() { - let toml = format!( - "{}\n[analytics]\nmodule = \"another\"\n", - crate_test_settings_str() - ); - - let err = Settings::from_toml(&toml).expect_err("no such module"); - let message = format!("{err:?}"); - - assert!( - message.contains( - "[analytics] selects `another`, which no module this deployment supplies" - ), - "should name what was written: {message}" - ); - assert!( - message.contains("The analytics modules it supplies are [tinybird]"), - "should list what can be selected: {message}" - ); - } - - /// The token is a secret reference, and an empty or absent one is refused - /// as the settings load, before any deploy step sees the document. - #[test] - fn an_analytics_token_reference_that_is_empty_or_absent_is_refused_when_the_settings_load() { - for (case, table) in [ - ( - "an empty token", - "api_host = \"api.us-east.aws.tinybird.co\"\nauction_token_secret = \"\"", - ), - ("no token", "api_host = \"api.us-east.aws.tinybird.co\""), - ] { - let err = Settings::from_toml(&settings_str_with_analytics_module(table)) - .err() - .unwrap_or_else(|| panic!("{case}: the settings should not load")); - - assert!( - format!("{err:?}").contains("auction_token_secret"), - "{case}: should name the token: {err:?}" - ); - } - } - - #[test] - fn the_analytics_module_table_refuses_a_key_it_does_not_know() { - let toml = settings_str_with_analytics_module(&format!( - "{ANALYTICS_MODULE_LINES}\nenabled = true" - )); - - let err = Settings::from_toml(&toml).expect_err("the enabled line is gone"); - - assert!( - format!("{err:?}").contains("enabled"), - "should name the key it does not know: {err:?}" - ); - } - - #[test] - fn a_table_the_section_does_not_select_gives_no_analytics_module() { - let analytics = AnalyticsConfig { - module: None, - tinybird: Some(TinybirdSettings::default()), - }; - - assert!( - analytics.analytics_module().is_none(), - "a table nothing selects should give no settings" - ); - } - - #[test] - fn the_tinybird_table_is_refused_with_directions() { - for table in [ - "enabled = true\napi_host = \"api.us-east.aws.tinybird.co\"", - "enabled = false", - ] { - let toml = format!("{}\n[tinybird]\n{table}\n", crate_test_settings_str()); - - let err = Settings::from_toml(&toml).expect_err("the table has moved"); - - assert!( - format!("{err:?}") - .contains("Configuration table `[tinybird]` is now `[analytics.tinybird]`"), - "should say where the table went: {err:?}" - ); - } - } - #[test] fn settings_rejects_removed_consent_store_toml() { let toml = format!( diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 4442942be2..a63a1b3466 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -1918,10 +1918,10 @@ See [Asset Routes](/guide/asset-routes) for request flow, S3 auth details, and I ## Analytics Configuration `[analytics]` selects the module that receives auction telemetry, the way every -other section selects its module. `module = "tinybird"` sends auction events -directly to the Tinybird Events API, with its settings in -`[analytics.tinybird]`. Without the section nothing is sent. Access-log -emission is not wired. +other section selects its module. `module = "tinybird"` selects the module from +`crates/analytics/tinybird`, which sends auction events directly to the +Tinybird Events API, with its settings in `[analytics.tinybird]`. Without the +section nothing is sent. Access-log emission is not wired. ### `[analytics]` @@ -1929,14 +1929,19 @@ emission is not wired. | -------- | ------ | ------- | ------------------------------------------------------------------------------------------------ | | `module` | String | none | Required when the section is present. `tinybird`, or the same name in full, `analytics.tinybird` | -**Refused when the settings load**: a section that selects no module, a module -this build does not have, and a key `[analytics.tinybird]` does not know. +**Refused when the settings load**: a section that selects no module, a table +the section does not select, and a `[tinybird]` table at the top level, which +is read as the section of a module type nothing supplies. -A configuration that still carries `[tinybird]` is refused with directions. -Move its settings to `[analytics.tinybird]` unchanged, and select the module -with `[analytics] module = "tinybird"`, which replaces the `enabled` line. A -stored configuration written by an earlier version carries the `[tinybird]` -table even where telemetry was off, so push the configuration again with this +**Refused when the deployment starts and by `ts config validate`**: a module +this build does not have, a key `[analytics.tinybird]` does not know, and a +table missing a setting the module requires. + +A configuration written before the table moved carries `[tinybird]`. Move its +settings to `[analytics.tinybird]` unchanged, and select the module with +`[analytics] module = "tinybird"`, which replaces the `enabled` line. A stored +configuration written by an earlier version carries the `[tinybird]` table +even where telemetry was off, so push the configuration again with this version. ### `[analytics.tinybird]` @@ -2061,7 +2066,7 @@ On a configured Fastly asset-rehost route, a matching rule is authoritative over the third-party origin's cache defaults, including `no-store`, because Trusted Server owns the rehosted copy. A later Trusted Server or operator-applied `private` or `no-store` directive still vetoes public policy reapplication and -removes shared-cache headers. +removes the shared-cache headers Trusted Server writes. TS-owned validated hash URLs such as `/static/tsjs=...js?v=` use their built-in cache policy and do not require an asset rule. Shared-cache keys for @@ -3410,8 +3415,8 @@ positive shared freshness. `private`, `no-store`, `no-cache`, exhausted or malformed freshness, `Set-Cookie`, `Vary: *`, `Vary: Cookie`, uncovered `Vary` names, response-bound CSP nonces, pass-through or ambiguous authorization, diagnostics sessions, range or conditional requests, positive or malformed -request `max-age`, `min-fresh`, and unsupported CDN-specific cache policy fields -all bypass the template cache. Fastly +request `max-age`, `min-fresh`, and a policy in `CDN-Cache-Control` or in the +running platform's own cache header all bypass the template cache. Fastly `Surrogate-Control` is the narrow exception: the template cache accepts exactly one positive `max-age` plus optional valid `stale-while-revalidate` and `stale-if-error` delta-seconds. Restrictive, duplicated, malformed, or unknown directives fail diff --git a/docs/guide/integrations/didomi.md b/docs/guide/integrations/didomi.md index 0048c4b744..bda3c5860c 100644 --- a/docs/guide/integrations/didomi.md +++ b/docs/guide/integrations/didomi.md @@ -162,8 +162,8 @@ appended to API URLs. The incoming API path, query, supported method, headers, a body continue through the proxy. Every API request bypasses the platform outbound cache. Every API response is -returned with `Cache-Control: private, no-store`; freshness validators and -independent edge-cache headers are removed. +returned with `Cache-Control: private, no-store`; freshness validators, +`Surrogate-Control` and `CDN-Cache-Control` are removed. ## Forwarded data diff --git a/docs/guide/telemetry.md b/docs/guide/telemetry.md index cb55984879..04d8c251a8 100644 --- a/docs/guide/telemetry.md +++ b/docs/guide/telemetry.md @@ -1,9 +1,14 @@ # Auction Telemetry -Trusted Server can emit bounded auction observations directly to Tinybird. -Emission is currently wired only by the Fastly adapter. Other adapters use the -core no-op sink even if they can parse the shared settings, and log a warning -at startup when `[analytics]` selects a module. +Trusted Server can emit bounded auction observations to an analytics module. +`[analytics] module` selects the module, and core hands each settled auction's +rows to the sink that module supplies. Without the section nothing is sent, +through core's no-op sink. The one module a stock build ships is Tinybird, +`crates/analytics/tinybird`, which each adapter lists among the modules it +links. Only the Fastly adapter hands that sink to the auction. The Axum, +Cloudflare and Spin adapters link the module and check its settings, and use +core's no-op sink, so they send nothing, and log a warning at startup when +`[analytics]` selects a module. ## Adapter support @@ -20,14 +25,14 @@ The request entry point derives coarse `DeviceSignals`. Auction execution combines those signals with request and navigation facts in an `AuctionObservationContext`. The terminal auction outcome is converted into an `AuctionEventBatch` containing summary, provider-call, and bid rows. -Fastly serializes the batch as NDJSON and starts one Tinybird Events API +The Tinybird module serializes the batch as NDJSON and starts one Events API request to the configured auction datasource. Emission is best effort. Failure to construct or start telemetry is logged and must not change the customer response. The sink drops the in-flight handle after dispatch; it does not wait for Tinybird's response. Each batch is limited -to 512 rows and `analytics.tinybird.max_body_bytes` bytes. The Fastly backend -uses two-second first-byte and between-byte timeouts. +to 512 rows and `analytics.tinybird.max_body_bytes` bytes. The backend the +module registers uses two-second first-byte and between-byte timeouts. ## Privacy boundary diff --git a/docs/superpowers/specs/2026-10-14-last-names-in-core-design.md b/docs/superpowers/specs/2026-10-14-last-names-in-core-design.md new file mode 100644 index 0000000000..ac4d716d35 --- /dev/null +++ b/docs/superpowers/specs/2026-10-14-last-names-in-core-design.md @@ -0,0 +1,164 @@ +# Design Spec: Small vendor, standard and platform names leave core + +**Status:** Proposed, 2026-10-14. Raised as draft pull request IABTechLab/trusted-server#1389 (from jwrosewell/trusted-server#41), for Tech Lab review. +**Author:** 51Degrees (contributed), for Tech Lab review +**Related specs:** `2026-10-14-source-modules-design.md`, `2026-10-14-no-modules-crate-design.md` and `2026-10-13-request-addressing-design.md` +**Related PRs:** IABTechLab/trusted-server#1389, jwrosewell/trusted-server#41 +**Issues:** IABTechLab/trusted-server#1341 +**Last updated:** 2026-10-16 + +> **Why this spec exists.** Core is the interfaces, the routing, the origin fetch, the caches and their tests. After the series before this one it still names, in small pieces, a telemetry vendor, two hosting platforms' cache headers and stores, a TLS fingerprint format and a published key format. Each is a place where a change by one vendor or one platform is a change in core. + +## 1. Problem + +These names remain in core, each with the code that reads it: + +| What core names | Where | +| ------------------------------------------------------ | -------------------------------------------------------- | +| The Tinybird telemetry module's settings and name | `settings.rs`, and the module in the Fastly adapter | +| The platforms' shared cache header names | `cache_policy.rs` | +| The platforms' store names and the removed store ids | `settings.rs`, `request_signing/mod.rs` | +| The JA4 fingerprint class in the device classification | `ec/device.rs` | +| The JWKS key format in request signing | `request_signing/jwks.rs`, `discovery.rs`, `rotation.rs` | + +Each name is small. Each one means that adding a second analytics vendor, a third platform, another fingerprint format or another key format is an edit to core rather than a crate beside it. + +## 2. Design + +Each name moves to the crate that reads it, and core keeps an interface in its place. The settings a deployment writes do not change. + +### 2.1 Analytics is a module type + +`[analytics]` was a section core read itself, with one module's settings typed into core and the module built by the Fastly adapter alone. It is now a module type like `[cmp]` or `[geo]`: core reads the section as it reads every module type's section, and a module crate supplies the sink. + +**The interface.** A registration declares the analytics module it supplies with `with_analytics_module(name, sink)`, where the sink is an `AuctionTelemetrySink`, the trait core already handed each settled auction's rows to. The registry resolves `[analytics] module` against the modules on offer, as `[geo] module` and `[device] module` are resolved, and `IntegrationRegistry::analytics_module` gives the sink the section selected, or `None` when there is no section, so core's no-op sink stands. A section that selects more than one name is refused, because an auction's rows go to one sink. + +**The module.** `crates/analytics/tinybird` (`trusted-server-analytics-tinybird`) carries the settings of `[analytics.tinybird]`, their checks and the sink that posts each batch to the Events API. It reads its table through `Settings::module_config`, declares `auction_token_secret` as a secret setting through `IntegrationBuilder::with_secret_settings`, so the token is looked up as the settings load and checked at deploy as every module's secret is. Each adapter lists the module's builder among the modules it links, as `shipped_module_builders()` in its own `app.rs`, and the `ts` tool lists the same builder in `linked_builders()` in `crates/trusted-server-cli/src/app_config.rs`, as `2026-10-14-no-modules-crate-design.md` describes, so `ts config validate` and `ts config push` run the module's checks and check the key name of its token. + +**Which adapters send.** Only the Fastly adapter sends. The Fastly adapter asks the registry for the sink with `analytics_module()` in `crates/trusted-server-adapter-fastly/src/app.rs`, keeps the sink in its application state and hands it to the services it builds for each request. Before this change the Fastly adapter built the sink itself. The Axum, Cloudflare and Spin adapters link the module and check its settings when they build their registry, and none of them reads the sink from the registry, so each runs on core's no-op sink and sends nothing, as before. + +```toml +[analytics] +module = "tinybird" + +[analytics.tinybird] +api_host = "api.us-east.aws.tinybird.co" +auction_dataset = "auction_events_raw" +auction_token_secret = "tinybird_auction_append_token" +``` + +**What changes for an operator.** Nothing in the document above, and the form the document is stored in is the same, with the selection and the table under `[analytics]` and no table at the old name. Three refusals move out of core's settings and into the module's own checks. A table missing its host or token, a key the table does not know and a reserved setting turned on are each refused, naming the table and the setting. A refusal stops the deployment from starting, and `ts config validate` reports the same refusal. A `[tinybird]` table at the top level is still refused, as the section of a module type nothing reads, rather than with directions to the new place, because the directions named the vendor in core. + +**Defaults and bounds.** The settings are those in the configuration guide under "Analytics Configuration", with these defaults and checks. + +| Setting | Default | Check | +| ---------------------- | -------------------- | ---------------------------------------------------------------------- | +| `api_host` | none | Required, a regional host name with no scheme, port or path | +| `auction_token_secret` | none | Required, the name of the secret holding the datasource's append token | +| `auction_dataset` | `auction_events_raw` | 1 to 128 ASCII letters, digits or `_` | +| `max_body_bytes` | 1048576 | At least 1024 | +| `access_enabled` | `false` | `true` is refused, because no access-log emitter exists | +| `access_sample_rate` | `0.0` | Between 0.0 and 1.0, and read by nothing | +| `access_dataset` | `access_logs_raw` | Read by nothing | + +`secret_store` and `access_token_secret` are accepted and dropped, and any other key is refused. + +**What the module sends.** Core builds the rows, being the auction's summary, each provider call and each bid, from the same observation as before, and the module posts the batch it is handed without adding a field, so the privacy boundary in `docs/guide/telemetry.md` is unchanged and the rows carry no Edge Cookie identifier. Core hands rows only to the sink `[analytics] module` selected, so a deployment without the section sends nothing. A failure to build or start a send is logged as a warning and never changes the response, and the module drops the pending request without waiting for the Events API. + +**A module written with its type folder.** The guide lets a section's selector name a module in full or with its type folder left off (`docs/guide/configuration-rules.md`), as `analytics.tinybird` or `tinybird` in `[analytics]`. Four places in core now shorten a name written with the section's own type folder before they use it, in every section and not only `[analytics]`, so each takes the module's table to be `[analytics.tinybird]` whichever way the selector wrote the name. `check_tables` in `provider_table.rs` decides whether each table in a section belongs to a selected module, `Settings::module_config` in `settings.rs` finds the table a module reads, the deploy check of a module's secret settings in `config.rs` finds the table it checks, and `module_secrets::clear_unused`, which the load of a stored configuration runs, keeps the secret settings of that table in use. `ts config validate` and the load of a stored configuration therefore accept `module = "analytics.tinybird"` as they accept `tinybird`. A table written at the full name's parts, as `[analytics.analytics.tinybird]`, is refused, whereas a name under another type's folder, as `testing.testlight` in `[auction]`, keeps its table at the full name's parts. Other places still look a table up by the name as the selector wrote it, among them `source::selected` for a `[source]` module and the demand and ad server tables read in `auction/plan.rs`. + +**What leaves core.** The `Settings::analytics` field, `Settings::analytics_module`, `AnalyticsConfig` and core's `TinybirdSettings` are gone, `TinybirdSettings` is now the module crate's, and the Fastly adapter's own `tinybird` module is deleted. `CHANGELOG.md` says what crates outside core use instead. + +**Rollout.** A deployment that already writes `[analytics] module = "tinybird"` and `[analytics.tinybird]` changes nothing, in the document or in the form stored. A deployment that lists its own builders adds `trusted_server_analytics_tinybird::builder()` to keep sending, because the registry refuses an `[analytics]` name no listed builder supplies. A deployment on the Axum, Cloudflare or Spin adapter sends nothing before or after. + +### 2.2 The adapter declares its own platform's cache header + +Core named four shared cache headers itself. Two come from standards, `Surrogate-Control` from the Edge Architecture Specification and `CDN-Cache-Control` from RFC 9213, and the other two belong to one platform each, `fastly-surrogate-control` and `cloudflare-cdn-cache-control`. Core could write a policy in three of them, all but `fastly-surrogate-control`, and it stripped all four from every response it made private, on every adapter. It now names only the two standard headers, and section 2.3 says which cache headers it writes or removes. + +**The interface.** An adapter declares the headers its own platform reads a shared cache policy from with `platform::declare_platform_cache_headers`, once, as it builds its state. Core reads the declaration only in `origin_response_may_be_kept` in `publisher.rs`, which keeps an origin response with a policy in a declared header out of the stored copies. The header an adapter has core write its policy in is `EdgeCacheHeader::Platform(name)`, beside `SurrogateControl`, `CdnCacheControl`, the `s-maxage` fallback and none, and `CachePolicy::apply_to_response` records it on the response as a `WrittenPlatformCacheHeader`. A name that is not a valid header name is never written, and a warning is logged each time a policy is rendered for it. The declared list is held for the whole running process, so a name declared twice is listed once, nothing removes a name, and a test that declares an example name leaves it listed for every other test in the same test program. + +**What each adapter declares and writes.** Each adapter holds the header it writes its policy in as `EDGE_CACHE_HEADER` in its own `app.rs`. + +| Adapter | Declares | Writes its policy in | +| ---------- | ------------------------------ | ------------------------------ | +| Fastly | `fastly-surrogate-control` | `Surrogate-Control` | +| Cloudflare | `cloudflare-cdn-cache-control` | `cloudflare-cdn-cache-control` | +| Axum | nothing | `s-maxage` in `Cache-Control` | +| Spin | nothing | `s-maxage` in `Cache-Control` | + +**What leaves core.** `EdgeCacheHeader::CloudflareCdnCacheControl` and the four constants for the Fastly and Cloudflare header names (`HEADER_FASTLY_SURROGATE_CONTROL`, `HEADER_FASTLY_SURROGATE_CONTROL_NAME`, `HEADER_CLOUDFLARE_CDN_CACHE_CONTROL` and `HEADER_CLOUDFLARE_CDN_CACHE_CONTROL_NAME`) are gone from `cache_policy.rs`, and `EDGE_CACHE_HEADER_NAMES` lists only the two standard headers. `EdgeCacheHeader::Platform` takes the variant's place, and the Fastly and Cloudflare adapters hold their own header names as `FASTLY_SURROGATE_CONTROL_HEADER` and `CDN_CACHE_CONTROL_HEADER`. `CHANGELOG.md` says what crates outside core use instead. + +**What changes for an operator.** Each adapter writes its policy in the header it wrote before. Where core removed all four headers from every response it made private, it now removes `fastly-surrogate-control` from none and `cloudflare-cdn-cache-control` only where core wrote its own policy there, so either header stays where an origin or an operator's `response_headers` wrote it. An origin response with a policy in `fastly-surrogate-control` is kept out of the stored copies on Fastly alone, and one with a policy in `cloudflare-cdn-cache-control` on Cloudflare alone, where every adapter kept both out. Section 2.3 says what a deployment configures to remove such a header. + +### 2.3 Cache headers follow the stored item's policy + +Only a middleware changes an asset or a response. Everything else changes the model, and the cache policy of an item as stored is model. So removing what an origin sent is the job of a fetch middleware that a deployment chooses, and the cache headers a response is served with mirror the cache policy of the item as stored. Neither core nor an adapter names a platform's cache header in order to remove it. + +Core owns `Cache-Control` and the two standard headers, which it reads on every adapter and may write its policy in, and on one response it also owns a platform header it wrote its policy in at the adapter's request. Where the item's policy is private, the response says so in those headers and in no others. Each place the declared list was used becomes the following. + +| Where core used the declared list | What it does now | +| -------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Removing them from a response core makes private (`response_privacy.rs`) | Removes `Surrogate-Control`, `CDN-Cache-Control` and a platform header core wrote its policy in on that response, which core records on the response when it writes it. A platform header an origin or an operator wrote stays. | +| Removing them before core writes its own policy (`cache_policy.rs`) | Removes the two standard headers and writes the policy in the header the adapter asks for, replacing what was there. | +| Skipping an operator's `response_headers` entry on an uncacheable response | Skips `Cache-Control`, the two standard headers and a platform header core wrote its policy in. An entry naming another header is written as configured. | +| Not keeping an origin response with a policy core does not read (`publisher.rs`) | Reads `CDN-Cache-Control` and the header the adapter declares for its own platform, because whether an item is kept is model, and keeps a response with a policy in either out of the stored copies. | + +**What a deployment configures.** A deployment whose origin sends a platform's cache header on a page core may make private names a fetch middleware that removes the header, in a `[[fetch]]` entry for the page's media type. None exists in the stack, and adding one takes two things. Core has to let a fetch middleware remove a header from the origin's response before the item is stored, because today a fetch middleware changes only the body and only a serve middleware writes a header (`MiddlewareAction::response_headers`). Then one crate per platform supplies the middleware, `crates/middleware/fastly` for `fastly-surrogate-control` and `crates/middleware/cloudflare` for `cloudflare-cdn-cache-control`. Until both exist the deployer stops the origin sending the header on such pages. + +**Where core or an adapter still alters the response.** Each place below writes or removes cache headers as a step of its own rather than in a serve middleware writing them from the item's policy, and each is to move there. + +| Place | What it does to the response | +| ------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `response_privacy::enforce_private_no_store`, through `enforce_terminal_private_cache_privacy` and `enforce_synthesized_html_cache_privacy` | Writes `Cache-Control: no-store, private` and removes the headers core owns, `ETag`, `Last-Modified`, `Expires` and `Age`. Core calls it on what it answers for one reader, personalized and assembled pages among them, and so do the Google ad tag diagnostics and the Didomi module. | +| `response_privacy::enforce_set_cookie_cache_privacy` | On a response that sets a cookie, removes the headers core owns and writes `Cache-Control: private, max-age=0` unless it already forbids shared storage. | +| `response_privacy::enforce_uncacheable_cache_privacy` | Removes the headers core owns from a response whose `Cache-Control` is private or no-store. | +| `response_privacy::apply_response_headers_with_cache_privacy`, called by every adapter's `apply_finalize_headers` | Writes the operator's `response_headers` between the two steps above, skipping the headers core owns on an uncacheable response. | +| `response_privacy::apply_inactive_ad_stack_browser_cache_policy` | Writes `Cache-Control: private, max-age=60` on an inactive ad template. | +| `cache_policy::CachePolicy::apply_to_response` and `apply_to_headers` | Write `Cache-Control` and the adapter's header, or `s-maxage`, for `/static/tsjs=`, a `[cache]` asset rule and the asset proxy. | +| `cache_policy::apply_no_store_private_to_headers`, also reached through `CacheControlPolicy::NoStorePrivate` | Writes `Cache-Control: no-store, private` and removes the standard headers, for the asset proxy's private answers, the Prebid module's script and the first place in this table. | +| The Fastly adapter's entry point, `lib.rs` | Applies the asset proxy's policy in `Surrogate-Control` after finalization, and runs the first three again after request filter effects and the Edge Cookie step (`apply_terminal_response_effects`). | +| Every adapter's routes | Choose the header core writes the policy in, `Surrogate-Control` on Fastly, `cloudflare-cdn-cache-control` on Cloudflare and `s-maxage` on Axum and Spin. | + +### 2.4 Later rows + +The store names, the fingerprint class and the key format follow in the same series, each with its own section here as it lands. + +## 3. What is not in this change + +- A second analytics module. The interface takes one, and the tests prove a name no module supplies is refused with the names on offer. +- Access-log telemetry. The reserved settings stay reserved, and turning one on is still refused. +- Telemetry from the Axum, Cloudflare and Spin adapters. They link the module and run on core's no-op sink, as before. +- The store names and removed store ids, the JA4 fingerprint class and the JWKS key format, which the table above lists and which follow in the same series. +- Names the table does not list. Core also names `fastly-client-ip` in the check of `[trusted_client_ip]` (`settings.rs`), lists `fastly-ssl` and `fastly-client-ip` among the headers it strips from a request on entry (`http_util.rs`), reads `fastly-ssl` as a signal of the request's scheme (`http_util.rs`), and removes `fastly-ssl` from the request before calling the origin (`publisher.rs`). +- The vendor's name in two places in core, being a comment in `provider_table.rs` and a secret fixture in the tests of `config_payload.rs`. +- The fetch middleware that removes a platform's cache header and the core change it needs, and moving the places section 2.3 lists into a serve middleware. + +## 4. Tests + +- Core tests that a declared platform header stays on a private response and on a cookie response, whoever wrote it (`a_declared_platform_header_stays_on_a_private_response`, `a_platform_header_core_did_not_write_is_left_on_a_cookie_response`), that a policy core wrote in a platform header is removed with the headers core owns and an operator header does not restore it (`a_policy_written_in_a_platform_header_is_removed_with_the_owned_headers`, `a_policy_core_wrote_in_a_platform_header_leaves_a_cookie_response`), and that a private policy leaves nothing in the header it is written in (`a_private_policy_empties_the_platform_header_it_is_written_in`). A policy is written in the platform's own header when the adapter asks for it, an origin response with a policy in a declared header is not kept, and a name declared twice is listed once. +- The module's crate tests its sink (one Events API request per batch, no wait, the row limit, an empty batch sends nothing), its table (the host, the token, the reserved settings, the bounds, an unknown key, the deprecated fields, the documented block in the example configuration), its stored form (the selection and the table survive being stored, with no table at the old name) and its selection (the short name and the full name, no section, a table the section does not select, the old top-level table, a name the build does not have, the token's key name at deploy). The full name goes through the settings load and the module's registration, through the deploy check of its token (`the_full_name_s_token_is_checked_at_deploy_in_the_table_the_module_reads`) and through the load of a stored configuration (`the_full_name_loads_from_the_stored_form_with_its_token`). +- Core tests a module of another type selected by its full name, `testing.probe` in `[testing]`, through the deploy check of its secret (`deploy_validation_checks_the_secret_of_a_module_selected_by_its_full_name`), the clearing of secrets not in use (`a_module_selected_by_its_full_name_keeps_the_settings_in_the_table_it_reads`) and the load of a stored configuration (`the_load_looks_up_the_secret_of_a_module_selected_by_its_full_name`). +- Core tests the kind: a selected module that supplies no analytics module is refused naming it, a section selecting two names is refused, and no section resolves no sink. +- Each of the Axum, Cloudflare, Fastly and Spin adapters tests that its linked modules are listed in hook order with the analytics module among them (`the_linked_modules_are_offered_in_hook_order`), and the Axum adapter's shipped modules test proves the analytics module refuses a setting it does not know. The `ts` tool's list has no test of its own. +- Each of the Axum, Cloudflare, Fastly and Spin adapters builds its state, proves it declares its own platform's header and no other, and proves a response core makes private carries `no-store, private`, loses the policy core wrote in the adapter's header and keeps a platform's header the origin sent (`a_private_response_follows_its_policy_and_keeps_the_origins_header`). + +Not tested: + +- Any adapter's reading of the sink. No test builds an adapter's state with `[analytics]` selecting the module and checks which sink its services carry, so the Fastly adapter's wiring and the lack of it on the other three adapters are both untested. +- `Settings::module_config` given a name written with its type folder for a module type other than analytics, and the refusal by `check_tables` of a table at the full name's parts. No test covers either. +- A platform header that is not a valid header name, which no test writes. + +## Review log + +1. **Round 1, 2026-10-15.** A review in a fresh session, against `split/46-last-names` at `8110daa5b`. Issues found, four of them blocking. The page named `crates/trusted-server-modules`, which no longer exists, said nothing changes for an operator when each adapter now strips a different set of cache headers, said nothing of the fact that only the Fastly adapter sends auction telemetry, and left out the core change that lets a section name a module with its own type folder. This revision states how each adapter and the `ts` tool list the module, which headers each adapter declares and strips with the narrowing put as an open question, that only the Fastly adapter sends, and the core change with the places that still read a table by the name as written, together with the partly true and unsupported claims, the coverage gaps and the advice the round gave, and it corrects the same claims in the changelog and the telemetry guide. +2. **Round 2, 2026-10-15.** A lighter review in a fresh session, against this page at `3286cb534`, of the round 1 fixes and the new text. No blocking issues. This revision says plainly that the full name of the analytics module fails `ts config validate` and the load of a stored configuration today and lists the code fix with the open question, limits the selection test to the settings load and the module's registration, and adds to the names left in core that core reads `fastly-ssl` as a signal of the scheme and removes the header before calling the origin. + +## Revision record + +| Date | Change | +| ---------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 2026-10-14 | First draft. | +| 2026-10-15 | Fixes from review round 1. The module lists of each adapter and of the `ts` tool, the headers each adapter declares and strips with the narrowing as an open question, the adapters that send telemetry, the table change for a name written with its type folder, the defaults and bounds, what leaves core, the rollout, the names left in core, the untested paths and the review log. | +| 2026-10-15 | Corrections from review round 2. The full name of the analytics module failing the deploy check and the load of a stored configuration, the code fix under the open question, what the selection test reaches, and `fastly-ssl` among the names left in core. | +| 2026-10-15 | Code fixes. The deploy check of a module's secret settings and the load of a stored configuration read the table of a module selected by its full name at the name without its type folder, and the tests of the fix are listed. | +| 2026-10-16 | Cache headers follow the stored item's policy, in a new section 2.3 that closes the open question on the cache headers. Each adapter declares only its own platform's header, which core reads only to decide whether an origin response may be kept, core removes from a private response only the headers it owns, and the page says what a deployment configures and lists where core or an adapter still alters the response. |