diff --git a/.cargo/config.toml b/.cargo/config.toml index aea0df2d59..6da9679911 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -26,20 +26,22 @@ test_details = "test --target aarch64-apple-darwin" # native crate needs no change here. Axum (native), Cloudflare # (wasm32-unknown-unknown), Spin, the CLI (native), and integration-tests # (native) are simply not listed. -build-fastly = "build -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" -check-fastly = "check -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" -clippy-fastly = "clippy -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-js -p trusted-server-openrtb --all-targets --all-features --target wasm32-wasip1 -- -D warnings" -test-fastly = "test -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" +build-fastly = "build -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" +check-fastly = "check -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" +clippy-fastly = "clippy -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --all-targets --all-features --target wasm32-wasip1 -- -D warnings" +test-fastly = "test -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" # Feature-on counterpart of `test-fastly`. `test-fastly` does NOT pass # --all-features, so without this the reusable-sandbox code is linted by # `clippy-fastly` but its tests never execute. test-fastly-reuse = "test -p trusted-server-adapter-fastly --target wasm32-wasip1 --features reusable-sandbox" # --- Axum adapter (native dev server) --- +# Test and clippy also cover the permission signal crates, whose own tests +# run with the Axum adapter in CI. build-axum = "build -p trusted-server-adapter-axum" check-axum = "check -p trusted-server-adapter-axum" -clippy-axum = "clippy -p trusted-server-adapter-axum --all-targets --all-features -- -D warnings" -test-axum = "test -p trusted-server-adapter-axum" +clippy-axum = "clippy -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-loading-cache-51degrees -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb --all-targets --all-features -- -D warnings" +test-axum = "test -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-loading-cache-51degrees -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb" # --- Cloudflare adapter (native host + wasm32-unknown-unknown) --- # Build/check target the WASM runtime (requires the `cloudflare` feature); diff --git a/.claude/skills/planning-prebid-aws/references/configuration-and-secrets.md b/.claude/skills/planning-prebid-aws/references/configuration-and-secrets.md index f0ff876d6c..d8d53ae0e2 100644 --- a/.claude/skills/planning-prebid-aws/references/configuration-and-secrets.md +++ b/.claude/skills/planning-prebid-aws/references/configuration-and-secrets.md @@ -12,17 +12,17 @@ Before asking questions the repository can answer: 4. Add observations to the existing deployment decision record with source path/key, candidate requirement, confidence/status, and the remaining question. Report credential identifiers or presence only; keep raw TOML, credential values, and commercially sensitive publisher values out of reports and generated examples. 5. Verify candidate adapters against the selected PBS Go release and confirm partner authorization. Record each host-secret requirement as required, not needed, or unresolved. A bidder name alone proves neither a secret requirement nor permission to use the bidder. -| Trusted Server input | Discovery use | -| ------------------------------------------------------------------ | ------------------------------------------------------------------------------------------- | -| `auction.enabled` | Determine active versus disabled server-auction intent without expanding defaults | -| `auction.providers.` with `profile = "prebid-server"` | Identify each configured PBS provider; withhold endpoint values | -| Provider `timeout_ms` and `profile_config.test_mode` or `debug` | Record explicit timeout and diagnostic intent, not permission to contact bidders | -| Provider `profile_config.bid_param_override_rules` | Count conditional publisher or placement inputs without reporting their values | -| `auction.bidders..provider` | Group candidate server-side adapters under the referenced PBS provider | -| `integrations.prebid.enabled`, `account_id`, `timeout_ms`, `debug` | Record browser integration intent and compatibility questions separately from server demand | -| `integrations.prebid.client_side_bidders` | Browser-side participation; do not automatically enable these adapters in PBS | -| `integrations.prebid.bundle` | Browser bundle and identity capabilities, not proof of server-side use | -| Relevant privacy, format, and stored-request settings | Identify dependencies that need confirmation from the caller or request path | +| Trusted Server input | Discovery use | +| --------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------ | +| `auction.enabled` | Determine active versus disabled server-auction intent without expanding defaults | +| `demand.` with `implementation = "auction.prebid-server"` | Identify each configured PBS demand source and whether `demand.modules` selects it; withhold endpoint values | +| Demand source `timeout_ms`, `test_mode` and `debug` | Record explicit timeout and diagnostic intent, not permission to contact bidders | +| Demand source `bid_param_override_rules` | Count conditional publisher or placement inputs without reporting their values | +| `auction.bidders..module` | Group candidate server-side adapters under the referenced PBS demand source | +| `prebid` in `auction.modules`, and `auction.prebid` `account_id`, `timeout_ms`, `debug` | Record browser integration intent and compatibility questions separately from server demand | +| `auction.prebid.client_side_bidders` | Browser-side participation; do not automatically enable these adapters in PBS | +| `auction.prebid.bundle` | Browser bundle and identity capabilities, not proof of server-side use | +| Relevant privacy, format, and stored-request settings | Identify dependencies that need confirmation from the caller or request path | Use `ts prebid server inspect --config ` for redacted local discovery. It reports explicit values without expanding defaults or proving host-secret requirements. Resolve schema defaults and adapter metadata separately, recording unsupported details as unresolved rather than fetching credentials or inventing mappings. Read config without invoking commands that publish or rewrite it. Leave `trusted-server.toml` unchanged. Propose caller endpoint/account changes separately after integration approval; discovery must not activate bidders, publish configuration, or change traffic. diff --git a/.env.example b/.env.example index a7f5973cdb..8a874e292c 100644 --- a/.env.example +++ b/.env.example @@ -40,41 +40,36 @@ TRUSTED_SERVER__REQUEST_SIGNING__ENABLED=false # ============================================================================= # Integrations # ============================================================================= +# Which integrations run is set in the TOML file, in the section of each +# one's type, as `[auction] modules = ["prebid"]` and +# `[framework] module = "nextjs"`. There is no per-integration enabled flag. +# An overlay below changes a setting the file already carries, and it cannot +# add one the file leaves out or replace a list. An overlay names its section +# as the file writes it, so a section with a hyphen in its name, such as +# `[ad-tag]`, is set with `env 'NAME=value' command`, because a shell's +# `export` refuses the name. -# Prebid -TRUSTED_SERVER__INTEGRATIONS__PREBID__ENABLED=false -# TRUSTED_SERVER__INTEGRATIONS__PREBID__SERVER_URL=https://prebid-server.example.com/openrtb2/auction -# TRUSTED_SERVER__INTEGRATIONS__PREBID__TIMEOUT_MS=1000 -# TRUSTED_SERVER__INTEGRATIONS__PREBID__BIDDERS=kargo,rubicon,appnexus -# TRUSTED_SERVER__INTEGRATIONS__PREBID__BID_PARAM_OVERRIDES='{"bidder-name":{"param1":12345,"param2":"value"}}' -# Compatibility env shape for bidder -> zone -> params overrides -# TRUSTED_SERVER__INTEGRATIONS__PREBID__BID_PARAM_ZONE_OVERRIDES='{"kargo":{"header":{"placementId":"_abc"}}}' -# Preferred canonical env shape for future generic rules -# TRUSTED_SERVER__INTEGRATIONS__PREBID__BID_PARAM_OVERRIDE_RULES='[{"when":{"bidder":"kargo","zone":"header"},"set":{"placementId":"_abc"}}]' -# TRUSTED_SERVER__INTEGRATIONS__PREBID__SUPPRESS_NURL_BIDDERS=exampleBidder,anotherBidder -# TRUSTED_SERVER__INTEGRATIONS__PREBID__AUTO_CONFIGURE=false -# TRUSTED_SERVER__INTEGRATIONS__PREBID__DEBUG=false -# TRUSTED_SERVER__INTEGRATIONS__PREBID__TEST_MODE=false +# Prebid, the browser side. The Prebid Server endpoint belongs to a demand +# source, so its overlay is TRUSTED_SERVER__DEMAND____ENDPOINT. +# TRUSTED_SERVER__AUCTION__PREBID__ACCOUNT_ID= +# TRUSTED_SERVER__AUCTION__PREBID__TIMEOUT_MS=1000 +# TRUSTED_SERVER__AUCTION__PREBID__DEBUG=false # Next.js -TRUSTED_SERVER__INTEGRATIONS__NEXTJS__ENABLED=false -# TRUSTED_SERVER__INTEGRATIONS__NEXTJS__REWRITE_ATTRIBUTES=href,link,url +# TRUSTED_SERVER__FRAMEWORK__NEXTJS__MAX_COMBINED_PAYLOAD_BYTES=10485760 # Didomi -TRUSTED_SERVER__INTEGRATIONS__DIDOMI__ENABLED=false -# TRUSTED_SERVER__INTEGRATIONS__DIDOMI__SDK_ORIGIN=https://sdk.privacy-center.org -# TRUSTED_SERVER__INTEGRATIONS__DIDOMI__API_ORIGIN=https://api.privacy-center.org +# TRUSTED_SERVER__CMP__DIDOMI__SDK_ORIGIN=https://sdk.privacy-center.org +# TRUSTED_SERVER__CMP__DIDOMI__API_ORIGIN=https://api.privacy-center.org # Permutive -TRUSTED_SERVER__INTEGRATIONS__PERMUTIVE__ENABLED=false -# TRUSTED_SERVER__INTEGRATIONS__PERMUTIVE__ORGANIZATION_ID= -# TRUSTED_SERVER__INTEGRATIONS__PERMUTIVE__WORKSPACE_ID= -# TRUSTED_SERVER__INTEGRATIONS__PERMUTIVE__PROJECT_ID= +# TRUSTED_SERVER__AUDIENCE__PERMUTIVE__ORGANIZATION_ID= +# TRUSTED_SERVER__AUDIENCE__PERMUTIVE__WORKSPACE_ID= +# TRUSTED_SERVER__AUDIENCE__PERMUTIVE__PROJECT_ID= # Lockr -TRUSTED_SERVER__INTEGRATIONS__LOCKR__ENABLED=false -# TRUSTED_SERVER__INTEGRATIONS__LOCKR__APP_ID= -# TRUSTED_SERVER__INTEGRATIONS__LOCKR__CACHE_TTL_SECONDS=3600 +# TRUSTED_SERVER__IDENTITY__LOCKR__APP_ID= +# TRUSTED_SERVER__IDENTITY__LOCKR__CACHE_TTL_SECONDS=3600 # ============================================================================= # Rewrite Configuration (optional) diff --git a/.github/workflows/inspector.yml b/.github/workflows/inspector.yml new file mode 100644 index 0000000000..5d58b3d93e --- /dev/null +++ b/.github/workflows/inspector.yml @@ -0,0 +1,26 @@ +name: "Permissions Inspector" + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +jobs: + build-inspector-wasm: + name: build inspector wasm + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Retrieve Rust version + id: rust-version + run: echo "rust-version=$(grep -oP 'channel = "\K[^"]+' rust-toolchain.toml)" >> "$GITHUB_OUTPUT" + - name: Set up Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + toolchain: ${{ steps.rust-version.outputs.rust-version }} + target: wasm32-unknown-unknown + - name: Build the inspector engine + run: ./scripts/build-inspector-wasm.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 95654ecddf..37d0d2148f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -68,12 +68,23 @@ jobs: - name: Run inline control harness run: BID_DELAY=3 ./scripts/template-cache-local-test.sh inline - - name: Run cache purge harness - run: BID_DELAY=3 ./scripts/template-cache-local-test.sh purge + - name: Run config by host harness + run: ./scripts/config-by-host-local-test.sh + + - name: Run config by host harness (reusable-sandbox feature) + run: ./scripts/config-by-host-local-test.sh reuse + + # Last, because it builds core with a key schedule compiled in. + - name: Run attestation harness + run: ./scripts/attestation-local-test.sh test-axum: - name: cargo test (axum native) - runs-on: ubuntu-latest + name: cargo test (axum native, ${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] steps: - uses: actions/checkout@v4 @@ -122,12 +133,33 @@ jobs: run: | cargo test --package trusted-server-openrtb-codegen --target "$(rustc -vV | sed -n 's/host: //p')" + # The core library's unit tests otherwise run only on the WebAssembly + # targets, which build with panic=abort, so their harness stops at the + # first failing test and reports every later one as never run. A run that + # looks like a single failure can hide many more. This native run reports + # them all at once, which is what makes a red build readable. + - name: Run host-target core library tests + run: | + cargo test --package trusted-server-core --target "$(rustc -vV | sed -n 's/host: //p')" + + # The seam-probe fixture crate's own unit tests only run when invoked on + # the host target, because no adapter alias compiles its test target. Run + # them here so a fault in the seam fixture cannot sit unseen behind a + # green gate set. + - name: Run host-target seam-probe integration tests + run: | + cargo test --package trusted-server-integration-seam-probe --target "$(rustc -vV | sed -n 's/host: //p')" + - name: Verify Fastly WASM release build run: cargo build --package trusted-server-adapter-fastly --release --target wasm32-wasip1 test-cloudflare: - name: cargo check (cloudflare native + wasm32-unknown-unknown) - runs-on: ubuntu-latest + name: cargo check (cloudflare native + wasm32-unknown-unknown, ${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] steps: - uses: actions/checkout@v4 @@ -322,3 +354,9 @@ jobs: - name: Run unit tests run: npm test -- --run + + # A module that writes a script into pages tests it in its own folder, + # with Node's test runner and no packages. + - name: Run module page script tests + working-directory: . + run: node --test "crates/*/*/tests/*.test.mjs" diff --git a/.gitignore b/.gitignore index 24b9e06aa1..a4a0af36c7 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,9 @@ /spin /spin.sig +# logs +*.log + # EdgeZero local KV store (created by edgezero-adapter-axum framework) .edgezero/ /dist/prebid/ diff --git a/AGENTS.md b/AGENTS.md index dd1dd52337..172b96f237 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,12 +15,29 @@ JavaScript injection. ``` crates/ - trusted-server-core/ # Core library — shared logic, integrations, HTML processing + trusted-server-core/ # Core library — shared logic, the integration seam, HTML processing trusted-server-adapter-fastly/ # Fastly Compute entry point (wasm32-wasip1 binary) trusted-server-adapter-axum/ # Axum dev server entry point (native binary) trusted-server-adapter-cloudflare/ # Cloudflare Workers entry point (wasm32-unknown-unknown binary) trusted-server-adapter-spin/ # Fermyon Spin entry point (wasm32-wasip1 component) trusted-server-cli/ # Host-target `ts` operator CLI + trusted-server-modules/ # the modules a stock build ships, in the order their hooks run + ad-server/ # ad server implementation crates (mock) + ads-txt/ # ads.txt line module crates (sellers) + ad-tag/ # ad tag module crates (google, which holds its diagnostics module too) + auction/ # auction module crates (aps, prebid, prebid-server) + audience/ # audience module crates (permutive) + bot-protection/ # bot protection module crates (datadome) + cmp/ # consent tool module crates (didomi, inmobi, osano, sourcepoint) + framework/ # framework module crates (nextjs) + identity/ # identity module crates (lockr) + tag/ # tag manager module crates (google-tag-manager) + testing/ # test module crates (seam-probe, testlight) + device/ + fastly/ # trusted-server-device-fastly (opt-in TLS/H2 device module) + edgecookie/ # vendor Edge Cookie module crates (built-in HMAC module is in core) + geo/ # vendor geo module crates (host geo is injected by the adapter) + permission-signal/ # permission signal module crates, one per scheme (gpc, gpp, tcf, us-privacy), and core links none trusted-server-js/ # TypeScript/JS build — per-integration IIFE bundles lib/ # TS source, Vitest tests, esbuild pipeline ``` @@ -61,7 +78,9 @@ fastly compute serve # Deploy to Fastly fastly compute publish -# Run Axum dev server (native — no Viceroy) +# Run Axum dev server (native — no Viceroy). Settings load at runtime from the +# platform config store on every adapter; publish an operator config with +# `ts config push` (see trusted-server.example.toml for the template). cargo run -p trusted-server-adapter-axum # Test Axum adapter only @@ -160,6 +179,24 @@ cd crates/trusted-server-js/lib && node build-all.mjs cargo install viceroy --version 0.17.0 --locked --force ``` +### Windows (use WSL for the Linux-only tests) + +The Rust adapter tests run natively on Windows through the cargo aliases +(`cargo test-fastly` via Viceroy, `cargo test-axum`, `cargo test-cloudflare`), +and CI runs them on `ubuntu-latest`. + +The Docker-based integration suite (`scripts/integration-tests.sh`) and the +Cloudflare worker build (`crates/trusted-server-adapter-cloudflare/build.sh`, +which uses `worker-build` + `wrangler dev`) are Linux tools. On Windows run them +inside WSL (Ubuntu) with Docker Desktop's WSL integration enabled. Provision the +WSL distro with the same toolchain as `.tool-versions` (rustup + the +`wasm32-wasip1` / `wasm32-unknown-unknown` targets, Node, Viceroy, wrangler), then +run the scripts from a clone on the WSL native filesystem for fast builds. +The CLI crate (`trusted-server-cli`) does not build on a Windows host either, +because its dependency `edgezero-adapter-fastly` uses a standard library +feature that is unstable on Windows, so run its tests and the template cache +harness (`scripts/template-cache-local-test.sh`), which builds it, in WSL too. + --- ## Coding Conventions @@ -287,12 +324,48 @@ impl core::error::Error for MyError {} ## Other guidelines +- Use US English spelling everywhere: code, identifiers, comments, + documentation, tests, commit messages, and configuration. For example, write + `color`, `behavior`, and `optimize`, not `colour`, `behaviour`, or `optimise`. + Where a term comes from an external source (for example the IAB TCF purpose + names), match that source's spelling even when it is not US English. - Use only example or fictional information in comments, tests, docs, examples, and similar non-runtime materials. (eg. for urls use: example.com domains only) - Do not write or commit real domains, customer names, credentials, configuration values, or other potentially sensitive real-world information in comments, tests, docs, or examples. +### Permission model terminology + +Permissions are the primitive. A module declares the permissions it requires +(`required_permissions`) and the system decides whether each is _set_. Consent +is only one of many ways a permission may be established. Country or +jurisdiction rules (a `Granted` group baseline), legitimate interest, or +configuration can set a permission with no consent at all. + +- A module that needs nothing **requires no permission**. Never write that it + "runs without any consent". +- A gated module **runs once its required permissions are set**, by whatever + method. + +**Evidence is not rationed, use is.** Every module and every integration sees +all the evidence available for a request, including host signals such as the TLS +JA4 and HTTP/2 signals. The core never decides which vendor may see what, +because withholding a signal from one vendor and not another discriminates +between them, and the core stays neutral. What a vendor may _do_ with the +evidence is governed by the permissions it declares and the system sets. Access +is universal, use is gated. + +The practical consequence: never "fix" a vendor's access to a signal by hiding +the signal. If a use needs controlling, express it as a permission. A change +that removes evidence from a module's reach is working against the +architecture, not protecting it. + +- Reserve "consent" for the consent subsystem (`consent/`, `ConsentContext`, + GDPR and TCF strings) where it genuinely means a consent signal. In the + permission layer prefer "permission", "set" / "unset", and "signal" (consent + is one kind of signal, alongside privacy and opt-out signals). + --- ## Git Commit Conventions @@ -309,6 +382,54 @@ Bad: `"fix: added feature flags"` --- +## Module Architecture + +Each vendor-differentiated capability is pluggable behind its own trait, so a +deployment selects an implementation and the core stays neutral. Every +pluggable thing follows one configuration convention, which is a top-level +table named for the job, a `module` key that selects what runs (`modules` +where several run), and a `[.]` +table for a selected name that has settings. The rules and what +is checked when are in `docs/guide/configuration-rules.md`, which is the page +to read before changing any module configuration. + +| Capability | Trait | Selector | Built-in (core) | Vendor / host crates | +| --------------------- | ----------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------- | +| Edge Cookie identity | `EdgeCookieModule` (`ec/module.rs`) | `[ec] module` | `hmac`, `host_signals`, `client_fixed` (opt-in, no default) | `crates/edgecookie/` | +| Device detection | `DeviceModule` (`ec/device.rs`) | `[device] module` | `builtin`, User-Agent only (the default) | `crates/device/` | +| Geo / IP intelligence | `PlatformGeo` (`platform/traits.rs`) | `[geo] module` | None, no location (the default), or `platform` | `crates/geo/` | +| Permission signals | `PermissionSignalModule` (`permission_signal/mod.rs`) | `[permission-signal] modules` (an ordered list) | `gpc`, `gpp`, `us-privacy`, `tcf`, all of them with no list | `crates/permission-signal/` | +| Auction demand | `DemandImplementation` (`auction/demand.rs`) | `[demand] modules` (a list) | `auction-protocol.openrtb`, `auction.prebid-server`, `auction.aps` | an integration builder | +| Ad server | `AdServerImplementation` (`auction/demand.rs`) | `[ad-server] module` | `mock` (`ad-server.mock`) | an integration builder | +| Page integrations | `IntegrationBuilder` (`integrations/mod.rs`) | the section of the module's type: `[cmp] module`, `[tag] modules`, `[ad-tag] modules`, `[bot-protection] module`, `[identity] module`, `[audience] module`, `[framework] module`, `[auction] modules`, `[proxy] modules` | `cmp.didomi`, `cmp.sourcepoint`, `cmp.osano`, `tag.google-tag-manager`, `ad-tag.google`, `ad-tag.google.diagnostics`, `bot-protection.datadome`, `identity.lockr`, `audience.permutive`, `framework.nextjs`, `auction.prebid`, `testing.testlight`, `js_asset_proxy` | an adapter-supplied builder | + +`auction-protocol.openrtb`, `auction.prebid-server`, `auction.aps` and `ad-server.mock` supply implementations +only. No module section selects them. + +Principles for adding or changing a module: + +- **Core stays neutral.** The trait and the host-neutral default live in + `trusted-server-core`. Host-specific and vendor implementations live in their + own crates and are injected by the adapter (for example `build_device_module` + and `build_geo_module`), so core never depends on a host SDK or a vendor, and + the default request path makes no host-specific calls. +- **Modules read request evidence, not a fixed parameter set.** A module must + be able to see everything about the request it needs (User-Agent, headers, and + host signals such as the TLS JA4 and HTTP/2 signals) through an evidence + abstraction rather than a hard-coded struct of fields. Host signals come from + the host (the Fastly SDK) and are opt-in, so a neutral module triggers no + host signal calls. +- **Modules are separated by capability but composed per request, and one may + need another's output.** Geo resolves the country and region the permission + model uses, and the permission model gates whether the Edge Cookie module + runs. Device signals gate Edge Cookie writes (the browser / bot gate). When + multiple vendor modules share a backend (for example a vendor's Edge Cookie, + geo, and device module on one cloud pipeline) they share a single call per + request rather than calling independently. Give a module the inputs and + upstream results it needs explicitly, rather than having it reach into globals. + +--- + ## Integration System Integrations register in Rust via: @@ -316,13 +437,13 @@ Integrations register in Rust via: ```rust IntegrationRegistration::builder(ID) .with_proxy() - .with_attribute_rewriter() - .with_head_injector() + .with_middleware() .build() ``` - Integration IDs match JS directory names: `prebid` (deferred), `lockr`, `permutive`, `datadome`, `didomi`, `testlight`. -- `creative` is JS-only (no Rust registration); `nextjs`, `aps`, `adserver_mock` are Rust-only. +- An integration runs when the section of its type selects its module. A module with a crate of its own is named by that crate's path under `crates/` (`permission-signal.gpc`, `testing.seam-probe`). Each page integration lives in a crate of its own and holds that name in a `MODULE` constant (`cmp.didomi`, `tag.google-tag-manager`, `ad-tag.google`, `bot-protection.datadome`, `identity.lockr`, `audience.permutive`, `framework.nextjs`, `auction.prebid`, `testing.testlight`), which a test in the crate holds to the crate's folder. A name is written in its section with the type left off. There is no `enabled` flag and no `[integration]` table. +- `creative` is JS-only (no Rust registration), and `nextjs` is Rust-only. `auction-protocol.openrtb`, `auction.prebid-server`, `auction.aps` and `ad-server.mock` register demand or ad server implementations rather than page integrations. - Integrations opt into deferred loading via `.with_deferred_js()` on the registration builder. Deferred modules are served as separate `", - tsjs::tsjs_single_module_script_src(GPT_DIAGNOSTICS_INTEGRATION_ID) + tsjs::tsjs_single_module_script_src(module) ) }) } - /// An active decision, for tests in other modules that need one. - /// - /// The fields are private and built by `prepare_request` from a cookie or query - /// parameter; there is no other way to obtain an active decision across a module - /// boundary. - #[cfg(test)] - pub(crate) fn active_for_tests() -> Self { - Self { - active: true, - clean_browser_path_and_query: None, - cookie_action: GptDiagnosticsCookieAction::None, - } - } } #[cfg(test)] mod head_seam_invariant_tests { use super::*; + /// The compile-time diagnostics module, as the registry would serve it. + fn diagnostics_part() -> JsModulePart { + JsModulePart::compile_time(GPT_DIAGNOSTICS_INTEGRATION_ID) + .expect("should have compiled the diagnostics module in") + } + /// Every combination of the three fields the decision carries. fn all_decisions() -> Vec { let mut out = Vec::new(); @@ -162,9 +180,10 @@ mod head_seam_invariant_tests { // // If a future change makes a script emit without also requiring the stamp, // this fails here rather than silently in a cached template. + let module = diagnostics_part(); for decision in all_decisions() { - let injects = - decision.bootstrap_script().is_some() || decision.module_script_tag().is_some(); + let injects = decision.bootstrap_script().is_some() + || decision.module_script_tag(&module).is_some(); if injects { assert!( decision.requires_private_no_store(), @@ -184,7 +203,7 @@ mod head_seam_invariant_tests { "should not inject a bootstrap for an inert decision" ); assert_eq!( - decision.module_script_tag(), + decision.module_script_tag(&diagnostics_part()), None, "should not inject a module for an inert decision" ); @@ -209,7 +228,20 @@ struct ConsoleCookieState { canonical: bool, } -/// Register GPT diagnostics when explicitly enabled. +/// Validates the GPT diagnostics configuration for deployment and reports whether +/// a section selects the integration's module. +/// +/// # Errors +/// +/// Returns an error when the GPT diagnostics configuration cannot be parsed or fails +/// validation. +pub(crate) fn validate(settings: &Settings) -> Result> { + settings + .module_config::(MODULE) + .map(|config| config.is_some()) +} + +/// Register GPT diagnostics when a section selects it. /// /// # Errors /// @@ -218,32 +250,59 @@ struct ConsoleCookieState { pub fn register( settings: &Settings, ) -> Result, Report> { - let Some(_config) = - settings.integration_config::(GPT_DIAGNOSTICS_INTEGRATION_ID)? - else { + let Some(_config) = settings.module_config::(MODULE)? else { return Ok(None); }; Ok(Some( IntegrationRegistration::builder(GPT_DIAGNOSTICS_INTEGRATION_ID) - .without_js() + .with_standalone_js() + .with_middleware(Arc::new(DiagnosticsHead)) .build(), )) } -/// Whether the diagnostics integration is present and enabled in configuration. +/// Writes what an active decision adds to a reader's copy of a document, on +/// the pages a `[[serve]]` entry names [`MODULE`] for. /// -/// This is the deployment-level switch, not the per-document activation state: -/// callers that only need to know whether diagnostics could consume a value use -/// this, while document behaviour uses [`GptDiagnosticsRequestDecision::active`]. -/// A configuration that cannot be parsed reads as disabled. -#[must_use] -pub fn is_enabled(settings: &Settings) -> bool { - settings - .integration_config::(GPT_DIAGNOSTICS_INTEGRATION_ID) - .ok() - .flatten() - .is_some() +/// It runs for each reader, and not once for a stored page, because the +/// decision is made for one request. +struct DiagnosticsHead; + +impl DiagnosticsHead { + /// The decision the preparer left for this document, when it left one. + fn decision( + document_state: &IntegrationDocumentState, + ) -> Option> { + document_state.get::(GPT_DIAGNOSTICS_INTEGRATION_ID) + } +} + +impl Middleware for DiagnosticsHead { + fn middleware_id(&self) -> &'static str { + MODULE + } + + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Serve] + } + + /// The bootstrap goes ahead of the bundle. The module loads synchronously + /// after the bundle, so its listeners are in place before the publisher's + /// scripts in the origin head run. + fn create(&self, context: &MiddlewareContext<'_>) -> MiddlewareAction { + let Some(decision) = Self::decision(context.document_state) else { + return MiddlewareAction::pass(); + }; + MiddlewareAction { + head_inserts: decision.bootstrap_script().into_iter().collect(), + after_bundle_inserts: JsModulePart::compile_time(GPT_DIAGNOSTICS_INTEGRATION_ID) + .and_then(|module| decision.module_script_tag(&module)) + .into_iter() + .collect(), + ..MiddlewareAction::pass() + } + } } /// Evaluate activation and sanitize the request before generic cookie handling. @@ -264,15 +323,15 @@ pub fn prepare_request( return Ok(existing.clone()); } - let integration_enabled = settings - .integration_config::(GPT_DIAGNOSTICS_INTEGRATION_ID)? + let integration_runs = settings + .module_config::(MODULE)? .is_some(); let (directive, clean_path, had_reserved_query) = console_query(request.uri()); let cookie_state = console_cookie_state(request); let eligible_navigation = request.method() == Method::GET && is_navigation_request(request) - && !crate::publisher::is_prefetch_request(request) - && !crate::publisher::is_bot_user_agent(request); + && !trusted_server_core::publisher::is_prefetch_request(request) + && !trusted_server_core::publisher::is_bot_user_agent(request); sanitize_console_cookie(request); if had_reserved_query { @@ -280,7 +339,7 @@ pub fn prepare_request( } let mut decision = GptDiagnosticsRequestDecision::default(); - if integration_enabled && eligible_navigation && had_reserved_query { + if integration_runs && eligible_navigation && had_reserved_query { decision.clean_browser_path_and_query = Some(clean_path); match directive { QueryDirective::Enable => { @@ -292,7 +351,7 @@ pub fn prepare_request( } QueryDirective::Invalid | QueryDirective::Absent => {} } - } else if integration_enabled + } else if integration_runs && directive == QueryDirective::Absent && cookie_state.occurrences == 1 && cookie_state.canonical @@ -302,17 +361,38 @@ pub fn prepare_request( } request.extensions_mut().insert(decision.clone()); + // Only a decision that changes the document or the response is left for + // the page path, so an ordinary request stays on the shared template path. + if decision.requires_private_no_store() { + IntegrationRequestState::insert(request, GPT_DIAGNOSTICS_INTEGRATION_ID, decision.clone()); + } Ok(decision) } -/// Read the request decision, defaulting to inactive when not prepared. -#[must_use] -pub fn request_decision(request: &Request) -> GptDiagnosticsRequestDecision { - request - .extensions() - .get::() - .cloned() - .unwrap_or_default() +/// Builder hook: prepares the request and discards the decision, which stays +/// in the request extensions for the publisher path to read. +/// +/// # Errors +/// +/// Returns the error from [`prepare_request`]. +pub(crate) fn prepare_request_hook( + settings: &Settings, + request: &mut Request, +) -> Result<(), Report> { + prepare_request(settings, request).map(|_| ()) +} + +/// Builder hook: finalizes the response from the decision the preparer left +/// for the page path, and leaves the response alone when it left none. +pub(crate) fn finalize_response_hook( + request_state: &IntegrationRequestState, + response: &mut Response, +) { + if let Some(decision) = + request_state.get::(GPT_DIAGNOSTICS_INTEGRATION_ID) + { + finalize_response(&decision, response); + } } /// Apply activation cookie and strict cache privacy to an origin response. @@ -339,7 +419,7 @@ pub fn finalize_response( // `RequestFilterEffects` mutation such as `Cache-Control: public` replaced it, // and the adapter's terminal guard had no marker to re-enforce from, so // request-scoped diagnostics HTML became shared-cacheable. - enforce_synthesized_html_cache_privacy(response); + enforce_terminal_private_cache_privacy(response); } } @@ -436,19 +516,20 @@ fn replace_path_and_query( #[cfg(test)] mod tests { use super::*; - use crate::integrations::IntegrationRegistry; - use crate::test_support::tests::create_test_settings; use serde_json::json; + use trusted_server_core::integrations::IntegrationRegistry; + use trusted_server_core::test_support::tests::create_test_settings; - fn settings(enabled: bool) -> Settings { + /// Settings that select the module and run its middleware on every + /// reader's copy of a page. + fn settings() -> Settings { let mut settings = create_test_settings(); - settings - .integrations - .insert_config( - GPT_DIAGNOSTICS_INTEGRATION_ID, - &json!({ "enabled": enabled }), - ) - .expect("should insert diagnostics config"); + settings.select_module("ad-tag", "ad-tag.google.diagnostics"); + trusted_server_core::html_processor::test_support::place_on_every_page( + &mut settings, + MiddlewarePhase::Serve, + &[MODULE], + ); settings } @@ -467,14 +548,11 @@ mod tests { #[test] fn register_excludes_diagnostics_from_unified_and_deferred_bundles() { - let settings = settings(true); - let plan = std::sync::Arc::new( - crate::auction::compile_auction_plan(&settings).expect("should compile auction plan"), - ); - let registry = - IntegrationRegistry::with_plan(&settings, plan).expect("should build registry"); + let settings = settings(); + let registry = IntegrationRegistry::with_registrations(&settings, &[builder()]) + .expect("should build registry"); - assert!(registry.integration_enabled(GPT_DIAGNOSTICS_INTEGRATION_ID)); + assert!(registry.integration_runs(GPT_DIAGNOSTICS_INTEGRATION_ID)); assert!( !registry .js_module_ids_immediate() @@ -485,6 +563,10 @@ mod tests { .js_module_ids_deferred() .contains(&GPT_DIAGNOSTICS_INTEGRATION_ID) ); + assert!( + registry.js_part(GPT_DIAGNOSTICS_INTEGRATION_ID).is_some(), + "should serve the diagnostics module standalone" + ); } #[test] @@ -494,7 +576,7 @@ mod tests { Some("other=value; __Host-ts-console=1"), ); - let decision = prepare_request(&settings(true), &mut request).expect("should prepare"); + let decision = prepare_request(&settings(), &mut request).expect("should prepare"); assert!(decision.active()); assert_eq!( @@ -518,7 +600,7 @@ mod tests { .headers_mut() .insert("purpose", HeaderValue::from_static("prefetch")); - let decision = prepare_request(&settings(true), &mut request).expect("should prepare"); + let decision = prepare_request(&settings(), &mut request).expect("should prepare"); assert!( !decision.active(), @@ -534,7 +616,7 @@ mod tests { "https://publisher.example/page", Some("__Host-ts-console=1; other=value"), ); - let decision = prepare_request(&settings(true), &mut active).expect("should prepare"); + let decision = prepare_request(&settings(), &mut active).expect("should prepare"); assert!(decision.active()); assert_eq!(active.headers()[header::COOKIE], "other=value"); @@ -542,7 +624,7 @@ mod tests { "https://publisher.example/page", Some("__Host-ts-console=1; __Host-ts-console=1; other=value"), ); - let decision = prepare_request(&settings(true), &mut duplicate).expect("should prepare"); + let decision = prepare_request(&settings(), &mut duplicate).expect("should prepare"); assert!(!decision.active()); assert_eq!(duplicate.headers()[header::COOKIE], "other=value"); } @@ -558,7 +640,7 @@ mod tests { &format!("https://publisher.example/page?{query}&keep=1"), Some("__Host-ts-console=1"), ); - let decision = prepare_request(&settings(true), &mut request).expect("should prepare"); + let decision = prepare_request(&settings(), &mut request).expect("should prepare"); assert!(!decision.active(), "{query} should fail closed"); assert_eq!(decision.cookie_action, GptDiagnosticsCookieAction::None); assert_eq!(request.uri().query(), Some("keep=1")); @@ -568,7 +650,7 @@ mod tests { "https://publisher.example/page?ts_console=false&keep=1", Some("__Host-ts-console=1"), ); - let decision = prepare_request(&settings(true), &mut request).expect("should prepare"); + let decision = prepare_request(&settings(), &mut request).expect("should prepare"); assert!(!decision.active()); assert_eq!( decision.cookie_action, @@ -579,7 +661,7 @@ mod tests { #[test] fn finalization_sets_cookie_and_strips_shared_cache_headers() { let mut request = navigation("https://publisher.example/?ts_console=1", None); - let decision = prepare_request(&settings(true), &mut request).expect("should prepare"); + let decision = prepare_request(&settings(), &mut request).expect("should prepare"); let mut response = Response::builder() .header(header::CACHE_CONTROL, "public, max-age=60") .header(header::ETAG, "\"origin\"") @@ -622,7 +704,7 @@ mod tests { // because the adapter's terminal guard keys on the marker, not on the stamp, and // the `Set-Cookie` privacy net never sees a response that sets no cookie. let mut request = navigation("https://publisher.example/", Some("__Host-ts-console=1")); - let decision = prepare_request(&settings(true), &mut request).expect("should prepare"); + let decision = prepare_request(&settings(), &mut request).expect("should prepare"); assert!(decision.active(), "the session cookie should activate"); assert_eq!( decision.cookie_action, @@ -642,7 +724,7 @@ mod tests { assert!( response .extensions() - .get::() + .get::() .is_some(), "should mark request-scoped diagnostics HTML for terminal re-enforcement" ); @@ -651,7 +733,7 @@ mod tests { #[test] fn an_inactive_decision_leaves_the_origin_cache_policy_alone() { let mut request = navigation("https://publisher.example/", None); - let decision = prepare_request(&settings(true), &mut request).expect("should prepare"); + let decision = prepare_request(&settings(), &mut request).expect("should prepare"); assert!(!decision.requires_private_no_store()); let mut response = Response::builder() .header(header::CACHE_CONTROL, "public, max-age=60") @@ -669,7 +751,7 @@ mod tests { assert!( response .extensions() - .get::() + .get::() .is_none(), "should not mark an untouched response terminal-private" ); @@ -679,17 +761,188 @@ mod tests { fn config_rejects_unknown_fields() { let mut settings = create_test_settings(); settings - .integrations - .insert_config( - GPT_DIAGNOSTICS_INTEGRATION_ID, - &json!({ "enabled": true, "typo": true }), + .insert_module_config( + "ad-tag", + "ad-tag.google.diagnostics", + &json!({"typo": true }), ) .expect("should insert diagnostics config"); let error = settings - .integration_config::(GPT_DIAGNOSTICS_INTEGRATION_ID) + .module_config::(MODULE) .expect_err("should reject unknown diagnostics config fields"); let error_text = format!("{error:?}"); assert!(error_text.contains("typo") || error_text.contains("unknown field")); } + + #[test] + fn module_constant_names_the_crate_folder_and_the_module_within_it() { + assert_eq!( + MODULE, + format!("{}.diagnostics", trusted_server_core::module_name!()), + "should be named by the folder this crate lives in" + ); + } + + #[test] + fn only_a_decision_that_acts_is_left_for_the_page_path() { + let mut ordinary = navigation("https://publisher.example/page", None); + prepare_request(&settings(), &mut ordinary).expect("should prepare"); + assert!( + IntegrationRequestState::of(&ordinary).is_empty(), + "an ordinary request should stay on the shared template path" + ); + + let mut activating = navigation("https://publisher.example/page?ts_console=1", None); + let decision = prepare_request(&settings(), &mut activating).expect("should prepare"); + assert_eq!( + IntegrationRequestState::of(&activating) + .get::(GPT_DIAGNOSTICS_INTEGRATION_ID) + .as_deref(), + Some(&decision), + "should leave the decision that acts for the page path" + ); + } + + #[test] + fn the_finalizer_acts_only_on_a_decision_left_for_the_page_path() { + let mut untouched = Response::builder() + .header(header::CACHE_CONTROL, "public, max-age=60") + .body(EdgeBody::empty()) + .expect("should build response"); + finalize_response_hook(&IntegrationRequestState::default(), &mut untouched); + assert_eq!( + untouched.headers()[header::CACHE_CONTROL], + "public, max-age=60", + "should leave a response alone when nothing was left for it" + ); + assert!( + !untouched.headers().contains_key(header::SET_COOKIE), + "should set no cookie when nothing was left for it" + ); + + let mut request = navigation("https://publisher.example/?ts_console=1", None); + prepare_request(&settings(), &mut request).expect("should prepare"); + let mut response = Response::builder() + .header(header::CACHE_CONTROL, "public, max-age=60") + .body(EdgeBody::empty()) + .expect("should build response"); + finalize_response_hook(&IntegrationRequestState::of(&request), &mut response); + assert_eq!( + response.headers()[header::CACHE_CONTROL], + "no-store, private", + "should make the activating response private" + ); + assert_eq!( + response.headers()[header::SET_COOKIE], + SET_CONSOLE_COOKIE, + "should establish the session" + ); + } + + #[test] + fn an_active_document_loads_the_module_after_the_unified_bundle_once() { + use trusted_server_core::html_processor::HtmlProcessorConfig; + use trusted_server_core::html_processor::test_support::create_page_processor; + use trusted_server_core::streaming_processor::StreamProcessor as _; + + let settings = settings(); + let registry = IntegrationRegistry::with_registrations(&settings, &[builder()]) + .expect("should build registry"); + let mut request = navigation("https://publisher.example/page?ts_console=1", None); + let decision = prepare_request(&settings, &mut request).expect("should prepare"); + assert!( + decision.active(), + "should activate diagnostics for the query" + ); + let config = HtmlProcessorConfig::from_settings( + &settings, + ®istry, + "origin.example.com", + "publisher.example", + "https", + ) + .with_request_state(IntegrationRequestState::of(&request)); + let mut processor = create_page_processor(&settings, ®istry, config); + + let output = processor + .process_chunk( + b"Test", + true, + ) + .expect("should process HTML"); + let processed = String::from_utf8(output).expect("should produce valid UTF-8"); + let bootstrap_marker = "__tsjs_gpt_diagnostics_active"; + let bundle_marker = "id=\"trustedserver-js\""; + let diagnostics_marker = "tsjs-gpt_diagnostics.min.js"; + + assert_eq!( + processed.matches(bootstrap_marker).count(), + 1, + "should inject the diagnostics bootstrap once" + ); + assert_eq!( + processed.matches(bundle_marker).count(), + 1, + "should inject the immediate TSJS bundle once" + ); + assert_eq!( + processed.matches(diagnostics_marker).count(), + 1, + "should inject one standalone diagnostics module" + ); + let bootstrap_index = processed + .find(bootstrap_marker) + .expect("should include diagnostics bootstrap"); + let bundle_index = processed + .find(bundle_marker) + .expect("should include immediate TSJS bundle"); + let diagnostics_index = processed + .find(diagnostics_marker) + .expect("should include standalone diagnostics module"); + assert!( + bootstrap_index < bundle_index, + "should activate before core executes" + ); + assert!( + bundle_index < diagnostics_index, + "should load diagnostics after core" + ); + } + + #[test] + fn an_inactive_document_gets_neither_script() { + use trusted_server_core::html_processor::HtmlProcessorConfig; + use trusted_server_core::html_processor::test_support::create_page_processor; + use trusted_server_core::streaming_processor::StreamProcessor as _; + + let settings = settings(); + let registry = IntegrationRegistry::with_registrations(&settings, &[builder()]) + .expect("should build registry"); + let mut request = navigation("https://publisher.example/page", None); + prepare_request(&settings, &mut request).expect("should prepare"); + let config = HtmlProcessorConfig::from_settings( + &settings, + ®istry, + "origin.example.com", + "publisher.example", + "https", + ) + .with_request_state(IntegrationRequestState::of(&request)); + let mut processor = create_page_processor(&settings, ®istry, config); + + let output = processor + .process_chunk( + b"Test", + true, + ) + .expect("should process HTML"); + let processed = String::from_utf8(output).expect("should produce valid UTF-8"); + + assert!( + !processed.contains("__tsjs_gpt_diagnostics_active") + && !processed.contains("tsjs-gpt_diagnostics.min.js"), + "should write nothing into a document diagnostics is not active for: {processed}" + ); + } } diff --git a/crates/ad-tag/google/src/fixtures/page-change.input.html b/crates/ad-tag/google/src/fixtures/page-change.input.html new file mode 100644 index 0000000000..6a9f667691 --- /dev/null +++ b/crates/ad-tag/google/src/fixtures/page-change.input.html @@ -0,0 +1,17 @@ + + + + +Google Publisher Tag + + + + + + + +An article on the origin + +
+ + diff --git a/crates/ad-tag/google/src/fixtures/page-change.recorded.html b/crates/ad-tag/google/src/fixtures/page-change.recorded.html new file mode 100644 index 0000000000..3d3a347ffd --- /dev/null +++ b/crates/ad-tag/google/src/fixtures/page-change.recorded.html @@ -0,0 +1,947 @@ + + + + +Google Publisher Tag + + + + + + + +An article on the origin + +
+ + diff --git a/crates/ad-tag/google/src/fixtures/page-change.settings.toml b/crates/ad-tag/google/src/fixtures/page-change.settings.toml new file mode 100644 index 0000000000..50ef4c9659 --- /dev/null +++ b/crates/ad-tag/google/src/fixtures/page-change.settings.toml @@ -0,0 +1,13 @@ +[ad-tag] +modules = ["google", "google.diagnostics"] + +[ad-tag.google] +gam_attribution_enabled = false + +[[fetch]] +media_type = "text/html" +middleware = ["ad-tag.google"] + +[[serve]] +media_type = "text/html" +middleware = ["ad-tag.google.diagnostics"] diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/ad-tag/google/src/gpt_bootstrap.js similarity index 100% rename from crates/trusted-server-core/src/integrations/gpt_bootstrap.js rename to crates/ad-tag/google/src/gpt_bootstrap.js diff --git a/crates/trusted-server-core/src/integrations/gpt.rs b/crates/ad-tag/google/src/lib.rs similarity index 83% rename from crates/trusted-server-core/src/integrations/gpt.rs rename to crates/ad-tag/google/src/lib.rs index 3292cc5b3a..8017393fc4 100644 --- a/crates/trusted-server-core/src/integrations/gpt.rs +++ b/crates/ad-tag/google/src/lib.rs @@ -18,7 +18,7 @@ //! //! ## How It Works //! -//! 1. **HTML rewriting** – The [`IntegrationAttributeRewriter`] swaps `src`/`href` +//! 1. **HTML rewriting** – The module's middleware swaps `src`/`href` //! attributes pointing at Google's GPT script with a first-party URL //! (`/integrations/gpt/script`). //! 2. **Script proxy** – [`IntegrationProxy`] endpoints serve `gpt.js` @@ -31,7 +31,25 @@ //! `` must be neutralised: the only // `` left is the tag's own legitimate closer. @@ -1540,15 +1525,7 @@ mod tests { #[test] fn head_inserts_omits_slim_prebid_url_when_not_configured() { let integration = GptIntegration::new(test_config()); - let doc_state = IntegrationDocumentState::default(); - let ctx = IntegrationHtmlContext { - request_host: "edge.example.com", - request_scheme: "https", - origin_host: "example.com", - document_state: &doc_state, - }; - - let inserts = integration.head_inserts(&ctx); + let inserts = integration.head_markup(); assert_eq!( inserts.len(), @@ -1562,4 +1539,30 @@ mod tests { "should not emit slim-Prebid URL tag when not configured" ); } + + #[test] + fn module_constant_is_the_crate_folder() { + assert_eq!( + super::MODULE, + trusted_server_core::module_name!(), + "should be named by the folder this crate lives in" + ); + } + + /// The page a reader receives with this module running, kept as a file + /// so that changing how the page change is made can be shown to leave + /// the page as it was. + #[test] + fn the_page_a_reader_receives_is_the_recorded_one() { + trusted_server_core::html_processor::test_support::assert_page_is_recorded( + include_str!("fixtures/page-change.settings.toml"), + &[super::builder(), super::diagnostics::builder()], + include_str!("fixtures/page-change.input.html"), + include_str!("fixtures/page-change.recorded.html"), + concat!( + env!("CARGO_MANIFEST_DIR"), + "/src/fixtures/page-change.recorded.html" + ), + ); + } } diff --git a/crates/ads-txt/sellers/Cargo.toml b/crates/ads-txt/sellers/Cargo.toml new file mode 100644 index 0000000000..48f16ca1f8 --- /dev/null +++ b/crates/ads-txt/sellers/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "trusted-server-ads-txt-sellers" +description = "Adds a publisher's sellers to the origin's ads.txt from the lines the settings give, or makes the file from them where the origin has none." +authors = { workspace = true } +edition = { workspace = true } +license = { workspace = true } +publish = { workspace = true } +version = { workspace = true } + +[lib] +doctest = false + +[lints] +workspace = true + +[dependencies] +error-stack = { workspace = true } +serde = { workspace = true } +trusted-server-core = { workspace = true } +validator = { workspace = true } + +[dev-dependencies] +trusted-server-core = { workspace = true, features = ["test-utils"] } + +# The visible owner of this crate. +[package.metadata.maintainers] +owner = "Trusted Server maintainers" +status = "project owned" diff --git a/crates/ads-txt/sellers/src/fixtures/sellers.settings.toml b/crates/ads-txt/sellers/src/fixtures/sellers.settings.toml new file mode 100644 index 0000000000..1d4b71bcd8 --- /dev/null +++ b/crates/ads-txt/sellers/src/fixtures/sellers.settings.toml @@ -0,0 +1,10 @@ +[ads-txt] +modules = ["sellers"] + +[ads-txt.sellers] +lines = ["ssp.example, 12345, DIRECT, f08c47fec0942fa0", "CONTACT=ads@publisher.example"] + +[[fetch]] +media_type = "text/plain" +path = "/ads.txt" +middleware = ["ads-txt.sellers"] diff --git a/crates/ads-txt/sellers/src/lib.rs b/crates/ads-txt/sellers/src/lib.rs new file mode 100644 index 0000000000..d70b652ccb --- /dev/null +++ b/crates/ads-txt/sellers/src/lib.rs @@ -0,0 +1,544 @@ +//! The sellers a deployment adds to the publisher's `ads.txt`. +//! +//! The publisher's own file at the origin stays authoritative. Where the +//! origin has none, the file is made of the lines alone. + +use std::sync::Arc; + +use error_stack::Report; +use serde::Deserialize; +use validator::Validate; + +use trusted_server_core::error::TrustedServerError; +use trusted_server_core::integrations::{IntegrationBuilder, IntegrationRegistration}; +use trusted_server_core::middleware::{ + Middleware, MiddlewareAction, MiddlewareContext, MiddlewarePhase, +}; +use trusted_server_core::settings::{IntegrationConfig, Settings}; +use trusted_server_core::streaming_processor::StreamProcessor; + +/// The id the module's registration carries. +const SELLERS_INTEGRATION_ID: &str = "ads_txt_sellers"; + +/// The name `[ads-txt] modules` selects the module by and a `[[fetch]]` entry +/// names its middleware by, from this crate's folder. Its settings are the +/// table `[ads-txt.sellers]`. +pub const MODULE: &str = "ads-txt.sellers"; + +/// The media type an `ads.txt` is served as, and the one the middleware +/// works on. +pub const MEDIA_TYPE: &str = "text/plain"; + +/// The variables the specification names, each written as `NAME=value`. +const VARIABLES: [&str; 5] = [ + "CONTACT", + "SUBDOMAIN", + "INVENTORYPARTNERDOMAIN", + "OWNERDOMAIN", + "MANAGERDOMAIN", +]; + +/// Configuration for the sellers added to `ads.txt`. +#[derive(Debug, Clone, Deserialize, Validate)] +#[serde(deny_unknown_fields)] +pub struct SellersConfig { + /// The lines written at the end of the publisher's file, each a data + /// record or a variable, as the file's grammar has them. + pub lines: Vec, +} + +impl IntegrationConfig for SellersConfig {} + +/// Refuses a line that is not a data record or a variable, saying which part +/// is wrong. +/// +/// A comment or an empty line is refused too, because the settings carry +/// what the deployment declares and a comment declares nothing. +fn check_line(line: &str) -> Result<(), String> { + let trimmed = line.trim(); + if trimmed.is_empty() { + return Err("is empty".to_owned()); + } + if trimmed.contains('#') { + return Err("carries a comment, and a line in the settings declares something".to_owned()); + } + if trimmed.contains(['\n', '\r']) { + return Err("is more than one line".to_owned()); + } + if let Some((name, value)) = trimmed.split_once('=') + && !name.contains(',') + { + let name = name.trim(); + if VARIABLES + .iter() + .any(|variable| variable.eq_ignore_ascii_case(name)) + { + if value.trim().is_empty() { + return Err(format!("gives `{name}` no value")); + } + return Ok(()); + } + return Err(format!( + "names the variable `{name}`, which is not one of {}", + VARIABLES.join(", ") + )); + } + let fields: Vec<&str> = trimmed.split(',').map(str::trim).collect(); + if fields.len() < 3 || fields.len() > 4 { + return Err(format!( + "has {} fields, and a data record has three or four: the domain, the account id, \ + DIRECT or RESELLER, and the certification authority id", + fields.len() + )); + } + if fields[0].is_empty() { + return Err("has no domain where its first field should be one".to_owned()); + } + if fields[0].contains(char::is_whitespace) { + return Err(format!( + "has `{}` as its domain, and a domain holds no space", + fields[0] + )); + } + if fields[1].is_empty() { + return Err("has no account id in its second field".to_owned()); + } + if !fields[2].eq_ignore_ascii_case("DIRECT") && !fields[2].eq_ignore_ascii_case("RESELLER") { + return Err(format!( + "says the relationship is `{}`, and it is DIRECT or RESELLER", + fields[2] + )); + } + if fields.len() == 4 && fields[3].is_empty() { + return Err("ends with a comma and no certification authority id".to_owned()); + } + Ok(()) +} + +/// Refuses settings whose lines are not each a record or a variable, or that +/// give no line. +fn validate_lines(config: &SellersConfig) -> Result<(), Report> { + if config.lines.is_empty() { + return Err(Report::new(TrustedServerError::Settings { + message: "`lines` in [ads-txt.sellers] is empty, so the module would add nothing. \ + Give the lines to add, or do not select the module" + .to_owned(), + })); + } + for (index, line) in config.lines.iter().enumerate() { + if let Err(problem) = check_line(line) { + return Err(Report::new(TrustedServerError::Settings { + message: format!( + "`lines` in [ads-txt.sellers], line {}, `{}`, {problem}", + index + 1, + line.trim() + ), + })); + } + } + Ok(()) +} + +/// Refuses a document that selects the module but names it in no entry, where +/// the lines would never be written. +fn require_an_entry(settings: &Settings) -> Result<(), Report> { + let placed = settings.fetch.entries().iter().any(|entry| { + entry.media_type == MEDIA_TYPE && entry.middleware.iter().any(|name| name == MODULE) + }); + if placed { + return Ok(()); + } + Err(Report::new(TrustedServerError::Settings { + message: format!( + "[ads-txt] selects `sellers`, but no [[fetch]] entry for `{MEDIA_TYPE}` names \ + `{MODULE}`, so the lines would never be written. Add an entry with media_type = \ + `{MEDIA_TYPE}`, path = `/ads.txt` and the middleware. The path is a prefix, so \ + the entry covers every path that starts with `/ads.txt`" + ), + })) +} + +/// Validates the sellers' configuration and reports whether a section selects +/// the module. +/// +/// # Errors +/// +/// Returns an error when the table cannot be read, a line is not a record or +/// a variable, no line is given, or no entry places the middleware. +pub fn validate(settings: &Settings) -> Result> { + let Some(config) = settings.module_config::(MODULE)? else { + return Ok(false); + }; + validate_lines(&config)?; + require_an_entry(settings)?; + Ok(true) +} + +/// Registers the sellers' middleware when a section selects the module. +/// +/// # Errors +/// +/// Returns the errors [`validate`] returns. +pub fn register( + settings: &Settings, +) -> Result, Report> { + let Some(config) = settings.module_config::(MODULE)? else { + return Ok(None); + }; + validate_lines(&config)?; + require_an_entry(settings)?; + Ok(Some( + IntegrationRegistration::builder(SELLERS_INTEGRATION_ID) + .without_js() + .with_middleware(Arc::new(Sellers::new(&config.lines))) + .build(), + )) +} + +/// The builder a deployment installs to offer this module. +#[must_use] +pub fn builder() -> IntegrationBuilder { + IntegrationBuilder::new( + SELLERS_INTEGRATION_ID, + env!("CARGO_PKG_NAME"), + register, + validate, + ) + .with_module_name(MODULE) +} + +/// The lines written at the end of the file, as the middleware adds them. +#[derive(Debug)] +pub struct Sellers { + /// The lines, each ending in a line break. + tail: Arc, +} + +impl Sellers { + /// The middleware for `lines`, which the caller has checked. + #[must_use] + pub fn new(lines: &[String]) -> Self { + let mut tail = String::new(); + for line in lines { + tail.push_str(line.trim()); + tail.push('\n'); + } + Self { + tail: Arc::from(tail), + } + } +} + +impl Middleware for Sellers { + fn middleware_id(&self) -> &'static str { + MODULE + } + + fn handles_media_type(&self, media_type: &str) -> bool { + media_type == MEDIA_TYPE + } + + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Fetch] + } + + fn create(&self, _context: &MiddlewareContext<'_>) -> MiddlewareAction { + MiddlewareAction { + stream: Some(Box::new(AppendLines { + tail: Arc::clone(&self.tail), + ended_with_break: true, + })), + ..MiddlewareAction::pass() + } + } +} + +/// Passes the file through and writes the lines after its last byte, on a +/// line of their own. +struct AppendLines { + tail: Arc, + /// Whether the file so far ends with a line break. + ended_with_break: bool, +} + +impl StreamProcessor for AppendLines { + fn process_chunk(&mut self, chunk: &[u8], is_last: bool) -> Result, std::io::Error> { + let mut output = chunk.to_vec(); + if let Some(last) = chunk.last() { + self.ended_with_break = *last == b'\n'; + } + if is_last { + if !self.ended_with_break { + output.push(b'\n'); + } + output.extend_from_slice(self.tail.as_bytes()); + } + Ok(output) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use trusted_server_core::html_processor::test_support::page_settings; + use trusted_server_core::integrations::{IntegrationDocumentState, IntegrationRegistry}; + use trusted_server_core::middleware::MiddlewareChain; + use trusted_server_core::middleware::test_support::context; + + const RECORD: &str = "ssp.example, 12345, DIRECT, f08c47fec0942fa0"; + + /// A document selecting the module with `lines`, placed on the file. + fn settings_with(lines: &[&str]) -> Settings { + let quoted: Vec = lines.iter().map(|line| format!("\"{line}\"")).collect(); + page_settings(&format!( + "[ads-txt]\nmodules = [\"sellers\"]\n\n[ads-txt.sellers]\nlines = [{}]\n\n\ + [[fetch]]\nmedia_type = \"text/plain\"\npath = \"/ads.txt\"\n\ + middleware = [\"ads-txt.sellers\"]\n", + quoted.join(", ") + )) + } + + /// What `body` becomes through the middleware, read `chunk` bytes at a + /// time. + fn through(lines: &[&str], body: &str, chunk: usize) -> String { + let lines: Vec = lines.iter().map(|line| (*line).to_owned()).collect(); + let document_state = IntegrationDocumentState::default(); + let plan = MiddlewareChain::new( + MiddlewarePhase::Fetch, + MEDIA_TYPE, + vec![Arc::new(Sellers::new(&lines))], + ) + .plan(&context(MiddlewarePhase::Fetch, &document_state)) + .expect("the chain should plan"); + let mut processors = plan.processors; + assert_eq!(processors.len(), 1, "the middleware adds one processor"); + let processor = &mut processors[0]; + let bytes = body.as_bytes(); + let mut output = Vec::new(); + if bytes.is_empty() { + output.extend(processor.process_chunk(&[], true).expect("should process")); + } else { + let chunks: Vec<&[u8]> = bytes.chunks(chunk).collect(); + for (index, piece) in chunks.iter().enumerate() { + let is_last = index + 1 == chunks.len(); + output.extend( + processor + .process_chunk(piece, is_last) + .expect("should process"), + ); + } + } + String::from_utf8(output).expect("text") + } + + #[test] + fn module_constant_is_the_crate_folder() { + assert_eq!( + MODULE, + trusted_server_core::module_name!(), + "should be named by the folder this crate lives in" + ); + } + + #[test] + fn an_unselected_module_does_not_register() { + let settings = page_settings(""); + assert!(register(&settings).expect("should read").is_none()); + assert!(!validate(&settings).expect("should read")); + } + + #[test] + fn the_lines_are_offered_as_the_modules_middleware_on_text() { + let settings = settings_with(&[RECORD]); + let registry = IntegrationRegistry::with_registrations(&settings, &[builder()]) + .expect("should build the registry"); + + assert_eq!( + registry + .middleware_chain( + &settings.fetch, + MiddlewarePhase::Fetch, + MEDIA_TYPE, + "/ads.txt" + ) + .ids(), + [MODULE], + "should run on the file" + ); + assert!( + registry + .middleware_chain( + &settings.fetch, + MiddlewarePhase::Fetch, + MEDIA_TYPE, + "/robots.txt" + ) + .is_empty(), + "should run on no other path" + ); + let registration = register(&settings) + .expect("should parse") + .expect("should register"); + assert!( + registration.js_disabled, + "nothing of this module runs in the browser" + ); + } + + #[test] + fn a_document_naming_the_module_in_no_entry_is_refused() { + let settings = page_settings(&format!( + "[ads-txt]\nmodules = [\"sellers\"]\n\n[ads-txt.sellers]\nlines = [\"{RECORD}\"]\n" + )); + + let refused = validate(&settings).expect_err("should refuse a module no entry places"); + assert!( + format!("{refused:?}") + .contains("no [[fetch]] entry for `text/plain` names `ads-txt.sellers`"), + "{refused:?}" + ); + assert!(register(&settings).is_err()); + } + + #[test] + fn a_line_that_is_not_a_record_or_a_variable_is_refused_for_its_reason() { + let cases = [ + ("", "line 2, ``, is empty"), + ("# a comment", "carries a comment"), + ("ssp.example, 12345", "has 2 fields"), + ("ssp.example, 12345, DIRECT, abc, extra", "has 5 fields"), + (", 12345, DIRECT", "has no domain"), + ( + "ssp example, 12345, DIRECT", + "has `ssp example` as its domain, and a domain holds no space", + ), + ("ssp.example, , DIRECT", "has no account id"), + ( + "ssp.example, 12345, PARTNER", + "says the relationship is `PARTNER`", + ), + ("ssp.example, 12345, DIRECT, ", "ends with a comma"), + ("CONTACT=", "gives `CONTACT` no value"), + ("EMAIL=ads@publisher.example", "names the variable `EMAIL`"), + ]; + for (line, reason) in cases { + let settings = settings_with(&[RECORD, line]); + for (call, refusal) in [ + ("validate", validate(&settings).err()), + ("register", register(&settings).err()), + ] { + let message = refusal + .unwrap_or_else(|| panic!("{call} should refuse {line:?}")) + .current_context() + .to_string(); + assert!( + message.contains("`lines` in [ads-txt.sellers], line 2") + && message.contains(reason), + "{call} should say {reason:?} of {line:?}, and said {message:?}" + ); + } + } + } + + #[test] + fn no_line_at_all_is_refused() { + let settings = settings_with(&[]); + let message = validate(&settings) + .expect_err("should refuse an empty list") + .current_context() + .to_string(); + assert!( + message.contains("is empty, so the module would add nothing"), + "{message}" + ); + } + + #[test] + fn records_and_variables_are_accepted_as_the_grammar_has_them() { + for line in [ + "ssp.example, 12345, DIRECT", + "ssp.example,12345,reseller,f08c47fec0942fa0", + " ssp.example , 12345 , Direct ", + "CONTACT=ads@publisher.example", + "contact=ads@publisher.example", + "Subdomain=news.publisher.example", + "ssp.example, pub=12345, DIRECT", + "SUBDOMAIN=news.publisher.example", + "INVENTORYPARTNERDOMAIN=partner.example", + "OWNERDOMAIN=publisher.example", + "MANAGERDOMAIN=manager.example, GB", + ] { + let settings = settings_with(&[line]); + assert!(validate(&settings).unwrap_or_else(|error| panic!("{line}: {error:?}"))); + } + } + + #[test] + fn the_lines_follow_the_publishers_file_on_a_line_of_their_own() { + let lines = [RECORD, "CONTACT=ads@publisher.example"]; + let added = format!("{RECORD}\nCONTACT=ads@publisher.example\n"); + + assert_eq!( + through(&lines, "own.example, 1, DIRECT\n", 8192), + format!("own.example, 1, DIRECT\n{added}"), + "a file ending with a line break takes the lines straight after it" + ); + assert_eq!( + through(&lines, "own.example, 1, DIRECT", 8192), + format!("own.example, 1, DIRECT\n{added}"), + "a file without a final line break gets one before the lines" + ); + assert_eq!( + through(&lines, "", 8192), + added, + "an empty file is the lines" + ); + } + + #[test] + fn a_file_whose_lines_end_with_a_carriage_return_takes_the_lines_after_its_last_one() { + assert_eq!( + through(&[RECORD], "own.example, 1, DIRECT\r\n", 8192), + format!("own.example, 1, DIRECT\r\n{RECORD}\n"), + "a file ending with a carriage return and a line feed takes no second break" + ); + assert_eq!( + through(&[RECORD], "a.example, 1, DIRECT\r\nb.example, 2, DIRECT", 3), + format!("a.example, 1, DIRECT\r\nb.example, 2, DIRECT\n{RECORD}\n"), + "a file without a final line break gets one before the lines" + ); + } + + #[test] + fn a_file_read_in_pieces_is_the_same_file() { + let lines = [RECORD]; + let body = "a.example, 1, DIRECT\nb.example, 2, RESELLER"; + let whole = through(&lines, body, 8192); + + for chunk in [1, 3, 7, 20] { + assert_eq!( + through(&lines, body, chunk), + whole, + "read {chunk} bytes at a time" + ); + } + assert_eq!(whole, format!("{body}\n{RECORD}\n")); + } + + #[test] + fn the_lines_are_written_as_given_and_trimmed() { + assert_eq!( + through(&[" ssp.example, 1, direct "], "x\n", 8192), + "x\nssp.example, 1, direct\n", + "the relationship is written as the deployment wrote it" + ); + } + + #[test] + fn the_middleware_works_on_text_alone_in_the_fetch_phase() { + let sellers = Sellers::new(&[RECORD.to_owned()]); + assert!(sellers.handles_media_type(MEDIA_TYPE)); + assert!(!sellers.handles_media_type("text/html")); + assert_eq!(sellers.phases(), [MiddlewarePhase::Fetch]); + } +} diff --git a/crates/auction-protocol/openrtb/Cargo.toml b/crates/auction-protocol/openrtb/Cargo.toml new file mode 100644 index 0000000000..83cb18ec64 --- /dev/null +++ b/crates/auction-protocol/openrtb/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "trusted-server-auction-protocol-openrtb" +description = "The plain OpenRTB 2.6 demand implementation, for an exchange that needs no code of its own." +authors = { workspace = true } +edition = { workspace = true } +license = { workspace = true } +publish = { workspace = true } +version = { workspace = true } + +[lib] +doctest = false + +[lints] +workspace = true + +[dependencies] +async-trait = { workspace = true } +error-stack = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +trusted-server-core = { workspace = true } + +[dev-dependencies] +trusted-server-core = { workspace = true, features = ["test-utils"] } + +# The visible owner of this crate, the way Prebid.js requires a named +# maintainer of every adapter. The plain OpenRTB implementation is the +# project's own, so the Trusted Server maintainers own it. +[package.metadata.maintainers] +owner = "Trusted Server maintainers" +status = "project owned" diff --git a/crates/auction-protocol/openrtb/src/lib.rs b/crates/auction-protocol/openrtb/src/lib.rs new file mode 100644 index 0000000000..42a1a9c6fb --- /dev/null +++ b/crates/auction-protocol/openrtb/src/lib.rs @@ -0,0 +1,407 @@ +//! The plain `OpenRTB` 2.6 demand implementation, `auction-protocol.openrtb`. +//! +//! A standards-compliant exchange needs no code of its own. It takes the +//! shared request baseline, adds the static extensions its table sets, and has +//! its response read the ordinary `OpenRTB` way. + +#![cfg_attr( + test, + allow( + clippy::print_stdout, + clippy::print_stderr, + clippy::panic, + clippy::dbg_macro, + clippy::unwrap_used, + reason = "tests use direct diagnostics and panic-on-failure helpers" + ) +)] + +use core::any::Any; +use std::sync::Arc; + +use async_trait::async_trait; +use error_stack::Report; +use serde::Deserialize; +use serde_json::{Map, Value}; + +use trusted_server_core::auction::demand::{ + CONSERVATIVE_LANGUAGE_MAX_BYTES, CompiledDemand, DemandFieldPolicy, DemandImplementation, + DemandResponse, DemandTimeoutDefault, ProviderAuctionInput, RegsPolicy, RequestExtensions, + accept_endpoint, +}; +use trusted_server_core::auction::openrtb::parse_openrtb_response; +use trusted_server_core::auction::types::AuctionResponse; +use trusted_server_core::error::TrustedServerError; +use trusted_server_core::platform::PlatformResponse; + +/// The name an `implementation` line gives this implementation, its module +/// path, which also serves as its builder's id. +pub const MODULE: &str = "auction-protocol.openrtb"; + +/// The plain `OpenRTB` demand implementation. +pub static DEMAND: DemandImplementation = DemandImplementation { + id: MODULE, + default_timeout: DemandTimeoutDefault::Auction, + allows_all_eligible: true, + serves_stored_requests: false, + canonicalize_endpoint: accept_endpoint, + compile, +}; + +/// The builder a deployment hands to an adapter, which offers the plain +/// `OpenRTB` implementation to `[demand]`. +#[must_use] +pub fn builder() -> trusted_server_core::integrations::IntegrationBuilder { + trusted_server_core::integrations::IntegrationBuilder::implementations( + MODULE, + env!("CARGO_PKG_NAME"), + ) + .with_demand(&DEMAND) +} + +const STATIC_EXTENSION_MAX_BYTES: usize = 16 * 1024; +const STATIC_EXTENSION_MAX_DEPTH: usize = 8; +const STATIC_EXTENSION_MAX_KEYS: usize = 256; + +/// A validated static extension object. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct StaticExtension(Map); + +impl StaticExtension { + /// Borrow the validated extension object. + #[must_use] + pub fn as_object(&self) -> &Map { + &self.0 + } +} + +/// One compiled plain `OpenRTB` demand source. +#[derive(Debug, Clone, Default)] +pub struct OpenRtbDemand { + /// Static request-level extension fields. + pub request_ext: StaticExtension, + /// Static impression-level extension fields. + pub imp_ext: StaticExtension, +} + +#[derive(Debug, Deserialize, Default)] +#[serde(deny_unknown_fields)] +struct OpenRtbSettings { + #[serde(default)] + request_ext: Option, + #[serde(default)] + imp_ext: Option, +} + +fn compile( + settings: &Map, +) -> Result, Report> { + let settings = OpenRtbSettings::deserialize(Value::Object(settings.clone())) + .map_err(|error| configuration_error(format!("invalid `{MODULE}` settings: {error}")))?; + Ok(Arc::new(OpenRtbDemand { + request_ext: validate_static_extension("request_ext", settings.request_ext)?, + imp_ext: validate_static_extension("imp_ext", settings.imp_ext)?, + })) +} + +#[async_trait(?Send)] +impl CompiledDemand for OpenRtbDemand { + fn field_policy(&self) -> DemandFieldPolicy { + DemandFieldPolicy { + language_max_bytes: Some(CONSERVATIVE_LANGUAGE_MAX_BYTES), + regs: RegsPolicy::Jurisdiction, + accept_json: true, + ..DemandFieldPolicy::default() + } + } + + fn augment_request( + &self, + extensions: &mut RequestExtensions<'_>, + _input: &ProviderAuctionInput, + ) -> Result<(), Report> { + *extensions.request = nonempty_map(self.request_ext.as_object().clone()); + for impression in &mut extensions.impressions { + *impression.ext = nonempty_map(self.imp_ext.as_object().clone()); + } + Ok(()) + } + + async fn parse_response( + &self, + context: DemandResponse<'_>, + response: PlatformResponse, + ) -> Result> { + parse_openrtb_response(context, response).await + } + + fn as_any(&self) -> &dyn Any { + self + } +} + +fn nonempty_map(value: Map) -> Option> { + (!value.is_empty()).then_some(value) +} + +fn configuration_error(message: impl Into) -> Report { + Report::new(TrustedServerError::Configuration { + message: message.into(), + }) +} + +fn validate_static_extension( + field: &str, + value: Option, +) -> Result> { + let Some(value) = value else { + return Ok(StaticExtension::default()); + }; + let object = value + .as_object() + .ok_or_else(|| configuration_error(format!("`{MODULE}` {field} must be an object")))?; + let size = serde_json::to_vec(&value) + .map_err(|error| configuration_error(format!("cannot serialize {field}: {error}")))? + .len(); + if size > STATIC_EXTENSION_MAX_BYTES { + return Err(configuration_error(format!( + "`{MODULE}` {field} exceeds {STATIC_EXTENSION_MAX_BYTES} bytes" + ))); + } + validate_extension_value(field, &value, 0)?; + reject_reserved_fields(field, object)?; + Ok(StaticExtension(object.clone())) +} + +fn validate_extension_value( + field: &str, + value: &Value, + container_depth: usize, +) -> Result<(), Report> { + match value { + Value::Object(object) => { + let container_depth = container_depth + 1; + if container_depth > STATIC_EXTENSION_MAX_DEPTH { + return Err(configuration_error(format!( + "`{MODULE}` {field} exceeds nesting depth {STATIC_EXTENSION_MAX_DEPTH}" + ))); + } + if object.len() > STATIC_EXTENSION_MAX_KEYS { + return Err(configuration_error(format!( + "`{MODULE}` {field} object exceeds {STATIC_EXTENSION_MAX_KEYS} keys" + ))); + } + for nested in object.values() { + validate_extension_value(field, nested, container_depth)?; + } + } + Value::Array(array) => { + let container_depth = container_depth + 1; + if container_depth > STATIC_EXTENSION_MAX_DEPTH { + return Err(configuration_error(format!( + "`{MODULE}` {field} exceeds nesting depth {STATIC_EXTENSION_MAX_DEPTH}" + ))); + } + for nested in array { + validate_extension_value(field, nested, container_depth)?; + } + } + _ => {} + } + Ok(()) +} + +fn reject_reserved_fields( + field: &str, + object: &Map, +) -> Result<(), Report> { + let reserved: &[&str] = match field { + "request_ext" => &["trusted_server"], + _ => &[], + }; + if let Some(key) = reserved.iter().find(|key| object.contains_key(**key)) { + return Err(configuration_error(format!( + "`{MODULE}` {field} cannot claim reserved field `{key}`" + ))); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use serde_json::{Map, Value, json}; + use trusted_server_core::auction::plan::AuctionPlan; + use trusted_server_core::auction::test_support::{ + build_for_first_source, canonical_parity_auction_request, demand_table, + deterministic_signer, golden_inbound_request, golden_plan_config, plan_config_with, + }; + use trusted_server_core::error::TrustedServerError; + use trusted_server_core::openrtb::OpenRtbRequest; + use trusted_server_core::request_signing::RequestSigner; + + use super::{MODULE, OpenRtbDemand, builder}; + + /// One `[demand.]` table running this implementation. + fn table() -> Map { + demand_table(MODULE, "https://bid.example/openrtb2/auction") + } + + /// The plan a `[demand]` selecting one source with `table` compiles to. + fn compile( + table: Map, + ) -> Result> { + AuctionPlan::compile(plan_config_with(vec![("one", table)], &[builder()])) + } + + /// The request core's driver builds for one source running this + /// implementation with `settings`, signed when `signer` is given. + fn build(settings: Value, signer: Option<&RequestSigner>) -> OpenRtbRequest { + let plan = AuctionPlan::compile(golden_plan_config(MODULE, settings, true, &[builder()])) + .expect("should compile plan"); + build_for_first_source( + &plan, + canonical_parity_auction_request(), + &golden_inbound_request(), + 321, + signer, + ) + .expect("should build request") + .expect("should keep the impression") + } + + fn nested_object(levels: usize) -> Value { + let mut value = Value::String("leaf".to_string()); + for level in 0..levels { + value = Value::Object(Map::from_iter([(format!("level_{level}"), value)])); + } + value + } + + fn nested_array(levels: usize) -> Value { + let mut value = Value::String("leaf".to_string()); + for _ in 0..levels { + value = Value::Array(vec![value]); + } + value + } + + #[test] + fn static_extensions_are_bounded_and_cannot_claim_reserved_fields() { + let mut valid = table(); + valid.insert( + "request_ext".to_string(), + json!({"fictional_account": "example"}), + ); + valid.insert("imp_ext".to_string(), json!({"placement_group": "display"})); + let plan = compile(valid).expect("should compile static extensions"); + let openrtb = plan.providers()[0] + .demand + .as_any() + .downcast_ref::() + .expect("should compile the OpenRTB implementation"); + assert_eq!( + openrtb.request_ext.as_object()["fictional_account"], + "example" + ); + + let mut reserved = table(); + reserved.insert( + "request_ext".to_string(), + json!({"trusted_server": {"signature": "forged"}}), + ); + assert!( + compile(reserved).is_err(), + "should refuse an extension claiming a reserved field" + ); + + let mut too_large = table(); + too_large.insert( + "request_ext".to_string(), + json!({"padding": "x".repeat(17 * 1024)}), + ); + assert!( + compile(too_large).is_err(), + "should refuse an extension over the size bound" + ); + + let mut too_deep = table(); + too_deep.insert("request_ext".to_string(), nested_object(9)); + assert!( + compile(too_deep).is_err(), + "should refuse an extension over the depth bound" + ); + + let mut deep_array = table(); + deep_array.insert( + "request_ext".to_string(), + json!({"levels": nested_array(8)}), + ); + assert!( + compile(deep_array).is_err(), + "should count array levels toward the depth bound" + ); + + let mut not_an_object = table(); + not_an_object.insert("request_ext".to_string(), json!("string")); + assert!( + compile(not_an_object).is_err(), + "should refuse an extension that is not an object" + ); + } + + #[test] + fn static_extensions_have_no_invented_bidder_param_location() { + let request = build( + json!({ + "request_ext": {"fictional_request": {"enabled": true}}, + "imp_ext": {"fictional_imp": "value"} + }), + None, + ); + let value = serde_json::to_value(request).expect("should serialize request"); + assert_eq!(value["ext"]["fictional_request"]["enabled"], true); + assert_eq!(value["imp"][0]["ext"]["fictional_imp"], "value"); + assert!( + !value.to_string().contains("exampleBidder"), + "the plain implementation must not invent bidder params placement" + ); + } + + #[test] + fn signed_and_unsigned_requests_have_exact_full_goldens() { + let settings = json!({"request_ext": {"fictional": true}}); + let signer = deterministic_signer(); + assert_eq!( + serde_json::to_string(&build(settings.clone(), Some(&signer))) + .expect("should serialize signed request"), + r#"{"id":"fictional-auction","imp":[{"id":"fictional-slot","banner":{"format":[{"w":300,"h":250},{"w":728,"h":90}]},"bidfloor":1.0,"bidfloorcur":"USD","secure":1}],"site":{"domain":"publisher.example","page":"https://publisher.example/article","publisher":{"domain":"publisher.example"}},"device":{"geo":{"type":2,"country":"US","region":"CA","metro":"501","city":"Example City"},"dnt":1,"ua":"Fictional Browser","ip":"192.0.2.10","language":"en"},"user":{"id":"fictional-user","consent":"fictional-tcf","ext":{"consent":"fictional-tcf","eids":[{"source":"identity.example","uids":[{"atype":1,"id":"fictional-uid"}]}]}},"tmax":321,"cur":["USD"],"regs":{"gdpr":1,"us_privacy":"1YNN","gpp":"fictional-gpp","gpp_sid":[2,6],"ext":{"gdpr":1,"gpp":"fictional-gpp","gpp_sid":[2,6],"us_privacy":"1YNN"}},"ext":{"fictional":true,"trusted_server":{"kid":"fictional-kid","request_host":"publisher.example","request_scheme":"https","signature":"LU_JUIA1BT80ShZNjSa4PIF5T-uMjEeodwKrV_6bXgh0hi1SYVtCKn9g_DTW62krmjCOFgoFYPHsu6L0nAcuDg","ts":1706900000,"version":"1.1"}}}"#, + "signed wire fixture should stay exact" + ); + assert_eq!( + serde_json::to_string(&build(settings, None)) + .expect("should serialize unsigned request"), + r#"{"id":"fictional-auction","imp":[{"id":"fictional-slot","banner":{"format":[{"w":300,"h":250},{"w":728,"h":90}]},"bidfloor":1.0,"bidfloorcur":"USD","secure":1}],"site":{"domain":"publisher.example","page":"https://publisher.example/article","publisher":{"domain":"publisher.example"}},"device":{"geo":{"type":2,"country":"US","region":"CA","metro":"501","city":"Example City"},"dnt":1,"ua":"Fictional Browser","ip":"192.0.2.10","language":"en"},"user":{"id":"fictional-user","consent":"fictional-tcf","ext":{"consent":"fictional-tcf","eids":[{"source":"identity.example","uids":[{"atype":1,"id":"fictional-uid"}]}]}},"tmax":321,"cur":["USD"],"regs":{"gdpr":1,"us_privacy":"1YNN","gpp":"fictional-gpp","gpp_sid":[2,6],"ext":{"gdpr":1,"gpp":"fictional-gpp","gpp_sid":[2,6],"us_privacy":"1YNN"}},"ext":{"fictional":true}}"#, + "unsigned wire fixture should stay exact" + ); + } + + #[test] + fn an_endpoint_is_left_as_written() { + let mut root = table(); + root.insert("endpoint".to_string(), json!("https://bid.example/")); + let plan = compile(root).expect("should compile a root endpoint unchanged"); + assert_eq!( + plan.providers()[0].endpoint.as_str(), + "https://bid.example/" + ); + } + + #[test] + fn module_constant_is_the_crate_folder() { + assert_eq!( + MODULE, + trusted_server_core::module_name!(), + "should be named by the folder this crate lives in" + ); + } +} diff --git a/crates/auction/aps/.gitattributes b/crates/auction/aps/.gitattributes new file mode 100644 index 0000000000..fad78ffdb5 --- /dev/null +++ b/crates/auction/aps/.gitattributes @@ -0,0 +1,4 @@ +# The recording is compared byte for byte with the page made from the input +# beside it, so both must arrive with the line endings they were committed +# with on every platform. +src/fixtures/page-change.*.html text eol=lf diff --git a/crates/auction/aps/Cargo.toml b/crates/auction/aps/Cargo.toml new file mode 100644 index 0000000000..7156f62a3e --- /dev/null +++ b/crates/auction/aps/Cargo.toml @@ -0,0 +1,39 @@ +[package] +name = "trusted-server-auction-aps" +description = "Amazon Publisher Services as a demand implementation, with the page support its renderer needs." +authors = { workspace = true } +edition = { workspace = true } +license = { workspace = true } +publish = { workspace = true } +version = { workspace = true } + +[lib] +doctest = false + +[lints] +workspace = true + +[dependencies] +async-trait = { workspace = true } +base64 = { workspace = true } +edgezero-core = { workspace = true } +error-stack = { workspace = true } +http = { workspace = true } +log = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +trusted-server-core = { workspace = true } +url = { workspace = true } +validator = { workspace = true } + +[dev-dependencies] +futures = { workspace = true } +tokio = { workspace = true } +trusted-server-core = { workspace = true, features = ["test-utils"] } + +# The visible owner of this vendor crate, the way Prebid.js requires a named +# maintainer of every adapter. Amazon has not yet adopted this crate, so the +# Trusted Server maintainers own it until a vendor owner steps up. +[package.metadata.maintainers] +owner = "Trusted Server maintainers" +status = "seeking vendor owner" diff --git a/crates/auction/aps/src/fixtures/page-change.input.html b/crates/auction/aps/src/fixtures/page-change.input.html new file mode 100644 index 0000000000..6a58b86ede --- /dev/null +++ b/crates/auction/aps/src/fixtures/page-change.input.html @@ -0,0 +1,13 @@ + + + + +APS + + + +An article on the origin + +
+ + diff --git a/crates/auction/aps/src/fixtures/page-change.recorded.html b/crates/auction/aps/src/fixtures/page-change.recorded.html new file mode 100644 index 0000000000..c33ea015d7 --- /dev/null +++ b/crates/auction/aps/src/fixtures/page-change.recorded.html @@ -0,0 +1,13 @@ + + + + +APS + + + +An article on the origin + +
+ + diff --git a/crates/auction/aps/src/fixtures/page-change.settings.toml b/crates/auction/aps/src/fixtures/page-change.settings.toml new file mode 100644 index 0000000000..0bc023ecf3 --- /dev/null +++ b/crates/auction/aps/src/fixtures/page-change.settings.toml @@ -0,0 +1,9 @@ +[demand] +modules = ["aps_main"] + +[demand.aps_main] +implementation = "auction.aps" +endpoint = "https://aps.example.com/e/pb/bid" +routing = "all_eligible" +account_id = "example-aps-account-id" +rendering_mode = "publisher_native" diff --git a/crates/trusted-server-core/src/integrations/aps.rs b/crates/auction/aps/src/lib.rs similarity index 60% rename from crates/trusted-server-core/src/integrations/aps.rs rename to crates/auction/aps/src/lib.rs index 4f45fbeb9c..3d7b8b11f1 100644 --- a/crates/trusted-server-core/src/integrations/aps.rs +++ b/crates/auction/aps/src/lib.rs @@ -1,5 +1,17 @@ //! Amazon Publisher Services (APS/TAM) `OpenRTB` integration. +#![cfg_attr( + test, + allow( + clippy::print_stdout, + clippy::print_stderr, + clippy::panic, + clippy::dbg_macro, + clippy::unwrap_used, + reason = "tests use direct diagnostics and panic-on-failure helpers" + ) +)] + use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::Arc; #[cfg(test)] @@ -15,48 +27,76 @@ use serde::de::{self, Visitor}; use serde::{Deserialize, Serialize}; use serde_json::{Value as Json, json}; use url::Url; -use validator::{Validate, ValidationError}; - -use crate::auction::openrtb::ignored_bidder_params_count; -use crate::auction::orchestrator::ERROR_TYPE_HTTP_STATUS; #[cfg(test)] -use crate::auction::plan::{AuctionPlanConfig, NotificationConfig, ProviderConfig, RoutingMode}; -use crate::auction::profile::ApsProfilePlan; +use validator::Validate; +use validator::ValidationError; + +use trusted_server_core::auction::demand::{ + CONSERVATIVE_LANGUAGE_MAX_BYTES, CompiledDemand, DemandFieldPolicy, DemandImplementation, + DemandResponse, DemandTimeoutDefault, ProviderAuctionInput, RegsPolicy, RequestExtensions, +}; +use trusted_server_core::auction::openrtb::ignored_bidder_params_count; +use trusted_server_core::auction::orchestrator::ERROR_TYPE_HTTP_STATUS; #[cfg(test)] -use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; -use crate::auction::routing::ProviderAuctionInput; +use trusted_server_core::auction::provider::{AuctionProvider, ProviderRequestOutcome}; #[cfg(test)] -use crate::auction::types::{AdSlot, AuctionContext, AuctionRequest}; -use crate::auction::types::{ - ApsRendererV1, ApsTagType, AuctionResponse, Bid, BidRenderer, MediaType, -}; -use crate::error::TrustedServerError; -use crate::integrations::{ - IntegrationEndpoint, IntegrationHeadInjector, IntegrationHtmlContext, IntegrationProxy, - IntegrationRegistration, UPSTREAM_RTB_MAX_RESPONSE_BYTES, collect_response_bounded, +use trusted_server_core::auction::types::{AdSlot, AuctionContext, AuctionRequest}; +use trusted_server_core::auction::types::{AuctionResponse, Bid, BidRenderer, MediaType}; +use trusted_server_core::error::TrustedServerError; +use trusted_server_core::integrations::{ + IntegrationEndpoint, IntegrationProxy, IntegrationRegistration, + UPSTREAM_RTB_MAX_RESPONSE_BYTES, collect_response_bounded, }; #[cfg(test)] -use crate::integrations::{ +use trusted_server_core::integrations::{ ensure_integration_backend_with_timeout, predict_integration_backend_name, }; #[cfg(test)] -use crate::openrtb::ToExt; +use trusted_server_core::openrtb::ToExt; #[cfg(test)] -use crate::openrtb::{ +use trusted_server_core::openrtb::{ Banner, Device, Format, Geo, Imp, OpenRtbRequest, Publisher, Regs, RegsExt, Site, User, UserExt, to_openrtb_i32, }; +use trusted_server_core::platform::PlatformResponse; #[cfg(test)] -use crate::platform::PlatformHttpRequest; -use crate::platform::{PlatformResponse, RuntimeServices}; -use crate::settings::{IntegrationConfig, Settings}; +use trusted_server_core::platform::{PlatformHttpRequest, RuntimeServices}; +use trusted_server_core::settings::Settings; + +pub(crate) const APS_INTEGRATION_ID: &str = "aps"; + +/// The builder a deployment hands to an adapter. It offers APS to `[demand]` +/// and registers the page support its renderer needs when the auction plan +/// selects an APS source. +#[must_use] +pub fn builder() -> trusted_server_core::integrations::IntegrationBuilder { + trusted_server_core::integrations::IntegrationBuilder::implementations( + APS_INTEGRATION_ID, + env!("CARGO_PKG_NAME"), + ) + .with_demand(&DEMAND) + .with_plan_registration(register_for_plan) +} + +/// The name an `implementation` line gives this implementation, its module +/// path. +pub const MODULE: &str = "auction.aps"; +/// Renderer type tag carried on the wire by an APS bid, read by the browser to +/// select the APS renderer. +pub const APS_RENDERER_TYPE: &str = "aps"; -const APS_INTEGRATION_ID: &str = "aps"; +/// Wire key carrying [`ApsRendererV1::bid_id`], for callers that read the bid +/// identifier out of a renderer payload without deserializing the rest. +/// +/// `ApsRendererV1` renames its fields to camelCase, so this is `bidId` rather +/// than the Rust field name. `renderer_bid_id_key_matches_the_serialized_form` +/// pins the two together. +pub const APS_RENDERER_BID_ID_KEY: &str = "bidId"; const APS_RENDERER_ROUTE: &str = "/integrations/aps/renderer"; const DEFAULT_CURRENCY: &str = "USD"; -#[cfg(test)] +/// The SDK source APS expects from a Prebid-shaped caller. const APS_SDK_SOURCE: &str = "prebid"; -#[cfg(test)] +/// The SDK version APS expects from a Prebid-shaped caller. const APS_SDK_VERSION: &str = "2.2.0"; const MAX_ACCOUNT_ID_BYTES: usize = 1024; const MAX_CREATIVE_ID_BYTES: usize = 1024; @@ -69,6 +109,45 @@ const MAX_PAGE_URL_BYTES: usize = 8192; const MAX_RENDER_ENVELOPE_BYTES: usize = 256 * 1024; const APS_RENDERER_CSP: &str = "default-src 'none'; sandbox allow-forms allow-pointer-lock allow-popups allow-popups-to-escape-sandbox allow-scripts allow-top-navigation-by-user-activation; script-src 'unsafe-inline' https:; connect-src https:; frame-src https:; img-src https: data:; media-src https: blob:; style-src 'unsafe-inline' https:; font-src https: data:;"; +/// APS creative tag type accepted by the Trusted Server renderer. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum ApsTagType { + /// APS loads the creative URL in a nested iframe. + Iframe, + /// APS fetches creative HTML and executes it in its nested renderer frame. + Script, +} + +/// Version 1 APS renderer descriptor shared with browser clients. +/// +/// Carried by a bid as the payload of a [`BidRenderer`] tagged +/// [`APS_RENDERER_TYPE`], so it travels with the APS integration rather than +/// with the neutral auction types. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ApsRendererV1 { + /// Renderer contract version. + pub version: u8, + /// APS account identifier used to initialize the fixed runner. + pub account_id: String, + /// Selected `OpenRTB` bid identifier. + pub bid_id: String, + /// Optional `OpenRTB` creative identifier. + #[serde(skip_serializing_if = "Option::is_none")] + pub creative_id: Option, + /// APS creative delivery mode. + pub tag_type: ApsTagType, + /// HTTPS creative URL consumed by the fixed APS runner. + pub creative_url: String, + /// Base64-encoded exact one-bid APS response envelope. + pub aax_response: String, + /// Creative width. + pub width: u32, + /// Creative height. + pub height: u32, +} + const APS_RENDERER_DOCUMENT: &str = r#" @@ -374,80 +453,217 @@ impl Default for LegacyApsProviderConfig { } } -/// Browser integration toggle retained independently from APS server providers. -#[derive(Debug, Clone, Default, Deserialize, Serialize, Validate)] +/// The APS demand implementation. +pub static DEMAND: DemandImplementation = DemandImplementation { + id: MODULE, + default_timeout: DemandTimeoutDefault::Fixed(800), + allows_all_eligible: true, + serves_stored_requests: false, + canonicalize_endpoint: check_aps_endpoint, + compile: compile_demand, +}; + +/// One compiled APS demand source, from its `[demand.]` table. +#[derive(Debug, Clone, Deserialize)] #[serde(deny_unknown_fields)] -pub struct ApsConfig { - /// Whether browser-side APS integration behavior is enabled. +pub struct ApsDemand { + /// APS account identifier. + #[serde(deserialize_with = "deserialize_account_id")] + pub account_id: String, + /// Include APS request and response diagnostics. + /// + /// This default-off metadata is unredacted and client-visible, so it can + /// carry identity, consent, page, account, bid and creative data. Set it + /// only on a controlled test site and never in production. #[serde(default)] - pub enabled: bool, + pub debug: bool, + /// Permit APS script creatives. + #[serde(default)] + pub allow_script_creatives: bool, + /// APS-authorized inventory domain used instead of the deployment hostname. + #[serde(default)] + pub inventory_domain: Option, + /// Canonical inventory page origin, keeping the sanitized path. + #[serde(default)] + pub inventory_page_origin: Option, /// Rendering owner for selected APS bids. #[serde(default)] pub rendering_mode: ApsRenderingMode, } -#[cfg(test)] -impl IntegrationConfig for LegacyApsProviderConfig { - fn is_enabled(&self) -> bool { - self.enabled - } -} - -impl IntegrationConfig for ApsConfig { - fn is_enabled(&self) -> bool { - self.enabled +/// Refuse the legacy APS bid path, which this implementation does not speak. +fn check_aps_endpoint(endpoint: &mut Url) -> Result<(), String> { + if endpoint + .path() + .trim_end_matches('/') + .ends_with("/e/dtb/bid") + { + return Err("names the unsupported legacy APS path `/e/dtb/bid`".to_string()); } + Ok(()) } -/// Typed server-side APS profile configuration used by the auction compiler. -#[derive(Debug, Clone, Deserialize)] -#[serde(deny_unknown_fields)] -pub(crate) struct ApsProfileConfig { - #[serde(deserialize_with = "deserialize_account_id")] - pub(crate) account_id: String, - #[serde(default)] - pub(crate) debug: bool, - #[serde(default)] - pub(crate) allow_script_creatives: bool, - #[serde(default)] - pub(crate) inventory_domain: Option, - #[serde(default)] - pub(crate) inventory_page_origin: Option, +fn compile_demand( + settings: &serde_json::Map, +) -> Result, Report> { + Ok(Arc::new(compile_aps_settings(Json::Object( + settings.clone(), + ))?)) } -/// Parse and validate server-owned APS profile fields without browser enablement. -pub(crate) fn compile_profile_config( - value: serde_json::Value, -) -> Result> { - let profile: ApsProfileConfig = serde_json::from_value(value).map_err(|error| { +/// Parse and validate the settings of one APS demand source. +/// +/// # Errors +/// +/// Returns a configuration error when a setting is missing, unknown or invalid. +pub(crate) fn compile_aps_settings(value: Json) -> Result> { + let demand: ApsDemand = serde_json::from_value(value).map_err(|error| { Report::new(TrustedServerError::Configuration { - message: format!("invalid `aps` profile_config: {error}"), + message: format!("invalid `aps` settings: {error}"), }) })?; - if let Some(domain) = profile.inventory_domain.as_deref() { + if let Some(domain) = demand.inventory_domain.as_deref() { validate_inventory_domain(domain).map_err(|error| { Report::new(TrustedServerError::Configuration { - message: format!("invalid `aps` profile_config inventory_domain: {error}"), + message: format!("invalid `aps` inventory_domain: {error}"), }) })?; } - if let Some(origin) = profile.inventory_page_origin.as_deref() { + if let Some(origin) = demand.inventory_page_origin.as_deref() { validate_inventory_page_origin(origin).map_err(|error| { Report::new(TrustedServerError::Configuration { - message: format!("invalid `aps` profile_config inventory_page_origin: {error}"), + message: format!("invalid `aps` inventory_page_origin: {error}"), }) })?; } validate_inventory_identity_override_values( - profile.inventory_domain.as_deref(), - profile.inventory_page_origin.as_deref(), + demand.inventory_domain.as_deref(), + demand.inventory_page_origin.as_deref(), ) .map_err(|error| { Report::new(TrustedServerError::Configuration { - message: format!("invalid `aps` profile_config inventory identity: {error}"), + message: format!("invalid `aps` inventory identity: {error}"), }) })?; - Ok(profile) + if demand.rendering_mode == ApsRenderingMode::PublisherNative && demand.allow_script_creatives { + log::warn!( + "APS publisher-native rendering with script creatives is ON; selected bidder scripts execute with publisher-origin privileges" + ); + } + if demand.debug { + log::warn!( + "APS debug mode is ON. Raw request and response data, including creative markup, is included in client-visible /auction responses" + ); + } + Ok(demand) +} + +#[async_trait(?Send)] +impl CompiledDemand for ApsDemand { + fn field_policy(&self) -> DemandFieldPolicy { + DemandFieldPolicy { + primary_banner_size: true, + language_max_bytes: Some(CONSERVATIVE_LANGUAGE_MAX_BYTES), + regs: RegsPolicy::ApplicabilityBit, + ..DemandFieldPolicy::default() + } + } + + fn site_domain(&self, publisher_domain: &str) -> String { + self.inventory_domain + .clone() + .unwrap_or_else(|| publisher_domain.to_owned()) + } + + fn site_page(&self, publisher_page: Option<&str>, site_domain: &str) -> Option { + let fallback = publisher_page + .and_then(valid_aps_page_url) + .unwrap_or_else(|| format!("https://{site_domain}")); + let Some(origin) = self.inventory_page_origin.as_deref() else { + return Some(fallback); + }; + let (Ok(mut canonical), Ok(current)) = (Url::parse(origin), Url::parse(&fallback)) else { + return Some(fallback); + }; + canonical.set_path(current.path()); + canonical.set_query(current.query()); + canonical.set_fragment(None); + Some(canonical.to_string()) + } + + fn augment_request( + &self, + extensions: &mut RequestExtensions<'_>, + _input: &ProviderAuctionInput, + ) -> Result<(), Report> { + *extensions.request = Some(serde_json::Map::from_iter([ + ("account".to_string(), Json::String(self.account_id.clone())), + ( + "sdk".to_string(), + json!({"source": APS_SDK_SOURCE, "version": APS_SDK_VERSION}), + ), + ])); + Ok(()) + } + + fn capture_request( + &self, + body: &[u8], + headers: &HeaderMap, + ) -> Option> { + self.debug + .then(|| Box::new(ApsDebugRequest::capture(body, headers)) as Box<_>) + } + + async fn parse_response( + &self, + context: DemandResponse<'_>, + response: PlatformResponse, + ) -> Result> { + let provider_id = context.provider_id; + let response_time_ms = context.response_time_ms; + let debug_request = context + .captured + .and_then(|captured| captured.downcast_ref::()) + .cloned(); + match parse_planned_aps_response( + provider_id, + self, + context.endpoint, + context.input, + response, + response_time_ms, + debug_request, + ) + .await + { + Ok(parsed) => Ok(parsed), + Err(error) => { + log::warn!("Provider '{provider_id}' APS response parse failed: {error:?}"); + Ok(AuctionResponse::error(provider_id, response_time_ms) + .with_metadata("error_type", json!("parse_response"))) + } + } + } + + fn as_any(&self) -> &dyn core::any::Any { + self + } +} + +/// Accept only a page URL APS can be given. +fn valid_aps_page_url(value: &str) -> Option { + const MAX_APS_PAGE_URL_BYTES: usize = 8192; + + if value.len() > MAX_APS_PAGE_URL_BYTES { + return None; + } + let parsed = Url::parse(value).ok()?; + (matches!(parsed.scheme(), "http" | "https") + && parsed.host_str().is_some() + && parsed.username().is_empty() + && parsed.password().is_none()) + .then(|| parsed.to_string()) } #[cfg(test)] @@ -603,7 +819,7 @@ fn planned_aps_renderer( if serialized.len() > MAX_RENDER_ENVELOPE_BYTES { return None; } - Some(BidRenderer::Aps(ApsRendererV1 { + let descriptor = ApsRendererV1 { version: 1, account_id: policy.account_id.to_string(), bid_id: input.bid_id.to_string(), @@ -613,7 +829,10 @@ fn planned_aps_renderer( aax_response: BASE64_STANDARD.encode(serialized), width: input.width, height: input.height, - })) + }; + BidRenderer::from_typed(APS_RENDERER_TYPE, &descriptor) + .ok() + .map(|renderer| renderer.picking_bid_by(APS_RENDERER_BID_ID_KEY)) } fn planned_aps_valid_creative_url(value: &str, publisher_domain: &str) -> bool { @@ -866,7 +1085,7 @@ fn parse_planned_aps_value( /// Parse one APS-profile response using only provider-local routed state. pub(crate) async fn parse_planned_aps_response( provider_id: &str, - profile: &ApsProfilePlan, + demand: &ApsDemand, endpoint: &str, input: &ProviderAuctionInput, response: PlatformResponse, @@ -876,9 +1095,9 @@ pub(crate) async fn parse_planned_aps_response( let policy = PlannedApsResponsePolicy { provider_id, endpoint, - account_id: &profile.account_id, - debug: profile.debug, - allow_script_creatives: profile.allow_script_creatives, + account_id: &demand.account_id, + debug: demand.debug, + allow_script_creatives: demand.allow_script_creatives, publisher_domain: &input.common_request().publisher.domain, }; let response = response.response; @@ -984,7 +1203,7 @@ impl ApsAuctionProvider { Self { config } } - fn build_regs(consent: Option<&crate::consent::ConsentContext>) -> Option { + fn build_regs(consent: Option<&trusted_server_core::consent::ConsentContext>) -> Option { let consent = consent?; let ext = RegsExt { gdpr: Some(u8::from(consent.gdpr_applies)), @@ -1318,7 +1537,7 @@ impl ApsAuctionProvider { if serialized.len() > MAX_RENDER_ENVELOPE_BYTES { return None; } - Some(BidRenderer::Aps(ApsRendererV1 { + let descriptor = ApsRendererV1 { version: 1, account_id: self.config.account_id.clone(), bid_id: input.bid_id.to_string(), @@ -1328,7 +1547,17 @@ impl ApsAuctionProvider { aax_response: BASE64_STANDARD.encode(serialized), width: input.width, height: input.height, - })) + }; + match BidRenderer::from_typed(APS_RENDERER_TYPE, &descriptor) { + Ok(renderer) => Some(renderer.picking_bid_by(APS_RENDERER_BID_ID_KEY)), + Err(error) => { + log::warn!( + "Dropping APS bid '{}': its renderer descriptor could not be built: {error:?}", + input.bid_id + ); + None + } + } } fn increment_reason(reasons: &mut BTreeMap, reason: &'static str) { @@ -1819,10 +2048,11 @@ impl IntegrationProxy for ApsRendererIntegration { .collect() } + // The renderer page is the same for every request, so the route takes + // nothing from its module call. async fn handle( &self, - _settings: &Settings, - _services: &RuntimeServices, + _call: trusted_server_core::module_context::ModuleCall<'_>, request: http::Request, ) -> Result, Report> { if request.method() != Method::GET || request.uri().path() != APS_RENDERER_ROUTE { @@ -1848,137 +2078,73 @@ impl IntegrationProxy for ApsRendererIntegration { } } -impl IntegrationHeadInjector for ApsRendererIntegration { - fn integration_id(&self) -> &'static str { - APS_INTEGRATION_ID - } - - fn head_inserts(&self, _ctx: &IntegrationHtmlContext<'_>) -> Vec { - Vec::new() - } - - fn tsjs_script_tag_attributes(&self) -> Vec<(&'static str, &'static str)> { - (self.rendering_mode == ApsRenderingMode::PublisherNative) - .then_some(("data-ts-aps-rendering-mode", "publisher_native")) - .into_iter() - .collect() - } -} +/// The attribute on the script bundle's tag that lets the browser render an +/// APS creative itself. It is written by the server alone, so a page cannot +/// claim the mode for itself. +const NATIVE_RENDERING_ATTRIBUTE: (&str, &str) = ("data-ts-aps-rendering-mode", "publisher_native"); -/// Register renderer support when the auction plan contains an APS provider. +/// Register renderer support when the plan selects an APS demand source. /// -/// Browser integration enablement does not control server-side APS rendering. -/// An absent or disabled browser block uses trusted-server rendering. An enabled -/// browser block may select publisher-native rendering. +/// The rendering owner comes from the demand table itself, so a deployment +/// sets it where it sets the rest of that source's settings. Two APS sources +/// that disagree is a configuration error, because one page can only be +/// rendered one way. /// /// # Errors /// -/// Returns an error when APS browser configuration is invalid. +/// Returns an error when two selected APS sources set different rendering +/// modes. pub fn register_for_plan( - settings: &Settings, - plan: &crate::auction::AuctionPlan, + _settings: &Settings, + plan: &trusted_server_core::auction::AuctionPlan, ) -> Result, Report> { - if !plan.has_profile(APS_INTEGRATION_ID) { - return Ok(None); + let mut selected: Option<(&str, ApsRenderingMode)> = None; + for provider in plan.providers() { + if provider.implementation.id != MODULE { + continue; + } + let Some(demand) = provider.demand.as_any().downcast_ref::() else { + continue; + }; + match selected { + Some((first, mode)) if mode != demand.rendering_mode => { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "[demand.{}] and [demand.{first}] set different rendering_mode values, and one page can be rendered only one way", + provider.id.as_str() + ), + })); + } + Some(_) => {} + None => selected = Some((provider.id.as_str(), demand.rendering_mode)), + } } - let rendering_mode = settings - .integration_config::(APS_INTEGRATION_ID)? - .map(|config| config.rendering_mode) - .unwrap_or_default(); - let integration = Arc::new(ApsRendererIntegration { rendering_mode }); - let registration = IntegrationRegistration::builder(APS_INTEGRATION_ID) - .without_js() - .with_head_injector(integration.clone()); - let registration = if rendering_mode == ApsRenderingMode::TrustedServer { - registration.with_proxy(integration) - } else { - registration - }; - Ok(Some(registration.build())) -} - -/// Register the APS auction provider when enabled. -/// -/// # Errors -/// -/// Returns an error when enabled APS configuration is invalid. -#[cfg(test)] -#[allow(clippy::missing_panics_doc)] -pub fn register( - settings: &Settings, -) -> Result, Report> { - let Some(config) = - settings.integration_config::(APS_INTEGRATION_ID)? - else { + let Some((_, rendering_mode)) = selected else { return Ok(None); }; - let mut browser_settings = settings.clone(); - browser_settings.integrations.insert_config( - APS_INTEGRATION_ID, - &ApsConfig { - enabled: true, - rendering_mode: config.rendering_mode, - }, - )?; - register_for_plan( - &browser_settings, - &crate::auction::AuctionPlan::compile(AuctionPlanConfig { - timeout_ms: 1000, - providers: BTreeMap::from([( - "aps".parse().expect("should parse APS provider ID"), - ProviderConfig { - protocol: "openrtb-2.6".to_string(), - profile: "aps".to_string(), - endpoint: default_endpoint(), - timeout_ms: None, - routing: RoutingMode::AllEligible, - notifications: NotificationConfig::default(), - profile_config: serde_json::json!({"account_id":"example-account"}), - }, - )]), - ..AuctionPlanConfig::default() - }) - .expect("should compile APS renderer test plan"), - ) -} - -#[cfg(test)] -#[allow(clippy::missing_errors_doc)] -pub fn register_providers( - settings: &Settings, -) -> Result>, Report> { - let Some(config) = - settings.integration_config::(APS_INTEGRATION_ID)? - else { - return Ok(Vec::new()); + let registration = IntegrationRegistration::builder(APS_INTEGRATION_ID).without_js(); + let registration = match rendering_mode { + ApsRenderingMode::TrustedServer => { + registration.with_proxy(Arc::new(ApsRendererIntegration { rendering_mode })) + } + ApsRenderingMode::PublisherNative => registration + .with_bundle_tag_attribute(NATIVE_RENDERING_ATTRIBUTE.0, NATIVE_RENDERING_ATTRIBUTE.1), }; - log::info!("Registering APS OpenRTB provider"); - if config.debug { - log::warn!( - "APS debug mode is ON — raw request and response data, including creative markup, will be included in client-visible /auction responses" - ); - } - if config.rendering_mode == ApsRenderingMode::PublisherNative && config.allow_script_creatives { - log::warn!( - "APS publisher-native rendering with script creatives is ON; selected bidder scripts execute with publisher-origin privileges" - ); - } - Ok(vec![Arc::new(ApsAuctionProvider::new(config))]) + Ok(Some(registration.build())) } #[cfg(test)] mod tests { use super::*; - use crate::auction::test_support::canonical_parity_auction_request; - use crate::auction::types::{ + use serde_json::json; + use trusted_server_core::auction::test_support::canonical_parity_auction_request; + use trusted_server_core::auction::types::{ AdFormat, AdSlot, AuctionContext, AuctionRequest, BidStatus, PublisherInfo, UserInfo, }; - use crate::integrations::IntegrationDocumentState; - use crate::platform::test_support::{ + use trusted_server_core::platform::test_support::{ StubHttpClient, build_services_with_http_client, noop_services, }; - use crate::test_support::tests::create_test_settings; - use serde_json::json; + use trusted_server_core::test_support::tests::create_test_settings; fn config() -> LegacyApsProviderConfig { LegacyApsProviderConfig { @@ -2185,8 +2351,8 @@ mod tests { .is_err() ); assert!( - serde_json::from_value::(json!({ - "enabled": true, + compile_aps_settings(json!({ + "account_id": "example-account", "rendering_mode": "unsupported" })) .is_err(), @@ -2455,7 +2621,7 @@ mod tests { .renderer .as_ref() .expect("should include renderer") - .as_aps() + .payload_as::(APS_RENDERER_TYPE) .expect("should be APS renderer"); let decoded = BASE64_STANDARD .decode(&renderer.aax_response) @@ -2509,7 +2675,7 @@ mod tests { bid.renderer .as_ref() .expect("should retain renderer") - .as_aps() + .payload_as::(APS_RENDERER_TYPE) .expect("should be APS renderer") .creative_id .is_none() @@ -2591,7 +2757,7 @@ mod tests { let stub = Arc::new(StubHttpClient::new()); stub.push_response(200, br#"{"seatbid":[]}"#.to_vec()); let services = build_services_with_http_client( - Arc::clone(&stub) as Arc + Arc::clone(&stub) as Arc ); let settings = create_test_settings(); let downstream = http::Request::builder() @@ -2922,7 +3088,7 @@ mod tests { .renderer .as_ref() .expect("should keep script renderer") - .as_aps() + .payload_as::(APS_RENDERER_TYPE) .expect("should be APS renderer"); assert_eq!(renderer.tag_type, ApsTagType::Script); } @@ -3027,16 +3193,16 @@ mod tests { assert_eq!(routes[0].method, Method::GET); assert_eq!(routes[0].path, APS_RENDERER_ROUTE); - let settings = create_test_settings(); - let services = noop_services(); let request = http::Request::builder() .method(Method::GET) .uri(APS_RENDERER_ROUTE) .body(EdgeBody::empty()) .expect("should build renderer request"); - let response = - futures::executor::block_on(integration.handle(&settings, &services, request)) - .expect("should serve renderer"); + let response = futures::executor::block_on(integration.handle( + trusted_server_core::module_context::ModuleCall::empty(), + request, + )) + .expect("should serve renderer"); assert_eq!(response.status(), StatusCode::OK); assert_eq!( response.headers()[header::CONTENT_TYPE], @@ -3054,129 +3220,96 @@ mod tests { .uri(APS_RENDERER_ROUTE) .body(EdgeBody::empty()) .expect("should build method rejection request"); - let response = futures::executor::block_on(integration.handle(&settings, &services, post)) - .expect("should reject unsupported method"); + let response = futures::executor::block_on(integration.handle( + trusted_server_core::module_context::ModuleCall::empty(), + post, + )) + .expect("should reject unsupported method"); assert_eq!(response.status(), StatusCode::NOT_FOUND); } - fn plan_with_aps_profile() -> crate::auction::AuctionPlan { - crate::auction::AuctionPlan::compile(AuctionPlanConfig { - timeout_ms: 1_000, - providers: BTreeMap::from([( - "aps".parse().expect("should parse APS provider ID"), - ProviderConfig { - protocol: "openrtb-2.6".to_string(), - profile: "aps".to_string(), - endpoint: default_endpoint(), - timeout_ms: None, - routing: RoutingMode::AllEligible, - notifications: NotificationConfig::default(), - profile_config: serde_json::json!({"account_id":"example-account"}), - }, - )]), - ..AuctionPlanConfig::default() - }) + fn aps_plan(rendering_modes: &[Option<&str>]) -> trusted_server_core::auction::AuctionPlan { + let tables = rendering_modes + .iter() + .enumerate() + .map(|(index, mode)| { + let mut table = trusted_server_core::auction::test_support::demand_table( + MODULE, + &default_endpoint(), + ); + table.insert("account_id".to_string(), json!("example-account")); + table.insert("routing".to_string(), json!("all_eligible")); + if let Some(mode) = mode { + table.insert("rendering_mode".to_string(), json!(mode)); + } + (if index == 0 { "aps_main" } else { "aps_second" }, table) + }) + .collect::>(); + trusted_server_core::auction::AuctionPlan::compile( + trusted_server_core::auction::test_support::plan_config_with(tables, &[builder()]), + ) .expect("should compile APS plan") } #[test] - fn aps_plan_registers_trusted_server_renderer_without_enabled_browser_config() { - for disabled_browser_config in [false, true] { - let mut settings = create_test_settings(); - if disabled_browser_config { - settings - .integrations - .insert_config( - APS_INTEGRATION_ID, - &json!({ - "enabled": false, - "rendering_mode": "publisher_native" - }), - ) - .expect("should insert disabled APS browser config"); - } - - let registration = register_for_plan(&settings, &plan_with_aps_profile()) - .expect("should register APS renderer support") - .expect("should return APS renderer registration"); - - assert_eq!( - registration.proxies.len(), - 1, - "APS plan should register trusted-server renderer route" - ); - assert!( - registration.head_injectors[0] - .tsjs_script_tag_attributes() - .is_empty(), - "disabled or absent browser config should not select publisher-native rendering" - ); - } - } + fn a_selected_aps_source_registers_the_trusted_server_renderer() { + let registration = register_for_plan(&create_test_settings(), &aps_plan(&[None])) + .expect("should register APS renderer support") + .expect("should return APS renderer registration"); - #[test] - fn enabled_config_registers_renderer_proxy() { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config( - APS_INTEGRATION_ID, - &json!({"enabled": true, "account_id": "example-account"}), - ) - .expect("should insert APS config"); - - let registration = register(&settings) - .expect("should register APS") - .expect("should return enabled registration"); - - assert_eq!(registration.integration_id, APS_INTEGRATION_ID); - assert_eq!(registration.proxies.len(), 1); - assert_eq!(registration.head_injectors.len(), 1); - let document_state = IntegrationDocumentState::default(); - let context = IntegrationHtmlContext { - request_host: "publisher.example", - request_scheme: "https", - origin_host: "origin.example", - document_state: &document_state, - }; - assert!( - registration.head_injectors[0] - .head_inserts(&context) - .is_empty(), - "should not inject a native-mode head marker by default" + assert_eq!( + registration.integration_id, APS_INTEGRATION_ID, + "the renderer registers under the APS id" + ); + assert_eq!( + registration.proxies.len(), + 1, + "a selected APS source should register the trusted-server renderer route" ); assert!( - registration.head_injectors[0] - .tsjs_script_tag_attributes() - .is_empty(), - "should not authorize native rendering by default" + registration.bundle_tag_attributes.is_empty(), + "the default rendering mode should not authorize publisher-native rendering" ); assert!(registration.js_disabled); } #[test] - fn publisher_native_config_registers_runner_mode_without_renderer_route() { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config( - APS_INTEGRATION_ID, - &json!({ - "enabled": true, - "account_id": "example-account", - "rendering_mode": "publisher_native" - }), - ) - .expect("should insert native APS config"); + fn no_aps_source_registers_nothing() { + let plan = trusted_server_core::auction::AuctionPlan::compile( + trusted_server_core::auction::test_support::plan_config(Vec::new()), + ) + .expect("should compile an empty plan"); + + assert!( + register_for_plan(&create_test_settings(), &plan) + .expect("should evaluate renderer registration") + .is_none(), + "a plan with no APS source should register no renderer" + ); + } - let registration = register(&settings) - .expect("should register APS") - .expect("should return enabled registration"); + #[test] + fn publisher_native_rendering_drops_the_renderer_route_and_marks_the_bundle_tag() { + let registration = register_for_plan( + &create_test_settings(), + &aps_plan(&[Some("publisher_native")]), + ) + .expect("should register APS renderer support") + .expect("should return APS renderer registration"); assert!( registration.proxies.is_empty(), - "should not register the static renderer" + "publisher-native rendering should not register the static renderer" + ); + assert!( + registration.middleware.is_empty(), + "should change no page, so a page cannot be made to carry a forged native-mode \ + marker" + ); + assert_eq!( + registration.bundle_tag_attributes, + [("data-ts-aps-rendering-mode", "publisher_native")], + "should authorize native mode on the publisher bundle tag" ); - assert_eq!(registration.head_injectors.len(), 1); let integration = ApsRendererIntegration { rendering_mode: ApsRenderingMode::PublisherNative, @@ -3185,94 +3318,45 @@ mod tests { integration.routes().is_empty(), "should expose no renderer route" ); - let document_state = IntegrationDocumentState::default(); - let context = IntegrationHtmlContext { - request_host: "publisher.example", - request_scheme: "https", - origin_host: "origin.example", - document_state: &document_state, + } + + #[test] + fn two_aps_sources_that_disagree_on_rendering_are_refused() { + let plan = aps_plan(&[None, Some("publisher_native")]); + let error = match register_for_plan(&create_test_settings(), &plan) { + Ok(_) => panic!("should refuse two rendering modes"), + Err(error) => error, }; + let message = error.to_string(); assert!( - integration.head_inserts(&context).is_empty(), - "should not inject a forgeable native-mode marker" - ); - assert_eq!( - integration.tsjs_script_tag_attributes(), - vec![("data-ts-aps-rendering-mode", "publisher_native")], - "should authorize native mode on the publisher bundle tag" + message.contains("rendering_mode") && message.contains("aps_second"), + "should name the setting and the source that disagrees: {error:?}" ); } #[test] fn publisher_native_script_creatives_remain_available_for_controlled_validation() { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config( - APS_INTEGRATION_ID, - &json!({ - "enabled": true, - "account_id": "example-account", - "allow_script_creatives": true, - "rendering_mode": "publisher_native" - }), - ) - .expect("should insert native APS script config"); - - let providers = register_providers(&settings).expect("should register APS provider"); + let demand = compile_aps_settings(json!({ + "account_id": "example-account", + "allow_script_creatives": true, + "rendering_mode": "publisher_native" + })) + .expect("should compile the controlled experiment"); - assert_eq!( - providers.len(), - 1, - "should retain the controlled experiment" - ); + assert!(demand.allow_script_creatives); + assert_eq!(demand.rendering_mode, ApsRenderingMode::PublisherNative); } #[test] - fn config_without_enabled_does_not_register_provider_or_renderer() { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config( - APS_INTEGRATION_ID, - &json!({"account_id": "example-account"}), - ) - .expect("should insert default-disabled APS config"); - + fn an_aps_source_needs_an_account() { + let error = compile_aps_settings(json!({"debug": true})) + .expect_err("should refuse an APS source with no account"); assert!( - register(&settings) - .expect("should evaluate renderer registration") - .is_none(), - "omitted enabled should not register the renderer route" - ); - assert!( - register_providers(&settings) - .expect("should evaluate provider registration") - .is_empty(), - "omitted enabled should not register the auction provider" + format!("{error:?}").contains("account_id"), + "should name the missing setting: {error:?}" ); } - #[test] - fn enabled_invalid_config_fails_provider_registration() { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config( - APS_INTEGRATION_ID, - &json!({ - "enabled": true, - "account_id": "example-account", - "endpoint": "http://insecure.example/openrtb" - }), - ) - .expect("should insert invalid APS config for startup validation"); - - let _error = register_providers(&settings) - .err() - .expect("should reject invalid enabled APS configuration"); - } - #[test] fn renderer_document_is_static_and_nonce_bound() { assert!(APS_RENDERER_DOCUMENT.contains("^#tsaps=")); @@ -3308,4 +3392,1244 @@ mod tests { assert!(APS_RENDERER_CSP.contains("sandbox allow-forms")); assert!(!APS_RENDERER_CSP.contains("allow-same-origin")); } + + #[test] + fn renderer_bid_id_key_matches_the_serialized_form() { + let descriptor = ApsRendererV1 { + version: 1, + account_id: "example-account".to_string(), + bid_id: "fictional-bid-id".to_string(), + creative_id: None, + tag_type: ApsTagType::Iframe, + creative_url: "https://creative.example/render".to_string(), + aax_response: "fictional-base64".to_string(), + width: 300, + height: 250, + }; + let renderer = BidRenderer::from_typed(APS_RENDERER_TYPE, &descriptor) + .expect("should build APS renderer descriptor"); + + assert_eq!( + renderer + .payload_field(APS_RENDERER_TYPE, APS_RENDERER_BID_ID_KEY) + .and_then(serde_json::Value::as_str), + Some(descriptor.bid_id.as_str()), + "should name the wire key `ApsRendererV1` serializes `bid_id` to" + ); + assert_eq!( + renderer + .payload_field(APS_RENDERER_TYPE, APS_RENDERER_BID_ID_KEY) + .and_then(serde_json::Value::as_str), + renderer + .payload_as::(APS_RENDERER_TYPE) + .as_ref() + .map(|full| full.bid_id.as_str()), + "should read the same value as deserializing the whole descriptor" + ); + } + + #[test] + fn aps_renderer_serializes_to_versioned_camel_case_contract() { + let renderer = BidRenderer::from_typed( + APS_RENDERER_TYPE, + &ApsRendererV1 { + version: 1, + account_id: "example-account-id".to_string(), + bid_id: "fictional-bid-id".to_string(), + creative_id: Some("fictional-creative-id".to_string()), + tag_type: ApsTagType::Iframe, + creative_url: "https://creative.example/render".to_string(), + aax_response: "base64-data".to_string(), + width: 300, + height: 250, + }, + ) + .expect("should build APS renderer descriptor"); + + let serialized = serde_json::to_value(&renderer).expect("should serialize renderer"); + + assert_eq!( + serialized, + json!({ + "type": "aps", + "version": 1, + "accountId": "example-account-id", + "bidId": "fictional-bid-id", + "creativeId": "fictional-creative-id", + "tagType": "iframe", + "creativeUrl": "https://creative.example/render", + "aaxResponse": "base64-data", + "width": 300, + "height": 250 + }), + "should match renderer wire contract" + ); + } + + #[test] + fn aps_renderer_omits_absent_creative_id() { + let renderer = BidRenderer::from_typed( + APS_RENDERER_TYPE, + &ApsRendererV1 { + version: 1, + account_id: "example-account-id".to_string(), + bid_id: "fictional-bid-id".to_string(), + creative_id: None, + tag_type: ApsTagType::Iframe, + creative_url: "https://creative.example/render".to_string(), + aax_response: "base64-data".to_string(), + width: 300, + height: 250, + }, + ) + .expect("should build APS renderer descriptor"); + + let serialized = serde_json::to_value(&renderer).expect("should serialize renderer"); + + assert!( + serialized.get("creativeId").is_none(), + "should omit absent creative ID" + ); + } + + /// Rewrites every object in `value` with its keys in sorted order, so + /// serializing the result gives one fixed key order. + /// + /// `serde_json::Map` is a `BTreeMap`, which serializes keys in sorted + /// order, only while the crate's `preserve_order` feature is off. With the + /// feature on it is an `IndexMap` and the order follows insertion instead. + /// Nothing in this crate asks for the feature, but Cargo unifies features + /// across everything built for one target, and `trusted-server-cli` pulls + /// it in through `edgezero-cli` and then `handlebars`. A maintainer + /// running `cargo test --workspace --target ` therefore builds this + /// crate with `preserve_order` on, and a test that pinned insertion order + /// would fail there for no reason. Sorting both sides removes the + /// dependence on which map `serde_json` was built with. + fn with_sorted_keys(value: &serde_json::Value) -> serde_json::Value { + match value { + serde_json::Value::Object(map) => { + let mut keys = map.keys().collect::>(); + keys.sort_unstable(); + let mut sorted = serde_json::Map::with_capacity(keys.len()); + for key in keys { + let child = map.get(key).expect("should find a key the map just listed"); + sorted.insert(key.clone(), with_sorted_keys(child)); + } + serde_json::Value::Object(sorted) + } + serde_json::Value::Array(items) => { + serde_json::Value::Array(items.iter().map(with_sorted_keys).collect()) + } + scalar => scalar.clone(), + } + } + + #[test] + fn the_open_renderer_serializes_to_the_same_bytes_as_the_aps_variant_did() { + // Literal strings captured from the closed-enum form before this + // change, through the same `serde_json::to_value` path production + // uses: `BidExt::to_ext` for the OpenRTB response extension, and + // `build_bid_map` for `window.tsjs.bids`. + // + // Both sides go through `with_sorted_keys` first, because the key + // order `serde_json` emits is not ours to pin, and that function + // explains why. Sorting settles the order without weakening what is + // pinned, since two objects serialize to the same sorted bytes only + // when they carry exactly the same keys with exactly the same values. + let full = BidRenderer::from_typed( + APS_RENDERER_TYPE, + &ApsRendererV1 { + version: 1, + account_id: "example-account-id".to_string(), + bid_id: "fictional-bid-id".to_string(), + creative_id: Some("fictional-creative-id".to_string()), + tag_type: ApsTagType::Iframe, + creative_url: "https://creative.example/render".to_string(), + aax_response: "base64-data".to_string(), + width: 300, + height: 250, + }, + ) + .expect("should build APS renderer descriptor"); + let absent = BidRenderer::from_typed( + APS_RENDERER_TYPE, + &ApsRendererV1 { + version: 1, + account_id: "example-account-id".to_string(), + bid_id: "fictional-bid-id".to_string(), + creative_id: None, + tag_type: ApsTagType::Script, + creative_url: "https://creative.example/render".to_string(), + aax_response: "base64-data".to_string(), + width: 300, + height: 250, + }, + ) + .expect("should build APS renderer descriptor"); + + let full_bytes = serde_json::to_string(&with_sorted_keys( + &serde_json::to_value(&full).expect("should convert renderer to a JSON value"), + )) + .expect("should serialize renderer"); + let absent_bytes = serde_json::to_string(&with_sorted_keys( + &serde_json::to_value(&absent).expect("should convert renderer to a JSON value"), + )) + .expect("should serialize renderer"); + + assert_eq!( + full_bytes, + "{\"aaxResponse\":\"base64-data\",\"accountId\":\"example-account-id\",\"bidId\":\"fictional-bid-id\",\"creativeId\":\"fictional-creative-id\",\"creativeUrl\":\"https://creative.example/render\",\"height\":250,\"tagType\":\"iframe\",\"type\":\"aps\",\"version\":1,\"width\":300}", + "should serialize to the bytes the closed enum produced" + ); + assert_eq!( + absent_bytes, + "{\"aaxResponse\":\"base64-data\",\"accountId\":\"example-account-id\",\"bidId\":\"fictional-bid-id\",\"creativeUrl\":\"https://creative.example/render\",\"height\":250,\"tagType\":\"script\",\"type\":\"aps\",\"version\":1,\"width\":300}", + "should serialize to the bytes the closed enum produced with no creative ID" + ); + } + + #[test] + fn module_constant_is_the_crate_folder() { + assert_eq!( + super::MODULE, + trusted_server_core::module_name!(), + "should be named by the folder this crate lives in" + ); + } +} + +/// APS run through core's auction engine: the orchestrator, and the reading +/// of an APS response as the driver hands it over. +#[cfg(test)] +mod engine_tests { + use std::collections::HashMap; + use std::sync::Arc; + + use base64::Engine as _; + + use trusted_server_core::auction::orchestrator::AuctionOrchestrator; + use trusted_server_core::auction::plan::{AuctionPlan, AuctionPlanConfig, NotificationConfig}; + use trusted_server_core::auction::test_support::{ + build_for_first_source, canonical_parity_auction_request, demand_table, + deterministic_signer, golden_inbound_request, golden_plan_config, parse_as_first_source, + plan_config_with, + }; + use trusted_server_core::auction::types::{ + AdFormat, AdSlot, AuctionContext, AuctionRequest, BidStatus, MediaType, PublisherInfo, + UserInfo, + }; + use trusted_server_core::platform::test_support::{ + NamingBackend, StubHttpClient, build_services_with_backend_and_http_client, + }; + use trusted_server_core::platform::{BackendNamingPolicy, PlatformResponse}; + use trusted_server_core::test_support::tests::create_test_settings; + + use super::{APS_RENDERER_TYPE, ApsRendererV1, MODULE, builder}; + + /// A plan configuration of APS sources, each with the settings and the + /// notification policy given. + fn aps_instances_config( + providers: &[(&str, serde_json::Value, NotificationConfig)], + ) -> AuctionPlanConfig { + let tables = providers + .iter() + .map(|(id, settings, notifications)| { + let mut entry = demand_table(MODULE, "https://aps.example/e/pb/bid"); + entry.insert("timeout_ms".to_string(), serde_json::json!(1_000)); + entry.insert("routing".to_string(), serde_json::json!("all_eligible")); + entry.insert( + "notifications".to_string(), + serde_json::to_value(notifications).expect("should serialize notifications"), + ); + if let serde_json::Value::Object(settings) = settings { + entry.extend(settings.clone()); + } + (*id, entry) + }) + .collect::>(); + let mut config = plan_config_with(tables, &[builder()]); + config.timeout_ms = 777; + config + } + + fn aps_config() -> AuctionPlanConfig { + aps_instances_config(&[( + "aps_instance", + serde_json::json!({"account_id": "example-account"}), + NotificationConfig::default(), + )]) + } + + fn planned_request() -> AuctionRequest { + AuctionRequest { + id: "fictional-auction".to_string(), + slots: vec![AdSlot { + id: "fictional-slot".to_string(), + formats: vec![AdFormat { + media_type: MediaType::Banner, + width: 300, + height: 250, + }], + floor_price: Some(1.0), + targeting: HashMap::new(), + bidders: HashMap::new(), + }], + publisher: PublisherInfo { + domain: "publisher.example".to_string(), + page_url: Some("https://publisher.example/article".to_string()), + }, + user: UserInfo { + id: None, + consent: None, + eids: None, + }, + device: None, + site: None, + context: HashMap::new(), + } + } + + /// One APS source as a `[demand.]` table, with nothing set that APS + /// defaults. + fn bare_aps_table() -> serde_json::Map { + let mut table = demand_table(MODULE, "https://aps.example/e/pb/bid"); + table.insert( + "account_id".to_string(), + serde_json::json!("example-account"), + ); + table + } + + fn compile_one( + table: serde_json::Map, + ) -> Result> + { + AuctionPlan::compile(plan_config_with(vec![("aps_one", table)], &[builder()])) + } + + #[test] + fn the_plan_compiles_an_aps_source_with_its_own_defaults() { + let plan = compile_one(bare_aps_table()).expect("should compile an APS source"); + + assert!( + plan.has_implementation(MODULE), + "a compiled plan should say it selected APS" + ); + let source = &plan.providers()[0]; + assert!( + source.demand.as_any().is::(), + "the APS source should compile its own settings" + ); + assert_eq!( + source.timeout_ms, 800, + "should default to the timeout APS declares" + ); + assert_eq!( + source.endpoint.as_str(), + "https://aps.example/e/pb/bid", + "should keep the endpoint as written" + ); + } + + #[test] + fn the_plan_refuses_what_aps_does_not_accept() { + let mut legacy_path = bare_aps_table(); + legacy_path.insert( + "endpoint".to_string(), + serde_json::json!("https://aps.example/e/dtb/bid"), + ); + assert!( + compile_one(legacy_path).is_err(), + "should refuse the legacy APS path" + ); + + let mut domain_alone = bare_aps_table(); + domain_alone.insert( + "inventory_domain".to_string(), + serde_json::json!("publisher.example"), + ); + assert!( + compile_one(domain_alone).is_err(), + "should refuse an APS inventory domain without its page origin" + ); + + let mut without_account = bare_aps_table(); + without_account.remove("account_id"); + assert!( + compile_one(without_account).is_err(), + "should refuse an APS source with no account" + ); + } + + #[test] + fn an_aps_demand_source_registers_the_renderer_with_no_integration_table() { + use trusted_server_core::auction::test_support::demand_selection; + use trusted_server_core::integrations::IntegrationRegistry; + + let mut settings = create_test_settings(); + let mut table = bare_aps_table(); + table.insert("routing".to_string(), serde_json::json!("all_eligible")); + settings.demand = demand_selection(vec![("aps_main", table)]); + let plan = Arc::new( + trusted_server_core::auction::compile_auction_plan_with(&settings, &[builder()]) + .expect("should compile APS plan"), + ); + let registry = + IntegrationRegistry::with_plan_and_registrations(&settings, plan, &[builder()]) + .expect("should build APS renderer registry"); + + assert!(registry.has_route(&http::Method::GET, "/integrations/aps/renderer")); + } + + #[test] + fn two_aps_sources_that_disagree_on_rendering_are_refused() { + use trusted_server_core::auction::test_support::demand_selection; + use trusted_server_core::integrations::IntegrationRegistry; + + let mut settings = create_test_settings(); + let mut publisher_native = bare_aps_table(); + publisher_native.insert( + "rendering_mode".to_string(), + serde_json::json!("publisher_native"), + ); + settings.demand = demand_selection(vec![ + ("aps_one", bare_aps_table()), + ("aps_two", publisher_native), + ]); + let plan = Arc::new( + trusted_server_core::auction::compile_auction_plan_with(&settings, &[builder()]) + .expect("should compile APS plan"), + ); + let error = + match IntegrationRegistry::with_plan_and_registrations(&settings, plan, &[builder()]) { + Ok(_) => panic!("should refuse two rendering modes"), + Err(error) => error, + }; + assert!( + error.to_string().contains("rendering_mode"), + "should name the setting that disagrees: {error:?}" + ); + } + + /// The request the driver builds for one APS source over the canonical + /// request, the way the goldens were captured. + fn golden_request( + settings: serde_json::Value, + signer: Option<&trusted_server_core::request_signing::RequestSigner>, + ) -> trusted_server_core::openrtb::OpenRtbRequest { + let plan = AuctionPlan::compile(golden_plan_config(MODULE, settings, true, &[builder()])) + .expect("should compile plan"); + build_for_first_source( + &plan, + canonical_parity_auction_request(), + &golden_inbound_request(), + 321, + signer, + ) + .expect("should build request") + .expect("should retain impression") + } + + /// The request the driver builds for one APS source over `request`. + fn request_for( + settings: serde_json::Value, + request: AuctionRequest, + accept_language: Option<&str>, + ) -> trusted_server_core::openrtb::OpenRtbRequest { + let plan = AuctionPlan::compile(golden_plan_config(MODULE, settings, true, &[builder()])) + .expect("should compile plan"); + let mut inbound = http::Request::builder().uri("https://publisher.example/auction"); + if let Some(language) = accept_language { + inbound = inbound.header(http::header::ACCEPT_LANGUAGE, language); + } + let inbound = inbound + .body(edgezero_core::body::Body::empty()) + .expect("should build inbound request"); + build_for_first_source(&plan, request, &inbound, 321, None) + .expect("should build request") + .expect("should retain impression") + } + + #[test] + fn consent_fields_keep_an_empty_admitted_context() { + use trusted_server_core::consent::ConsentContext; + use trusted_server_core::consent::jurisdiction::Jurisdiction; + + let cases = [ + ("empty", ConsentContext::default()), + ( + "gdpr", + ConsentContext { + gdpr_applies: true, + raw_tc_string: Some("tc-string".to_string()), + jurisdiction: Jurisdiction::Gdpr, + ..Default::default() + }, + ), + ( + "unknown-gpc", + ConsentContext { + gpc: true, + jurisdiction: Jurisdiction::Unknown, + ..Default::default() + }, + ), + ( + "nonregulated-gpc", + ConsentContext { + gpc: true, + jurisdiction: Jurisdiction::NonRegulated, + ..Default::default() + }, + ), + ( + "usp-gpp", + ConsentContext { + raw_us_privacy: Some("1YNN".to_string()), + raw_gpp_string: Some("gpp-string".to_string()), + gpp_section_ids: Some(vec![7, 8]), + jurisdiction: Jurisdiction::NonRegulated, + ..Default::default() + }, + ), + ]; + for (name, consent) in cases { + let mut canonical = canonical_parity_auction_request(); + canonical.user.consent = Some(consent.clone()); + let value = serde_json::to_value(request_for( + serde_json::json!({"account_id": "example-account-id"}), + canonical, + None, + )) + .expect("should serialize request"); + let regs = value + .get("regs") + .expect("APS should preserve empty admitted context"); + assert_eq!( + regs["gdpr"], + serde_json::json!(u8::from(consent.gdpr_applies)), + "{name}" + ); + assert!( + !value.to_string().contains("1YYY"), + "must never synthesize USP from GPC" + ); + if name == "usp-gpp" { + assert_eq!(regs["us_privacy"], "1YNN"); + assert_eq!(regs["gpp"], "gpp-string"); + assert_eq!(regs["gpp_sid"], serde_json::json!([7, 8])); + assert_eq!(regs["ext"]["us_privacy"], "1YNN"); + assert_eq!(regs["ext"]["gpp"], "gpp-string"); + assert_eq!(regs["ext"]["gpp_sid"], serde_json::json!([7, 8])); + } + } + } + + #[test] + fn language_is_kept_only_within_the_conservative_limit() { + let settings = serde_json::json!({"account_id": "example-account-id"}); + + let long = request_for( + settings.clone(), + canonical_parity_auction_request(), + Some("abcdefghijk"), + ); + assert_eq!( + long.device.and_then(|device| device.language).as_deref(), + None, + "should drop a language tag over the limit" + ); + + let ordinary = request_for( + settings, + canonical_parity_auction_request(), + Some("en-US,en;q=0.9"), + ); + assert_eq!( + ordinary + .device + .and_then(|device| device.language) + .as_deref(), + Some("en") + ); + } + + #[test] + fn inventory_identity_and_page_fallback_preserve_legacy_policy() { + let mut request = canonical_parity_auction_request(); + request.publisher.domain = "deployment.example".to_string(); + request.publisher.page_url = + Some("https://deployment.example/news/story?edition=fictional#section".to_string()); + let built = request_for( + serde_json::json!({ + "account_id": "example-account-id", + "inventory_domain": "publisher.example", + "inventory_page_origin": "https://www.publisher.example" + }), + request, + None, + ); + let site = built.site.expect("should include APS site"); + assert_eq!(site.domain.as_deref(), Some("publisher.example")); + assert_eq!( + site.page.as_deref(), + Some("https://www.publisher.example/news/story?edition=fictional") + ); + assert_eq!( + site.publisher + .and_then(|publisher| publisher.domain) + .as_deref(), + Some("publisher.example") + ); + + for unsafe_page in [ + "https://user:password@publisher.example/private", + "data:text/html,fictional", + ] { + let mut request = canonical_parity_auction_request(); + request.publisher.page_url = Some(unsafe_page.to_string()); + let built = request_for( + serde_json::json!({"account_id":"example-account-id"}), + request, + None, + ); + assert_eq!( + built.site.and_then(|site| site.page).as_deref(), + Some("https://publisher.example"), + "unsafe page should fall back to publisher domain" + ); + } + } + + #[test] + fn the_driver_request_matches_the_exact_golden() { + let request = golden_request( + serde_json::json!({"account_id": "example-account-id"}), + None, + ); + assert_eq!( + serde_json::to_string(&request).expect("should serialize APS driver request"), + r#"{"id":"fictional-auction","imp":[{"id":"fictional-slot","banner":{"format":[{"w":300,"h":250},{"w":728,"h":90}],"w":300,"h":250,"topframe":0},"bidfloor":1.0,"bidfloorcur":"USD","secure":1}],"site":{"domain":"publisher.example","page":"https://publisher.example/article","publisher":{"domain":"publisher.example"}},"device":{"geo":{"type":2,"country":"US","region":"CA","metro":"501","city":"Example City"},"dnt":1,"ua":"Fictional Browser","ip":"192.0.2.10","language":"en"},"user":{"id":"fictional-user","consent":"fictional-tcf","ext":{"consent":"fictional-tcf","eids":[{"source":"identity.example","uids":[{"atype":1,"id":"fictional-uid"}]}]}},"tmax":321,"cur":["USD"],"regs":{"gdpr":1,"us_privacy":"1YNN","gpp":"fictional-gpp","gpp_sid":[2,6],"ext":{"gdpr":1,"gpp":"fictional-gpp","gpp_sid":[2,6],"us_privacy":"1YNN"}},"ext":{"account":"example-account-id","sdk":{"source":"prebid","version":"2.2.0"}}}"#, + "should preserve APS parity differences" + ); + } + + #[test] + fn signing_is_applied_after_the_request_is_built_and_sets_every_owned_key() { + let settings = serde_json::json!({"account_id": "example-account-id"}); + let unsigned = serde_json::to_value(golden_request(settings.clone(), None)) + .expect("should serialize unsigned request"); + assert!( + unsigned["ext"].get("trusted_server").is_none(), + "should omit the extension when unsigned" + ); + + let signer = deterministic_signer(); + let signed = serde_json::to_value(golden_request(settings, Some(&signer))) + .expect("should serialize signed request"); + let extension = &signed["ext"]["trusted_server"]; + assert_eq!(extension["version"], "1.1", "should set signing version"); + assert_eq!(extension["kid"], "fictional-kid", "should set key ID"); + assert_eq!( + extension["request_host"], "publisher.example", + "should set host" + ); + assert_eq!(extension["request_scheme"], "https", "should set scheme"); + assert_eq!( + extension["ts"], 1_706_900_000_u64, + "should set explicit time" + ); + assert!( + extension["signature"] + .as_str() + .is_some_and(|value| !value.is_empty()), + "should set signature" + ); + } + + #[test] + fn the_signed_driver_request_matches_the_exact_golden() { + let signer = deterministic_signer(); + let request = golden_request( + serde_json::json!({"account_id": "example-account-id"}), + Some(&signer), + ); + assert_eq!( + serde_json::to_string(&request).expect("should serialize signed request"), + r#"{"id":"fictional-auction","imp":[{"id":"fictional-slot","banner":{"format":[{"w":300,"h":250},{"w":728,"h":90}],"w":300,"h":250,"topframe":0},"bidfloor":1.0,"bidfloorcur":"USD","secure":1}],"site":{"domain":"publisher.example","page":"https://publisher.example/article","publisher":{"domain":"publisher.example"}},"device":{"geo":{"type":2,"country":"US","region":"CA","metro":"501","city":"Example City"},"dnt":1,"ua":"Fictional Browser","ip":"192.0.2.10","language":"en"},"user":{"id":"fictional-user","consent":"fictional-tcf","ext":{"consent":"fictional-tcf","eids":[{"source":"identity.example","uids":[{"atype":1,"id":"fictional-uid"}]}]}},"tmax":321,"cur":["USD"],"regs":{"gdpr":1,"us_privacy":"1YNN","gpp":"fictional-gpp","gpp_sid":[2,6],"ext":{"gdpr":1,"gpp":"fictional-gpp","gpp_sid":[2,6],"us_privacy":"1YNN"}},"ext":{"account":"example-account-id","sdk":{"source":"prebid","version":"2.2.0"},"trusted_server":{"kid":"fictional-kid","request_host":"publisher.example","request_scheme":"https","signature":"LU_JUIA1BT80ShZNjSa4PIF5T-uMjEeodwKrV_6bXgh0hi1SYVtCKn9g_DTW62krmjCOFgoFYPHsu6L0nAcuDg","ts":1706900000,"version":"1.1"}}}"#, + "signed wire fixture should stay exact" + ); + } + + /// Settings whose `[demand]` selects one APS source with `account_id` + /// and the settings `extra` adds. + fn settings_with_aps_source( + account_id: &str, + extra: &[(&str, serde_json::Value)], + ) -> trusted_server_core::settings::Settings { + let mut settings = create_test_settings(); + let mut table = demand_table(MODULE, "https://aps.example.com/e/pb/bid"); + table.insert("account_id".to_string(), serde_json::json!(account_id)); + for (key, value) in extra { + table.insert((*key).to_string(), value.clone()); + } + settings.demand = + trusted_server_core::auction::test_support::demand_selection(vec![("aps_main", table)]); + settings + } + + fn validate_for_deploy( + settings: &trusted_server_core::settings::Settings, + ) -> Result<(), error_stack::Report> { + trusted_server_core::config::validate_settings_for_deploy_with(settings, &[builder()]) + } + + #[test] + fn deploy_validation_rejects_a_blank_account_id() { + for (label, account_id) in [("empty", ""), ("whitespace-only", " ")] { + let err = validate_for_deploy(&settings_with_aps_source(account_id, &[])) + .expect_err("should reject blank APS account_id"); + + assert!( + format!("{err:?}").contains("account_id"), + "should mention the APS account_id for {label}: {err:?}" + ); + } + } + + #[test] + fn deploy_validation_normalizes_a_padded_account_id() { + validate_for_deploy(&settings_with_aps_source(" example-account ", &[])) + .expect("should accept a padded APS account_id after trimming it"); + } + + #[test] + fn deploy_validation_rejects_a_setting_aps_does_not_know() { + let settings = + settings_with_aps_source("example-account", &[("enabled", serde_json::json!(false))]); + + let error = validate_for_deploy(&settings) + .expect_err("should reject a setting the APS implementation does not know"); + let rendered = format!("{error:?}"); + assert!( + rendered.contains("aps_main"), + "should identify the demand source: {rendered}" + ); + assert!( + rendered.contains("enabled"), + "should identify the setting it does not know: {rendered}" + ); + } + + #[tokio::test] + async fn planned_aps_transport_omits_accept_header() { + let http = Arc::new(StubHttpClient::new()); + http.push_response(400, Vec::new()); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Fastly)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = AuctionPlan::compile(aps_config()).expect("should compile planned APS auction"); + let orchestrator = AuctionOrchestrator::from_plan(Arc::new(plan), None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::builder() + .uri("https://publisher.example/auction") + .body(edgezero_core::body::Body::empty()) + .expect("should build inbound request"); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + orchestrator + .run_auction(&request, &context) + .await + .expect("should execute planned APS auction"); + + let headers = http.recorded_request_headers(); + assert_eq!(headers.len(), 1); + assert!( + headers[0].iter().all(|(name, _)| name != "accept"), + "planned APS transport must not add Accept beyond legacy headers" + ); + } + + #[tokio::test] + async fn planned_aps_profile_normalizes_renderer_reduction_and_metadata() { + let http = Arc::new(StubHttpClient::new()); + http.push_response_with_headers( + 200, + serde_json::to_vec(&serde_json::json!({ + "cur": "USD", + "seatbid": [ + {"seat": "returned-seat", "bid": [ + {"id": "z-high", "impid": "fictional-slot", "price": 2.0, "w": 300, "h": 250, + "nurl": "https://notice.example/win", "burl": "https://notice.example/bill", + "crid": "fictional-creative", "adomain": ["advertiser.example"], + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, + {"id": "a-high", "impid": "fictional-slot", "price": 2.0, "w": 300, "h": 250, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, + {"id": "bad-script", "impid": "fictional-slot", "price": 9.0, "w": 300, "h": 250, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "script"}}, + {"id": "bad-domain", "impid": "fictional-slot", "price": 8.0, "w": 300, "h": 250, + "ext": {"creativeurl": "https://publisher.example/render", "tagtype": "iframe"}}, + {"id": "bad-credentials", "impid": "fictional-slot", "price": 8.0, "w": 300, "h": 250, + "ext": {"creativeurl": "https://user:password@creative.example/render", "tagtype": "iframe"}}, + {"id": "bad-imp", "impid": "unknown-slot", "price": 8.0, "w": 300, "h": 250, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, + {"id": "bad-dimensions", "impid": "fictional-slot", "price": 8.0, "w": 320, "h": 50, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, + {"id": "bad-price", "impid": "fictional-slot", "price": "high", "w": 300, "h": 250, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, + {"id": "bad-mtype", "impid": "fictional-slot", "price": 8.0, "mtype": 2, "w": 300, "h": 250, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, + {"id": "bad-tag", "impid": "fictional-slot", "price": 8.0, "w": 300, "h": 250, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "native"}}, + {"id": "bad-crid", "impid": "fictional-slot", "price": 8.0, "w": 300, "h": 250, + "crid": "x".repeat(1025), + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, + {"impid": "fictional-slot", "price": 8.0, "w": 300, "h": 250, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}} + ]}, + {"seat": 7, "bid": "bad-shape"} + ] + })) + .expect("should serialize APS profile response"), + vec![ + ("content-type", "application/json"), + ("authorization", "fictional-secret"), + ], + ); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Fastly)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = AuctionPlan::compile(aps_instances_config(&[( + "aps_instance", + serde_json::json!({"account_id": "example-account", "debug": true}), + NotificationConfig { + suppress_all: false, + suppress_seats: vec!["returned-seat".to_string()], + }, + )])) + .expect("should compile planned APS profile"); + let orchestrator = AuctionOrchestrator::from_plan(Arc::new(plan), None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should execute planned APS profile"); + + let response = &result.provider_responses[0]; + assert_eq!(response.provider, "aps_instance"); + assert_eq!(response.status, BidStatus::Success); + assert_eq!( + response.bids.len(), + 1, + "should retain one bid per impression" + ); + let bid = &response.bids[0]; + assert_eq!(bid.bidder, "aps"); + assert_eq!(bid.returned_seat.as_deref(), Some("returned-seat")); + assert_eq!( + bid.bid_id.as_deref(), + Some("a-high"), + "lexical ID should break equal-price tie" + ); + assert!(bid.creative.is_none()); + assert!( + bid.nurl.is_none() && bid.burl.is_none(), + "APS must discard notification URLs" + ); + let renderer = bid + .renderer + .as_ref() + .and_then(|renderer| renderer.payload_as::(APS_RENDERER_TYPE)) + .expect("should construct typed APS renderer"); + assert_eq!(renderer.account_id, "example-account"); + let decoded = base64::engine::general_purpose::STANDARD + .decode(&renderer.aax_response) + .expect("should decode minimized APS response"); + assert_eq!( + serde_json::from_slice::(&decoded) + .expect("should parse minimized APS response"), + serde_json::json!({"seatbid":[{"bid":[{ + "id":"a-high","price":2.0,"w":300,"h":250, + "ext":{"creativeurl":"https://creative.example/render","tagtype":"iframe"} + }]}]}) + ); + assert_eq!(response.metadata["seatbid_count"], 2); + assert_eq!(response.metadata["accepted_bid_count"], 1); + assert_eq!(response.metadata["dropped_bid_count"], 12); + for reason in [ + "lost_to_higher_bid", + "script_rendering_disabled", + "unknown_impid", + "invalid_dimensions", + "invalid_price", + "unsupported_media_type", + "unsupported_tagtype", + "creative_id_too_large", + "missing_render_source", + "empty_seatbid_bids", + ] { + assert_eq!(response.metadata["drop_reasons"][reason], 1, "{reason}"); + } + assert_eq!( + response.metadata["drop_reasons"]["invalid_creative_url"], 2, + "same-publisher and credentialed URLs should both be rejected" + ); + assert_eq!( + response.metadata["routing"]["unused_bidder_params_count"], + 0 + ); + let debug = &response.metadata["debug"]["httpcalls"]["aps"][0]; + assert_eq!(debug["uri"], "https://aps.example/e/pb/bid"); + assert_eq!( + debug["responseheaders"], + serde_json::json!({"content-type": ["application/json"]}), + "async stub should preserve allowlisted response headers" + ); + assert!( + debug["requestbody"] + .as_str() + .is_some_and(|body| body.contains("example-account")) + ); + assert!(debug["requestheaders"].get("authorization").is_none()); + assert!(debug["responseheaders"].get("authorization").is_none()); + } + + #[tokio::test] + async fn two_planned_aps_instances_correlate_independently() { + let http = Arc::new(StubHttpClient::new()); + for (seat, id, price) in [("seat-a", "bid-a", 1.0), ("seat-b", "bid-b", 2.0)] { + http.push_response( + 200, + serde_json::to_vec(&serde_json::json!({"seatbid":[{"seat":seat,"bid":[{ + "id":id,"impid":"fictional-slot","price":price,"w":300,"h":250, + "ext":{"creativeurl":"https://creative.example/render","tagtype":"iframe"} + }]}]})) + .expect("should serialize APS instance response"), + ); + } + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Fastly)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = AuctionPlan::compile(aps_instances_config(&[ + ( + "aps_a", + serde_json::json!({"account_id":"account-a"}), + NotificationConfig::default(), + ), + ( + "aps_b", + serde_json::json!({"account_id":"account-b"}), + NotificationConfig::default(), + ), + ])) + .expect("should compile two APS instances"); + let orchestrator = AuctionOrchestrator::from_plan(Arc::new(plan), None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should execute two APS instances"); + + assert_eq!(result.provider_responses.len(), 2); + assert_eq!(result.provider_responses[0].provider, "aps_a"); + assert_eq!( + result.provider_responses[0].bids[0].bid_id.as_deref(), + Some("bid-a") + ); + assert_eq!(result.provider_responses[1].provider, "aps_b"); + assert_eq!( + result.provider_responses[1].bids[0].bid_id.as_deref(), + Some("bid-b") + ); + assert_eq!(http.recorded_request_bodies().len(), 2); + assert_eq!( + result.winning_bids["fictional-slot"].bid_id.as_deref(), + Some("bid-b"), + "global ranking should remain orchestrator-owned" + ); + let specs = backend.specs.lock().expect("should lock specs"); + assert_eq!(specs.len(), 2); + assert_ne!(specs[0].discriminator, specs[1].discriminator); + } + + #[tokio::test] + async fn planned_aps_returned_seat_accepts_only_valid_nonempty_strings() { + let plan = AuctionPlan::compile(aps_config()).expect("should compile APS plan"); + for (seat, expected) in [ + (serde_json::Value::Null, None), + (serde_json::json!(7), None), + (serde_json::json!(""), None), + (serde_json::json!("exact-seat"), Some("exact-seat")), + ] { + let response = PlatformResponse::new( + edgezero_core::http::response_builder() + .status(200) + .body(edgezero_core::body::Body::from( + serde_json::to_vec(&serde_json::json!({"seatbid":[{"seat":seat,"bid":[{ + "id":"bid","impid":"fictional-slot","price":1.0,"w":300,"h":250, + "nurl":"https://notice.example/win","burl":"https://notice.example/bill", + "ext":{"creativeurl":"https://creative.example/render","tagtype":"iframe"} + }]}]})) + .expect("should serialize seat identity response"), + )) + .expect("should build seat identity response"), + ); + let parsed = parse_as_first_source(&plan, planned_request(), response, 4) + .await + .expect("should parse seat identity response"); + assert_eq!(parsed.bids[0].returned_seat.as_deref(), expected); + assert!(parsed.bids[0].nurl.is_none() && parsed.bids[0].burl.is_none()); + } + } + + #[tokio::test] + async fn planned_aps_response_status_shape_and_currency_matrix() { + let plan = AuctionPlan::compile(aps_config()).expect("should compile APS plan"); + let cases = [ + (204, Vec::new(), BidStatus::NoBid, None, None), + (400, Vec::new(), BidStatus::Error, None, Some("http_status")), + ( + 200, + b"not-json".to_vec(), + BidStatus::Error, + Some("unexpected_response_shape"), + Some("parse_response"), + ), + ( + 200, + b"[]".to_vec(), + BidStatus::Error, + Some("unexpected_response_shape"), + Some("parse_response"), + ), + ( + 200, + br#"{"contextual":true}"#.to_vec(), + BidStatus::Error, + Some("unexpected_response_shape"), + Some("parse_response"), + ), + ( + 200, + br#"{"cur":"EUR","seatbid":[]}"#.to_vec(), + BidStatus::NoBid, + Some("unsupported_currency"), + None, + ), + ]; + for (status, body, expected, reason, error_type) in cases { + let response = PlatformResponse::new( + edgezero_core::http::response_builder() + .status(status) + .body(edgezero_core::body::Body::from(body)) + .expect("should build APS matrix response"), + ); + let parsed = parse_as_first_source(&plan, planned_request(), response, 4) + .await + .expect("should classify APS matrix response"); + assert_eq!(parsed.status, expected, "status {status}"); + if let Some(reason) = reason { + assert_eq!( + parsed.metadata["drop_reasons"][reason], 1, + "status {status}" + ); + } + if let Some(error_type) = error_type { + assert_eq!(parsed.metadata["error_type"], error_type, "status {status}"); + } + } + } + + #[tokio::test] + async fn planned_aps_script_opt_in_matches_shared_renderer_fixture() { + let plan = AuctionPlan::compile(aps_instances_config(&[( + "aps_instance", + serde_json::json!({ + "account_id":"example-account-id", + "allow_script_creatives":true + }), + NotificationConfig::default(), + )])) + .expect("should compile script-enabled APS plan"); + let response = PlatformResponse::new( + edgezero_core::http::response_builder() + .status(200) + .body(edgezero_core::body::Body::from( + serde_json::to_vec(&serde_json::json!({"seatbid":[{"bid":[{ + "id":"fictional-selected-bid-id","impid":"fictional-slot","price":1.23, + "w":300,"h":250,"crid":"fictional-creative", + "ext":{"creativeurl":"https://creative.example/render","tagtype":"iframe"} + },{ + "id":"script-bid","impid":"fictional-slot","price":1.0, + "w":300,"h":250, + "ext":{"creativeurl":"https://creative.example/script","tagtype":"script"} + }]}]})) + .expect("should serialize APS renderer fixture response"), + )) + .expect("should build APS renderer fixture response"), + ); + + let parsed = parse_as_first_source(&plan, planned_request(), response, 3) + .await + .expect("should parse APS renderer fixture response"); + + assert_eq!(parsed.status, BidStatus::Success); + assert_eq!( + parsed.metadata["drop_reasons"]["lost_to_higher_bid"], 1, + "enabled script creative should be eligible before reduction" + ); + let renderer = parsed.bids[0] + .renderer + .as_ref() + .and_then(|renderer| renderer.payload_as::(APS_RENDERER_TYPE)) + .expect("should construct APS renderer"); + let decoded = base64::engine::general_purpose::STANDARD + .decode(&renderer.aax_response) + .expect("should decode APS fixture envelope"); + let fixture: serde_json::Value = serde_json::from_str(include_str!( + "../../../trusted-server-js/lib/test/fixtures/aps-renderer-v1.json" + )) + .expect("should parse shared APS renderer fixture"); + assert_eq!( + serde_json::from_slice::(&decoded) + .expect("should parse decoded APS renderer"), + fixture + ); + } + + #[tokio::test] + async fn planned_aps_debug_response_headers_are_allowlisted() { + let plan = AuctionPlan::compile(aps_instances_config(&[( + "aps_instance", + serde_json::json!({"account_id":"example-account","debug":true}), + NotificationConfig::default(), + )])) + .expect("should compile debug APS plan"); + let response = PlatformResponse::new( + edgezero_core::http::response_builder() + .status(200) + .header("content-type", "application/json") + .header("authorization", "fictional-secret") + .body(edgezero_core::body::Body::from("{}")) + .expect("should build debug APS response"), + ); + + let parsed = parse_as_first_source(&plan, planned_request(), response, 3) + .await + .expect("should parse debug APS response"); + + let headers = &parsed.metadata["debug"]["httpcalls"]["aps"][0]["responseheaders"]; + assert_eq!( + headers, + &serde_json::json!({"content-type":["application/json"]}) + ); + assert!(headers.get("authorization").is_none()); + } + + #[tokio::test] + async fn planned_routing_count_survives_a_bounded_body_failure() { + let (profile, provider_id) = ("aps", "aps_instance"); + let http = Arc::new(StubHttpClient::new()); + http.push_response(200, vec![b'x'; 1024 * 1024 + 1]); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let mut config = aps_config(); + config.bidders.insert( + "example-bidder" + .parse() + .expect("should parse fictional bidder ID"), + trusted_server_core::auction::plan::BidderRouteConfig { + module: provider_id + .parse() + .expect("should parse fictional provider ID"), + }, + ); + let plan = AuctionPlan::compile(config).expect("should compile bounded-body plan"); + let orchestrator = AuctionOrchestrator::from_plan(Arc::new(plan), None); + let mut request = planned_request(); + request.slots[0].bidders.insert( + "example-bidder".to_string(), + serde_json::json!({"private": "value"}), + ); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should materialize bounded-body failure"); + let response = &result.provider_responses[0]; + assert_eq!(response.status, BidStatus::Error, "{profile}"); + assert_eq!( + response.metadata["routing"]["unused_bidder_params_count"], 1, + "{profile} bounded-body failure should retain the input-derived count" + ); + let routing = serde_json::to_string(&response.metadata["routing"]) + .expect("should serialize routing metadata"); + assert!(!routing.contains("example-bidder") && !routing.contains("private")); + } + + /// The page a reader receives with this module running, kept as a file + /// so that changing how the page change is made can be shown to leave + /// the page as it was. + #[test] + fn the_page_a_reader_receives_is_the_recorded_one() { + trusted_server_core::html_processor::test_support::assert_page_is_recorded( + include_str!("fixtures/page-change.settings.toml"), + &[super::builder()], + include_str!("fixtures/page-change.input.html"), + include_str!("fixtures/page-change.recorded.html"), + concat!( + env!("CARGO_MANIFEST_DIR"), + "/src/fixtures/page-change.recorded.html" + ), + ); + } } diff --git a/crates/auction/prebid-server/Cargo.toml b/crates/auction/prebid-server/Cargo.toml new file mode 100644 index 0000000000..c1bfa1cefd --- /dev/null +++ b/crates/auction/prebid-server/Cargo.toml @@ -0,0 +1,41 @@ +[package] +name = "trusted-server-auction-prebid-server" +description = "Prebid Server as a demand implementation: its request extensions, bidder parameter overrides, transport headers and response reader." +authors = { workspace = true } +edition = { workspace = true } +license = { workspace = true } +publish = { workspace = true } +version = { workspace = true } + +[lib] +doctest = false + +[lints] +workspace = true + +[dependencies] +async-trait = { workspace = true } +error-stack = { workspace = true } +http = { workspace = true } +log = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +trusted-server-core = { workspace = true } +url = { workspace = true } + +[dev-dependencies] +base64 = { workspace = true } +bytes = { workspace = true } +edgezero-core = { workspace = true } +futures = { workspace = true } +tokio = { workspace = true } +validator = { workspace = true } +web-time = { workspace = true } +trusted-server-core = { workspace = true, features = ["test-utils"] } + +# The visible owner of this vendor crate, the way Prebid.js requires a named +# maintainer of every adapter. Prebid.org has not yet adopted this crate, so the +# Trusted Server maintainers own it until a vendor owner steps up. +[package.metadata.maintainers] +owner = "Trusted Server maintainers" +status = "seeking vendor owner" diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/auction/prebid-server/src/lib.rs similarity index 67% rename from crates/trusted-server-core/src/integrations/prebid.rs rename to crates/auction/prebid-server/src/lib.rs index f1d1d0b509..35fd0b3e6d 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/auction/prebid-server/src/lib.rs @@ -1,78 +1,324 @@ -use std::collections::HashMap; +//! The Prebid Server demand implementation, `auction.prebid-server`. +//! +//! Prebid Server takes the shared `OpenRTB` baseline with its own impression +//! and request extensions, forwards the raw headers its auction needs, and +//! answers with Prebid's own response shape. + +#![cfg_attr( + test, + allow( + clippy::print_stdout, + clippy::print_stderr, + clippy::panic, + clippy::dbg_macro, + clippy::unwrap_used, + reason = "tests use direct diagnostics and panic-on-failure helpers" + ) +)] + +use core::any::Any; +#[cfg(test)] use std::collections::HashSet; -use std::sync::{Arc, LazyLock}; +use std::collections::{BTreeMap, HashMap}; +use std::sync::Arc; #[cfg(test)] use std::time::Duration; use async_trait::async_trait; -use base64::{ - Engine as _, - engine::general_purpose::{ - STANDARD as BASE64_STANDARD, STANDARD_NO_PAD as BASE64_STANDARD_NO_PAD, - }, -}; +#[cfg(test)] +use base64::Engine as _; +#[cfg(test)] use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt}; +#[cfg(test)] +use http::StatusCode; use http::header::HeaderValue; -use http::{Method, StatusCode, header}; -use regex::Regex; +use http::{HeaderMap, header}; use serde::{Deserialize, Serialize}; -use serde_json::Value as Json; +use serde_json::{Map, Value, Value as Json, json}; use url::{Url, Url as ParsedUrl}; -use validator::{Validate, ValidationError}; +#[cfg(test)] +use validator::Validate; -use crate::auction::openrtb::{ +use trusted_server_core::auction::demand::{ + CompiledDemand, DemandFieldPolicy, DemandImplementation, DemandResponse, DemandTimeoutDefault, + DemandTransport, ProviderAuctionInput, RegsPolicy, RequestExtensions, TransportHeaders, +}; +use trusted_server_core::auction::openrtb::{ BidDimensionIndex, BidRejectionReason, ResponseAdmissionDiagnostics, build_bid_dimension_index, parse_optional_bid_dimension, resolve_bid_dimensions, }; -use crate::auction::orchestrator::ERROR_TYPE_HTTP_STATUS; -use crate::auction::plan::AuctionPlan; -use crate::auction::profile::PrebidProfilePlan; +use trusted_server_core::auction::orchestrator::ERROR_TYPE_HTTP_STATUS; #[cfg(test)] -use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; -use crate::auction::routing::{PrebidTransportHeaders, ProviderAuctionInput}; +use trusted_server_core::auction::provider::{AuctionProvider, ProviderRequestOutcome}; #[cfg(test)] -use crate::auction::types::{AuctionContext, AuctionRequest, MediaType}; -use crate::auction::types::{AuctionResponse, Bid as AuctionBid}; -use crate::cache_policy::{CacheControlPolicy, EdgeCacheHeader}; -use crate::consent_config::ConsentForwardingMode; -use crate::cookies::{CONSENT_COOKIE_NAMES, strip_cookies}; -use crate::error::TrustedServerError; +use trusted_server_core::auction::types::{AuctionContext, AuctionRequest, MediaType}; +use trusted_server_core::auction::types::{AuctionResponse, Bid as AuctionBid}; +use trusted_server_core::consent::{ConsentContext, ConsentSource}; +use trusted_server_core::consent_config::ConsentForwardingMode; +use trusted_server_core::cookies::{CONSENT_COOKIE_NAMES, strip_cookies}; +use trusted_server_core::error::TrustedServerError; #[cfg(test)] -use crate::http_util::RequestInfo; -use crate::integrations::{ - AttributeRewriteAction, IntegrationAttributeContext, IntegrationAttributeRewriter, - IntegrationEndpoint, IntegrationHeadInjector, IntegrationHtmlContext, IntegrationProxy, - IntegrationRegistration, UPSTREAM_RTB_MAX_RESPONSE_BYTES, collect_response_bounded, +use trusted_server_core::http_util::RequestInfo; +use trusted_server_core::integrations::{ + UPSTREAM_RTB_MAX_RESPONSE_BYTES, collect_response_bounded, }; #[cfg(test)] -use crate::integrations::{ +use trusted_server_core::integrations::{ ensure_integration_backend_with_timeout, predict_integration_backend_name, }; #[cfg(test)] -use crate::openrtb::{ - Banner, ConsentedProvidersSettings, Device, Format, Geo, Imp, ImpExt, ImpStoredRequest, - OpenRtbRequest, PrebidExt, PrebidImpExt, Publisher, Regs, RegsExt, RequestExt, Site, ToExt, - TrustedServerExt, User, UserExt, to_openrtb_i32, +use trusted_server_core::openrtb::{ + Banner, ConsentedProvidersSettings, Device, Format, Geo, Imp, OpenRtbRequest, Publisher, Regs, + RegsExt, Site, ToExt, TrustedServerExt, User, UserExt, to_openrtb_i32, }; #[cfg(test)] -use crate::platform::PlatformHttpRequest; -use crate::platform::{PlatformResponse, RuntimeServices}; -use crate::proxy::{ProxyRequestConfig, is_host_allowed, proxy_request}; +use trusted_server_core::platform::PlatformHttpRequest; +use trusted_server_core::platform::PlatformResponse; +#[cfg(test)] +use trusted_server_core::platform::RuntimeServices; +#[cfg(test)] +use trusted_server_core::request_signing::{RequestSigner, SIGNING_VERSION, SigningParams}; #[cfg(test)] -use crate::request_signing::{RequestSigner, SIGNING_VERSION, SigningParams}; -use crate::settings::{IntegrationConfig, Settings}; +use trusted_server_core::settings::{IntegrationConfig, Settings}; + +/// The name an `implementation` line gives this implementation, its module +/// path, which also serves as its builder's id. +pub const MODULE: &str = "auction.prebid-server"; + +/// The canonical Prebid Server auction path. +const AUCTION_PATH: &str = "/openrtb2/auction"; + +/// The Prebid Server demand implementation. +pub static DEMAND: DemandImplementation = DemandImplementation { + id: MODULE, + default_timeout: DemandTimeoutDefault::Fixed(1000), + allows_all_eligible: false, + serves_stored_requests: true, + canonicalize_endpoint, + compile, +}; + +/// The builder a deployment hands to an adapter, which offers Prebid Server +/// to `[demand]`. +#[must_use] +pub fn builder() -> trusted_server_core::integrations::IntegrationBuilder { + trusted_server_core::integrations::IntegrationBuilder::implementations( + MODULE, + env!("CARGO_PKG_NAME"), + ) + .with_demand(&DEMAND) +} + +/// One compiled Prebid Server demand source. +#[derive(Debug, Clone)] +pub struct PrebidServerDemand { + /// Include Prebid HTTP exchange diagnostics. + pub debug: bool, + /// Set `OpenRTB` test mode. + pub test_mode: bool, + /// Query fragment appended to the page URL for a test deployment. + pub debug_query_params: Option, + /// Compiled override matching and merge index. + pub(crate) override_engine: BidParamOverrideEngine, + /// Consent transport policy. + pub consent_forwarding: ConsentForwardingMode, +} + +/// The operator settings `[demand.]` holds for this implementation. +#[derive(Debug, Deserialize, Default)] +#[serde(deny_unknown_fields)] +struct PrebidServerSettings { + #[serde(default)] + debug: bool, + #[serde(default)] + test_mode: bool, + #[serde(default)] + debug_query_params: Option, + #[serde(default)] + bid_param_zone_overrides: BTreeMap>>, + #[serde(default)] + bid_param_overrides: BTreeMap>, + #[serde(default)] + bid_param_override_rules: Vec, + #[serde(default)] + consent_forwarding: ConsentForwardingMode, +} + +fn compile( + settings: &Map, +) -> Result, Report> { + let settings = + PrebidServerSettings::deserialize(Value::Object(settings.clone())).map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: format!("invalid `{MODULE}` settings: {error}"), + }) + })?; + let override_engine = compile_profile_override_rules( + &settings.bid_param_zone_overrides, + &settings.bid_param_overrides, + &settings.bid_param_override_rules, + )?; + Ok(Arc::new(PrebidServerDemand { + debug: settings.debug, + test_mode: settings.test_mode, + debug_query_params: settings.debug_query_params, + override_engine, + consent_forwarding: settings.consent_forwarding, + })) +} + +/// Complete an endpoint that names the host alone with the auction path. +fn canonicalize_endpoint(endpoint: &mut Url) -> Result<(), String> { + match endpoint.path() { + "" | "/" | "/openrtb2/auction/" => endpoint.set_path(AUCTION_PATH), + _ => {} + } + Ok(()) +} + +#[async_trait(?Send)] +impl CompiledDemand for PrebidServerDemand { + fn field_policy(&self) -> DemandFieldPolicy { + DemandFieldPolicy { + imp_tagid: true, + site_ref: true, + precise_geo: true, + additional_consent: true, + test: self.test_mode, + unsigned_request_identity: true, + consumes_bidder_params: true, + regs: RegsPolicy::Jurisdiction, + ..DemandFieldPolicy::default() + } + } + + fn site_page(&self, publisher_page: Option<&str>, site_domain: &str) -> Option { + let _ = site_domain; + publisher_page.map(|page| { + self.debug_query_params.as_deref().map_or_else( + || page.to_owned(), + |query| append_query_fragment(page, query), + ) + }) + } + + fn body_consent<'c>(&self, consent: Option<&'c ConsentContext>) -> Option<&'c ConsentContext> { + consent.filter(|value| { + self.consent_forwarding.includes_body_consent() + || !matches!(value.source, ConsentSource::Cookie) + }) + } + + fn augment_request( + &self, + extensions: &mut RequestExtensions<'_>, + _input: &ProviderAuctionInput, + ) -> Result<(), Report> { + for impression in &mut extensions.impressions { + let slot = impression.slot; + let bidder = slot + .bidder_params() + .iter() + .filter_map(|(bidder, params)| { + let mut params = params.clone(); + self.override_engine + .apply_routed(bidder.as_str(), slot.zone(), &mut params); + params + .as_object() + .is_some_and(|params| !params.is_empty()) + .then(|| (bidder.as_str().to_string(), params)) + }) + .collect::>(); + // Usable inline parameters win. Otherwise only a stored request the + // slot's intent permits is sent, and a slot left with neither has + // no demand for Prebid Server and is left out of the request. + let mut prebid = Map::new(); + if !bidder.is_empty() { + prebid.insert("bidder".to_string(), Value::Object(bidder)); + } else if slot.allows_stored_fallback() { + prebid.insert("storedrequest".to_string(), json!({"id": slot.slot().id})); + } + if prebid.is_empty() { + impression.omitted = true; + continue; + } + *impression.ext = Some(Map::from_iter([( + "prebid".to_string(), + Value::Object(prebid), + )])); + } + let mut prebid_request = Map::new(); + if self.debug { + prebid_request.insert("debug".to_string(), Value::Bool(true)); + prebid_request.insert("returnallbidstatus".to_string(), Value::Bool(true)); + } + *extensions.request = Some(Map::from_iter([( + "prebid".to_string(), + Value::Object(prebid_request), + )])); + Ok(()) + } + + fn prepare_outbound(&self, headers: &mut HeaderMap, transport: DemandTransport<'_>) { + apply_prebid_transport_headers( + transport.headers, + headers, + self.consent_forwarding, + transport.attested_client_ip, + ); + } + + async fn parse_response( + &self, + context: DemandResponse<'_>, + response: PlatformResponse, + ) -> Result> { + let provider_id = context.provider_id; + let response_time_ms = context.response_time_ms; + let auction_id = context.input.common_request().id.clone(); + match parse_planned_prebid_response( + provider_id, + self, + context.input, + response, + response_time_ms, + &auction_id, + ) + .await + { + Ok(parsed) => Ok(parsed), + Err(error) => { + log::warn!("Provider '{provider_id}' PBS response parse failed: {error:?}"); + Ok(AuctionResponse::error(provider_id, response_time_ms) + .with_metadata("error_type", json!("parse_response"))) + } + } + } + + fn as_any(&self) -> &dyn Any { + self + } +} + +/// Append a query fragment the deployment set, without repeating one the page +/// already carries. +fn append_query_fragment(url: &str, query: &str) -> String { + if query.is_empty() || url.contains(query) { + return url.to_string(); + } + let separator = if url.contains('?') { '&' } else { '?' }; + format!("{url}{separator}{query}") +} +/// The provider id [`PrebidAuctionProvider`] reports. +#[cfg(test)] const PREBID_INTEGRATION_ID: &str = "prebid"; -const PREBID_BUNDLE_ROUTE: &str = "/integrations/prebid/bundle.js"; -const PREBID_BUNDLE_CONTENT_TYPE: &str = "application/javascript; charset=utf-8"; -const PREBID_BUNDLE_IMMUTABLE_CACHE_CONTROL: &str = "public, max-age=31536000, immutable"; -const PREBID_BUNDLE_REVALIDATION_CACHE_CONTROL: &str = - "public, max-age=300, s-maxage=300, stale-while-revalidate=60, stale-if-error=86400"; -const PREBID_BUNDLE_ERROR_CACHE_CONTROL: &str = "no-store"; -const PREBID_BUNDLE_ERROR_CONTENT_TYPE: &str = "text/plain; charset=utf-8"; -const PREBID_BUNDLE_NOSNIFF_HEADER: &str = "x-content-type-options"; -const PREBID_BUNDLE_NOSNIFF_VALUE: &str = "nosniff"; + +/// The settings table [`LegacyPrebidServerConfig`] is read from. +#[cfg(test)] +const LEGACY_TABLE: &str = "auction.prebid"; + #[cfg(test)] const TRUSTED_SERVER_BIDDER: &str = "trustedServer"; #[cfg(test)] @@ -221,143 +467,19 @@ fn extract_prebid_error_message( #[cfg(test)] const GPC_US_PRIVACY: &str = "1YYN"; -/// Rejects a Prebid User ID identifier that Prebid.js could not address. -/// -/// Applies only the constraints Prebid itself imposes on a `userSync.userIds` -/// entry name and on a storage key: a non-empty ASCII token with no -/// surrounding whitespace. Anything narrower would encode one vendor's rules -/// into core. -fn validate_prebid_user_id_token(value: &str) -> Result<(), ValidationError> { - let is_valid = !value.is_empty() - && value.trim() == value - && value - .bytes() - .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-' | b'.')); - if is_valid { - return Ok(()); - } - - let mut error = ValidationError::new("invalid_prebid_user_id_token"); - error.message = Some( - "must be a non-empty ASCII token of letters, digits, `_`, `-`, or `.` without surrounding whitespace" - .into(), - ); - Err(error) -} - -/// Browser storage mechanism for an operator-managed Prebid User ID module. -#[derive(Debug, Clone, Copy, Default, Deserialize, Eq, PartialEq, Serialize)] -#[serde(rename_all = "lowercase")] -pub enum PrebidUserIdStorageType { - /// Store the module's value in a browser cookie. - #[default] - Cookie, - /// Store the module's value in browser local storage. - Html5, -} - -/// Browser storage settings forwarded verbatim to a Prebid User ID module. -#[derive(Debug, Clone, Deserialize, Serialize, Validate)] -#[serde(deny_unknown_fields)] -pub struct PrebidManagedUserIdStorage { - /// Browser storage mechanism. - #[serde(default, rename = "type")] - pub storage_type: PrebidUserIdStorageType, - /// Cookie or local-storage key the module reads and writes. - #[validate(custom(function = "validate_prebid_user_id_token"))] - pub name: String, - /// Number of days the browser retains the stored value. - /// - /// Omitted leaves Prebid's own default in place. Core applies no upper - /// bound: the ceiling is a property of the selected module, not of Trusted - /// Server. - #[serde(default, skip_serializing_if = "Option::is_none")] - #[validate(range(min = 1))] - pub expires: Option, - /// Number of seconds before the module may refresh the stored value. - /// - /// Omitted leaves Prebid's own default in place. - #[serde(default, skip_serializing_if = "Option::is_none")] - #[validate(range(min = 1))] - pub refresh_in_seconds: Option, -} - -/// Rejects a managed User ID list whose names address one module twice. -/// -/// Prebid matches `userSync.userIds` entry names to submodules -/// case-insensitively and takes the first matching entry, so two entries whose -/// names differ only by case give one submodule two conflicting configurations -/// with no defined winner. -fn validate_unique_managed_user_id_names( - entries: &[PrebidManagedUserIdConfig], -) -> Result<(), ValidationError> { - let mut seen = HashSet::with_capacity(entries.len()); - let Some(duplicate) = entries - .iter() - .find(|entry| !seen.insert(entry.name.to_ascii_lowercase())) - else { - return Ok(()); - }; - - let mut error = ValidationError::new("duplicate_managed_user_id_name"); - // Name the matching rule: for a collision that differs only by case, the - // printed name alone does not look repeated in the operator's config. - error.message = Some( - format!( - "managed Prebid User ID module `{}` is configured more than once (names are matched case-insensitively)", - duplicate.name - ) - .into(), - ); - Err(error) -} - -/// Operator-owned Prebid User ID module entry that Trusted Server manages. -/// -/// Core treats every entry as opaque: it validates only what Prebid.js needs to -/// address the module, then forwards the entry to the browser unchanged. Which -/// identity vendor an entry selects is an operator configuration choice, not a -/// property of core. -#[derive(Debug, Clone, Deserialize, Serialize, Validate)] -#[serde(deny_unknown_fields)] -pub struct PrebidManagedUserIdConfig { - /// Prebid `userSync.userIds` entry name, for example `sharedId`. - #[validate(custom(function = "validate_prebid_user_id_token"))] - pub name: String, - /// Module-specific parameters, forwarded to Prebid without inspection. - #[serde(default, skip_serializing_if = "serde_json::Map::is_empty")] - pub params: serde_json::Map, - /// Optional browser storage settings for the module. - #[serde(default, skip_serializing_if = "Option::is_none")] - #[validate(nested)] - pub storage: Option, -} - +/// The configuration of [`PrebidAuctionProvider`], the provider the tests +/// compare this implementation with. #[cfg(test)] #[derive(Debug, Clone, Deserialize, Serialize, Validate)] pub struct LegacyPrebidServerConfig { - #[serde(default = "default_enabled")] - pub enabled: bool, #[validate(url)] pub server_url: String, - /// Prebid Server account ID, injected into the client-side bundle via - /// `window.__tsjs_prebid.accountId` so publishers don't need to configure - /// it in JavaScript. - #[serde(default)] - pub account_id: Option, - /// Prebid User ID modules that Trusted Server installs and keeps installed. - /// - /// Each entry is forwarded to Prebid.js verbatim; publisher-configured - /// entries with other names are preserved. Names must be unique. - #[serde(default)] - #[validate(nested, custom(function = "validate_unique_managed_user_id_names"))] - pub managed_user_ids: Vec, #[serde(default = "default_timeout_ms")] #[validate(range(min = 1, max = 60000))] pub timeout_ms: u32, #[serde( default = "default_bidders", - deserialize_with = "crate::settings::vec_from_seq_or_map" + deserialize_with = "trusted_server_core::settings::vec_from_seq_or_map" )] pub bidders: Vec, #[serde(default)] @@ -370,29 +492,6 @@ pub struct LegacyPrebidServerConfig { pub test_mode: bool, #[serde(default)] pub debug_query_params: Option, - /// Patterns to match Prebid script URLs for serving empty JS. - /// Supports suffix matching (e.g., "/prebid.min.js" matches any path ending with that) - /// and wildcard patterns (e.g., "/static/prebid/*" matches paths under that prefix). - #[serde( - default = "default_script_patterns", - deserialize_with = "crate::settings::vec_from_seq_or_map" - )] - pub script_patterns: Vec, - /// Absolute HTTPS URL of the generated external Prebid bundle. - #[serde(default)] - #[validate(custom(function = "validate_external_bundle_url"))] - pub external_bundle_url: Option, - /// Optional hex SHA-256 of the exact external bundle bytes. - #[serde(default)] - #[validate(regex( - path = *EXTERNAL_BUNDLE_SHA256_PATTERN, - message = "external_bundle_sha256 must be a 64-character hex SHA-256" - ))] - pub external_bundle_sha256: Option, - /// Optional browser Subresource Integrity value for the first-party script. - #[serde(default)] - #[validate(custom(function = "validate_external_bundle_sri"))] - pub external_bundle_sri: Option, /// Bidders that should run client-side in the browser via native Prebid.js /// adapters instead of being routed through the server-side auction. /// @@ -403,15 +502,11 @@ pub struct LegacyPrebidServerConfig { /// /// This list is independent of [`bidders`](Self::bidders) — the operator /// manages both lists explicitly. - #[serde(default, deserialize_with = "crate::settings::vec_from_seq_or_map")] + #[serde( + default, + deserialize_with = "trusted_server_core::settings::vec_from_seq_or_map" + )] pub client_side_bidders: Vec, - /// GAM ad-unit-path suffixes excluded from Trusted Server refresh auctions. - /// - /// Matching is exact and case-sensitive. Excluded slots still refresh through - /// GAM, but are not included in synthetic Prebid refresh ad units. - #[serde(default, deserialize_with = "crate::settings::vec_from_seq_or_map")] - #[validate(custom(function = "validate_excluded_gam_ad_unit_path_suffixes"))] - pub excluded_gam_ad_unit_path_suffixes: Vec, /// Compatibility sugar for per-bidder, per-zone param overrides. /// /// This preserves the natural `bidder -> zone -> params` config shape for @@ -421,7 +516,7 @@ pub struct LegacyPrebidServerConfig { /// /// Example in TOML: /// ```toml - /// [integrations.prebid.bid_param_zone_overrides.kargo] + /// [auction.prebid.bid_param_zone_overrides.kargo] /// header = {placementId = "_s2sHeaderId"} /// in_content = {placementId = "_s2sContentId"} /// fixed_bottom = {placementId = "_s2sBottomId"} @@ -437,7 +532,7 @@ pub struct LegacyPrebidServerConfig { /// /// Example in TOML: /// ```toml - /// [integrations.prebid.bid_param_overrides.bidder-name] + /// [auction.prebid.bid_param_overrides.bidder-name] /// param1 = 12345 /// param2 = "value" /// ``` @@ -453,7 +548,7 @@ pub struct LegacyPrebidServerConfig { /// /// Example in TOML: /// ```toml - /// [[integrations.prebid.bid_param_override_rules]] + /// [[auction.prebid.bid_param_override_rules]] /// when.bidder = "kargo" /// when.zone = "header" /// set = { placementId = "_abc" } @@ -480,135 +575,15 @@ pub struct LegacyPrebidServerConfig { /// Use this when only specific PBS seats fire win/billing notifications /// internally. The global [`suppress_nurl`](Self::suppress_nurl) switch still /// suppresses every bidder when set. - #[serde(default, deserialize_with = "crate::settings::vec_from_seq_or_map")] - pub suppress_nurl_bidders: Vec, -} - -#[cfg(test)] -impl IntegrationConfig for LegacyPrebidServerConfig { - fn is_enabled(&self) -> bool { - self.enabled - } -} - -/// CLI build inputs retained in app config but ignored safely by the runtime. -#[derive(Debug, Clone, Default, Deserialize, Serialize)] -#[serde(deny_unknown_fields)] -pub struct PrebidBundleBuildConfig { - /// Typed Prebid.js module selections consumed by `ts prebid client`. - #[serde(default)] - pub modules: PrebidBundleModulesConfig, -} - -/// Exact Prebid.js module stems selected by `ts prebid client`. -/// -/// The CLI validates these values. The runtime only parses them so app config -/// carrying build inputs remains loadable. -#[derive(Debug, Clone, Default, Deserialize, Serialize)] -#[serde(deny_unknown_fields)] -pub struct PrebidBundleModulesConfig { - /// Bidder adapter module stems. - #[serde(default)] - pub bidder: Vec, - /// User ID module stems; omission selects the generator's curated preset. - #[serde(default)] - pub user_id: Option>, - /// Analytics adapter module stems; omission selects no analytics adapters. - #[serde(default)] - pub analytics: Option>, -} - -/// Browser-only Prebid integration settings. -#[derive(Debug, Clone, Deserialize, Serialize, Validate)] -#[serde(deny_unknown_fields)] -pub struct PrebidIntegrationConfig { - #[serde(default = "default_enabled")] - pub enabled: bool, - #[serde(default)] - pub account_id: Option, - #[serde(default = "default_timeout_ms")] - pub timeout_ms: u32, - #[serde(default)] - pub debug: bool, #[serde( - default = "default_script_patterns", - deserialize_with = "crate::settings::vec_from_seq_or_map" + default, + deserialize_with = "trusted_server_core::settings::vec_from_seq_or_map" )] - pub script_patterns: Vec, - #[serde(default)] - #[validate(custom(function = "validate_external_bundle_url"))] - pub external_bundle_url: Option, - #[serde(default)] - #[validate(regex( - path = *EXTERNAL_BUNDLE_SHA256_PATTERN, - message = "external_bundle_sha256 must be a 64-character hex SHA-256" - ))] - pub external_bundle_sha256: Option, - #[serde(default)] - #[validate(custom(function = "validate_external_bundle_sri"))] - pub external_bundle_sri: Option, - #[serde(default, deserialize_with = "crate::settings::vec_from_seq_or_map")] - pub client_side_bidders: Vec, - #[serde(default, deserialize_with = "crate::settings::vec_from_seq_or_map")] - #[validate(custom(function = "validate_excluded_gam_ad_unit_path_suffixes"))] - pub excluded_gam_ad_unit_path_suffixes: Vec, - /// Prebid User ID modules that Trusted Server installs and keeps installed. - /// - /// Each entry is forwarded to Prebid.js verbatim; publisher-configured - /// entries with other names are preserved. Names must be unique, and no two - /// names may resolve to the same Prebid User ID submodule. - #[serde(default)] - #[validate(nested, custom(function = "validate_unique_managed_user_id_names"))] - pub managed_user_ids: Vec, - /// CLI-only external bundle build inputs; runtime registration ignores these fields. - #[serde(default)] - pub bundle: PrebidBundleBuildConfig, -} - -impl Default for PrebidIntegrationConfig { - fn default() -> Self { - Self { - enabled: default_enabled(), - account_id: None, - timeout_ms: default_timeout_ms(), - debug: false, - script_patterns: default_script_patterns(), - external_bundle_url: None, - external_bundle_sha256: None, - external_bundle_sri: None, - client_side_bidders: Vec::new(), - excluded_gam_ad_unit_path_suffixes: Vec::new(), - managed_user_ids: Vec::new(), - bundle: PrebidBundleBuildConfig::default(), - } - } -} - -impl IntegrationConfig for PrebidIntegrationConfig { - fn is_enabled(&self) -> bool { - self.enabled - } + pub suppress_nurl_bidders: Vec, } #[cfg(test)] -impl From<&LegacyPrebidServerConfig> for PrebidIntegrationConfig { - fn from(config: &LegacyPrebidServerConfig) -> Self { - Self { - enabled: config.enabled, - account_id: config.account_id.clone(), - timeout_ms: config.timeout_ms, - debug: config.debug, - script_patterns: config.script_patterns.clone(), - external_bundle_url: config.external_bundle_url.clone(), - external_bundle_sha256: config.external_bundle_sha256.clone(), - external_bundle_sri: config.external_bundle_sri.clone(), - client_side_bidders: config.client_side_bidders.clone(), - excluded_gam_ad_unit_path_suffixes: config.excluded_gam_ad_unit_path_suffixes.clone(), - managed_user_ids: config.managed_user_ids.clone(), - bundle: PrebidBundleBuildConfig::default(), - } - } -} +impl IntegrationConfig for LegacyPrebidServerConfig {} #[cfg(test)] fn remove_aps_bidders(config: &mut LegacyPrebidServerConfig) { @@ -620,97 +595,23 @@ fn remove_aps_bidders(config: &mut LegacyPrebidServerConfig) { bidders.retain(|bidder| !bidder.eq_ignore_ascii_case("aps")); if bidders.len() != original_len { log::warn!( - "prebid: ignoring APS in integrations.prebid.{field}; configure APS under [integrations.aps]" + "prebid: ignoring APS in auction.prebid.{field}; configure APS as a [demand] source with implementation = \"auction.aps\"" ); } } } -fn excluded_gam_ad_unit_path_suffix_validation_error(message: &'static str) -> ValidationError { - let mut error = ValidationError::new("invalid_gam_ad_unit_path_suffix"); - error.message = Some(message.into()); - error -} - -fn validate_excluded_gam_ad_unit_path_suffix(value: &str) -> Result<(), ValidationError> { - if value.trim() != value { - return Err(excluded_gam_ad_unit_path_suffix_validation_error( - "excluded_gam_ad_unit_path_suffixes entries must not have surrounding whitespace", - )); - } - - if value.is_empty() { - return Err(excluded_gam_ad_unit_path_suffix_validation_error( - "excluded_gam_ad_unit_path_suffixes entries must not be empty", - )); - } - - if !value.starts_with('/') { - return Err(excluded_gam_ad_unit_path_suffix_validation_error( - "excluded_gam_ad_unit_path_suffixes entries must start with '/'", - )); - } - - if value == "/" { - return Err(excluded_gam_ad_unit_path_suffix_validation_error( - "excluded_gam_ad_unit_path_suffixes entries must identify a non-root path suffix", - )); - } - - Ok(()) -} - -fn validate_excluded_gam_ad_unit_path_suffixes(values: &[String]) -> Result<(), ValidationError> { - for value in values { - validate_excluded_gam_ad_unit_path_suffix(value)?; - } - - Ok(()) -} - -#[cfg(test)] -fn canonicalize_excluded_gam_ad_unit_path_suffixes(config: &mut LegacyPrebidServerConfig) { - let mut canonical = Vec::with_capacity(config.excluded_gam_ad_unit_path_suffixes.len()); - for suffix in std::mem::take(&mut config.excluded_gam_ad_unit_path_suffixes) { - if !canonical.contains(&suffix) { - canonical.push(suffix); - } - } - config.excluded_gam_ad_unit_path_suffixes = canonical; -} - #[cfg(test)] fn load_config( settings: &Settings, ) -> Result, Report> { - let Some(mut config) = - settings.integration_config::(PREBID_INTEGRATION_ID)? - else { + let Some(mut config) = settings.module_config::(LEGACY_TABLE)? else { return Ok(None); }; - canonicalize_excluded_gam_ad_unit_path_suffixes(&mut config); remove_aps_bidders(&mut config); Ok(Some(config)) } -/// Validate enabled Prebid config using the same startup-only checks as runtime registration. -/// -/// # Errors -/// -/// Returns a configuration error if enabled Prebid settings fail typed parsing, -/// schema validation, or bidder-param override compilation. -#[cfg(test)] -pub fn validate_config_for_startup( - settings: &Settings, -) -> Result, Report> { - let Some(config) = load_config(settings)? else { - return Ok(None); - }; - BidParamOverrideEngine::try_from_config(&config)?; - validate_external_bundle_config(&config, &settings.proxy.allowed_domains)?; - Ok(Some(config)) -} - /// Canonical bidder-param override rule. /// /// A rule matches against the request-time facts in [`BidParamOverrideWhen`] @@ -740,6 +641,7 @@ pub struct BidParamOverrideWhen { pub zone: Option, } +#[cfg(test)] fn default_timeout_ms() -> u32 { 1000 } @@ -749,5012 +651,2436 @@ fn default_bidders() -> Vec { vec!["mocktioneer".to_string()] } -fn default_enabled() -> bool { - true +/// Returns `true` when `params` is not a usable PBS bidder-params object — an +/// empty object `{}` or any non-object value such as `null`. +/// +/// PBS rejects an imp whose `bidder.` carries such a value, and it cannot +/// tell a fabricated empty from an explicitly supplied one — they are identical +/// bytes on the wire. The merge uses this to stop an unusable value from +/// clobbering real params, and the final pass uses it to drop whatever remains. +#[cfg(test)] +fn is_unusable_bidder_params(params: &Json) -> bool { + // `None` covers non-object values (e.g. `null`); an empty map covers `{}`. + params.as_object().is_none_or(serde_json::Map::is_empty) } -/// Default suffixes that identify Prebid scripts -const PREBID_SCRIPT_SUFFIXES: &[&str] = &[ - "/prebid.js", - "/prebid.min.js", - "/prebidjs.js", - "/prebidjs.min.js", -]; +#[cfg(test)] +fn expand_trusted_server_bidders( + configured_bidders: &[String], + params: &Json, +) -> HashMap { + let per_bidder = params.get(BIDDER_PARAMS_KEY).and_then(Json::as_object); + + if configured_bidders.is_empty() { + return per_bidder + .map(|m| m.iter().map(|(k, v)| (k.clone(), v.clone())).collect()) + .unwrap_or_default(); + } -fn default_script_patterns() -> Vec { - PREBID_SCRIPT_SUFFIXES + configured_bidders .iter() - .map(|&s| s.to_owned()) + .map(|bidder| { + let value = per_bidder + .and_then(|m| m.get(bidder).cloned()) + .unwrap_or_else(|| { + // No per-bidder map → use entire params as shared config + if per_bidder.is_some() { + Json::Object(Default::default()) + } else { + params.clone() + } + }); + (bidder.clone(), value) + }) .collect() } -fn validate_external_bundle_url(value: &str) -> Result<(), ValidationError> { - let url = Url::parse(value).map_err(|_| { - let mut err = ValidationError::new("invalid_external_bundle_url"); - err.message = Some("external_bundle_url must be a valid absolute URL".into()); - err - })?; - - if url.scheme() != "https" { - let mut err = ValidationError::new("invalid_external_bundle_scheme"); - err.message = Some("external_bundle_url must use https".into()); - return Err(err); - } - - if url.host_str().is_none() { - let mut err = ValidationError::new("missing_external_bundle_host"); - err.message = Some("external_bundle_url must include a host".into()); - return Err(err); - } - - Ok(()) -} - -/// Exact hex SHA-256: 64 hex digits. Used by the built-in `regex` validator on -/// [`PrebidIntegrationConfig::external_bundle_sha256`]. -static EXTERNAL_BUNDLE_SHA256_PATTERN: LazyLock = - LazyLock::new(|| Regex::new(r"^[0-9a-fA-F]{64}$").expect("SHA-256 hex regex should compile")); - -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -enum ExternalBundleSriAlgorithm { - Sha256, - Sha384, - Sha512, -} - -impl ExternalBundleSriAlgorithm { - fn parse(value: &str) -> Option { - match value { - "sha256" => Some(Self::Sha256), - "sha384" => Some(Self::Sha384), - "sha512" => Some(Self::Sha512), - _ => None, +/// Shallow-merges `override_obj` into `params`. +/// +/// When `params` is a JSON object, each top-level key in `override_obj` is +/// inserted or replaced in `params`. Nested objects are replaced wholesale — +/// they are not recursed into. When `params` is not an object, it is left +/// untouched to preserve pre-existing behavior. +/// +/// Returns whether `params` was an object and the merge path ran. +fn merge_bidder_param_object( + params: &mut Json, + override_obj: &serde_json::Map, +) -> bool { + if let Json::Object(base) = params { + for (k, v) in override_obj { + base.insert(k.clone(), v.clone()); } - } - fn expected_digest_len(self) -> usize { - match self { - Self::Sha256 => 32, - Self::Sha384 => 48, - Self::Sha512 => 64, - } + true + } else { + false } } -fn external_bundle_sri_validation_error(message: &'static str) -> ValidationError { - let mut err = ValidationError::new("invalid_external_bundle_sri"); - err.message = Some(message.into()); - err -} - -fn parse_external_bundle_sri(value: &str) -> Result<(), ValidationError> { - let trimmed = value.trim(); - if trimmed.is_empty() || trimmed != value { - return Err(external_bundle_sri_validation_error( - "external_bundle_sri must be non-empty with no surrounding whitespace", - )); - } - - for token in trimmed.split_ascii_whitespace() { - let Some((algorithm_raw, digest_raw)) = token.split_once('-') else { - return Err(external_bundle_sri_validation_error( - "external_bundle_sri entries must use algorithm-digest format", - )); - }; - - let Some(algorithm) = ExternalBundleSriAlgorithm::parse(algorithm_raw) else { - return Err(external_bundle_sri_validation_error( - "external_bundle_sri must use sha256, sha384, or sha512", - )); - }; - - if digest_raw.is_empty() { - return Err(external_bundle_sri_validation_error( - "external_bundle_sri digest must be non-empty", - )); - } - - let digest = BASE64_STANDARD - .decode(digest_raw) - .or_else(|_| BASE64_STANDARD_NO_PAD.decode(digest_raw)) - .map_err(|_| { - external_bundle_sri_validation_error("external_bundle_sri digest must be base64") - })?; +// ============================================================================ +// Generic bid-parameter override engine +// ============================================================================ - if digest.len() != algorithm.expected_digest_len() { - return Err(external_bundle_sri_validation_error( - "external_bundle_sri digest length does not match its algorithm", - )); +#[cfg(test)] +fn warn_unconfigured_bidder(config: &LegacyPrebidServerConfig, bidder: &str, field: &str) { + if !config.bidders.iter().any(|b| b == bidder) { + if config.client_side_bidders.iter().any(|b| b == bidder) { + log::warn!( + "prebid: {field} entry targets client-side-only bidder \ + '{bidder}' — server-side override will never apply" + ); + } else { + log::warn!( + "prebid: {field} entry references unconfigured bidder \ + '{bidder}' — rule will never fire" + ); } } - - Ok(()) -} - -fn validate_external_bundle_sri(value: &str) -> Result<(), ValidationError> { - parse_external_bundle_sri(value) -} - -fn validate_external_bundle_url_allowed( - external_bundle_url: Option<&str>, - allowed_domains: &[String], -) -> Result<(), Report> { - let url = external_bundle_url.ok_or_else(|| { - Report::new(TrustedServerError::Configuration { - message: "integrations.prebid.external_bundle_url is required when prebid is enabled" - .to_string(), - }) - })?; - - let parsed = Url::parse(url).map_err(|_| { - Report::new(TrustedServerError::Configuration { - message: "integrations.prebid.external_bundle_url must be a valid absolute URL" - .to_string(), - }) - })?; - - if parsed.scheme() != "https" { - return Err(Report::new(TrustedServerError::Configuration { - message: "integrations.prebid.external_bundle_url must use https".to_string(), - })); - } - - let host = parsed.host_str().ok_or_else(|| { - Report::new(TrustedServerError::Configuration { - message: "integrations.prebid.external_bundle_url must include a host".to_string(), - }) - })?; - - if allowed_domains.is_empty() { - return Err(Report::new(TrustedServerError::Configuration { - message: - "proxy.allowed_domains must include the external Prebid bundle host when integrations.prebid.external_bundle_url is configured" - .to_string(), - })); - } - - if !allowed_domains - .iter() - .any(|pattern| is_host_allowed(host, pattern)) - { - return Err(Report::new(TrustedServerError::Configuration { - message: format!( - "integrations.prebid.external_bundle_url host `{host}` is not permitted by proxy.allowed_domains" - ), - })); - } - - Ok(()) -} - -pub(crate) fn validate_browser_config_for_startup( - config: &PrebidIntegrationConfig, - allowed_domains: &[String], -) -> Result<(), Report> { - validate_external_bundle_url_allowed(config.external_bundle_url.as_deref(), allowed_domains) } -pub(crate) fn validate_browser_bidder_ownership( - config: &PrebidIntegrationConfig, - plan: &AuctionPlan, -) -> Result<(), Report> { - if !plan.enabled() { - return Ok(()); - } - - let server_side = plan - .browser_bidder_codes() - .collect::>(); - let conflicts = config - .client_side_bidders - .iter() - .filter(|bidder| server_side.contains(bidder.as_str())) - .cloned() - .collect::>(); - if conflicts.is_empty() { - return Ok(()); - } - - Err(Report::new(TrustedServerError::Configuration { - message: format!( - "Prebid bidders must have exactly one browser owner; configured as both client-side and server-side: {}", - conflicts.into_iter().collect::>().join(", ") - ), - })) +#[derive(Debug, Default, Clone)] +pub(crate) struct BidParamOverrideEngine { + rules: Vec, + // Maps bidder name to the indices (into `rules`) of rules that constrain on that bidder. + // Rules with no bidder constraint (zone-only or catch-all) are kept in `wildcard_indices`. + // Both slices are in declaration order; `apply` merges them without allocation. + bidder_index: HashMap>, + wildcard_indices: Vec, } -#[cfg(test)] -fn validate_external_bundle_config( - config: &LegacyPrebidServerConfig, - allowed_domains: &[String], -) -> Result<(), Report> { - validate_external_bundle_url_allowed(config.external_bundle_url.as_deref(), allowed_domains) +#[derive(Debug, Clone, PartialEq, Eq)] +struct CompiledBidParamOverrideRule { + bidder: Option, + zone: Option, + set: serde_json::Map, } -pub struct PrebidIntegration { - config: PrebidIntegrationConfig, - planned_head_inserts: Option>, - #[cfg(test)] - legacy_config: Option, - #[cfg(test)] - engine: Arc, +#[derive(Debug, Copy, Clone)] +struct BidParamOverrideFacts<'a> { + bidder: &'a str, + zone: Option<&'a str>, } -impl PrebidIntegration { - #[cfg(test)] - fn try_new(config: LegacyPrebidServerConfig) -> Result, Report> { - let engine = Arc::new(BidParamOverrideEngine::try_from_config(&config)?); - Ok(Arc::new(Self { - config: PrebidIntegrationConfig::from(&config), - planned_head_inserts: None, - legacy_config: Some(config), - engine, - })) - } - +impl BidParamOverrideEngine { #[cfg(test)] - fn new(config: LegacyPrebidServerConfig) -> Arc { - Self::try_new(config).expect("should compile prebid bid param overrides") - } + fn try_from_config( + config: &LegacyPrebidServerConfig, + ) -> Result> { + let mut rules = Vec::new(); - fn for_browser_plan(config: &PrebidIntegrationConfig, plan: &AuctionPlan) -> Arc { - let mut integration = Self { - config: config.clone(), - planned_head_inserts: None, - #[cfg(test)] - legacy_config: None, - #[cfg(test)] - engine: Arc::new(BidParamOverrideEngine::default()), - }; - integration.planned_head_inserts = Some(integration.head_inserts_for_plan(config, plan)); - Arc::new(integration) - } + let mut bidder_overrides = config.bid_param_overrides.iter().collect::>(); + bidder_overrides.sort_by(|(left, _), (right, _)| left.as_str().cmp(right.as_str())); - #[cfg(test)] - fn auction_provider(&self) -> PrebidAuctionProvider { - PrebidAuctionProvider { - config: self - .legacy_config - .clone() - .expect("should retain legacy config for legacy provider tests"), - bid_param_override_engine: Arc::clone(&self.engine), + for (bidder, set) in bidder_overrides { + warn_unconfigured_bidder(config, bidder, "bid_param_overrides"); + rules.push(CompiledBidParamOverrideRule::from_bidder_override( + bidder.as_str(), + set, + )?); } - } - fn matches_script_url(&self, attr_value: &str) -> bool { - let trimmed = attr_value.trim(); - let without_query = trimmed.split(['?', '#']).next().unwrap_or(trimmed); + let mut zone_overrides = config.bid_param_zone_overrides.iter().collect::>(); + zone_overrides.sort_by(|(left, _), (right, _)| left.as_str().cmp(right.as_str())); + + for (bidder, zone_override_sets) in zone_overrides { + warn_unconfigured_bidder(config, bidder, "bid_param_zone_overrides"); + let mut sorted_zone_overrides = zone_override_sets.iter().collect::>(); + sorted_zone_overrides + .sort_by(|(left, _), (right, _)| left.as_str().cmp(right.as_str())); - if self.matches_script_pattern(without_query) { - return true; + for (zone, set) in sorted_zone_overrides { + rules.push(CompiledBidParamOverrideRule::from_zone_override( + bidder.as_str(), + zone.as_str(), + set, + )?); + } } - if !without_query.starts_with('/') - && !without_query.starts_with("//") - && !without_query.contains("://") - { - let with_slash = format!("/{without_query}"); - if self.matches_script_pattern(&with_slash) { - return true; + for rule in &config.bid_param_override_rules { + if let Some(bidder) = &rule.when.bidder { + warn_unconfigured_bidder(config, bidder, "bid_param_override_rules"); } + rules.push(CompiledBidParamOverrideRule::try_from(rule)?); } - let parsed = if without_query.starts_with("//") { - ParsedUrl::parse(&format!("https:{without_query}")) - } else { - ParsedUrl::parse(without_query) - }; + let mut bidder_index: HashMap> = HashMap::new(); + let mut wildcard_indices: Vec = Vec::new(); + for (idx, rule) in rules.iter().enumerate() { + match &rule.bidder { + Some(bidder) => bidder_index.entry(bidder.clone()).or_default().push(idx), + None => wildcard_indices.push(idx), + } + } - parsed - .ok() - .is_some_and(|url| self.matches_script_pattern(url.path())) + Ok(Self { + rules, + bidder_index, + wildcard_indices, + }) } - fn matches_script_pattern(&self, path: &str) -> bool { - // Normalize path to lowercase for case-insensitive matching - let path_lower = path.to_ascii_lowercase(); - - // Check if path matches any configured pattern - for pattern in &self.config.script_patterns { - let pattern_lower = pattern.to_ascii_lowercase(); + fn try_from_profile_config( + bid_param_zone_overrides: &std::collections::BTreeMap< + String, + std::collections::BTreeMap>, + >, + bid_param_overrides: &std::collections::BTreeMap>, + bid_param_override_rules: &[BidParamOverrideRule], + ) -> Result> { + let mut rules = Vec::new(); + for (bidder, set) in bid_param_overrides { + rules.push(CompiledBidParamOverrideRule::from_bidder_override( + bidder, set, + )?); + } + for (bidder, zone_override_sets) in bid_param_zone_overrides { + for (zone, set) in zone_override_sets { + rules.push(CompiledBidParamOverrideRule::from_zone_override( + bidder, zone, set, + )?); + } + } + for rule in bid_param_override_rules { + rules.push(CompiledBidParamOverrideRule::try_from(rule)?); + } + let mut bidder_index: HashMap> = HashMap::new(); + let mut wildcard_indices = Vec::new(); + for (index, rule) in rules.iter().enumerate() { + match &rule.bidder { + Some(bidder) => bidder_index.entry(bidder.clone()).or_default().push(index), + None => wildcard_indices.push(index), + } + } + Ok(Self { + rules, + bidder_index, + wildcard_indices, + }) + } - // Check for wildcard patterns: /* or {*name} - if pattern_lower.ends_with("/*") || pattern_lower.contains("{*") { - // Extract prefix before the wildcard - let prefix = if pattern_lower.ends_with("/*") { - &pattern_lower[..pattern_lower.len() - 1] // Remove trailing * + fn apply(&self, facts: BidParamOverrideFacts<'_>, params: &mut Json) { + let bidder_indices = self.bidder_index.get(facts.bidder).map(Vec::as_slice); + for idx in merged_rule_indices(&self.wildcard_indices, bidder_indices) { + let rule = &self.rules[idx]; + if rule.matches(facts) { + if merge_bidder_param_object(params, &rule.set) { + log::debug!( + "prebid: applying bidder param override for bidder '{}' zone {:?}: keys {:?}", + facts.bidder, + facts.zone, + rule.set.keys().collect::>() + ); } else { - // Find {* and extract prefix before it - pattern_lower.split("{*").next().unwrap_or("") - }; - - if path_lower.starts_with(prefix) { - // Check if it ends with a known Prebid script suffix - if PREBID_SCRIPT_SUFFIXES - .iter() - .any(|suffix| path_lower.ends_with(suffix)) - { - return true; - } - } - } else { - // Exact match or suffix match - if path_lower.ends_with(&pattern_lower) { - return true; + log::debug!( + "prebid: skipping bidder param override for bidder '{}' zone {:?}: params is not a JSON object", + facts.bidder, + facts.zone + ); } } } - false - } - - fn handle_script_handler( - &self, - ) -> Result, Report> { - let body = "// Script overridden by Trusted Server\n"; - - let mut response = http::Response::builder() - .status(StatusCode::OK) - .header(header::CONTENT_TYPE, PREBID_BUNDLE_CONTENT_TYPE) - .body(EdgeBody::from(body)) - .change_context(TrustedServerError::Prebid { - message: "Failed to build Prebid script handler response".to_string(), - })?; - CacheControlPolicy::NoStorePrivate - .apply_to_headers(response.headers_mut(), EdgeCacheHeader::None); - Ok(response) - } - - fn external_bundle_script_tag(&self) -> String { - external_bundle_script_tag( - self.config.external_bundle_sha256.as_deref(), - self.config.external_bundle_sri.as_deref(), - ) - } - - /// Build the prepared browser injection from browser settings and validated routes. - pub(crate) fn head_inserts_for_plan( - &self, - browser_config: &PrebidIntegrationConfig, - plan: &AuctionPlan, - ) -> Vec { - let payload = InjectedPrebidClientConfig { - server_side_bidders: Some(if plan.enabled() { - plan.browser_bidder_codes().collect() - } else { - Vec::new() - }), - ..InjectedPrebidClientConfig::from(browser_config) - }; - let config_json = serialize_injected_prebid_config(&payload); - - vec![ - injected_prebid_config_script(&config_json), - external_bundle_script_tag( - browser_config.external_bundle_sha256.as_deref(), - browser_config.external_bundle_sri.as_deref(), - ), - ] } - fn is_managed_external(&self) -> bool { - self.config.external_bundle_url.is_some() + /// Apply compiled profile rules to already centrally routed bidder params. + pub(crate) fn apply_routed(&self, bidder: &str, zone: Option<&str>, params: &mut Json) { + self.apply(BidParamOverrideFacts { bidder, zone }, params); } +} - fn external_bundle_request_cache_mode( - &self, - req: &http::Request, - ) -> Result, Report> { - let versions = req - .uri() - .query() - .map(|query| { - url::form_urlencoded::parse(query.as_bytes()) - .filter(|(key, _)| key == "v") - .map(|(_, value)| value.into_owned()) - .collect::>() - }) - .unwrap_or_default(); +/// Validate and compile server-side Prebid profile override fields. +/// +/// This narrow hook shares the existing override compiler without coupling +/// auction-profile availability to whether the browser integration runs. +pub(crate) fn compile_profile_override_rules( + bid_param_zone_overrides: &std::collections::BTreeMap< + String, + std::collections::BTreeMap>, + >, + bid_param_overrides: &std::collections::BTreeMap>, + bid_param_override_rules: &[BidParamOverrideRule], +) -> Result> { + BidParamOverrideEngine::try_from_profile_config( + bid_param_zone_overrides, + bid_param_overrides, + bid_param_override_rules, + ) +} - if versions.len() > 1 { - return Ok(None); - } +fn merged_rule_indices<'a>( + wildcard_indices: &'a [usize], + bidder_indices: Option<&'a [usize]>, +) -> impl Iterator + 'a { + let mut wildcard = wildcard_indices.iter().copied().peekable(); + let mut bidder = bidder_indices.unwrap_or(&[]).iter().copied().peekable(); - let requested_version = versions.first().map(String::as_str); - match ( - self.config.external_bundle_sha256.as_deref(), - requested_version, - ) { - (None, Some(_)) => Ok(None), - (Some(expected), Some(actual)) if expected != actual => Ok(None), - (Some(_), Some(_)) => Ok(Some(ExternalBundleCacheMode::Immutable)), - _ => Ok(Some(ExternalBundleCacheMode::Revalidate)), - } - } + std::iter::from_fn(move || match (wildcard.peek(), bidder.peek()) { + (Some(wildcard_idx), Some(bidder_idx)) if wildcard_idx <= bidder_idx => wildcard.next(), + (Some(_), Some(_)) => bidder.next(), + (Some(_), None) => wildcard.next(), + (None, Some(_)) => bidder.next(), + (None, None) => None, + }) +} - fn apply_external_bundle_headers( - &self, - response: &mut http::Response, - mode: ExternalBundleCacheMode, - ) { - response.headers_mut().insert( - header::CONTENT_TYPE, - HeaderValue::from_static(PREBID_BUNDLE_CONTENT_TYPE), - ); - response.headers_mut().insert( - header::HeaderName::from_static(PREBID_BUNDLE_NOSNIFF_HEADER), - HeaderValue::from_static(PREBID_BUNDLE_NOSNIFF_VALUE), - ); +impl CompiledBidParamOverrideRule { + fn from_bidder_override( + bidder: &str, + set: &serde_json::Map, + ) -> Result> { + Self::new( + Some(bidder), + None, + set, + &format!("auction.prebid.bid_param_overrides.{bidder}"), + false, + ) + } - match mode { - ExternalBundleCacheMode::Immutable => { - response.headers_mut().insert( - header::CACHE_CONTROL, - HeaderValue::from_static(PREBID_BUNDLE_IMMUTABLE_CACHE_CONTROL), - ); - if let Some(sha256) = self.config.external_bundle_sha256.as_deref() { - response.headers_mut().insert( - header::ETAG, - HeaderValue::from_str(&format!("\"sha256:{sha256}\"")) - .expect("should build etag header"), - ); - } - } - ExternalBundleCacheMode::Revalidate => { - response.headers_mut().insert( - header::CACHE_CONTROL, - HeaderValue::from_static(PREBID_BUNDLE_REVALIDATION_CACHE_CONTROL), - ); - if let Some(sha256) = self.config.external_bundle_sha256.as_deref() { - response.headers_mut().insert( - header::ETAG, - HeaderValue::from_str(&format!("\"sha256:{sha256}\"")) - .expect("should build etag header"), - ); - } - } - } + fn from_zone_override( + bidder: &str, + zone: &str, + set: &serde_json::Map, + ) -> Result> { + Self::new( + Some(bidder), + Some(zone), + set, + &format!("auction.prebid.bid_param_zone_overrides.{bidder}.{zone}"), + false, + ) } - fn sanitize_external_bundle_response( - &self, - response: http::Response, - mode: ExternalBundleCacheMode, - ) -> http::Response { - let status = response.status(); - let content_encoding = response.headers().get(header::CONTENT_ENCODING).cloned(); - let body = response.into_body(); - - let mut sanitized = http::Response::builder() - .status(status) - .body(body) - .expect("should build sanitized response"); - - if let Some(content_encoding) = content_encoding { - sanitized - .headers_mut() - .insert(header::CONTENT_ENCODING, content_encoding); - } + fn new( + bidder: Option<&str>, + zone: Option<&str>, + set: &serde_json::Map, + source: &str, + is_canonical: bool, + ) -> Result> { + let bidder = bidder + .map(|value| validate_override_matcher_string(value, "when.bidder", source)) + .transpose()?; + let zone = zone + .map(|value| validate_override_matcher_string(value, "when.zone", source)) + .transpose()?; - if status == StatusCode::OK { - self.apply_external_bundle_headers(&mut sanitized, mode); - } else { - sanitized.headers_mut().insert( - header::CONTENT_TYPE, - HeaderValue::from_static(PREBID_BUNDLE_ERROR_CONTENT_TYPE), - ); - sanitized.headers_mut().insert( - header::CACHE_CONTROL, - HeaderValue::from_static(PREBID_BUNDLE_ERROR_CACHE_CONTROL), - ); - sanitized.headers_mut().insert( - header::HeaderName::from_static(PREBID_BUNDLE_NOSNIFF_HEADER), - HeaderValue::from_static(PREBID_BUNDLE_NOSNIFF_VALUE), - ); + if bidder.is_none() && zone.is_none() { + return Err(Report::new(TrustedServerError::Configuration { + message: format!("{source} must include at least one matcher"), + })); } - sanitized + Ok(Self { + bidder, + zone, + set: non_empty_override_object(set, source, is_canonical)?, + }) } - async fn handle_external_bundle( - &self, - settings: &Settings, - services: &RuntimeServices, - req: http::Request, - ) -> Result, Report> { - let Some(cache_mode) = self.external_bundle_request_cache_mode(&req)? else { - return Ok(http::Response::builder() - .status(StatusCode::NOT_FOUND) - .body(EdgeBody::from("Not Found")) - .expect("should build not found response")); - }; - - let target_url = self.config.external_bundle_url.as_deref().ok_or_else(|| { - Report::new(TrustedServerError::Configuration { - message: - "integrations.prebid.external_bundle_url is required when prebid is enabled" - .to_string(), - }) - })?; + fn matches(&self, facts: BidParamOverrideFacts<'_>) -> bool { + if let Some(expected_bidder) = self.bidder.as_deref() + && expected_bidder != facts.bidder + { + return false; + } - let proxy_config = ProxyRequestConfig::new(target_url) - .without_ec_id() - .without_forward_headers() - .with_streaming() - .with_allowed_domains(&settings.proxy.allowed_domains) - .with_https_only(); + if let Some(expected_zone) = self.zone.as_deref() + && facts.zone != Some(expected_zone) + { + return false; + } - let response = proxy_request(settings, req, proxy_config, services).await?; - Ok(self.sanitize_external_bundle_response(response, cache_mode)) + true } } -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -enum ExternalBundleCacheMode { - Immutable, - Revalidate, -} - -fn escape_html_attr(value: &str) -> String { - value - .replace('&', "&") - .replace('"', """) - .replace('<', "<") - .replace('>', ">") -} +impl TryFrom for CompiledBidParamOverrideRule { + type Error = Report; -fn external_bundle_script_src(sha256: Option<&str>) -> String { - match sha256 { - Some(sha256) => format!("{PREBID_BUNDLE_ROUTE}?v={sha256}"), - None => PREBID_BUNDLE_ROUTE.to_string(), + fn try_from(rule: BidParamOverrideRule) -> Result { + Self::try_from(&rule) } } -fn external_bundle_script_tag(sha256: Option<&str>, sri: Option<&str>) -> String { - let src = external_bundle_script_src(sha256); - let integrity = sri - .map(|value| format!(" integrity=\"{}\"", escape_html_attr(value))) - .unwrap_or_default(); +impl TryFrom<&BidParamOverrideRule> for CompiledBidParamOverrideRule { + type Error = Report; - format!("") + fn try_from(rule: &BidParamOverrideRule) -> Result { + Self::new( + rule.when.bidder.as_deref(), + rule.when.zone.as_deref(), + &rule.set, + "auction.prebid.bid_param_override_rules[*]", + true, + ) + } } -#[cfg(test)] -fn build( - settings: &Settings, -) -> Result>, Report> { - let Some(config) = load_config(settings)? else { - return Ok(None); - }; - - validate_external_bundle_config(&config, &settings.proxy.allowed_domains)?; +// Trims leading/trailing whitespace from config-side matcher strings so that +// accidental padding in TOML/JSON does not produce a silently non-matching rule. +// Runtime facts (bidder name, zone) are not trimmed — they come from controlled +// internal sources (bidder-map keys, trustedServer.zone) where whitespace is +// never expected. Matching is exact and case-sensitive after this normalisation. +fn validate_override_matcher_string( + value: &str, + field: &str, + source: &str, +) -> Result> { + let trimmed = value.trim(); - // Warn about bidders that appear in both lists — this is likely a config - // mistake. A bidder should be in either `bidders` (server-side) or - // `client_side_bidders` (browser-side), not both. - for bidder in &config.client_side_bidders { - if config.bidders.iter().any(|b| b == bidder) { - log::warn!( - "prebid: bidder \"{}\" is in both bidders and client_side_bidders — \ - it will run server-side AND be left for client-side, which is likely unintended", - bidder - ); - } + if trimmed.is_empty() { + return Err(Report::new(TrustedServerError::Configuration { + message: format!("{source}.{field} must not be empty"), + })); } - Ok(Some(PrebidIntegration::try_new(config)?)) + Ok(trimmed.to_string()) } -/// Register the Prebid integration when enabled. -/// -/// # Errors -/// -/// Returns an error when the Prebid integration is enabled with invalid -/// configuration. -pub fn register_for_plan( - settings: &Settings, - plan: &AuctionPlan, -) -> Result, Report> { - let Some(mut config) = - settings.integration_config::(PREBID_INTEGRATION_ID)? - else { - return Ok(None); - }; - let mut canonical = Vec::with_capacity(config.excluded_gam_ad_unit_path_suffixes.len()); - for suffix in std::mem::take(&mut config.excluded_gam_ad_unit_path_suffixes) { - if !canonical.contains(&suffix) { - canonical.push(suffix); - } +fn non_empty_override_object( + value: &serde_json::Map, + source: &str, + is_canonical: bool, +) -> Result, Report> { + if value.is_empty() { + let message = if is_canonical { + format!("{source}.set must not be empty") + } else { + format!("{source} entry must not be empty") + }; + return Err(Report::new(TrustedServerError::Configuration { message })); } - config.excluded_gam_ad_unit_path_suffixes = canonical; - validate_browser_config_for_startup(&config, &settings.proxy.allowed_domains)?; - validate_browser_bidder_ownership(&config, plan)?; - let integration = PrebidIntegration::for_browser_plan(&config, plan); - Ok(Some( - IntegrationRegistration::builder(PREBID_INTEGRATION_ID) - .with_proxy(integration.clone()) - .with_attribute_rewriter(integration.clone()) - .with_head_injector(integration) - .with_deferred_js() - .build(), - )) + + Ok(value.clone()) } +/// Copies browser headers to the outgoing Prebid Server request. +/// +/// The inbound `X-Forwarded-For` header is never copied: on the server-side +/// publisher auction path `from` is the browser navigation request, so its +/// XFF value is client-supplied and spoofable. Instead the header is +/// synthesized from `client_ip` — the platform-attested client address — +/// matching the trusted IP already sent in `OpenRTB` `device.ip`. +/// +/// In [`ConsentForwardingMode::OpenrtbOnly`] mode, consent cookies are +/// stripped from the `Cookie` header since consent travels exclusively +/// through the `OpenRTB` body. #[cfg(test)] -#[allow(clippy::missing_errors_doc)] -pub fn register( - settings: &Settings, -) -> Result, Report> { - let Some(integration) = build(settings)? else { - return Ok(None); - }; - - Ok(Some( - IntegrationRegistration::builder(PREBID_INTEGRATION_ID) - .with_proxy(integration.clone()) - .with_attribute_rewriter(integration.clone()) - .with_head_injector(integration) - .with_deferred_js() - .build(), - )) +fn copy_request_headers( + from: &http::Request, + to: &mut http::Request, + consent_forwarding: ConsentForwardingMode, + client_ip: Option, +) { + apply_prebid_header_values( + from.headers().get(header::COOKIE), + from.headers().get(header::USER_AGENT), + from.headers().get(header::REFERER), + from.headers().get(header::ACCEPT_LANGUAGE), + to.headers_mut(), + consent_forwarding, + client_ip, + ); } -#[async_trait(?Send)] -impl IntegrationProxy for PrebidIntegration { - fn integration_name(&self) -> &'static str { - PREBID_INTEGRATION_ID - } - - fn routes(&self) -> Vec { - let mut routes = vec![]; - - routes.push(self.get("/bundle.js")); +/// Apply the common raw-header transport policy to a planned PBS request's +/// outbound headers. +pub(crate) fn apply_prebid_transport_headers( + from: &TransportHeaders, + to: &mut http::HeaderMap, + consent_forwarding: ConsentForwardingMode, + client_ip: Option, +) { + apply_prebid_header_values( + from.cookie(), + from.user_agent(), + from.referer(), + from.accept_language(), + to, + consent_forwarding, + client_ip, + ); +} - // Register routes for script removal patterns - // Patterns can be exact paths (e.g., "/prebid.min.js") or use matchit wildcards - // (e.g., "/static/prebid/{*rest}") - for pattern in &self.config.script_patterns { - // Intentional leak: runs once at startup and patterns are small. - // `IntegrationEndpoint` requires `&'static str`. - let static_path: &'static str = Box::leak(pattern.clone().into_boxed_str()); - routes.push(IntegrationEndpoint::get(static_path)); +#[allow( + clippy::too_many_arguments, + reason = "the helper preserves four independently optional raw headers plus transport policy" +)] +fn apply_prebid_header_values( + cookie: Option<&HeaderValue>, + user_agent: Option<&HeaderValue>, + referer: Option<&HeaderValue>, + accept_language: Option<&HeaderValue>, + to: &mut http::HeaderMap, + consent_forwarding: ConsentForwardingMode, + client_ip: Option, +) { + for (name, value) in [ + (header::USER_AGENT, user_agent), + (header::REFERER, referer), + (header::ACCEPT_LANGUAGE, accept_language), + ] { + if let Some(value) = value { + to.insert(name, value.clone()); } + } - routes + if let Some(ip) = client_ip + && let Ok(value) = HeaderValue::from_str(&ip.to_string()) + { + to.insert(header::HeaderName::from_static("x-forwarded-for"), value); } - async fn handle( - &self, - settings: &Settings, - services: &RuntimeServices, - req: http::Request, - ) -> Result, Report> { - let path = req.uri().path().to_string(); - let method = req.method().clone(); - - match method { - Method::GET if self.is_managed_external() && path == PREBID_BUNDLE_ROUTE => { - self.handle_external_bundle(settings, services, req).await + let Some(cookie_value) = cookie else { + return; + }; + if !consent_forwarding.strips_consent_cookies() { + to.insert(header::COOKIE, cookie_value.clone()); + return; + } + match cookie_value.to_str() { + Ok(value) => { + let stripped = strip_cookies(value, CONSENT_COOKIE_NAMES); + if !stripped.is_empty() + && let Ok(cookie_header) = HeaderValue::from_str(&stripped) + { + to.insert(header::COOKIE, cookie_header); } - // Serve empty JS for matching script patterns - Method::GET if self.matches_script_pattern(&path) => self.handle_script_handler(), - _ => http::Response::builder() - .status(StatusCode::NOT_FOUND) - .body(EdgeBody::from("Not Found")) - .change_context(TrustedServerError::Prebid { - message: "Failed to build Prebid not found response".to_string(), - }), + } + Err(_) => { + to.insert(header::COOKIE, cookie_value.clone()); } } } -impl IntegrationAttributeRewriter for PrebidIntegration { - fn integration_id(&self) -> &'static str { - PREBID_INTEGRATION_ID - } - - fn handles_attribute(&self, attribute: &str) -> bool { - matches!(attribute, "src" | "href") +/// Appends query parameters to a URL, handling both URLs with and without existing query strings. +/// Returns the original URL unchanged if params are empty or already present. +#[cfg(test)] +fn append_query_params(url: &str, params: &str) -> String { + if params.is_empty() || url.contains(params) { + return url.to_string(); } - - fn rewrite( - &self, - _attr_name: &str, - attr_value: &str, - _ctx: &IntegrationAttributeContext<'_>, - ) -> AttributeRewriteAction { - if self.matches_script_url(attr_value) { - AttributeRewriteAction::remove_element() - } else { - AttributeRewriteAction::keep() - } + if url.contains('?') { + format!("{}&{}", url, params) + } else { + format!("{}?{}", url, params) } } -fn serialize_injected_prebid_config(payload: &impl Serialize) -> String { - // JSON appears in script raw-text, where every less-than sign must be escaped. - serde_json::to_string(payload) - .unwrap_or_else(|error| { - log::warn!("Prebid: failed to serialize client config: {error}"); - "{}".to_string() - }) - .replace('<', "\\u003c") -} - -fn injected_prebid_config_script(config_json: &str) -> String { - format!( - r#""# +/// Parse a planned PBS response with the configured profile behavior. +/// +/// This preserves the legacy PBS status, body, bid, cache, and metadata +/// semantics while allowing each planned provider to retain its own identity. +pub(crate) async fn parse_planned_prebid_response( + provider_id: &str, + demand: &PrebidServerDemand, + input: &ProviderAuctionInput, + response: PlatformResponse, + response_time_ms: u64, + auction_id: &str, +) -> Result> { + let response = response.response; + let status = response.status(); + let content_type = response + .headers() + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .map(str::to_owned); + let body_bytes = collect_response_bounded( + response.into_body(), + UPSTREAM_RTB_MAX_RESPONSE_BYTES, + "prebid", ) -} - -impl IntegrationHeadInjector for PrebidIntegration { - fn integration_id(&self) -> &'static str { - PREBID_INTEGRATION_ID - } + .await + .change_context(TrustedServerError::Integration { + integration: MODULE.to_string(), + message: "Failed to read Prebid response body".to_string(), + }); - fn head_inserts(&self, _ctx: &IntegrationHtmlContext<'_>) -> Vec { - if let Some(inserts) = &self.planned_head_inserts { - return inserts.clone(); - } - let payload = InjectedPrebidClientConfig { - bidders: Some({ - #[cfg(test)] - { - self.legacy_config - .as_ref() - .map_or(&[][..], |config| config.bidders.as_slice()) - } - #[cfg(not(test))] - { - &[] - } - }), - ..InjectedPrebidClientConfig::from(&self.config) + if !status.is_success() { + log::warn!("Prebid auction {auction_id:?} returned non-success status: {status}"); + let body_bytes = match body_bytes { + Ok(body_bytes) => Some(body_bytes), + Err(error) => { + log::warn!( + "Prebid auction {auction_id:?} failed to read non-success response body: {error:?}" + ); + None + } }; + if demand.debug + && let Some(body_bytes) = body_bytes.as_deref() + { + match prebid_body_preview(body_bytes) { + Some(preview) => { + let truncation = if preview.truncated { + " (truncated)" + } else { + "" + }; + log::warn!( + "Prebid auction {auction_id:?} error response body preview{truncation}: {}", + preview.text + ); + } + None => log::warn!( + "Prebid auction {auction_id:?} returned an empty error response body" + ), + } + } - let config_json = serialize_injected_prebid_config(&payload); - let mut inserts = vec![injected_prebid_config_script(&config_json)]; - - inserts.push(self.external_bundle_script_tag()); - - inserts + let status_code = status.as_u16(); + let mut parsed = AuctionResponse::error(provider_id, response_time_ms) + .with_metadata("error_type", serde_json::json!(ERROR_TYPE_HTTP_STATUS)) + .with_metadata("http_status", serde_json::json!(status_code)) + .with_metadata( + "message", + serde_json::json!(format!("Prebid Server returned HTTP {status_code}")), + ); + if demand.debug + && let Some(message) = body_bytes + .as_deref() + .and_then(|body| extract_prebid_error_message(body, content_type.as_deref())) + { + parsed.metadata.insert( + "upstream_message".to_string(), + serde_json::json!(message.text), + ); + parsed.metadata.insert( + "upstream_message_truncated".to_string(), + serde_json::json!(message.truncated), + ); + } + return Ok(parsed); } -} - -#[derive(Serialize)] -#[serde(rename_all = "camelCase")] -struct InjectedManagedUserIdStorage<'a> { - #[serde(rename = "type")] - storage_type: PrebidUserIdStorageType, - name: &'a str, - #[serde(skip_serializing_if = "Option::is_none")] - expires: Option, - #[serde(skip_serializing_if = "Option::is_none")] - refresh_in_seconds: Option, -} -impl<'a> From<&'a PrebidManagedUserIdStorage> for InjectedManagedUserIdStorage<'a> { - fn from(storage: &'a PrebidManagedUserIdStorage) -> Self { - Self { - storage_type: storage.storage_type, - name: &storage.name, - expires: storage.expires, - refresh_in_seconds: storage.refresh_in_seconds, + let body_bytes = body_bytes?; + let response_json: Json = + serde_json::from_slice(&body_bytes).change_context(TrustedServerError::Integration { + integration: MODULE.to_string(), + message: "Failed to parse Prebid response".to_string(), + })?; + if demand.debug && log::log_enabled!(log::Level::Trace) { + match serde_json::to_string_pretty(&response_json) { + Ok(json) => log::trace!("Prebid OpenRTB response:\n{json}"), + Err(error) => log::warn!("Prebid: failed to serialize response for logging: {error}"), } } -} -#[derive(Serialize)] -#[serde(rename_all = "camelCase")] -struct InjectedManagedUserId<'a> { - name: &'a str, - #[serde(skip_serializing_if = "serde_json::Map::is_empty")] - params: &'a serde_json::Map, - #[serde(skip_serializing_if = "Option::is_none")] - storage: Option>, + let mut parsed = + parse_planned_prebid_openrtb(provider_id, input, &response_json, response_time_ms); + enrich_planned_prebid_metadata(demand, &response_json, &mut parsed); + log::info!( + "Prebid provider {provider_id} returned {} bids in {}ms", + parsed.bids.len(), + response_time_ms + ); + Ok(parsed) } -impl<'a> From<&'a PrebidManagedUserIdConfig> for InjectedManagedUserId<'a> { - fn from(config: &'a PrebidManagedUserIdConfig) -> Self { - Self { - name: &config.name, - params: &config.params, - storage: config - .storage - .as_ref() - .map(InjectedManagedUserIdStorage::from), +fn parse_planned_prebid_openrtb( + provider_id: &str, + input: &ProviderAuctionInput, + response_json: &Json, + response_time_ms: u64, +) -> AuctionResponse { + let Some(response) = response_json.as_object() else { + return AuctionResponse::error(provider_id, response_time_ms) + .with_metadata("error_type", serde_json::json!("parse_response")); + }; + match response.get("cur") { + None => {} + Some(Json::String(currency)) if currency.eq_ignore_ascii_case(DEFAULT_CURRENCY) => {} + Some(Json::String(currency)) => { + return AuctionResponse::no_bid(provider_id, response_time_ms) + .with_metadata("unsupported_currency", serde_json::json!(currency)); + } + Some(_) => { + return AuctionResponse::error(provider_id, response_time_ms) + .with_metadata("error_type", serde_json::json!("parse_response")); } } -} - -#[derive(Serialize)] -#[serde(rename_all = "camelCase")] -struct InjectedPrebidClientConfig<'a> { - account_id: &'a str, - #[serde(skip_serializing_if = "Vec::is_empty")] - managed_user_ids: Vec>, - timeout: u32, - debug: bool, - #[serde(skip_serializing_if = "Option::is_none")] - bidders: Option<&'a [String]>, - #[serde(skip_serializing_if = "Option::is_none")] - server_side_bidders: Option>, - #[serde(skip_serializing_if = "<[String]>::is_empty")] - client_side_bidders: &'a [String], - #[serde(skip_serializing_if = "<[String]>::is_empty")] - excluded_gam_ad_unit_path_suffixes: &'a [String], -} -impl<'a> From<&'a PrebidIntegrationConfig> for InjectedPrebidClientConfig<'a> { - fn from(config: &'a PrebidIntegrationConfig) -> Self { - Self { - account_id: config.account_id.as_deref().unwrap_or_default(), - managed_user_ids: config - .managed_user_ids - .iter() - .map(InjectedManagedUserId::from) - .collect(), - timeout: config.timeout_ms, - debug: config.debug, - bidders: None, - server_side_bidders: None, - client_side_bidders: &config.client_side_bidders, - excluded_gam_ad_unit_path_suffixes: &config.excluded_gam_ad_unit_path_suffixes, + let dimensions_by_slot = build_bid_dimension_index(input); + let mut diagnostics = ResponseAdmissionDiagnostics::default(); + let mut bids = Vec::new(); + if let Some(seatbids) = response_json.get("seatbid").and_then(Json::as_array) { + for seatbid in seatbids { + let returned_seat = seatbid + .get("seat") + .and_then(Json::as_str) + .filter(|seat| !seat.is_empty()); + let delivery_bidder = returned_seat.unwrap_or("unknown"); + if let Some(entries) = seatbid.get("bid").and_then(Json::as_array) { + for entry in entries { + match parse_planned_prebid_bid( + entry, + delivery_bidder, + returned_seat, + &dimensions_by_slot, + ) { + Ok(bid) => bids.push(bid), + Err(reason) => { + diagnostics.record(reason); + if reason == BidRejectionReason::InvalidBid { + let impression = entry + .get("impid") + .and_then(Json::as_str) + .unwrap_or(""); + log::warn!( + "Prebid: failed to parse bid from seat '{delivery_bidder}' for imp '{impression}'" + ); + } + } + } + } + } } } + let mut parsed = if bids.is_empty() { + AuctionResponse::no_bid(provider_id, response_time_ms) + } else { + AuctionResponse::success(provider_id, bids, response_time_ms) + }; + diagnostics.attach_to(&mut parsed); + parsed } -/// Returns `true` when `params` is not a usable PBS bidder-params object — an -/// empty object `{}` or any non-object value such as `null`. -/// -/// PBS rejects an imp whose `bidder.` carries such a value, and it cannot -/// tell a fabricated empty from an explicitly supplied one — they are identical -/// bytes on the wire. The merge uses this to stop an unusable value from -/// clobbering real params, and the final pass uses it to drop whatever remains. -#[cfg(test)] -fn is_unusable_bidder_params(params: &Json) -> bool { - // `None` covers non-object values (e.g. `null`); an empty map covers `{}`. - params.as_object().is_none_or(serde_json::Map::is_empty) +fn enrich_planned_prebid_metadata( + demand: &PrebidServerDemand, + response_json: &Json, + parsed: &mut AuctionResponse, +) { + let ext = response_json.get("ext"); + for key in ["responsetimemillis", "errors", "warnings"] { + if let Some(value) = ext.and_then(|ext| ext.get(key)) { + parsed.metadata.insert(key.to_string(), value.clone()); + } + } + if demand.debug { + if let Some(value) = ext.and_then(|ext| ext.get("debug")) { + parsed.metadata.insert("debug".to_string(), value.clone()); + } + if let Some(value) = ext + .and_then(|ext| ext.get("prebid")) + .and_then(|prebid| prebid.get("bidstatus")) + { + parsed + .metadata + .insert("bidstatus".to_string(), value.clone()); + } + } } -#[cfg(test)] -fn expand_trusted_server_bidders( - configured_bidders: &[String], - params: &Json, -) -> HashMap { - let per_bidder = params.get(BIDDER_PARAMS_KEY).and_then(Json::as_object); - - if configured_bidders.is_empty() { - return per_bidder - .map(|m| m.iter().map(|(k, v)| (k.clone(), v.clone())).collect()) - .unwrap_or_default(); - } - - configured_bidders - .iter() - .map(|bidder| { - let value = per_bidder - .and_then(|m| m.get(bidder).cloned()) - .unwrap_or_else(|| { - // No per-bidder map → use entire params as shared config - if per_bidder.is_some() { - Json::Object(Default::default()) - } else { - params.clone() - } - }); - (bidder.clone(), value) +fn parse_planned_prebid_bid( + bid: &Json, + delivery_bidder: &str, + returned_seat: Option<&str>, + dimensions_by_slot: &BidDimensionIndex, +) -> Result { + let slot_id = bid + .get("impid") + .and_then(Json::as_str) + .filter(|slot_id| !slot_id.is_empty()) + .ok_or(BidRejectionReason::InvalidBid)? + .to_string(); + let width = parse_optional_bid_dimension(bid, "w")?; + let height = parse_optional_bid_dimension(bid, "h")?; + let (width, height) = resolve_bid_dimensions(dimensions_by_slot, &slot_id, width, height)?; + let price = bid + .get("price") + .and_then(Json::as_f64) + .filter(|price| price.is_finite() && *price >= 0.0) + .ok_or(BidRejectionReason::InvalidBid)?; + let creative = bid.get("adm").and_then(Json::as_str).map(String::from); + let cache_entry = bid + .get("ext") + .and_then(|ext| ext.get("prebid")) + .and_then(|prebid| prebid.get("cache")) + .and_then(|cache| cache.get("bids")); + let cache_id = cache_entry + .and_then(|cache| cache.get("cacheId")) + .and_then(Json::as_str) + .map(String::from); + let (cache_host, cache_path) = cache_entry + .and_then(|cache| cache.get("url")) + .and_then(Json::as_str) + .and_then(|value| { + ParsedUrl::parse(value) + .map_err(|error| log::debug!("PBS cache URL parse failed: {error}")) + .ok() }) - .collect() -} - -/// Shallow-merges `override_obj` into `params`. -/// -/// When `params` is a JSON object, each top-level key in `override_obj` is -/// inserted or replaced in `params`. Nested objects are replaced wholesale — -/// they are not recursed into. When `params` is not an object, it is left -/// untouched to preserve pre-existing behavior. -/// -/// Returns whether `params` was an object and the merge path ran. -fn merge_bidder_param_object( - params: &mut Json, - override_obj: &serde_json::Map, -) -> bool { - if let Json::Object(base) = params { - for (k, v) in override_obj { - base.insert(k.clone(), v.clone()); - } - - true - } else { - false + .map(|url| { + let host = url.host_str().map(String::from); + let path = url.path().to_string(); + let path = (!path.is_empty() && path != "/").then_some(path); + (host, path) + }) + .unwrap_or((None, None)); + if cache_id.is_some() && cache_host.is_none() { + log::warn!( + "PBS bid has cache UUID but cache URL could not be parsed — creative will fail to render for slot '{slot_id}'" + ); } + + Ok(AuctionBid { + slot_id, + price: Some(price), + currency: DEFAULT_CURRENCY.to_string(), + creative, + adomain: bid.get("adomain").and_then(Json::as_array).map(|domains| { + domains + .iter() + .filter_map(Json::as_str) + .map(String::from) + .collect() + }), + bidder: delivery_bidder.to_string(), + returned_seat: returned_seat.map(str::to_string), + width, + height, + nurl: bid.get("nurl").and_then(Json::as_str).map(String::from), + burl: bid.get("burl").and_then(Json::as_str).map(String::from), + bid_id: bid + .get("id") + .and_then(Json::as_str) + .filter(|value| !value.is_empty()) + .map(String::from), + ad_id: bid.get("adid").and_then(Json::as_str).map(String::from), + creative_id: bid.get("crid").and_then(Json::as_str).map(String::from), + renderer: None, + cache_id, + cache_host, + cache_path, + metadata: HashMap::new(), + }) } // ============================================================================ -// Generic bid-parameter override engine +// Prebid Auction Provider // ============================================================================ +/// The request extension [`PrebidAuctionProvider`] writes. #[cfg(test)] -fn warn_unconfigured_bidder(config: &LegacyPrebidServerConfig, bidder: &str, field: &str) { - if !config.bidders.iter().any(|b| b == bidder) { - if config.client_side_bidders.iter().any(|b| b == bidder) { - log::warn!( - "prebid: {field} entry targets client-side-only bidder \ - '{bidder}' — server-side override will never apply" - ); - } else { - log::warn!( - "prebid: {field} entry references unconfigured bidder \ - '{bidder}' — rule will never fire" - ); - } - } +#[derive(Debug, Serialize)] +struct RequestExt { + #[serde(skip_serializing_if = "Option::is_none")] + prebid: Option, + #[serde(skip_serializing_if = "Option::is_none")] + trusted_server: Option, } -#[derive(Debug, Default, Clone)] -pub(crate) struct BidParamOverrideEngine { - rules: Vec, - // Maps bidder name to the indices (into `rules`) of rules that constrain on that bidder. - // Rules with no bidder constraint (zone-only or catch-all) are kept in `wildcard_indices`. - // Both slices are in declaration order; `apply` merges them without allocation. - bidder_index: HashMap>, - wildcard_indices: Vec, -} +#[cfg(test)] +impl ToExt for RequestExt {} -#[derive(Debug, Clone, PartialEq, Eq)] -struct CompiledBidParamOverrideRule { - bidder: Option, - zone: Option, - set: serde_json::Map, +/// The `ext.prebid` object of a request. +#[cfg(test)] +#[derive(Debug, Serialize, Deserialize)] +struct PrebidExt { + #[serde(skip_serializing_if = "Option::is_none")] + debug: Option, + #[serde(skip_serializing_if = "Option::is_none")] + returnallbidstatus: Option, } -#[derive(Debug, Copy, Clone)] -struct BidParamOverrideFacts<'a> { - bidder: &'a str, - zone: Option<&'a str>, +/// The impression extension [`PrebidAuctionProvider`] writes. +#[cfg(test)] +#[derive(Debug, Serialize)] +struct ImpExt { + prebid: PrebidImpExt, } -impl BidParamOverrideEngine { - #[cfg(test)] - fn try_from_config( - config: &LegacyPrebidServerConfig, - ) -> Result> { - let mut rules = Vec::new(); - - let mut bidder_overrides = config.bid_param_overrides.iter().collect::>(); - bidder_overrides.sort_by(|(left, _), (right, _)| left.as_str().cmp(right.as_str())); +#[cfg(test)] +impl ToExt for ImpExt {} - for (bidder, set) in bidder_overrides { - warn_unconfigured_bidder(config, bidder, "bid_param_overrides"); - rules.push(CompiledBidParamOverrideRule::from_bidder_override( - bidder.as_str(), - set, - )?); - } +/// The `imp.ext.prebid` object: inline bidder parameters, or a stored +/// request reference. +#[cfg(test)] +#[derive(Debug, Default, Serialize)] +struct PrebidImpExt { + #[serde(skip_serializing_if = "HashMap::is_empty")] + bidder: HashMap, + #[serde(skip_serializing_if = "Option::is_none")] + storedrequest: Option, +} - let mut zone_overrides = config.bid_param_zone_overrides.iter().collect::>(); - zone_overrides.sort_by(|(left, _), (right, _)| left.as_str().cmp(right.as_str())); +/// PBS imp-level stored request reference. +/// +/// PBS merges the stored imp JSON (keyed by `id`) into the outgoing request, +/// populating bidder params that are not sent inline. +#[cfg(test)] +#[derive(Debug, Serialize)] +struct ImpStoredRequest { + id: String, +} - for (bidder, zone_override_sets) in zone_overrides { - warn_unconfigured_bidder(config, bidder, "bid_param_zone_overrides"); - let mut sorted_zone_overrides = zone_override_sets.iter().collect::>(); - sorted_zone_overrides - .sort_by(|(left, _), (right, _)| left.as_str().cmp(right.as_str())); +/// Legacy Prebid Server auction provider retained only for parity tests. +#[cfg(test)] +pub struct PrebidAuctionProvider { + config: LegacyPrebidServerConfig, + bid_param_override_engine: Arc, +} - for (zone, set) in sorted_zone_overrides { - rules.push(CompiledBidParamOverrideRule::from_zone_override( - bidder.as_str(), - zone.as_str(), - set, - )?); - } - } +#[cfg(test)] +#[derive(Default)] +struct PrebidImpressionDisposition { + aps_only: usize, + invalid: usize, +} - for rule in &config.bid_param_override_rules { - if let Some(bidder) = &rule.when.bidder { - warn_unconfigured_bidder(config, bidder, "bid_param_override_rules"); - } - rules.push(CompiledBidParamOverrideRule::try_from(rule)?); - } +#[cfg(test)] +struct PrebidRequestBuild { + request: OpenRtbRequest, + disposition: PrebidImpressionDisposition, +} - let mut bidder_index: HashMap> = HashMap::new(); - let mut wildcard_indices: Vec = Vec::new(); - for (idx, rule) in rules.iter().enumerate() { - match &rule.bidder { - Some(bidder) => bidder_index.entry(bidder.clone()).or_default().push(idx), - None => wildcard_indices.push(idx), - } - } +#[cfg(test)] +impl PrebidAuctionProvider { + #[cfg(test)] + fn new(config: LegacyPrebidServerConfig) -> Self { + Self::try_new(config).expect("should compile prebid bid param overrides") + } + /// Create a new Prebid auction provider with validated override rules. + /// + /// # Errors + /// + /// Returns an error when the configured bidder-param override rules are invalid. + pub fn try_new(config: LegacyPrebidServerConfig) -> Result> { Ok(Self { - rules, - bidder_index, - wildcard_indices, + bid_param_override_engine: Arc::new(BidParamOverrideEngine::try_from_config(&config)?), + config, }) } - fn try_from_profile_config( - bid_param_zone_overrides: &std::collections::BTreeMap< - String, - std::collections::BTreeMap>, - >, - bid_param_overrides: &std::collections::BTreeMap>, - bid_param_override_rules: &[BidParamOverrideRule], - ) -> Result> { - let mut rules = Vec::new(); - for (bidder, set) in bid_param_overrides { - rules.push(CompiledBidParamOverrideRule::from_bidder_override( - bidder, set, - )?); - } - for (bidder, zone_override_sets) in bid_param_zone_overrides { - for (zone, set) in zone_override_sets { - rules.push(CompiledBidParamOverrideRule::from_zone_override( - bidder, zone, set, - )?); - } - } - for rule in bid_param_override_rules { - rules.push(CompiledBidParamOverrideRule::try_from(rule)?); - } - let mut bidder_index: HashMap> = HashMap::new(); - let mut wildcard_indices = Vec::new(); - for (index, rule) in rules.iter().enumerate() { - match &rule.bidder { - Some(bidder) => bidder_index.entry(bidder.clone()).or_default().push(index), - None => wildcard_indices.push(index), - } - } - Ok(Self { - rules, - bidder_index, - wildcard_indices, - }) - } - - fn apply(&self, facts: BidParamOverrideFacts<'_>, params: &mut Json) { - let bidder_indices = self.bidder_index.get(facts.bidder).map(Vec::as_slice); - for idx in merged_rule_indices(&self.wildcard_indices, bidder_indices) { - let rule = &self.rules[idx]; - if rule.matches(facts) { - if merge_bidder_param_object(params, &rule.set) { - log::debug!( - "prebid: applying bidder param override for bidder '{}' zone {:?}: keys {:?}", - facts.bidder, - facts.zone, - rule.set.keys().collect::>() - ); - } else { - log::debug!( - "prebid: skipping bidder param override for bidder '{}' zone {:?}: params is not a JSON object", - facts.bidder, - facts.zone - ); - } - } - } - } - - /// Apply compiled profile rules to already centrally routed bidder params. - pub(crate) fn apply_routed(&self, bidder: &str, zone: Option<&str>, params: &mut Json) { - self.apply(BidParamOverrideFacts { bidder, zone }, params); - } -} - -/// Validate and compile server-side Prebid profile override fields. -/// -/// This narrow hook shares the existing override compiler without coupling -/// auction-profile availability to the browser integration's enablement. -pub(crate) fn compile_profile_override_rules( - bid_param_zone_overrides: &std::collections::BTreeMap< - String, - std::collections::BTreeMap>, - >, - bid_param_overrides: &std::collections::BTreeMap>, - bid_param_override_rules: &[BidParamOverrideRule], -) -> Result> { - BidParamOverrideEngine::try_from_profile_config( - bid_param_zone_overrides, - bid_param_overrides, - bid_param_override_rules, - ) -} - -fn merged_rule_indices<'a>( - wildcard_indices: &'a [usize], - bidder_indices: Option<&'a [usize]>, -) -> impl Iterator + 'a { - let mut wildcard = wildcard_indices.iter().copied().peekable(); - let mut bidder = bidder_indices.unwrap_or(&[]).iter().copied().peekable(); - - std::iter::from_fn(move || match (wildcard.peek(), bidder.peek()) { - (Some(wildcard_idx), Some(bidder_idx)) if wildcard_idx <= bidder_idx => wildcard.next(), - (Some(_), Some(_)) => bidder.next(), - (Some(_), None) => wildcard.next(), - (None, Some(_)) => bidder.next(), - (None, None) => None, - }) -} - -impl CompiledBidParamOverrideRule { - fn from_bidder_override( - bidder: &str, - set: &serde_json::Map, - ) -> Result> { - Self::new( - Some(bidder), - None, - set, - &format!("integrations.prebid.bid_param_overrides.{bidder}"), - false, - ) - } - - fn from_zone_override( - bidder: &str, - zone: &str, - set: &serde_json::Map, - ) -> Result> { - Self::new( - Some(bidder), - Some(zone), - set, - &format!("integrations.prebid.bid_param_zone_overrides.{bidder}.{zone}"), - false, - ) - } - - fn new( - bidder: Option<&str>, - zone: Option<&str>, - set: &serde_json::Map, - source: &str, - is_canonical: bool, - ) -> Result> { - let bidder = bidder - .map(|value| validate_override_matcher_string(value, "when.bidder", source)) - .transpose()?; - let zone = zone - .map(|value| validate_override_matcher_string(value, "when.zone", source)) - .transpose()?; - - if bidder.is_none() && zone.is_none() { - return Err(Report::new(TrustedServerError::Configuration { - message: format!("{source} must include at least one matcher"), - })); - } - - Ok(Self { - bidder, - zone, - set: non_empty_override_object(set, source, is_canonical)?, - }) - } - - fn matches(&self, facts: BidParamOverrideFacts<'_>) -> bool { - if let Some(expected_bidder) = self.bidder.as_deref() - && expected_bidder != facts.bidder - { - return false; - } - - if let Some(expected_zone) = self.zone.as_deref() - && facts.zone != Some(expected_zone) - { - return false; - } - - true - } -} - -impl TryFrom for CompiledBidParamOverrideRule { - type Error = Report; - - fn try_from(rule: BidParamOverrideRule) -> Result { - Self::try_from(&rule) - } -} - -impl TryFrom<&BidParamOverrideRule> for CompiledBidParamOverrideRule { - type Error = Report; - - fn try_from(rule: &BidParamOverrideRule) -> Result { - Self::new( - rule.when.bidder.as_deref(), - rule.when.zone.as_deref(), - &rule.set, - "integrations.prebid.bid_param_override_rules[*]", - true, - ) - } -} - -// Trims leading/trailing whitespace from config-side matcher strings so that -// accidental padding in TOML/JSON does not produce a silently non-matching rule. -// Runtime facts (bidder name, zone) are not trimmed — they come from controlled -// internal sources (bidder-map keys, trustedServer.zone) where whitespace is -// never expected. Matching is exact and case-sensitive after this normalisation. -fn validate_override_matcher_string( - value: &str, - field: &str, - source: &str, -) -> Result> { - let trimmed = value.trim(); - - if trimmed.is_empty() { - return Err(Report::new(TrustedServerError::Configuration { - message: format!("{source}.{field} must not be empty"), - })); - } - - Ok(trimmed.to_string()) -} - -fn non_empty_override_object( - value: &serde_json::Map, - source: &str, - is_canonical: bool, -) -> Result, Report> { - if value.is_empty() { - let message = if is_canonical { - format!("{source}.set must not be empty") - } else { - format!("{source} entry must not be empty") - }; - return Err(Report::new(TrustedServerError::Configuration { message })); - } - - Ok(value.clone()) -} - -/// Copies browser headers to the outgoing Prebid Server request. -/// -/// The inbound `X-Forwarded-For` header is never copied: on the server-side -/// publisher auction path `from` is the browser navigation request, so its -/// XFF value is client-supplied and spoofable. Instead the header is -/// synthesized from `client_ip` — the platform-attested client address — -/// matching the trusted IP already sent in `OpenRTB` `device.ip`. -/// -/// In [`ConsentForwardingMode::OpenrtbOnly`] mode, consent cookies are -/// stripped from the `Cookie` header since consent travels exclusively -/// through the `OpenRTB` body. -#[cfg(test)] -fn copy_request_headers( - from: &http::Request, - to: &mut http::Request, - consent_forwarding: ConsentForwardingMode, - client_ip: Option, -) { - apply_prebid_header_values( - from.headers().get(header::COOKIE), - from.headers().get(header::USER_AGENT), - from.headers().get(header::REFERER), - from.headers().get(header::ACCEPT_LANGUAGE), - to, - consent_forwarding, - client_ip, - ); -} - -/// Apply the common raw-header transport policy for a planned PBS request. -pub(crate) fn apply_prebid_transport_headers( - from: &PrebidTransportHeaders, - to: &mut http::Request, - consent_forwarding: ConsentForwardingMode, - client_ip: Option, -) { - apply_prebid_header_values( - from.cookie(), - from.user_agent(), - from.referer(), - from.accept_language(), - to, - consent_forwarding, - client_ip, - ); -} - -#[allow( - clippy::too_many_arguments, - reason = "the helper preserves four independently optional raw headers plus transport policy" -)] -fn apply_prebid_header_values( - cookie: Option<&HeaderValue>, - user_agent: Option<&HeaderValue>, - referer: Option<&HeaderValue>, - accept_language: Option<&HeaderValue>, - to: &mut http::Request, - consent_forwarding: ConsentForwardingMode, - client_ip: Option, -) { - for (name, value) in [ - (header::USER_AGENT, user_agent), - (header::REFERER, referer), - (header::ACCEPT_LANGUAGE, accept_language), - ] { - if let Some(value) = value { - to.headers_mut().insert(name, value.clone()); - } - } - - if let Some(ip) = client_ip - && let Ok(value) = HeaderValue::from_str(&ip.to_string()) - { - to.headers_mut() - .insert(header::HeaderName::from_static("x-forwarded-for"), value); - } - - let Some(cookie_value) = cookie else { - return; - }; - if !consent_forwarding.strips_consent_cookies() { - to.headers_mut() - .insert(header::COOKIE, cookie_value.clone()); - return; - } - match cookie_value.to_str() { - Ok(value) => { - let stripped = strip_cookies(value, CONSENT_COOKIE_NAMES); - if !stripped.is_empty() - && let Ok(cookie_header) = HeaderValue::from_str(&stripped) - { - to.headers_mut().insert(header::COOKIE, cookie_header); - } - } - Err(_) => { - to.headers_mut() - .insert(header::COOKIE, cookie_value.clone()); - } - } -} - -/// Appends query parameters to a URL, handling both URLs with and without existing query strings. -/// Returns the original URL unchanged if params are empty or already present. -#[cfg(test)] -fn append_query_params(url: &str, params: &str) -> String { - if params.is_empty() || url.contains(params) { - return url.to_string(); - } - if url.contains('?') { - format!("{}&{}", url, params) - } else { - format!("{}?{}", url, params) - } -} - -/// Parse a planned PBS response with the configured profile behavior. -/// -/// This preserves the legacy PBS status, body, bid, cache, and metadata -/// semantics while allowing each planned provider to retain its own identity. -pub(crate) async fn parse_planned_prebid_response( - provider_id: &str, - profile: &PrebidProfilePlan, - input: &ProviderAuctionInput, - response: PlatformResponse, - response_time_ms: u64, - auction_id: &str, -) -> Result> { - let response = response.response; - let status = response.status(); - let content_type = response - .headers() - .get(header::CONTENT_TYPE) - .and_then(|value| value.to_str().ok()) - .map(str::to_owned); - let body_bytes = collect_response_bounded( - response.into_body(), - UPSTREAM_RTB_MAX_RESPONSE_BYTES, - "prebid", - ) - .await - .change_context(TrustedServerError::Prebid { - message: "Failed to read Prebid response body".to_string(), - }); - - if !status.is_success() { - log::warn!("Prebid auction {auction_id:?} returned non-success status: {status}"); - let body_bytes = match body_bytes { - Ok(body_bytes) => Some(body_bytes), - Err(error) => { - log::warn!( - "Prebid auction {auction_id:?} failed to read non-success response body: {error:?}" - ); - None - } - }; - if profile.debug - && let Some(body_bytes) = body_bytes.as_deref() - { - match prebid_body_preview(body_bytes) { - Some(preview) => { - let truncation = if preview.truncated { - " (truncated)" - } else { - "" - }; - log::warn!( - "Prebid auction {auction_id:?} error response body preview{truncation}: {}", - preview.text - ); - } - None => log::warn!( - "Prebid auction {auction_id:?} returned an empty error response body" - ), - } - } - - let status_code = status.as_u16(); - let mut parsed = AuctionResponse::error(provider_id, response_time_ms) - .with_metadata("error_type", serde_json::json!(ERROR_TYPE_HTTP_STATUS)) - .with_metadata("http_status", serde_json::json!(status_code)) - .with_metadata( - "message", - serde_json::json!(format!("Prebid Server returned HTTP {status_code}")), - ); - if profile.debug - && let Some(message) = body_bytes - .as_deref() - .and_then(|body| extract_prebid_error_message(body, content_type.as_deref())) - { - parsed.metadata.insert( - "upstream_message".to_string(), - serde_json::json!(message.text), - ); - parsed.metadata.insert( - "upstream_message_truncated".to_string(), - serde_json::json!(message.truncated), - ); - } - return Ok(parsed); - } - - let body_bytes = body_bytes?; - let response_json: Json = - serde_json::from_slice(&body_bytes).change_context(TrustedServerError::Prebid { - message: "Failed to parse Prebid response".to_string(), - })?; - if profile.debug && log::log_enabled!(log::Level::Trace) { - match serde_json::to_string_pretty(&response_json) { - Ok(json) => log::trace!("Prebid OpenRTB response:\n{json}"), - Err(error) => log::warn!("Prebid: failed to serialize response for logging: {error}"), - } - } - - let mut parsed = - parse_planned_prebid_openrtb(provider_id, input, &response_json, response_time_ms); - enrich_planned_prebid_metadata(profile, &response_json, &mut parsed); - log::info!( - "Prebid provider {provider_id} returned {} bids in {}ms", - parsed.bids.len(), - response_time_ms - ); - Ok(parsed) -} - -fn parse_planned_prebid_openrtb( - provider_id: &str, - input: &ProviderAuctionInput, - response_json: &Json, - response_time_ms: u64, -) -> AuctionResponse { - let Some(response) = response_json.as_object() else { - return AuctionResponse::error(provider_id, response_time_ms) - .with_metadata("error_type", serde_json::json!("parse_response")); - }; - match response.get("cur") { - None => {} - Some(Json::String(currency)) if currency.eq_ignore_ascii_case(DEFAULT_CURRENCY) => {} - Some(Json::String(currency)) => { - return AuctionResponse::no_bid(provider_id, response_time_ms) - .with_metadata("unsupported_currency", serde_json::json!(currency)); - } - Some(_) => { - return AuctionResponse::error(provider_id, response_time_ms) - .with_metadata("error_type", serde_json::json!("parse_response")); - } - } - - let dimensions_by_slot = build_bid_dimension_index(input); - let mut diagnostics = ResponseAdmissionDiagnostics::default(); - let mut bids = Vec::new(); - if let Some(seatbids) = response_json.get("seatbid").and_then(Json::as_array) { - for seatbid in seatbids { - let returned_seat = seatbid - .get("seat") - .and_then(Json::as_str) - .filter(|seat| !seat.is_empty()); - let delivery_bidder = returned_seat.unwrap_or("unknown"); - if let Some(entries) = seatbid.get("bid").and_then(Json::as_array) { - for entry in entries { - match parse_planned_prebid_bid( - entry, - delivery_bidder, - returned_seat, - &dimensions_by_slot, - ) { - Ok(bid) => bids.push(bid), - Err(reason) => { - diagnostics.record(reason); - if reason == BidRejectionReason::InvalidBid { - let impression = entry - .get("impid") - .and_then(Json::as_str) - .unwrap_or(""); - log::warn!( - "Prebid: failed to parse bid from seat '{delivery_bidder}' for imp '{impression}'" - ); - } - } - } - } - } - } - } - let mut parsed = if bids.is_empty() { - AuctionResponse::no_bid(provider_id, response_time_ms) - } else { - AuctionResponse::success(provider_id, bids, response_time_ms) - }; - diagnostics.attach_to(&mut parsed); - parsed -} - -fn enrich_planned_prebid_metadata( - profile: &PrebidProfilePlan, - response_json: &Json, - parsed: &mut AuctionResponse, -) { - let ext = response_json.get("ext"); - for key in ["responsetimemillis", "errors", "warnings"] { - if let Some(value) = ext.and_then(|ext| ext.get(key)) { - parsed.metadata.insert(key.to_string(), value.clone()); - } - } - if profile.debug { - if let Some(value) = ext.and_then(|ext| ext.get("debug")) { - parsed.metadata.insert("debug".to_string(), value.clone()); - } - if let Some(value) = ext - .and_then(|ext| ext.get("prebid")) - .and_then(|prebid| prebid.get("bidstatus")) - { - parsed - .metadata - .insert("bidstatus".to_string(), value.clone()); - } - } -} - -fn parse_planned_prebid_bid( - bid: &Json, - delivery_bidder: &str, - returned_seat: Option<&str>, - dimensions_by_slot: &BidDimensionIndex, -) -> Result { - let slot_id = bid - .get("impid") - .and_then(Json::as_str) - .filter(|slot_id| !slot_id.is_empty()) - .ok_or(BidRejectionReason::InvalidBid)? - .to_string(); - let width = parse_optional_bid_dimension(bid, "w")?; - let height = parse_optional_bid_dimension(bid, "h")?; - let (width, height) = resolve_bid_dimensions(dimensions_by_slot, &slot_id, width, height)?; - let price = bid - .get("price") - .and_then(Json::as_f64) - .filter(|price| price.is_finite() && *price >= 0.0) - .ok_or(BidRejectionReason::InvalidBid)?; - let creative = bid.get("adm").and_then(Json::as_str).map(String::from); - let cache_entry = bid - .get("ext") - .and_then(|ext| ext.get("prebid")) - .and_then(|prebid| prebid.get("cache")) - .and_then(|cache| cache.get("bids")); - let cache_id = cache_entry - .and_then(|cache| cache.get("cacheId")) - .and_then(Json::as_str) - .map(String::from); - let (cache_host, cache_path) = cache_entry - .and_then(|cache| cache.get("url")) - .and_then(Json::as_str) - .and_then(|value| { - ParsedUrl::parse(value) - .map_err(|error| log::debug!("PBS cache URL parse failed: {error}")) - .ok() - }) - .map(|url| { - let host = url.host_str().map(String::from); - let path = url.path().to_string(); - let path = (!path.is_empty() && path != "/").then_some(path); - (host, path) - }) - .unwrap_or((None, None)); - if cache_id.is_some() && cache_host.is_none() { - log::warn!( - "PBS bid has cache UUID but cache URL could not be parsed — creative will fail to render for slot '{slot_id}'" - ); - } - - Ok(AuctionBid { - slot_id, - price: Some(price), - currency: DEFAULT_CURRENCY.to_string(), - creative, - adomain: bid.get("adomain").and_then(Json::as_array).map(|domains| { - domains - .iter() - .filter_map(Json::as_str) - .map(String::from) - .collect() - }), - bidder: delivery_bidder.to_string(), - returned_seat: returned_seat.map(str::to_string), - width, - height, - nurl: bid.get("nurl").and_then(Json::as_str).map(String::from), - burl: bid.get("burl").and_then(Json::as_str).map(String::from), - bid_id: bid - .get("id") - .and_then(Json::as_str) - .filter(|value| !value.is_empty()) - .map(String::from), - ad_id: bid.get("adid").and_then(Json::as_str).map(String::from), - creative_id: bid.get("crid").and_then(Json::as_str).map(String::from), - renderer: None, - cache_id, - cache_host, - cache_path, - metadata: HashMap::new(), - }) -} - -// ============================================================================ -// Prebid Auction Provider -// ============================================================================ - -/// Legacy Prebid Server auction provider retained only for parity tests. -#[cfg(test)] -pub struct PrebidAuctionProvider { - config: LegacyPrebidServerConfig, - bid_param_override_engine: Arc, -} - -#[cfg(test)] -#[derive(Default)] -struct PrebidImpressionDisposition { - aps_only: usize, - invalid: usize, -} - -#[cfg(test)] -struct PrebidRequestBuild { - request: OpenRtbRequest, - disposition: PrebidImpressionDisposition, -} - -#[cfg(test)] -impl PrebidAuctionProvider { - #[cfg(test)] - fn new(config: LegacyPrebidServerConfig) -> Self { - Self::try_new(config).expect("should compile prebid bid param overrides") - } - - /// Create a new Prebid auction provider with validated override rules. - /// - /// # Errors - /// - /// Returns an error when the configured bidder-param override rules are invalid. - pub fn try_new(config: LegacyPrebidServerConfig) -> Result> { - Ok(Self { - bid_param_override_engine: Arc::new(BidParamOverrideEngine::try_from_config(&config)?), - config, - }) - } - - /// Returns the full Prebid Server `OpenRTB2` auction endpoint URL. - /// - /// Backward-compatible normalization: `server_url` may be configured as - /// either the PBS origin (path is appended here) or the full endpoint - /// already ending in `/openrtb2/auction` (used as-is, ignoring a trailing - /// slash) — both shapes produce the same request URL. - fn auction_endpoint_url(&self) -> String { - let base = self.config.server_url.trim_end_matches('/'); - if base.ends_with("/openrtb2/auction") { - base.to_string() - } else { - format!("{base}/openrtb2/auction") - } - } - - /// Build an enriched `OpenRTB` request and retain impression drop provenance. - fn build_openrtb( - &self, - request: &AuctionRequest, - context: &AuctionContext<'_>, - signer: Option<(&RequestSigner, String, &SigningParams)>, - request_info: RequestInfo, - ) -> PrebidRequestBuild { - let mut disposition = PrebidImpressionDisposition::default(); - let imps = request - .slots - .iter() - .filter_map(|slot| { - let slot_context = format!("slot '{}'", slot.id); - let formats: Vec<_> = slot - .formats - .iter() - .filter(|f| f.media_type == MediaType::Banner) - .filter_map(|f| { - let width = to_openrtb_i32(f.width, "format.w", &slot_context); - let height = to_openrtb_i32(f.height, "format.h", &slot_context); - - match (width, height) { - (Some(width), Some(height)) => Some(Format { - w: Some(width), - h: Some(height), - ..Default::default() - }), - _ => None, - } - }) - .collect(); - - if formats.is_empty() { - disposition.invalid += 1; - log::warn!( - "prebid: dropping imp '{}' — no valid banner formats after filtering", - slot.id - ); - return None; - } - - // Extract zone from trustedServer params (sent by the JS - // adapter from `mediaTypes.banner.name`, e.g. "header", "fixed_bottom"). - let zone: Option<&str> = slot - .bidders - .get(TRUSTED_SERVER_BIDDER) - .and_then(|p| p.get(ZONE_KEY)) - .and_then(Json::as_str); - - // Build the bidder map for PBS from two sources: - // 1. the `trustedServer` orchestrator entry, expanded into the - // configured PBS bidders (`expand_trusted_server_bidders`); - // 2. direct entries naming a configured PBS bidder. - // The JS adapter sends "trustedServer" as the bidder (our - // orchestrator adapter name); provider-specific keys like "aps" - // run their own auction and are skipped here. - // - // The two sources are collected separately so the merge below is - // deterministic regardless of `slot.bidders` HashMap iteration - // order: direct entries win, but an unusable direct value (`{}` or - // a non-object) must not clobber real params from the expansion. - let mut expanded: HashMap = HashMap::new(); - let mut direct: Vec<(String, Json)> = Vec::new(); - let mut excluded_aps = false; - for (name, params) in &slot.bidders { - if name.eq_ignore_ascii_case("aps") { - // APS is a separate OpenRTB provider. Never send native - // APS demand through PBS for the same cohort. - excluded_aps = true; - } else if name == TRUSTED_SERVER_BIDDER { - for (bidder, params) in - expand_trusted_server_bidders(&self.config.bidders, params) - { - if bidder.eq_ignore_ascii_case("aps") { - excluded_aps = true; - } else { - expanded.insert(bidder, params); - } - } - } else if self.config.bidders.iter().any(|bidder| bidder == name) { - direct.push((name.clone(), params.clone())); - } else { - // Any unrecognized key is likely a misconfiguration (a - // slot bidder absent from `config.bidders`) that silently - // yields an empty bidder map and a stored-request no-bid — - // log it so the drop is diagnosable. - log::debug!( - "prebid: dropping slot '{}' bidder '{}' — not in config.bidders and not a known provider key", - slot.id, - name - ); - } - } - - let mut bidder = expanded; - // Bidders whose params came from the slot itself rather than from - // `trustedServer` expansion. Only these can represent a publisher - // misconfiguration if they are dropped below; a dropped fabricated - // entry is the designed stored-request path. - let mut slot_supplied: HashSet = HashSet::new(); - for (name, params) in direct { - // Direct entries win over the expansion — except when an - // unusable direct value (`{}` or a non-object) would overwrite - // real params the expansion already supplied. PBS cannot tell a - // fabricated empty from an explicit one, so real params must not - // be discarded in favour of either. - let clobbers_real = is_unusable_bidder_params(¶ms) - && bidder - .get(&name) - .is_some_and(|existing| !is_unusable_bidder_params(existing)); - if !clobbers_real { - slot_supplied.insert(name.clone()); - bidder.insert(name, params); - } - } - - // Apply canonical and compatibility-derived rules in normalized - // order. An override rule can populate a bidder that arrived with - // empty params, promoting a fabricated empty into a valid bidder. - for (name, params) in &mut bidder { - self.bid_param_override_engine - .apply(BidParamOverrideFacts { bidder: name, zone }, params); - } - - // Drop any bidder whose params are still unusable after overrides — - // an empty `{}` or a non-object like `null`. Both are byte-identical - // to PBS regardless of whether they were fabricated or supplied - // explicitly, and shipping either gets the imp rejected; a single - // non-2xx then collapses into an auction-wide bid wipeout (see - // `parse_response`), so one misconfigured slot must never reach the - // wire. - // - // The two drop sources get different levels, one line per slot. A - // slot that supplied unusable params inline is a publisher mistake - // an operator can act on, so it warns — the PBS 400 body is - // otherwise discarded unless trace logging is enabled. Fabricated - // empties that no override rule filled are the designed - // creative-opportunity path (see `CreativeOpportunitySlot::to_ad_slot`), - // so they log at debug and do not bury the warn. - let mut dropped_slot_supplied: Vec = Vec::new(); - let mut dropped_fabricated: Vec = Vec::new(); - bidder.retain(|name, params| { - if is_unusable_bidder_params(params) { - if slot_supplied.contains(name) { - dropped_slot_supplied.push(name.clone()); - } else { - dropped_fabricated.push(name.clone()); - } - return false; - } - true - }); - if !dropped_slot_supplied.is_empty() { - // Sorted so the line is stable across `HashMap` iteration order. - dropped_slot_supplied.sort(); - log::warn!( - "prebid: dropping slot '{}' bidders [{}] — slot supplied empty or non-object params; slot falls back to its stored request", - slot.id, - dropped_slot_supplied.join(", ") - ); - } - if !dropped_fabricated.is_empty() { - dropped_fabricated.sort(); - log::debug!( - "prebid: dropping slot '{}' bidders [{}] — expansion fabricated empty params and no override rule filled them; slot falls back to its stored request", - slot.id, - dropped_fabricated.join(", ") - ); - } - - if excluded_aps && bidder.is_empty() { - disposition.aps_only += 1; - log::warn!( - "prebid: dropping imp '{}' because it contains only APS demand; refusing PBS stored-request fallback", - slot.id - ); - return None; - } - - // When no eligible PBS bidder params remain, tell PBS to resolve - // bidder config from the stored request keyed by this slot ID. This - // covers creative-opportunity slots whose PBS params live in stored - // requests, and slots whose configured bidders all resolved to - // unusable params and were dropped above. - // - // Note the fallback set is wider than "slots with no inline params": - // a slot carrying real inline params for a bidder absent from - // `config.bidders` also lands here — that bidder is never expanded, - // the configured bidders fabricate empties, the drop clears them, and - // the real params are lost. That param loss is pre-existing (see the - // debug log in the build loop); it is called out here because it - // means an operator with `config.bidders` set but no stored request - // provisioned for such a slot gets a no-bid. - let storedrequest = if bidder.is_empty() { - Some(ImpStoredRequest { - id: slot.id.clone(), - }) - } else { - None - }; - - Some(Imp { - id: Some(slot.id.clone()), - banner: Some(Banner { - format: formats, - ..Default::default() - }), - bidfloor: slot.floor_price, - // NOTE: Currency defaults to DEFAULT_CURRENCY. If - // multi-currency support is needed, this should come from - // config or the AdSlot itself. - bidfloorcur: slot.floor_price.map(|_| DEFAULT_CURRENCY.to_string()), - secure: Some(true), // require HTTPS creatives - tagid: Some(slot.id.clone()), - ext: ImpExt { - prebid: PrebidImpExt { - bidder, - storedrequest, - }, - } - .to_ext(), - ..Default::default() - }) - }) - .collect(); - - // Build page URL with debug query params if configured - let page_url = request.publisher.page_url.as_ref().map(|url| { - if let Some(ref params) = self.config.debug_query_params { - append_query_params(url, params) - } else { - url.clone() - } - }); - - // Build user object — populate consent at both OpenRTB 2.6 top-level - // and Prebid ext-based locations (dual placement). - // In cookies_only mode, cookie-sourced consent travels through the - // forwarded Cookie header. Policy-sourced consent has no inbound cookie - // to forward, so carry it in the OpenRTB body instead. - let consent_ctx = request.user.consent.as_ref().filter(|ctx| { - self.config.consent_forwarding.includes_body_consent() - || !matches!(ctx.source, crate::consent::ConsentSource::Cookie) - }); - let raw_tc = consent_ctx.and_then(|c| c.raw_tc_string.clone()); - let user = Some(User { - id: request.user.id.clone(), - // OpenRTB 2.6 top-level consent field - consent: raw_tc.clone(), - ext: UserExt { - // Prebid ext-based consent field - consent: raw_tc, - consented_providers_settings: consent_ctx - .and_then(|c| c.raw_ac_string.as_ref()) - .map(|ac| ConsentedProvidersSettings { - consented_providers: Some(ac.clone()), - }), - // EIDs resolved from the KV identity graph and consent-gated - // in `handle_auction` via `gate_eids_by_consent`. - eids: request.user.eids.clone(), - } - .to_ext(), - ..Default::default() - }); - - // Extract DNT header and Accept-Language from the original request - let dnt = context - .request - .headers() - .get("DNT") - .and_then(|value| value.to_str().ok()) - .and_then(|value| { - if value.trim() == "1" { - Some(true) - } else { - None - } - }); - - let language = context - .request - .headers() - .get(header::ACCEPT_LANGUAGE) - .and_then(|value| value.to_str().ok()) - .and_then(|v| { - // Extract the primary ISO-639 language tag (e.g., "en" from - // "en-US,en;q=0.9"). Strip the region subtag so bidders get a - // normalised two-letter code that maximises match quality. - v.split(',') - .next() - .and_then(|tag| tag.split(';').next()) - .map(|tag| { - tag.split('-') - .next() - .expect("should have at least one split segment") - .trim() - .to_string() - }) - .filter(|s| !s.is_empty()) - }); - - // Build device object with user-agent, client IP, geo, DNT, and language. - // Forwarding the real client IP is critical: without it PBS infers the - // IP from the incoming connection (a data-center / edge IP), causing - // bidders like PubMatic to filter the traffic as non-human. - // - // When `request.device` is `None` we still construct a minimal `Device` - // if DNT or language were extracted from HTTP headers, so those signals - // are never silently discarded. - let device = request - .device - .as_ref() - .map(|d| Device { - ua: d.user_agent.clone(), - ip: d.ip.clone(), - geo: d.geo.as_ref().map(|geo| Geo { - country: Some(geo.country.clone()), - city: Some(geo.city.clone()), - region: geo.region.clone(), - lat: Some(geo.latitude), - lon: Some(geo.longitude), - // DMA/metro code: convert i64 to string for OpenRTB. - // Fastly returns 0 for "no metro code"; negative values - // are not realistic for DMA codes so the > 0 guard is - // sufficient. - metro: if geo.metro_code > 0 { - Some(geo.metro_code.to_string()) - } else { - None - }, - r#type: Some(2), - ..Default::default() - }), - dnt, - // Clone needed: `language` is also used in the `or_else` - // fallback below when `request.device` is `None`. - language: language.clone(), - ..Default::default() - }) - .or_else(|| { - if dnt.is_some() || language.is_some() { - Some(Device { - dnt, - language, - ..Default::default() - }) - } else { - None - } - }); - - // Build regs object from ConsentContext, populating both OpenRTB 2.6 - // top-level fields and `regs.ext` for Prebid Server compatibility. - let regs = Self::build_regs(consent_ctx); - - // Build ext object - let (version, signature, kid, ts) = signer - .map(|(s, sig, params)| { - ( - Some(SIGNING_VERSION.to_string()), - Some(sig), - Some(s.kid.clone()), - Some(params.timestamp), - ) - }) - .unwrap_or((None, None, None, None)); - - let debug_enabled = self.config.debug; - - let ext = RequestExt { - prebid: Some(PrebidExt { - debug: debug_enabled.then_some(true), - returnallbidstatus: debug_enabled.then_some(true), - }), - trusted_server: Some(TrustedServerExt { - version, - signature, - kid, - request_host: Some(request_info.host), - request_scheme: Some(request_info.scheme), - ts, - }), - } - .to_ext(); - - // Extract Referer header for site.ref - let referer = context - .request - .headers() - .get(header::REFERER) - .and_then(|value| value.to_str().ok()) - .map(std::string::ToString::to_string); - - // Advertise the effective auction budget, not the raw provider config: - // the orchestrator caps `context.timeout_ms` to the remaining auction - // budget, and the edge backend stops waiting after that long. Telling - // PBS it has more time than the edge will wait turns partial bids into - // edge timeouts. - let tmax = to_openrtb_i32(context.timeout_ms, "tmax", "request"); - - let request = OpenRtbRequest { - id: Some(request.id.clone()), - imp: imps, - site: Some(Site { - domain: Some(request.publisher.domain.clone()), - page: page_url, - r#ref: referer, - publisher: Some(Publisher { - domain: Some(request.publisher.domain.clone()), - ..Default::default() - }), - ..Default::default() - }), - user, - device, - regs, - test: self.config.test_mode.then_some(true), - tmax, - cur: vec![DEFAULT_CURRENCY.to_string()], - ext, - ..Default::default() - }; - - PrebidRequestBuild { - request, - disposition, - } - } - - /// Convert an auction request to `OpenRTB` format with all enrichments. - #[cfg(test)] - fn to_openrtb( - &self, - request: &AuctionRequest, - context: &AuctionContext<'_>, - signer: Option<(&RequestSigner, String, &SigningParams)>, - request_info: RequestInfo, - ) -> OpenRtbRequest { - self.build_openrtb(request, context, signer, request_info) - .request - } - - /// Builds the `regs` object from a [`ConsentContext`]. - /// - /// Populates consent fields at **both** `OpenRTB` 2.6 top-level locations - /// and the `regs.ext.*` locations that Prebid Server reads today. - /// - /// Returns [`None`] if no consent-relevant data is present (avoids sending - /// an empty `regs` object to Prebid Server). - fn build_regs(consent_ctx: Option<&crate::consent::ConsentContext>) -> Option { - let ctx = consent_ctx?; - - let has_data = ctx.gdpr_applies - || ctx.raw_us_privacy.is_some() - || ctx.raw_gpp_string.is_some() - || ctx.gpp_section_ids.is_some() - || ctx.gpc; - - if !has_data { - return None; - } - - // Use jurisdiction to inform the GDPR flag: if we know the user is in - // a GDPR jurisdiction, signal that even without a TCF string (e.g., - // GPC-only requests from EU users). When jurisdiction is unknown and - // no TCF signal exists, omit the field rather than falsely signaling - // "GDPR does not apply." - let in_gdpr_jurisdiction = matches!( - ctx.jurisdiction, - crate::consent::jurisdiction::Jurisdiction::Gdpr - ); - let gdpr = if ctx.gdpr_applies || in_gdpr_jurisdiction { - Some(true) - } else if matches!( - ctx.jurisdiction, - crate::consent::jurisdiction::Jurisdiction::Unknown - ) { - None - } else { - Some(false) - }; - - // Dual-placement: OpenRTB 2.6 top-level fields AND `regs.ext` for - // backward compatibility with older exchanges. Extract top-level - // values first, then clone once into the ext to avoid extra copies. - let us_privacy = ctx.raw_us_privacy.clone(); - let gpp = ctx.raw_gpp_string.clone(); - let gpp_sid = ctx.gpp_section_ids.clone(); - - // RegsExt uses u8 for GDPR (Prebid convention) while the top-level - // Regs uses bool (OpenRTB proto). Map accordingly. - let gdpr_u8 = gdpr.map(u8::from); - let gpp_sid_u16 = gpp_sid.clone(); - let ext = RegsExt { - gdpr: gdpr_u8, - us_privacy: us_privacy.clone(), - gpp: gpp.clone(), - gpp_sid: gpp_sid_u16, - }; - - Some(Regs { - coppa: None, - gdpr, - us_privacy, - gpp, - gpp_sid: gpp_sid - .map(|ids| ids.into_iter().map(i32::from).collect()) - .unwrap_or_default(), - ext: ext.to_ext(), - }) - } - - /// Parse `OpenRTB` response into auction response. - fn parse_openrtb_response(&self, json: &Json, response_time_ms: u64) -> AuctionResponse { - let mut bids = Vec::new(); - - if let Some(seatbids) = json.get("seatbid").and_then(|v| v.as_array()) { - for seatbid in seatbids { - let seat = seatbid - .get("seat") - .and_then(|v| v.as_str()) - .unwrap_or("unknown"); - - if let Some(bid_array) = seatbid.get("bid").and_then(|v| v.as_array()) { - for bid_obj in bid_array { - match self.parse_bid(bid_obj, seat) { - Ok(bid) => bids.push(bid), - Err(()) => { - let impid = bid_obj - .get("impid") - .and_then(|v| v.as_str()) - .unwrap_or(""); - log::warn!( - "Prebid: failed to parse bid from seat '{seat}' for imp '{impid}'" - ); - } - } - } - } - } - } - - if bids.is_empty() { - AuctionResponse::no_bid(PREBID_INTEGRATION_ID, response_time_ms) - } else { - AuctionResponse::success(PREBID_INTEGRATION_ID, bids, response_time_ms) - } - } - - /// Enrich an [`AuctionResponse`] with metadata extracted from the Prebid - /// Server response `ext`. - /// - /// Always-on fields: `responsetimemillis`, `errors`, `warnings`. - /// Debug-only fields (gated on `config.debug`): `debug`, `bidstatus`. - fn enrich_response_metadata( - &self, - response_json: &Json, - auction_response: &mut AuctionResponse, - ) { - let ext = response_json.get("ext"); - - // Always attach per-bidder timing and diagnostics. - if let Some(rtm) = ext.and_then(|e| e.get("responsetimemillis")) { - auction_response - .metadata - .insert("responsetimemillis".to_string(), rtm.clone()); - } - if let Some(errors) = ext.and_then(|e| e.get("errors")) { - auction_response - .metadata - .insert("errors".to_string(), errors.clone()); - } - if let Some(warnings) = ext.and_then(|e| e.get("warnings")) { - auction_response - .metadata - .insert("warnings".to_string(), warnings.clone()); - } - - // When debug is enabled, surface httpcalls, resolvedrequest, and - // per-bid status from the Prebid Server response. - if self.config.debug { - if let Some(debug) = ext.and_then(|e| e.get("debug")) { - auction_response - .metadata - .insert("debug".to_string(), debug.clone()); - } - if let Some(bidstatus) = ext - .and_then(|e| e.get("prebid")) - .and_then(|p| p.get("bidstatus")) - { - auction_response - .metadata - .insert("bidstatus".to_string(), bidstatus.clone()); - } - } - } - - async fn parse_response_inner( - &self, - response: PlatformResponse, - response_time_ms: u64, - auction_id: Option<&str>, - ) -> Result> { - let response = response.response; - let status = response.status(); - let content_type = response - .headers() - .get(header::CONTENT_TYPE) - .and_then(|value| value.to_str().ok()) - .map(str::to_owned); - - // Parse response — collect_response_bounded caps memory from misbehaving providers. - let body_bytes = collect_response_bounded( - response.into_body(), - UPSTREAM_RTB_MAX_RESPONSE_BYTES, - "prebid", - ) - .await - .change_context(TrustedServerError::Prebid { - message: "Failed to read Prebid response body".to_string(), - }); - - if !status.is_success() { - let auction_id = auction_id.unwrap_or(""); - log::warn!("Prebid auction {auction_id:?} returned non-success status: {status}"); - - let body_bytes = match body_bytes { - Ok(body_bytes) => Some(body_bytes), - Err(error) => { - log::warn!( - "Prebid auction {auction_id:?} failed to read non-success response body: {error:?}" - ); - None - } - }; - - if self.config.debug - && let Some(body_bytes) = body_bytes.as_deref() - { - match prebid_body_preview(body_bytes) { - Some(preview) => { - let truncation = if preview.truncated { - " (truncated)" - } else { - "" - }; - log::warn!( - "Prebid auction {auction_id:?} error response body preview{truncation}: {}", - preview.text - ); - } - None => log::warn!( - "Prebid auction {auction_id:?} returned an empty error response body" - ), - } - } - - let status_code = status.as_u16(); - let mut auction_response = - AuctionResponse::error(PREBID_INTEGRATION_ID, response_time_ms) - .with_metadata("error_type", serde_json::json!(ERROR_TYPE_HTTP_STATUS)) - .with_metadata("http_status", serde_json::json!(status_code)) - .with_metadata( - "message", - serde_json::json!(format!("Prebid Server returned HTTP {status_code}")), - ); - - let upstream_message = if self.config.debug { - body_bytes.as_deref().and_then(|body_bytes| { - extract_prebid_error_message(body_bytes, content_type.as_deref()) - }) - } else { - None - }; - if let Some(message) = upstream_message { - auction_response.metadata.insert( - "upstream_message".to_string(), - serde_json::json!(message.text), - ); - auction_response.metadata.insert( - "upstream_message_truncated".to_string(), - serde_json::json!(message.truncated), - ); - } - - return Ok(auction_response); - } - - let body_bytes = body_bytes?; - let response_json: Json = - serde_json::from_slice(&body_bytes).change_context(TrustedServerError::Prebid { - message: "Failed to parse Prebid response".to_string(), - })?; - - // Log the full response body when debug is enabled to surface - // ext.debug.httpcalls, resolvedrequest, bidstatus, errors, etc. - if self.config.debug && log::log_enabled!(log::Level::Trace) { - match serde_json::to_string_pretty(&response_json) { - Ok(json) => log::trace!("Prebid OpenRTB response:\n{json}"), - Err(e) => { - log::warn!("Prebid: failed to serialize response for logging: {e}"); - } - } - } - - let mut auction_response = self.parse_openrtb_response(&response_json, response_time_ms); - self.enrich_response_metadata(&response_json, &mut auction_response); - - log::info!( - "Prebid returned {} bids in {}ms", - auction_response.bids.len(), - response_time_ms - ); - - Ok(auction_response) - } - - fn should_suppress_bid_notifications(&self, bidder: &str) -> bool { - self.config.suppress_nurl - || self - .config - .suppress_nurl_bidders - .iter() - .any(|suppressed_bidder| suppressed_bidder == bidder) - } - - /// Parse a single bid from `OpenRTB` response. - fn parse_bid(&self, bid_obj: &Json, seat: &str) -> Result { - let slot_id = bid_obj - .get("impid") - .and_then(|v| v.as_str()) - .ok_or(())? - .to_string(); - - let price = bid_obj - .get("price") - .and_then(serde_json::Value::as_f64) - .ok_or(())?; - - let creative = bid_obj - .get("adm") - .and_then(|v| v.as_str()) - .map(std::string::ToString::to_string); - - let width = bid_obj - .get("w") - .and_then(serde_json::Value::as_u64) - .and_then(|v| u32::try_from(v).ok()) - .unwrap_or(0); - let height = bid_obj - .get("h") - .and_then(serde_json::Value::as_u64) - .and_then(|v| u32::try_from(v).ok()) - .unwrap_or(0); - - let suppress_bid_notifications = self.should_suppress_bid_notifications(seat); - let nurl = if suppress_bid_notifications { - None - } else { - bid_obj - .get("nurl") - .and_then(|v| v.as_str()) - .map(std::string::ToString::to_string) - }; - - let burl = if suppress_bid_notifications { - None - } else { - bid_obj - .get("burl") - .and_then(|v| v.as_str()) - .map(std::string::ToString::to_string) - }; - - // `adid` is the creative/ad identifier. The OpenRTB `id` is the bid ID, - // not an ad ID, so it is not used as a fallback: surfacing it as `ad_id` - // (which is exposed raw in the debug bid) would mislead any consumer that - // treats `ad_id` as a creative identifier. Absent `adid`, `ad_id` is None. - // The bid ID is carried separately in `bid_id` instead. An empty `id` is - // treated as absent — a blank hb_adid is falsey on the page, so it would - // be no better than omitting the key. - let bid_id = bid_obj - .get("id") - .and_then(|v| v.as_str()) - .filter(|id| !id.is_empty()) - .map(String::from); - let ad_id = bid_obj - .get("adid") - .and_then(|v| v.as_str()) - .map(String::from); - let creative_id = bid_obj - .get("crid") - .and_then(|v| v.as_str()) - .map(String::from); - - let adomain = bid_obj - .get("adomain") - .and_then(|v| v.as_array()) - .map(|arr| { - arr.iter() - .filter_map(|v| v.as_str().map(std::string::ToString::to_string)) - .collect() - }); - - // Extract PBS Cache coordinates from ext.prebid.cache.bids - let cache_entry = bid_obj - .get("ext") - .and_then(|e| e.get("prebid")) - .and_then(|p| p.get("cache")) - .and_then(|c| c.get("bids")); - - let cache_id = cache_entry - .and_then(|c| c.get("cacheId")) - .and_then(|v| v.as_str()) - .map(String::from); - - let (cache_host, cache_path) = cache_entry - .and_then(|c| c.get("url")) - .and_then(|v| v.as_str()) - .and_then(|url_str| { - ParsedUrl::parse(url_str) - .map_err(|e| log::debug!("PBS cache URL parse failed: {}", e)) - .ok() - }) - .map(|u| { - let host = u.host_str().map(String::from); - // path() returns "/" for root — only use if non-trivial - let path = u.path().to_string(); - let path = if path.is_empty() || path == "/" { - None - } else { - Some(path) - }; - (host, path) - }) - .unwrap_or((None, None)); - - // Guard: if we extracted a cache UUID but couldn't extract the host, - // the bid will have hb_adid set but no endpoint to fetch from — creative will fail. - if cache_id.is_some() && cache_host.is_none() { - log::warn!( - "PBS bid has cache UUID but cache URL could not be parsed — \ - creative will fail to render for slot '{}'", - slot_id - ); - } - - Ok(AuctionBid { - slot_id, - price: Some(price), // Prebid provides decoded prices - currency: DEFAULT_CURRENCY.to_string(), - creative, - adomain, - bidder: seat.to_string(), - returned_seat: None, - width, - height, - nurl, - burl, - bid_id, - ad_id, - creative_id, - renderer: None, - cache_id, - cache_host, - cache_path, - metadata: std::collections::HashMap::new(), - }) - } -} - -#[cfg(test)] -#[async_trait(?Send)] -impl AuctionProvider for PrebidAuctionProvider { - fn provider_name(&self) -> &str { - PREBID_INTEGRATION_ID - } - - async fn request_bids( - &self, - request: &AuctionRequest, - context: &AuctionContext<'_>, - ) -> Result> { - let request_start = web_time::Instant::now(); - log::info!("Prebid: requesting bids for {} slots", request.slots.len()); - - // `ext.trusted_server.request_host` must be the publisher's own domain - // (matching `site.domain`/`publisher.domain` below), not the raw edge - // `Host` header of the incoming `/auction` request. On the SSAT proxy - // path the browser calls `/auction` against the trusted-server edge - // domain (e.g. `ts.example.com`), which is not what PBS's - // `trusted_server` verification module expects — see the "Host header - // value expected by the receiving service" doc on `SigningParams`. - // `scheme` is canonical `"https"` rather than detected from the - // incoming request: the publisher origin is always https in - // production, and PBS verification is expected to match on that, - // not on the edge's local scheme (e.g. http under `fastly compute - // serve`). - let request_info = RequestInfo { - host: request.publisher.domain.clone(), - scheme: "https".to_owned(), - }; - - // Create signer and compute signature if request signing is enabled - let signer_with_signature = - if let Some(request_signing_config) = &context.settings.request_signing { - if request_signing_config.enabled { - let signer = RequestSigner::from_services(context.services)?; - let params = SigningParams::new( - request.id.clone(), - request_info.host.clone(), - request_info.scheme.clone(), - ); - let signature = signer.sign_request(¶ms)?; - Some((signer, signature, params)) - } else { - None - } - } else { - None - }; - - let PrebidRequestBuild { - request: openrtb, - disposition, - } = self.build_openrtb( - request, - context, - signer_with_signature - .as_ref() - .map(|(signer, signature, params)| (signer, signature.clone(), params)), - request_info, - ); - - if openrtb.imp.is_empty() { - let all_slots_are_aps_only = !request.slots.is_empty() - && disposition.invalid == 0 - && disposition.aps_only == request.slots.len(); - if all_slots_are_aps_only { - log::info!( - "Prebid: returning no-bid because all {} valid impressions are APS-only", - disposition.aps_only - ); - return Ok(ProviderRequestOutcome::Immediate(AuctionResponse::no_bid( - PREBID_INTEGRATION_ID, - request_start.elapsed().as_millis() as u64, - ))); - } - - log::info!("Prebid: skipping request — no valid impressions after filtering"); - return Err(Report::new(TrustedServerError::Prebid { - message: "No valid impressions after filtering".to_string(), - })); - } - - // Log the outgoing OpenRTB request for debugging. - if log::log_enabled!(log::Level::Debug) { - match serde_json::to_string_pretty(&openrtb) { - Ok(json) => log::debug!( - "Prebid OpenRTB request to {}:\n{}", - self.auction_endpoint_url(), - json - ), - Err(e) => { - log::warn!("Prebid: failed to serialize OpenRTB request for logging: {e}") - } - } - } - - // Create HTTP request - let mut pbs_req = http::Request::builder() - .method(http::Method::POST) - .uri(self.auction_endpoint_url()) - .body(EdgeBody::empty()) - .change_context(TrustedServerError::Prebid { - message: "Failed to build Prebid request".to_string(), - })?; - copy_request_headers( - context.request, - &mut pbs_req, - self.config.consent_forwarding, - context.services.client_info().client_ip, - ); - - let pbs_body = serde_json::to_vec(&openrtb).change_context(TrustedServerError::Prebid { - message: "Failed to serialize Prebid request body".to_string(), - })?; - pbs_req.headers_mut().insert( - header::CONTENT_TYPE, - HeaderValue::from_static("application/json"), - ); - *pbs_req.body_mut() = EdgeBody::from(pbs_body); - - // Uses context.timeout_ms (auction-scoped) rather than the 15 s fixed - // timeout in ensure_integration_backend, which is for proxy endpoints. - // Send request asynchronously with auction-scoped timeout - let backend_name = ensure_integration_backend_with_timeout( - context.services, - &self.config.server_url, - "prebid", - Duration::from_millis(u64::from(context.timeout_ms)), - ) - .change_context(TrustedServerError::Auction { - message: format!( - "Failed to resolve backend for Prebid Server endpoint: {}", - self.config.server_url - ), - })?; - let pending = context - .services - .http_client() - .send_async(PlatformHttpRequest::new(pbs_req, backend_name)) - .await - .change_context(TrustedServerError::Prebid { - message: "Failed to send async request to Prebid Server".to_string(), - })?; - - Ok(ProviderRequestOutcome::pending(pending)) - } - - async fn parse_response( - &self, - response: PlatformResponse, - response_time_ms: u64, - ) -> Result> { - self.parse_response_inner(response, response_time_ms, None) - .await - } - - async fn parse_response_with_context( - &self, - response: PlatformResponse, - response_time_ms: u64, - request: &AuctionRequest, - _context: &AuctionContext<'_>, - ) -> Result> { - self.parse_response_inner(response, response_time_ms, Some(request.id.as_str())) - .await - } - - fn supports_media_type(&self, media_type: &MediaType) -> bool { - matches!(media_type, MediaType::Banner) - } - - fn timeout_ms(&self) -> u32 { - self.config.timeout_ms - } - - fn is_enabled(&self) -> bool { - self.config.enabled - } - - fn backend_name(&self, services: &RuntimeServices, timeout_ms: u32) -> Option { - predict_integration_backend_name( - services, - &self.config.server_url, - PREBID_INTEGRATION_ID, - Duration::from_millis(u64::from(timeout_ms)), - ) - .inspect_err(|e| { - log::error!( - "Failed to predict backend name for Prebid server URL '{}': {e:?}", - self.config.server_url - ); - }) - .ok() - } -} - -// ============================================================================ -// Provider Auto-Registration -// ============================================================================ - -/// Auto-register Prebid provider based on settings configuration. -/// -/// This function checks the settings for Prebid configuration and returns -/// the provider if enabled. -/// -/// # Errors -/// -/// Returns an error when the Prebid provider is enabled with invalid -/// configuration. -#[cfg(test)] -pub fn register_auction_provider( - settings: &Settings, -) -> Result>, Report> { - let Some(integration) = build(settings)? else { - log::info!("Prebid auction provider not registered: integration not found or disabled"); - return Ok(Vec::new()); - }; - - if let Some(config) = integration.legacy_config.as_ref() { - log::info!( - "Registering Prebid auction provider (server_url={})", - config.server_url - ); - } - if integration.config.debug { - log::warn!( - "Prebid debug mode is ON — debug data (httpcalls, resolvedrequest, \ - bidstatus) will be included in /auction responses" - ); - } - - Ok(vec![Arc::new(integration.auction_provider())]) -} - -#[cfg(test)] -mod tests { - use std::sync::Arc; - - use super::*; - use crate::auction::formats::convert_to_openrtb_response; - use crate::auction::orchestrator::OrchestrationResult; - use crate::auction::plan::{ - AuctionPlanConfig, BidderId, BidderRouteConfig, NotificationConfig, ProviderConfig, - ProviderId, RoutingMode, - }; - use crate::auction::test_support::{ - canonical_parity_auction_request, - create_test_auction_context as shared_test_auction_context, - }; - use crate::auction::types::{ - AdFormat, AdSlot, AuctionContext, AuctionRequest, DeviceInfo, PublisherInfo, UserInfo, - }; - - use crate::consent::{ConsentContext, ConsentSource}; - use crate::geo::GeoInfo; - use crate::html_processor::{HtmlProcessorConfig, create_html_processor}; - use crate::integrations::{ - AttributeRewriteAction, IntegrationDocumentState, IntegrationRegistry, - }; - use crate::platform::test_support::{ - HashMapConfigStore, HashMapSecretStore, NoopConfigStore, NoopGeo, NoopHttpClient, - NoopSecretStore, StubHttpClient, build_services_with_config_secret_and_http_client, - build_services_with_http_client, build_services_with_http_client_and_client_ip, - }; - use crate::platform::{ - ClientInfo, PlatformBackend, PlatformBackendSpec, PlatformError, RuntimeServices, - }; - use crate::settings::Settings; - use crate::streaming_processor::{Compression, PipelineConfig, StreamingPipeline}; - use crate::test_support::tests::create_test_settings; - use base64::engine::general_purpose::STANDARD as TEST_BASE64_STANDARD; - use bytes::Bytes; - use http::Method; - use serde_json::json; - use std::collections::{BTreeMap, HashMap}; - use std::io::Cursor; - use std::str::FromStr as _; - - #[test] - fn external_bundle_sha256_validation_matches_hex_pattern() { - use validator::Validate as _; - - let config = |sha: &str| -> PrebidIntegrationConfig { - serde_json::from_value(serde_json::json!({ - "external_bundle_sha256": sha, - })) - .expect("should deserialize prebid config") - }; - - // Exactly 64 hex digits (either case) passes. - config(&"a".repeat(64)) - .validate() - .expect("64-char lowercase hex sha256 should pass"); - config("ABCDEF0123456789abcdef0123456789ABCDEF0123456789abcdef0123456789") - .validate() - .expect("mixed-case 64-char hex sha256 should pass"); - - // Wrong length or non-hex characters are rejected. - for bad in [ - "a".repeat(63), - "a".repeat(65), - "g".repeat(64), - String::new(), - ] { - config(&bad) - .validate() - .expect_err(&format!("invalid sha256 {bad:?} should be rejected")); - } - } - - fn make_settings() -> Settings { - create_test_settings() - } - - #[test] - fn injected_prebid_config_escapes_every_less_than_sign() { - let config_json = serialize_injected_prebid_config(&json!({ - "accountId": "x C[endpoints::convert_tsjs_to_auction_request] C --> D[routing::route_auction] D --> E[provider::GenericOpenRtbProvider builds requests] - E --> F[orchestrator::AuctionOrchestrator dispatches providers] - F --> G[Provider responses are normalized] - G --> H{Mediator configured?} - H -->|Yes| I[Mediator selects bids] + E --> F[orchestrator::AuctionOrchestrator dispatches demand sources] + F --> G[Demand responses are normalized] + G --> H{Ad server configured?} + H -->|Yes| I[Ad server selects bids] H -->|No| J[Orchestrator ranks bids locally] I --> K[formats::convert_to_openrtb_response] J --> K @@ -27,80 +29,80 @@ flowchart TB ``` Each adapter owns transport routing in its `app.rs`. Core request handling stays -in `auction::endpoints`, so no provider or profile depends on Fastly types. +in `auction::endpoints`, so no demand implementation depends on Fastly types. `handle_auction` performs these steps: 1. Enforce the body limit and parse the Trusted Server ad-unit request. -2. Apply the disabled-auction and consent gates before provider work. +2. Apply the disabled-auction and consent gates before any demand work. 3. Consume the request's existing EC and consent context. The endpoint does not generate an EC ID. 4. Convert the request with `convert_tsjs_to_auction_request`. 5. Route slots and bidder params through the compiled `AuctionPlan`. -6. Run the plan-backed orchestrator and optional mediator. +6. Run the plan-backed orchestrator and the optional ad server. 7. Build the OpenRTB response with `convert_to_openrtb_response`. ## Configuration boundary `auction::compile_auction_plan` is the single settings-to-plan boundary used by -startup and operator validation. It validates: - -- provider IDs, protocols, profiles, endpoints, timeouts, and routing modes; -- bidder-to-provider ownership; -- profile-specific configuration; -- mediator and request-signing references; -- bounded configuration values. +startup and operator validation. It reads `[demand]`, `[ad-server]` and +`[auction.bidders]` and validates: + +- demand source and ad server names, which must be snake_case +- a settings table its type's `modules` or `module` does not select +- an `implementation` no builder registered, naming the ones that are +- endpoints, which must be HTTPS or HTTP to a loopback host, with no + credentials or fragment +- timeouts, routing modes and notification bounds +- the settings each implementation accepts, since each one rejects unknown keys +- bidder-to-demand ownership, and +- request-signing structure. Adapters then call `AuctionPlan::validate_for_target` for backend naming, fan-out support, and target resource limits. -A plan-backed orchestrator contains generic OpenRTB providers compiled from the -plan. `AuctionOrchestrator::register_provider` and the old concrete Prebid +A plan-backed orchestrator contains generic OpenRTB demand sources compiled from +the plan. `AuctionOrchestrator::register_provider` and the old concrete Prebid provider remain test-only parity code. They are not extension APIs. ## Routing `routing::route_auction` normalizes the browser `trustedServer` envelope and -produces one `ProviderAuctionInput` per provider. +produces one `ProviderAuctionInput` per demand source. - `explicit` sends a slot only when it has bidder demand assigned to that - provider, or trusted stored-request demand where the profile supports it. + source, or trusted stored-request demand where the implementation supports it. - `all_eligible` sends every compatible banner slot without copying another - provider's bidder params. -- `prebid-server` requires `explicit`. PBS rejects impressions that have neither + source's bidder params. +- `auction.prebid-server` requires `explicit`. PBS rejects impressions that have neither bidder demand nor a stored-request reference. - APS normally uses `all_eligible` because APS participates across eligible inventory without browser bidder params. -Each `[auction.bidders.]` route has one provider owner. Unlisted page -bidders remain browser demand. +Each `[auction.bidders.]` route has one owner, named by its +`[demand]` table name. Unlisted page bidders remain browser demand. -## Provider execution +## Demand execution `provider::GenericOpenRtbProvider` owns the shared transport path for the -`standard`, `prebid-server`, and `aps` profiles. Profiles receive routed and -privacy-approved facts, not the raw inbound request. - -The orchestrator launches all eligible providers before collecting responses. -It uses adapter `PlatformHttpClient` handles and predicted backend names for -correlation. Provider launch, transport, HTTP, parse, and admission failures are -provider-local when another provider can continue. +`auction-protocol.openrtb`, `auction.prebid-server` and `auction.aps` implementations. An implementation receives +routed and privacy-approved facts, not the raw inbound request. -When no mediator is configured, the orchestrator selects the highest decoded -CPM per slot and applies floors locally. When a mediator is configured, it sends -normalized provider responses to the separately registered mediator and falls -back to local ranking when mediation cannot run. +The orchestrator launches all eligible demand sources before collecting +responses. It uses adapter `PlatformHttpClient` handles and predicted backend +names for correlation. Launch, transport, HTTP, parse and admission failures are +local to one demand source when another can continue. ## Response admission -Providers normalize successful upstream bids into `auction::types::Bid`. +Demand sources normalize successful upstream bids into `auction::types::Bid`. Admission checks keep malformed or unrequested bids out of ranking. Aggregate metadata reports bounded rejection counts without retaining raw upstream bid payloads. Notification suppression runs after normalization and matches exact returned -OpenRTB seats. Provider response identity uses the configured provider ID, such -as `pbs-main`. +OpenRTB seats. Response identity uses the configured demand source name, such +as `pbs_main`. ## Creative delivery @@ -121,55 +123,65 @@ response. enabled = true timeout_ms = 2000 -[auction.providers.pbs-main] -protocol = "openrtb-2.6" -profile = "prebid-server" +[demand] +modules = ["pbs_main", "aps_main"] + +[demand.pbs_main] +implementation = "auction.prebid-server" endpoint = "https://prebid.example.com/openrtb2/auction" timeout_ms = 900 routing = "explicit" - -[auction.providers.pbs-main.profile_config] debug = false test_mode = false consent_forwarding = "both" -[auction.providers.pbs-main.notifications] +[demand.pbs_main.notifications] suppress_all = false suppress_seats = ["example-seat"] -[auction.bidders.example-server] -provider = "pbs-main" - -[auction.providers.aps-main] -protocol = "openrtb-2.6" -profile = "aps" +[demand.aps_main] +implementation = "auction.aps" endpoint = "https://aps.example.com/e/pb/bid" routing = "all_eligible" -profile_config = { account_id = "example-account" } +account_id = "example-account" + +[auction.bidders.example-server] +module = "pbs_main" + +[ad-server] +module = "mock" + +[ad-server.mock] +endpoint = "https://adserver.example.com/decide" +timeout_ms = 500 ``` -Provider endpoints must be absolute HTTPS URLs. Replace all example values -before enabling an auction. +Every demand table carries an `implementation` line naming the +implementation by its module path, which is how two Prebid Servers run side +by side under names of their own, and `[ad-server]` writes `ad-server.mock` +as `mock`. Endpoints must be HTTPS, or HTTP to `127.0.0.1`, `::1` or +`localhost`. Replace all example values before enabling an auction. ## Code map - `mod.rs` compiles plans and builds the shared orchestrator. - `endpoints.rs` handles `POST /auction` and converts the browser request. - `plan.rs` owns plan validation and target capability checks. -- `profile.rs` owns typed OpenRTB profile configuration. -- `routing.rs` assigns slots and bidder params to providers. +- `demand.rs` owns the demand and ad server implementation seam. +- `routing.rs` assigns slots and bidder params to demand sources. - `openrtb.rs` builds shared requests and parses standard responses. -- `provider.rs` runs plan-backed provider requests and profile-specific parsing. -- `orchestrator.rs` owns fan-out, deadlines, mediation, and local ranking. +- `provider.rs` runs plan-backed demand requests and per-implementation parsing. +- `orchestrator.rs` owns fan-out, deadlines, the ad server call, and local + ranking. - `formats.rs` builds direct endpoint responses and processes creatives. - `types.rs` contains normalized auction request, response, slot, and bid types. ## Testing Use `compile_auction_plan` in tests, then construct the orchestrator and -integration registry from the same `Arc`. Profile tests should -cover typed configuration, exact request output, response admission, routing, -provider-local failures, and target validation. +integration registry from the same `Arc`. Tests for one +implementation should cover its typed settings, exact request output, response +admission, routing, local failures, and target validation. Run target-matched aliases rather than bare workspace tests: diff --git a/crates/trusted-server-core/src/auction/context.rs b/crates/trusted-server-core/src/auction/context.rs index f7fd782942..467801726f 100644 --- a/crates/trusted-server-core/src/auction/context.rs +++ b/crates/trusted-server-core/src/auction/context.rs @@ -1,6 +1,6 @@ //! Context query-parameter forwarding for auction providers. //! -//! Provides a config-driven mechanism for ad-server / mediator providers to +//! Provides a config-driven mechanism for ad server providers to //! forward integration-supplied data (e.g. audience segments) as URL query //! parameters without hard-coding integration-specific knowledge. @@ -26,12 +26,12 @@ pub enum ContextValue { /// Mapping from auction-request context keys to query-parameter names. /// -/// Used by ad-server / mediator providers to forward integration-supplied data +/// Used by ad server providers to forward integration-supplied data /// (e.g. audience segments) as URL query parameters without hard-coding /// integration-specific knowledge. /// /// ```toml -/// [integrations.adserver_mock.context_query_params] +/// [ad-server.mock.context_query_params] /// permutive_segments = "permutive" /// lockr_ids = "lockr" /// ``` diff --git a/crates/trusted-server-core/src/auction/demand.rs b/crates/trusted-server-core/src/auction/demand.rs new file mode 100644 index 0000000000..c9800bb36c --- /dev/null +++ b/crates/trusted-server-core/src/auction/demand.rs @@ -0,0 +1,290 @@ +//! Demand and ad server implementations, the seam through which an integration +//! supplies the providers an auction calls. +//! +//! A deployment selects its demand sources with `[demand] modules = [...]` and +//! its ad server with `[ad-server] module = "..."`. Each selected name resolves +//! to an implementation that an integration builder registers, so core names no +//! vendor. The common `OpenRTB` driver builds every standard request field, sends +//! the request and enforces privacy. An implementation decides only the choices +//! this module exposes, which are its field policy, its own extensions, its +//! outbound headers and how its responses are read. + +use core::any::Any; +use std::net::IpAddr; +use std::sync::Arc; + +use async_trait::async_trait; +use error_stack::Report; +use http::HeaderMap; +use serde_json::{Map, Value}; +use url::Url; + +use crate::consent::ConsentContext; +use crate::error::TrustedServerError; +use crate::platform::PlatformResponse; + +use super::provider::AuctionProvider; +pub use super::routing::{ProviderAuctionInput, ProviderSlotInput, TransportHeaders}; +use super::types::AuctionResponse; + +/// The longest primary `Accept-Language` tag a conservative source receives. +/// +/// A source that sets no limit receives the whole primary tag. +pub const CONSERVATIVE_LANGUAGE_MAX_BYTES: usize = 8; + +/// Where a demand source's timeout comes from when its table sets none. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DemandTimeoutDefault { + /// The whole-auction timeout. + Auction, + /// A fixed timeout, in milliseconds. + Fixed(u32), +} + +/// How the `regs` object is derived from the consent the auction admitted. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum RegsPolicy { + /// No `regs` without consent data, and GDPR set from the applicability bit + /// or from a GDPR jurisdiction. + #[default] + Jurisdiction, + /// `regs` for any admitted consent, and GDPR set from the applicability bit + /// alone. + ApplicabilityBit, +} + +/// The standard `OpenRTB` field choices one demand source makes. +/// +/// The common driver constructs every standard field and reads this policy +/// while it does. A policy can leave data out of what central privacy +/// enforcement approved, and it cannot add data that enforcement removed. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct DemandFieldPolicy { + /// Send the slot id as `imp.tagid`. + pub imp_tagid: bool, + /// Copy the first banner format into `banner.w` and `banner.h`, and send + /// `banner.topframe` as not in the top frame. + pub primary_banner_size: bool, + /// Send the accepted `Referer` header as `site.ref`. + pub site_ref: bool, + /// Send precise latitude and longitude with the coarse geo. + pub precise_geo: bool, + /// Send Google Additional Consent as `user.ext.ConsentedProvidersSettings`. + pub additional_consent: bool, + /// The longest primary `Accept-Language` tag sent, or `None` for no limit. + pub language_max_bytes: Option, + /// How `regs` is derived from consent. + pub regs: RegsPolicy, + /// Mark the request as a test request. + pub test: bool, + /// Keep the Trusted Server host and scheme in `ext.trusted_server` when the + /// request is not signed. + pub unsigned_request_identity: bool, + /// Send `Accept: application/json`. + pub accept_json: bool, + /// Treat routed bidder parameters as request data rather than reporting + /// them as parameters the source ignored. + pub consumes_bidder_params: bool, +} + +/// Facts an implementation may use when it adjusts the outbound request. +#[derive(Debug, Clone, Copy)] +pub struct DemandTransport<'a> { + /// The request headers admitted for forwarding. + pub headers: &'a TransportHeaders, + /// The client address the platform attested, never one a client supplied. + pub attested_client_ip: Option, +} + +/// Facts an implementation reads while it parses a response. +#[derive(Clone, Copy)] +pub struct DemandResponse<'a> { + /// The configured name of the demand source. + pub provider_id: &'a str, + /// The canonical endpoint the request was sent to. + pub endpoint: &'a str, + /// The routed input the request was built from. + pub input: &'a ProviderAuctionInput, + /// How long the request took, in milliseconds. + pub response_time_ms: u64, + /// State [`CompiledDemand::capture_request`] returned for this request. + pub captured: Option<&'a (dyn Any + Send + Sync)>, +} + +/// The extension objects an implementation may add to a request the driver +/// built, one for the request and one for each impression. +/// +/// This is the only part of the request an implementation writes. Every +/// standard field is the driver's, chosen through [`DemandFieldPolicy`], so an +/// implementation can add what its exchange needs and cannot change what +/// privacy enforcement decided. The `trusted_server` request extension is the +/// driver's too, and a request that claims it is refused. +pub struct RequestExtensions<'a> { + /// The request-level `ext` object. + pub request: &'a mut Option>, + /// The impressions the driver built, each beside the slot it came from. + pub impressions: Vec>, +} + +/// One impression's `ext` object beside the routed slot it was built from. +pub struct ImpressionExtension<'a> { + /// The routed slot, with its bidder parameters and zone. + pub slot: &'a ProviderSlotInput, + /// The impression's `ext` object. + pub ext: &'a mut Option>, + /// Set by an implementation that has no demand to send for this slot. + /// The driver leaves the impression out of the request, and makes no + /// request at all when every impression is left out. + pub omitted: bool, +} + +/// The request and response behavior of one selected demand source, compiled +/// from its `[demand.]` table at startup. +/// +/// Every method except [`augment_request`](Self::augment_request) and +/// [`parse_response`](Self::parse_response) has a default that suits a source +/// with no special handling. +#[async_trait(?Send)] +pub trait CompiledDemand: Send + Sync { + /// The standard field choices this source makes. + fn field_policy(&self) -> DemandFieldPolicy; + + /// The domain sent as `site.domain` and `site.publisher.domain`. + fn site_domain(&self, publisher_domain: &str) -> String { + publisher_domain.to_owned() + } + + /// The page sent as `site.page`, given the publisher's page and the site + /// domain [`site_domain`](Self::site_domain) chose. + fn site_page(&self, publisher_page: Option<&str>, site_domain: &str) -> Option { + let _ = site_domain; + publisher_page.map(str::to_owned) + } + + /// The consent that may travel in the request body. + fn body_consent<'c>(&self, consent: Option<&'c ConsentContext>) -> Option<&'c ConsentContext> { + consent + } + + /// Adds this source's own request and impression extensions. + /// + /// # Errors + /// + /// Returns an error when an extension cannot be built for this request. + fn augment_request( + &self, + extensions: &mut RequestExtensions<'_>, + input: &ProviderAuctionInput, + ) -> Result<(), Report>; + + /// Adds to the outbound headers before the request is sent, for example to + /// forward admitted headers. The driver has already set `Content-Type`, + /// and `Accept` when the field policy asks for it. + fn prepare_outbound(&self, headers: &mut HeaderMap, transport: DemandTransport<'_>) { + let _ = (headers, transport); + } + + /// Keeps request-local state the response parser needs, taken from the + /// serialized body and the outbound headers as they will be sent, after + /// [`prepare_outbound`](Self::prepare_outbound) has run. + fn capture_request( + &self, + body: &[u8], + headers: &HeaderMap, + ) -> Option> { + let _ = (body, headers); + None + } + + /// Reads the upstream response into an auction response. + /// + /// # Errors + /// + /// Returns an error when the response cannot be read at all. A response + /// that arrived but is not usable is reported as an error response rather + /// than as an `Err`. + async fn parse_response( + &self, + context: DemandResponse<'_>, + response: PlatformResponse, + ) -> Result>; + + /// This compiled source as [`Any`], so the implementation that created it + /// can read its own settings back, for example while it registers routes. + fn as_any(&self) -> &dyn Any; +} + +/// Compiles one demand source from its settings. +/// +/// The map holds the table's own settings, without `implementation` and the +/// common keys the driver reads, which are `endpoint`, `timeout_ms`, `routing` +/// and `notifications`. +pub type CompileDemandFn = + fn(&Map) -> Result, Report>; + +/// Builds an ad server from its configured name and its settings. +/// +/// The map holds the table's own settings without `implementation`, and the +/// name is what the ad server reports itself as in responses and telemetry. +pub type BuildAdServerFn = + fn(&str, &Map) -> Result, Report>; + +/// A demand implementation an integration registers, named by +/// `[demand] modules` or by an `implementation` line. +#[derive(Clone, Copy)] +pub struct DemandImplementation { + /// The implementation's name, its module path. + pub id: &'static str, + /// The timeout a source uses when its table sets none. + pub default_timeout: DemandTimeoutDefault, + /// Whether `routing = "all_eligible"` is allowed. + pub allows_all_eligible: bool, + /// Whether a slot that carries no bidder parameters is still sent, as a + /// stored request, with the slot's zone. + pub serves_stored_requests: bool, + /// Checks and canonicalizes the endpoint after the common checks passed. + pub canonicalize_endpoint: fn(&mut Url) -> Result<(), String>, + /// Compiles one source from its settings. + pub compile: CompileDemandFn, +} + +impl core::fmt::Debug for DemandImplementation { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + formatter + .debug_struct("DemandImplementation") + .field("id", &self.id) + .field("default_timeout", &self.default_timeout) + .field("allows_all_eligible", &self.allows_all_eligible) + .field("serves_stored_requests", &self.serves_stored_requests) + .finish_non_exhaustive() + } +} + +/// Accepts every endpoint the common checks accepted, unchanged. +/// +/// # Errors +/// +/// Never returns an error. +pub fn accept_endpoint(endpoint: &mut Url) -> Result<(), String> { + let _ = endpoint; + Ok(()) +} + +/// An ad server implementation an integration registers, named by +/// `[ad-server] module` or by an `implementation` line. +#[derive(Clone, Copy)] +pub struct AdServerImplementation { + /// The implementation's name, its module path. + pub id: &'static str, + /// Builds the ad server from its configured name and its settings. + pub build: BuildAdServerFn, +} + +impl core::fmt::Debug for AdServerImplementation { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + formatter + .debug_struct("AdServerImplementation") + .field("id", &self.id) + .finish_non_exhaustive() + } +} diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index f364464561..47d90edf69 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -9,8 +9,8 @@ use serde_json::Value as JsonValue; use crate::auction::formats::AdRequest; use crate::auction::orchestrator::OrchestrationResult; -use crate::consent::{consent_allows_server_side_auction, gate_eids_by_consent}; -use crate::constants::COOKIE_TS_EIDS; +use crate::consent::{consent_allows_server_side_auction, gate_eids_by_permissions}; +use crate::constants::COOKIE_EIDS; use crate::cookies::extract_cookie_value; use crate::ec::EcContext; use crate::ec::EcKvSnapshot; @@ -171,10 +171,11 @@ pub async fn handle_auction( // Story 5 middleware contract: auction is a read-only EC route. // It must not generate EC IDs; it only consumes pre-routed context. - // Only forward the EC ID to auction partners when consent allows it. - // Owned so the identity-graph snapshot can be stored back on `ec_context` - // below without holding a borrow of it across the mutation. - let ec_id = if ec_context.ec_allowed() { + // Forward the EC ID to auction partners only when sharing is permitted, + // being storage plus personalized-ad selection, the same pair that gates + // EIDs. Owned so the identity-graph snapshot can be stored back on + // `ec_context` below without holding a borrow of it across the mutation. + let ec_id = if ec_context.ec_sharing_allowed() { ec_context.ec_value().map(str::to_owned) } else { None @@ -211,7 +212,7 @@ pub async fn handle_auction( let empty_result = OrchestrationResult { provider_responses: Vec::new(), - mediator_response: None, + adserver_response: None, winning_bids: HashMap::new(), total_time_ms: 0, metadata: HashMap::new(), @@ -264,7 +265,7 @@ pub async fn handle_auction( let empty_result = OrchestrationResult { provider_responses: Vec::new(), - mediator_response: None, + adserver_response: None, winning_bids: HashMap::new(), total_time_ms: 0, metadata: HashMap::new(), @@ -282,8 +283,9 @@ pub async fn handle_auction( // `ts-eids` cookie so later requests can still forward the browser's // full OpenRTB-style EID structure. // - // Gate this on the same identity-consent condition as the EC ID - // (`ec_id.is_some()`, which is already filtered by `ec_context.ec_allowed()`). + // Gate this on the same identity condition as the EC ID + // (`ec_id.is_some()`, which is already filtered by the sharing pair via + // `ec_context.ec_sharing_allowed()`). // Otherwise a US/GPC or US-Privacy opt-out context — where EC identity use is // denied but a non-personalized auction may still run — could forward // persistent client EIDs from the body/cookie, since `gate_eids_by_consent` @@ -293,7 +295,11 @@ pub async fn handle_auction( let client_eids = if ec_id.is_some() { resolve_client_auction_eids( body.eids.as_ref(), - extract_cookie_value(&http_req, COOKIE_TS_EIDS).as_deref(), + extract_cookie_value( + &http_req, + &services.public_prefix().cookie_name(COOKIE_EIDS), + ) + .as_deref(), ) } else { None @@ -309,8 +315,13 @@ pub async fn handle_auction( // EC and both KV and partner stores are available. Gate the read on a // present registry: without one, `resolve_auction_eids` yields no // server-side EIDs, so the snapshot would be an unused billable KV read. + // The row is read under the owning module's canonical form of the + // identifier, the key it is stored under, rather than under the identifier + // as issued. let auction_kv_snapshot = match (kv, ec_id.as_deref(), registry) { - (Some(graph), Some(ec_id), Some(_)) => graph.load_snapshot(ec_id), + (Some(graph), Some(_), Some(_)) => ec_context + .ec_kv_key() + .map_or(EcKvSnapshot::NotRead, |kv_key| graph.load_snapshot(&kv_key)), _ => EcKvSnapshot::NotRead, }; // Hand the loaded row to the request context so response finalization — @@ -325,7 +336,8 @@ pub async fn handle_auction( // Look up geo for device info. let geo = services .geo() - .lookup(services.client_info().client_ip) + .lookup(services.client_info().client_ip, services) + .await .unwrap_or_else(|e| { log::warn!("geo lookup failed: {e}"); None @@ -346,10 +358,9 @@ pub async fn handle_auction( // consent gating before attaching them to the auction request. let merged_eids = merge_auction_eids(client_eids, eids); let had_eids = merged_eids.as_ref().is_some_and(|v| !v.is_empty()); - auction_request.user.eids = - gate_eids_by_consent(merged_eids, auction_request.user.consent.as_ref()); + auction_request.user.eids = gate_eids_by_permissions(merged_eids, ec_context.permissions()); if had_eids && auction_request.user.eids.is_none() { - log::warn!("Auction EIDs stripped by TCF consent gating"); + log::warn!("Auction EIDs stripped: bidstream permissions not set"); } // Create auction context @@ -457,7 +468,13 @@ pub(crate) fn resolve_auction_eids( let ec_id = ec_context.ec_value()?; - let Some(entry) = snapshot.entry_for(ec_id) else { + // Callers read the snapshot under the identity-graph key, the owning + // module's canonical form of the identifier, so the entry is looked up + // under that key rather than under the identifier as issued. + let Some(entry) = ec_context + .kv_key_for(ec_id) + .and_then(|kv_key| snapshot.entry_for(&kv_key)) + else { return Some(Vec::new()); }; @@ -630,6 +647,7 @@ mod tests { use crate::auction::types::{AuctionRequest, AuctionResponse}; use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::ConsentContext; + use crate::ec::tests::{CANONICAL_COOKIE_VALUE, CANONICAL_KV_KEY, CanonicalizingModule}; use crate::error::IntoHttpResponse as _; use crate::openrtb::Uid; use crate::platform::test_support::{ @@ -677,13 +695,25 @@ mod tests { .build() } - fn make_ec_context(jurisdiction: Jurisdiction, ec_value: Option<&str>) -> EcContext { - EcContext::new_for_test( + fn make_ec_context(ec_allowed: bool, ec_value: Option<&str>) -> EcContext { + EcContext::new_for_test_gated( + ec_value.map(str::to_owned), + ConsentContext::default(), + ec_allowed, + ) + } + + /// A context in a non-regulated jurisdiction with the permission gate open, + /// for tests whose subject is dispatch or identity reuse rather than the + /// jurisdiction gate. + fn make_non_regulated_ec_context(ec_value: Option<&str>) -> EcContext { + EcContext::new_for_test_gated( ec_value.map(str::to_owned), ConsentContext { - jurisdiction, + jurisdiction: Jurisdiction::NonRegulated, ..ConsentContext::default() }, + true, ) } @@ -716,9 +746,9 @@ mod tests { let settings = create_test_settings(); let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["eid_capturing_provider"]), + provider_names: vec!["eid_capturing_provider".to_string()], timeout_ms: 2000, - mediator: None, + adserver_name: None, ..Default::default() }); orchestrator.register_provider(Arc::new(EidCapturingProvider { @@ -735,7 +765,7 @@ mod tests { graph.create(&ec_id, &live).expect("should seed live row"); lookups.store(0, std::sync::atomic::Ordering::Relaxed); - let mut ec_context = make_ec_context(Jurisdiction::NonRegulated, Some(&ec_id)); + let mut ec_context = make_non_regulated_ec_context(Some(&ec_id)); let req = Request::builder() .method("POST") .uri("https://test-publisher.com/auction") @@ -785,7 +815,9 @@ mod tests { None, Some("shared-cookie-id"), &mut response, - ); + &crate::platform::test_support::noop_services(), + ) + .await; assert_eq!( lookups.load(std::sync::atomic::Ordering::Relaxed), @@ -803,6 +835,84 @@ mod tests { ); } + #[tokio::test] + async fn auction_endpoint_loads_the_row_under_the_canonical_key() { + // The identity graph stores a row under the owning module's + // canonical form of the identifier. Loaded and resolved under the + // identifier as issued, a module whose canonical form differs from + // the cookie value found no row, so the auction carried no server-side + // EIDs and the context kept a snapshot bound to the wrong key. + let settings = create_test_settings(); + let had_eids = Arc::new(std::sync::Mutex::new(None)); + let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { + enabled: true, + provider_names: vec!["eid_capturing_provider".to_string()], + timeout_ms: 2000, + adserver_name: None, + ..Default::default() + }); + orchestrator.register_provider(Arc::new(EidCapturingProvider { + had_eids: Arc::clone(&had_eids), + })); + let registry = PartnerRegistry::from_config(&[counting_test_partner("ssp.example.com")]) + .expect("should build partner registry"); + let graph = KvIdentityGraph::in_memory("canonical-auction-store"); + graph + .create( + CANONICAL_KV_KEY, + &crate::ec::kv_types::KvEntry::minimal( + "ssp.example.com", + "partner-uid-123", + 1_741_824_000, + ), + ) + .expect("should seed the row under the canonical key"); + let mut ec_context = make_non_regulated_ec_context(Some(CANONICAL_COOKIE_VALUE)) + .with_module_for_test(Arc::new(CanonicalizingModule)); + let req = Request::builder() + .method("POST") + .uri("https://test-publisher.com/auction") + .body(EdgeBody::from( + serde_json::to_vec(&json!({ + "adUnits": [ + { + "code": "div-gpt-ad-1", + "mediaTypes": { "banner": { "sizes": [[300, 250]] } } + } + ] + })) + .expect("should serialize body"), + )) + .expect("should build auction request"); + + // The capturing provider records whether the request carried EIDs and + // then fails its launch, which is all this test needs. The request + // carries no client EIDs, so any EID it records came from the graph. + let _ = handle_auction( + &settings, + &orchestrator, + Some(&graph), + Some(®istry), + &mut ec_context, + &noop_services(), + req, + ) + .await; + + assert!( + ec_context + .kv_snapshot() + .entry_for(CANONICAL_KV_KEY) + .is_some(), + "the endpoint should load the row stored under the canonical key" + ); + assert_eq!( + *had_eids.lock().expect("should lock captured eids"), + Some(true), + "the auction should carry the canonical row's partner ID as an EID" + ); + } + #[tokio::test] async fn auction_body_eids_reach_kv_even_when_the_ts_eids_cookie_is_absent() { // Regression test for #1184: `/auction` sends every EID it has in the @@ -814,9 +924,9 @@ mod tests { let settings = create_test_settings(); let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["eid_capturing_provider"]), + provider_names: vec!["eid_capturing_provider".to_string()], timeout_ms: 2000, - mediator: None, + adserver_name: None, ..Default::default() }); orchestrator.register_provider(Arc::new(EidCapturingProvider { @@ -831,7 +941,15 @@ mod tests { live.consent.ok = true; graph.create(&ec_id, &live).expect("should seed live row"); - let mut ec_context = make_ec_context(Jurisdiction::NonRegulated, Some(&ec_id)); + let mut ec_context = EcContext::new_for_test_gated( + Some(ec_id.clone()), + ConsentContext { + jurisdiction: Jurisdiction::NonRegulated, + ..ConsentContext::default() + }, + true, + ) + .with_module_for_test(crate::ec::tests::hmac_module()); ec_context.set_eid_sync_source(crate::ec::EidSyncSource::Auction); let req = Request::builder() .method("POST") @@ -883,7 +1001,9 @@ mod tests { None, // No `ts-eids` cookie on this request at all. None, &mut response, - ); + &noop_services(), + ) + .await; let (stored, _) = graph .get(&ec_id) @@ -940,7 +1060,7 @@ mod tests { #[async_trait::async_trait(?Send)] impl AuctionProvider for TemplateSwitchProbeProvider { fn provider_name(&self) -> &'static str { - "template-switch-probe" + "template_switch_probe" } async fn request_bids( @@ -992,13 +1112,15 @@ mod tests { #[tokio::test] async fn direct_auction_remains_available_when_templates_are_disabled() { let settings_toml = format!( - "{}\n[auction]\nenabled = true\n\n[auction.providers.template-switch-probe]\nprotocol = \"openrtb-2.6\"\nendpoint = \"https://bidder.example/auction\"\nrouting = \"all_eligible\"\n\n[creative_opportunities]\nenabled = false\ngam_network_id = \"12345\"\n", - crate_test_settings_str() + "{}\n[demand]\nmodules = [\"template_switch_probe\"]\n\n[demand.template_switch_probe]\nimplementation = \"auction.plain-fixture\"\nendpoint = \"https://bidder.example/auction\"\nrouting = \"all_eligible\"\n\n[creative_opportunities]\nenabled = false\ngam_network_id = \"12345\"\n", + crate_test_settings_str().replace("[auction]\n", "[auction]\nenabled = true\n") ); let settings = Settings::from_toml(&settings_toml) .expect("should parse settings with disabled templates"); let calls = Arc::new(Mutex::new(0)); - let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let mut orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); orchestrator.register_provider(Arc::new(TemplateSwitchProbeProvider { calls: Arc::clone(&calls), })); @@ -1014,7 +1136,11 @@ mod tests { .geo(Arc::new(NoopGeo)) .client_info(ClientInfo::default()) .build(); - let mut ec_context = make_ec_context(Jurisdiction::NonRegulated, None); + let consent = ConsentContext { + jurisdiction: Jurisdiction::NonRegulated, + ..ConsentContext::default() + }; + let mut ec_context = EcContext::new_for_test_gated(None, consent, true); let body = json!({ "adUnits": [{ "code": "div-gpt-ad-1", @@ -1054,16 +1180,16 @@ mod tests { let settings = create_test_settings(); let config = AuctionConfig { enabled: false, - providers: AuctionConfig::legacy_provider_map(&["panic_provider"]), + provider_names: vec!["panic_provider".to_string()], timeout_ms: 2000, - mediator: None, + adserver_name: None, ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); orchestrator.register_provider(Arc::new(PanicOnBidProvider)); let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); let services = services_with_telemetry(Arc::clone(&telemetry_sink)); - let mut ec_context = make_ec_context(Jurisdiction::NonRegulated, None); + let mut ec_context = make_non_regulated_ec_context(None); let body = json!({ "adUnits": [{ "code": "div-gpt-ad-1", @@ -1114,8 +1240,8 @@ mod tests { #[tokio::test] async fn all_planned_launch_failures_return_bad_gateway_and_execution_failed_telemetry() { let settings_toml = format!( - "{}\n[auction]\nenabled = true\n\n[auction.providers.launch-fail]\nprotocol = \"openrtb-2.6\"\nprofile = \"standard\"\nendpoint = \"https://bidder.example/auction\"\nrouting = \"all_eligible\"\n", - crate_test_settings_str() + "{}\n[demand]\nmodules = [\"launch_fail\"]\n\n[demand.launch_fail]\nimplementation = \"auction.plain-fixture\"\nendpoint = \"https://bidder.example/auction\"\nrouting = \"all_eligible\"\n", + crate_test_settings_str().replace("[auction]\n", "[auction]\nenabled = true\n") ); let settings = Settings::from_toml(&settings_toml).expect("should parse launch-failure settings"); @@ -1126,7 +1252,7 @@ mod tests { let orchestrator = AuctionOrchestrator::from_plan(plan, None); let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); let services = services_with_telemetry(Arc::clone(&telemetry_sink)); - let mut ec_context = make_ec_context(Jurisdiction::NonRegulated, None); + let mut ec_context = make_non_regulated_ec_context(None); let body = json!({ "adUnits": [{ "code": "div-gpt-ad-1", @@ -1178,9 +1304,9 @@ mod tests { let settings = create_test_settings(); let config = AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["panic_provider"]), + provider_names: vec!["panic_provider".to_string()], timeout_ms: 2000, - mediator: None, + adserver_name: None, ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); @@ -1188,7 +1314,9 @@ mod tests { let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); let services = services_with_telemetry(Arc::clone(&telemetry_sink)); let ec_id = format!("{}.ABC123", "a".repeat(64)); - let mut ec_context = make_ec_context(Jurisdiction::Unknown, Some(&ec_id)); + // The default consent context keeps the jurisdiction unknown, so the + // server-side auction gate fails closed; the EC gate is off to match. + let mut ec_context = make_ec_context(false, Some(&ec_id)); let body = json!({ "adUnits": [ @@ -1306,9 +1434,9 @@ mod tests { let settings = create_test_settings(); let config = AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["eid_capturing_provider"]), + provider_names: vec!["eid_capturing_provider".to_string()], timeout_ms: 2000, - mediator: None, + adserver_name: None, ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); @@ -1355,7 +1483,7 @@ mod tests { let req = Request::builder() .method("POST") .uri("https://test-publisher.com/auction") - .header("cookie", format!("{COOKIE_TS_EIDS}={encoded_cookie}")) + .header("cookie", format!("ts-{COOKIE_EIDS}={encoded_cookie}")) .body(EdgeBody::from( serde_json::to_vec(&body).expect("should serialize body"), )) @@ -1385,7 +1513,7 @@ mod tests { fn resolve_auction_eids_returns_empty_without_snapshot() { let registry = PartnerRegistry::empty(); let ec_id = format!("{}.ABC123", "a".repeat(64)); - let ec_context = make_ec_context(Jurisdiction::NonRegulated, Some(&ec_id)); + let ec_context = make_ec_context(true, Some(&ec_id)); let result = resolve_auction_eids(&EcKvSnapshot::NotRead, Some(®istry), &ec_context); assert!( @@ -1397,7 +1525,7 @@ mod tests { #[test] fn resolve_auction_eids_returns_none_without_registry() { let ec_id = format!("{}.ABC123", "a".repeat(64)); - let ec_context = make_ec_context(Jurisdiction::NonRegulated, Some(&ec_id)); + let ec_context = make_ec_context(true, Some(&ec_id)); let result = resolve_auction_eids(&EcKvSnapshot::NotRead, None, &ec_context); assert!( @@ -1410,7 +1538,7 @@ mod tests { fn resolve_auction_eids_returns_none_when_consent_denied() { let registry = PartnerRegistry::empty(); let ec_id = format!("{}.ABC123", "a".repeat(64)); - let ec_context = make_ec_context(Jurisdiction::Unknown, Some(&ec_id)); + let ec_context = make_ec_context(false, Some(&ec_id)); let result = resolve_auction_eids(&EcKvSnapshot::NotRead, Some(®istry), &ec_context); assert!( @@ -1422,7 +1550,11 @@ mod tests { #[test] fn resolve_auction_eids_returns_none_when_no_ec() { let registry = PartnerRegistry::empty(); - let ec_context = make_ec_context(Jurisdiction::NonRegulated, None); + let consent = ConsentContext { + jurisdiction: Jurisdiction::NonRegulated, + ..ConsentContext::default() + }; + let ec_context = EcContext::new_for_test_gated(None, consent, true); let result = resolve_auction_eids(&EcKvSnapshot::NotRead, Some(®istry), &ec_context); assert!( @@ -1435,7 +1567,7 @@ mod tests { fn resolve_auction_eids_returns_empty_on_kv_miss() { let registry = PartnerRegistry::empty(); let ec_id = format!("{}.ABC123", "a".repeat(64)); - let ec_context = make_ec_context(Jurisdiction::NonRegulated, Some(&ec_id)); + let ec_context = make_ec_context(true, Some(&ec_id)); let snapshot = EcKvSnapshot::Failed { ec_id: ec_id.clone(), diff --git a/crates/trusted-server-core/src/auction/formats.rs b/crates/trusted-server-core/src/auction/formats.rs index a0df63d94a..d64caa8a8f 100644 --- a/crates/trusted-server-core/src/auction/formats.rs +++ b/crates/trusted-server-core/src/auction/formats.rs @@ -481,11 +481,12 @@ pub(crate) fn convert_to_openrtb_response_with_report( delivery.delivered_winner_slots.insert(slot_id.clone()); } - // Determine strategy name for response metadata - let strategy_name = if settings.auction.has_mediator() { - "parallel_mediation" - } else { + // The strategy names what the deployment selected, so it reads the same + // whether or not the ad server answered this auction. + let strategy_name = if settings.adserver.is_empty() { "parallel_only" + } else { + "parallel_adserver" }; // Build per-provider summaries from the orchestration result @@ -554,20 +555,16 @@ pub(crate) fn convert_to_openrtb_response_with_report( #[cfg(test)] mod tests { use super::*; - use crate::auction::plan::{ - AuctionPlan, AuctionPlanConfig, NotificationConfig, ProviderConfig, ProviderId, RoutingMode, - }; + use crate::auction::plan::AuctionPlan; use crate::auction::routing::route_auction; - use crate::auction::types::{ - ApsRendererV1, ApsTagType, AuctionResponse, Bid, BidRenderer, BidStatus, - }; + use crate::auction::test_support::{demand_table, plan_config}; + use crate::auction::types::{AuctionResponse, Bid, BidRenderer, BidStatus}; use crate::openrtb::{Eid, Uid}; use crate::platform::test_support::noop_services; use crate::test_support::tests::create_test_settings; use http::Method; use serde_json::json; - use std::collections::{BTreeMap, BTreeSet, HashSet}; - use std::str::FromStr as _; + use std::collections::{BTreeSet, HashSet}; fn make_request() -> Request { Request::builder() @@ -582,26 +579,13 @@ mod tests { create_test_settings() } - fn single_prebid_plan() -> AuctionPlan { - AuctionPlan::compile(AuctionPlanConfig { - timeout_ms: 900, - providers: BTreeMap::from([( - ProviderId::from_str("pbs-primary").expect("should parse provider ID"), - ProviderConfig { - protocol: "openrtb-2.6".to_string(), - profile: "prebid-server".to_string(), - endpoint: "https://pbs.example.test/openrtb".to_string(), - timeout_ms: None, - routing: RoutingMode::Explicit, - notifications: NotificationConfig::default(), - profile_config: json!({}), - }, - )]), - bidders: BTreeMap::new(), - mediator: None, - request_signing: None, - }) - .expect("should compile plan") + fn single_stored_plan() -> AuctionPlan { + let mut config = plan_config(vec![( + "stored_primary", + demand_table("auction.fixture", "https://stored.example.test/openrtb"), + )]); + config.timeout_ms = 900; + AuctionPlan::compile(config).expect("should compile plan") } fn make_auction_request() -> AuctionRequest { @@ -636,7 +620,7 @@ mod tests { fn make_empty_result() -> OrchestrationResult { OrchestrationResult { provider_responses: Vec::new(), - mediator_response: None, + adserver_response: None, winning_bids: HashMap::new(), total_time_ms: 10, metadata: HashMap::new(), @@ -685,7 +669,7 @@ mod tests { response_time_ms: 42, metadata: HashMap::new(), }], - mediator_response: None, + adserver_response: None, winning_bids: HashMap::from([(bid.slot_id.clone(), bid)]), total_time_ms: 50, metadata: HashMap::new(), @@ -757,7 +741,7 @@ mod tests { "bids":[{"bidder":"trustedServer","params":{"bidderParams":{},"storedRequest":intent}}]}] })).expect("should deserialize wire request"); let request = convert_body_to_auction_request(&body, &make_settings()); - let routed = route_auction(request, &make_request(), &single_prebid_plan(), None); + let routed = route_auction(request, &make_request(), &single_stored_plan(), None); assert_eq!(routed.inputs().len(), expected_inputs); assert_eq!(routed.diagnostics().malformed_envelope_count(), malformed); } @@ -779,7 +763,7 @@ mod tests { eids: None, }; let request = convert_body_to_auction_request(&body, &make_settings()); - let routed = route_auction(request, &make_request(), &single_prebid_plan(), None); + let routed = route_auction(request, &make_request(), &single_stored_plan(), None); assert_eq!(routed.inputs().len(), 1); assert!( @@ -1503,7 +1487,7 @@ mod tests { response_time_ms: 42, metadata: HashMap::from([("debug".to_string(), debug.clone())]), }], - mediator_response: None, + adserver_response: None, winning_bids: HashMap::from([(bid.slot_id.clone(), bid)]), total_time_ms: 50, metadata: HashMap::new(), @@ -1536,24 +1520,25 @@ mod tests { rejected.creative = Some("".to_string()); let unpriced = make_bid("unpriced", "invalid", None); let ordinary = make_bid("ordinary", "appnexus", Some(2.75)); - let mut renderer = make_bid("renderer", "aps", Some(2.5)); + let mut renderer = make_bid("renderer", "example", Some(2.5)); renderer.creative = Some("".to_string()); renderer.bid_id = Some("upstream-renderer-bid".to_string()); renderer.creative_id = None; - renderer.renderer = Some(BidRenderer::Aps(ApsRendererV1 { - version: 1, - account_id: "example-account".to_string(), - bid_id: "upstream-renderer-bid".to_string(), - creative_id: None, - tag_type: ApsTagType::Iframe, - creative_url: "https://creative.example/render".to_string(), - aax_response: "fictional-base64".to_string(), - width: 300, - height: 250, - })); + renderer.renderer = Some( + BidRenderer::new( + "example", + json!({ + "version": 1, + "bidId": "upstream-renderer-bid", + "creativeUrl": "https://creative.example/render", + "envelope": "fictional-base64", + }), + ) + .expect("should build the example renderer descriptor"), + ); let result = OrchestrationResult { provider_responses: vec![], - mediator_response: None, + adserver_response: None, winning_bids: HashMap::from([ (missing.slot_id.clone(), missing), (whitespace.slot_id.clone(), whitespace), @@ -1637,18 +1622,19 @@ mod tests { fn convert_to_openrtb_response_prefers_creative_when_both_render_sources_exist() { let settings = make_settings(); let auction_request = make_auction_request(); - let mut bid = make_bid("div-gpt-top", "aps", Some(2.75)); - bid.renderer = Some(BidRenderer::Aps(ApsRendererV1 { - version: 1, - account_id: "example-account".to_string(), - bid_id: "fictional-bid".to_string(), - creative_id: None, - tag_type: ApsTagType::Iframe, - creative_url: "https://creative.example/render".to_string(), - aax_response: "fictional-base64".to_string(), - width: 300, - height: 250, - })); + let mut bid = make_bid("div-gpt-top", "example", Some(2.75)); + bid.renderer = Some( + BidRenderer::new( + "example", + json!({ + "version": 1, + "bidId": "fictional-bid", + "creativeUrl": "https://creative.example/render", + "envelope": "fictional-base64", + }), + ) + .expect("should build the example renderer descriptor"), + ); let result = make_result(bid); let response = convert_to_openrtb_response(&result, &settings, &auction_request, false) @@ -1670,29 +1656,30 @@ mod tests { } #[test] - fn convert_to_openrtb_response_emits_typed_aps_renderer_without_adm() { + fn convert_to_openrtb_response_emits_a_typed_renderer_without_adm() { let settings = make_settings(); let auction_request = make_auction_request(); - let mut bid = make_bid("div-gpt-top", "aps", Some(2.75)); + let mut bid = make_bid("div-gpt-top", "example", Some(2.75)); bid.creative = None; bid.bid_id = Some("fictional-bid".to_string()); bid.ad_id = Some("fictional-ad".to_string()); bid.creative_id = Some("fictional-creative".to_string()); - bid.renderer = Some(BidRenderer::Aps(ApsRendererV1 { - version: 1, - account_id: "example-account".to_string(), - bid_id: "fictional-bid".to_string(), - creative_id: Some("fictional-creative".to_string()), - tag_type: ApsTagType::Iframe, - creative_url: "https://creative.example/render".to_string(), - aax_response: "fictional-base64".to_string(), - width: 300, - height: 250, - })); + bid.renderer = Some( + BidRenderer::new( + "example", + json!({ + "version": 1, + "bidId": "fictional-bid", + "creativeUrl": "https://creative.example/render", + "envelope": "fictional-base64", + }), + ) + .expect("should build the example renderer descriptor"), + ); let result = make_result(bid); let response = convert_to_openrtb_response(&result, &settings, &auction_request, false) - .expect("should convert APS renderer bid"); + .expect("should convert a typed renderer bid"); let json = response_json(response); let bid = json["seatbid"][0]["bid"][0] .as_object() @@ -1703,12 +1690,12 @@ mod tests { "should omit adm for renderer bids" ); assert_eq!(bid["id"], json!("fictional-bid")); - assert_eq!(json["seatbid"][0]["seat"], json!("aps")); + assert_eq!(json["seatbid"][0]["seat"], json!("example")); assert_eq!(bid["adid"], json!("fictional-ad")); assert_eq!(bid["crid"], json!("fictional-creative")); assert_eq!( bid["ext"]["trusted_server"]["renderer"]["type"], - json!("aps") + json!("example") ); assert_eq!( bid["ext"]["trusted_server"]["renderer"]["bidId"], @@ -1744,7 +1731,7 @@ mod tests { let auction_request = make_auction_request(); let result = OrchestrationResult { provider_responses: vec![], - mediator_response: None, + adserver_response: None, winning_bids: HashMap::new(), total_time_ms: 50, metadata: HashMap::new(), @@ -1781,7 +1768,7 @@ mod tests { response_time_ms: 42, metadata: HashMap::new(), }], - mediator_response: None, + adserver_response: None, winning_bids: HashMap::from([ (top_bid.slot_id.clone(), top_bid), (sidebar_bid.slot_id.clone(), sidebar_bid), @@ -1868,9 +1855,12 @@ mod tests { } #[test] - fn convert_to_openrtb_response_uses_parallel_mediation_when_mediator_configured() { + fn convert_to_openrtb_response_uses_parallel_adserver_when_adserver_configured() { let mut settings = make_settings(); - settings.auction.mediator = Some("adserver_mock".to_string()); + settings.adserver = crate::provider_table::ProviderChoice::new( + Some("mock".to_string()), + std::collections::BTreeMap::new(), + ); let auction_request = make_auction_request(); let result = make_result(make_bid("div-gpt-top", "appnexus", Some(2.75))); @@ -1880,8 +1870,8 @@ mod tests { assert_eq!( json["ext"]["orchestrator"]["strategy"], - json!("parallel_mediation"), - "should use mediation strategy when mediator is configured" + json!("parallel_adserver"), + "should use ad server decision strategy when adserver is configured" ); } @@ -2028,8 +2018,11 @@ mod convert_tests { // (empty) applies — verify a key is NOT present rather than IS. // To test the allow-list, inject a key via a custom settings string. let settings_str = format!( - "{}\n[auction]\nallowed_context_keys = [\"permutive_segments\"]\n", - crate_test_settings_str() + "{}\n", + crate_test_settings_str().replace( + "[auction]\n", + "[auction]\nallowed_context_keys = [\"permutive_segments\"]\n" + ) ); let settings = Settings::from_toml(&settings_str).expect("should parse"); let services = noop_services(); diff --git a/crates/trusted-server-core/src/auction/mod.rs b/crates/trusted-server-core/src/auction/mod.rs index 4323039284..9cde111aed 100644 --- a/crates/trusted-server-core/src/auction/mod.rs +++ b/crates/trusted-server-core/src/auction/mod.rs @@ -1,38 +1,42 @@ -//! Auction orchestration module for managing multi-provider bidding. +//! Auction orchestration across the demand sources and the ad server a +//! deployment selects. //! -//! This module provides an extensible framework for running auctions across -//! multiple providers (Prebid, Amazon APS, Google GAM, etc.) with support for -//! parallel execution and mediation strategies. -//! -//! Note: Individual auction providers are located in the `integrations` module -//! (e.g., `crate::integrations::aps`, `crate::integrations::prebid`). +//! `[demand] modules` selects the demand sources and `[ad-server] module` +//! the optional ad server. Their implementations are registered by +//! integrations through [`demand`], so this module names no vendor. use error_stack::Report; use crate::error::TrustedServerError; +use crate::integrations::IntegrationBuilder; use crate::settings::Settings; use std::sync::Arc; pub mod config; pub mod context; +pub mod demand; pub mod endpoints; pub mod formats; -pub(crate) mod openrtb; +pub mod openrtb; pub mod orchestrator; pub mod plan; -pub(crate) mod profile; pub mod provider; pub(crate) mod routing; pub mod telemetry; -#[cfg(test)] -pub(crate) mod test_support; +#[cfg(any(test, feature = "test-utils"))] +#[allow( + clippy::must_use_candidate, + clippy::missing_panics_doc, + reason = "test helpers, offered to other crates' tests under the test-utils feature" +)] +pub mod test_support; pub mod types; pub use config::AuctionConfig; pub use context::{ContextQueryParams, ContextValue, build_url_with_context_params}; pub use orchestrator::AuctionOrchestrator; pub use plan::{ - AuctionPlan, BidderId, BidderRouteConfig, NotificationConfig, ProviderConfig, ProviderId, + AdServerPlan, AuctionPlan, BidderId, BidderRouteConfig, NotificationConfig, ProviderId, RoutingMode, }; pub use provider::AuctionProvider; @@ -45,66 +49,86 @@ pub use types::{ AdFormat, AuctionContext, AuctionRequest, AuctionResponse, Bid, BidStatus, MediaType, }; -/// Compile the canonical target-independent auction plan for [`Settings`]. +/// Compile the canonical target-independent auction plan for [`Settings`] with +/// the implementations the built-in integrations register. /// -/// This is the single settings-to-plan boundary used by deploy validation, -/// adapter startup, and operator tooling. Global request signing remains owned -/// by [`Settings`] and is copied into compiler input only at this boundary. +/// This is the settings-to-plan boundary used by deploy validation, adapter +/// startup and operator tooling. Global request signing remains owned by +/// [`Settings`] and is copied into compiler input only at this boundary. /// /// # Errors /// -/// Returns an error when auction provider, bidder route, signing, or mediator +/// Returns an error when the `[demand]`, `[ad-server]`, bidder route or signing /// configuration is invalid. pub fn compile_auction_plan( settings: &Settings, ) -> Result> { + compile_auction_plan_with(settings, &[]) +} + +/// Compile the auction plan with the implementations the built-in integrations +/// register followed by those the `extra` builders register, the builders an +/// adapter or a vendor crate supplies. +/// +/// # Errors +/// +/// Returns an error when the `[demand]`, `[ad-server]`, bidder route or signing +/// configuration is invalid, including a selected name whose implementation no +/// builder registers. +pub fn compile_auction_plan_with( + settings: &Settings, + extra: &[IntegrationBuilder], +) -> Result> { + let builders = crate::integrations::all_builders(extra).collect::>(); AuctionPlan::compile(plan::AuctionPlanConfig { timeout_ms: settings.auction.timeout_ms, - providers: settings.auction.providers.clone(), + demand: settings.demand.clone(), + adserver: settings.adserver.clone(), bidders: settings.auction.bidders.clone(), - mediator: settings.auction.mediator.clone(), request_signing: settings.request_signing.clone(), + demand_implementations: builders + .iter() + .filter_map(IntegrationBuilder::demand) + .collect(), + adserver_implementations: builders + .iter() + .filter_map(IntegrationBuilder::adserver) + .collect(), + }) + .map(|plan| { + plan.with_enabled(settings.auction.enabled) + .with_auction_token(builders.iter().any(|builder| { + builder.reads_auction_token() + && builder + .module_name() + .is_some_and(|name| settings.selects_module(name)) + })) }) - .map(|plan| plan.with_enabled(settings.auction.enabled)) } /// Build a new auction orchestrator from one shared compiled plan. /// -/// This constructor registers all auction providers discovered from the provided settings. -/// Callers can reuse the returned [`AuctionOrchestrator`] across requests. -/// -/// # Arguments -/// * `plan` - Shared immutable compiled plan -/// * `settings` - Application settings used only for the separately registered mediator +/// The demand sources come from the plan, and so does the ad server, which is +/// built here from its implementation and settings. Callers can reuse the +/// returned [`AuctionOrchestrator`] across requests. /// /// # Errors /// -/// Returns an error when an enabled auction provider has invalid configuration. +/// Returns an error when the selected ad server cannot be built from its +/// settings. pub fn build_orchestrator_with_plan( plan: Arc, - settings: &Settings, ) -> Result> { log::info!("Building plan-backed auction orchestrator"); - let mediator = if let Some(expected_id) = plan.mediator() { - let provider = crate::integrations::adserver_mock::register_providers(settings)? - .into_iter() - .find(|provider| provider.provider_name() == expected_id) - .ok_or_else(|| { - Report::new(TrustedServerError::Configuration { - message: format!( - "auction mediator `{expected_id}` must reference a separately registered enabled integration with the exact same ID" - ), - }) - })?; - Some(provider) - } else { - None - }; - let orchestrator = AuctionOrchestrator::from_plan(plan, mediator); + let adserver = plan + .adserver() + .map(|adserver| (adserver.implementation.build)(adserver.id.as_str(), &adserver.settings)) + .transpose()?; + let orchestrator = AuctionOrchestrator::from_plan(plan, adserver); log::info!( - "Auction orchestrator built with {} bidder providers", + "Auction orchestrator built with {} demand sources", orchestrator.provider_count() ); @@ -115,27 +139,97 @@ pub fn build_orchestrator_with_plan( /// /// # Errors /// -/// Returns an error when plan compilation or mediator construction fails. +/// Returns an error when plan compilation or ad server construction fails. #[cfg(test)] pub fn build_orchestrator( settings: &Settings, ) -> Result> { let plan = Arc::new(compile_auction_plan(settings)?); - build_orchestrator_with_plan(plan, settings) + build_orchestrator_with_plan(plan) +} + +#[cfg(test)] +mod auction_token_tests { + use super::*; + use crate::integrations::registry_test_support::{ + probe_registration, request_fixture, validate_nothing, + }; + use crate::test_support::tests::create_test_settings; + + #[test] + fn the_plan_publishes_a_token_only_where_a_selected_module_reads_one() { + let mut settings = create_test_settings(); + assert!( + !compile_auction_plan(&settings) + .expect("should compile the plan") + .publishes_auction_token(), + "should publish no token where no module is selected" + ); + + settings.select_module("testing", request_fixture::MODULE); + assert!( + compile_auction_plan(&settings) + .expect("should compile the plan") + .publishes_auction_token(), + "should publish a token once a module that reads one is selected" + ); + } + + #[test] + fn a_selected_module_that_reads_no_token_does_not_turn_one_on() { + let mut settings = create_test_settings(); + settings.select_module("testing", "testing.probe"); + let reads_nothing = + [ + IntegrationBuilder::new( + "probe", + "seam-probe", + probe_registration, + validate_nothing, + ) + .with_module_name("testing.probe"), + ]; + assert!( + !compile_auction_plan_with(&settings, &reads_nothing) + .expect("should compile the plan") + .publishes_auction_token(), + "should publish no token for a selected module that reads none" + ); + + let reads_one = [reads_nothing[0].with_auction_token()]; + assert!( + compile_auction_plan_with(&settings, &reads_one) + .expect("should compile the plan") + .publishes_auction_token(), + "should publish a token for a module a vendor crate supplies that reads one" + ); + } } #[cfg(test)] mod plan_sharing_tests { use super::*; + use crate::auction::test_support::demand_named; use crate::integrations::IntegrationRegistry; + use crate::provider_table::ProviderChoice; use crate::test_support::tests::create_test_settings; + use serde_json::{Map, json}; + use std::collections::BTreeMap; + + /// An `[ad-server]` table selecting one name with the settings given. + fn adserver(name: &str, settings: Map) -> ProviderChoice { + ProviderChoice::new( + Some(name.to_string()), + BTreeMap::from([(name.to_string(), settings)]), + ) + } #[test] fn orchestrator_and_registry_share_the_compiled_plan_allocation() { let settings = create_test_settings(); let plan = Arc::new(compile_auction_plan(&settings).expect("should compile auction plan")); - let orchestrator = build_orchestrator_with_plan(Arc::clone(&plan), &settings) - .expect("should build orchestrator"); + let orchestrator = + build_orchestrator_with_plan(Arc::clone(&plan)).expect("should build orchestrator"); let registry = IntegrationRegistry::with_plan(&settings, Arc::clone(&plan)) .expect("should build integration registry"); @@ -144,54 +238,99 @@ mod plan_sharing_tests { } #[test] - fn configured_mediator_requires_enabled_exact_registration() { - for mediator_config in [None, Some(serde_json::json!({"enabled": false}))] { - let mut settings = create_test_settings(); - settings.auction.mediator = Some("adserver_mock".to_string()); - if let Some(config) = mediator_config { - settings - .integrations - .insert_config("adserver_mock", &config) - .expect("should insert mediator config"); - } else { - settings.integrations.remove("adserver_mock"); - } - let plan = Arc::new(compile_auction_plan(&settings).expect("should compile plan")); - - let error = match build_orchestrator_with_plan(plan, &settings) { - Ok(_) => panic!("should require enabled mediator registration"), - Err(error) => error, - }; - assert!(error.to_string().contains("adserver_mock")); - } + fn an_ad_server_this_build_does_not_have_fails_the_plan() { + let mut settings = create_test_settings(); + settings.adserver = + ProviderChoice::new(Some("fictional_adserver".to_string()), BTreeMap::new()); + let error = compile_auction_plan(&settings) + .expect_err("should refuse an ad server no builder registers"); + assert!( + error.to_string().contains("fictional_adserver"), + "should name the ad server: {error:?}" + ); + } + + /// The ad server implementation a crate outside core supplies. + static EXTERNAL_ADSERVER: crate::auction::demand::AdServerImplementation = + crate::auction::demand::AdServerImplementation { + id: "ad-server.example", + build: build_external_adserver, + }; + + fn build_external_adserver( + name: &str, + _settings: &Map, + ) -> Result, Report> { + Ok(Arc::new( + crate::auction::test_support::adserver_fixture::FixtureAdServer::new(name, 500), + )) } + /// An ad server a builder outside core supplies resolves in the plan when + /// that builder is passed, and is a name no builder registers when it is + /// not. Both halves are asserted, so the test fails if either stops + /// holding. #[test] - fn configured_mediator_builds_when_exact_registration_is_enabled() { + fn an_ad_server_an_external_builder_supplies_compiles_only_when_it_is_passed() { let mut settings = create_test_settings(); - settings.auction.mediator = Some("adserver_mock".to_string()); - settings - .integrations - .insert_config( - "adserver_mock", - &serde_json::json!({ - "enabled": true, - "endpoint": "https://mediator.example/mediate" - }), - ) - .expect("should insert mediator config"); + settings.adserver = adserver("example", Map::new()); + let extra = [crate::integrations::IntegrationBuilder::implementations( + "example-adserver", + "example-crate", + ) + .with_adserver(&EXTERNAL_ADSERVER)]; + + let error = compile_auction_plan(&settings) + .expect_err("built-ins alone should not know this ad server"); + assert!( + error.to_string().contains("example"), + "should name the ad server: {error:?}" + ); + + let plan = compile_auction_plan_with(&settings, &extra) + .expect("the external builder's ad server should compile"); + assert_eq!( + plan.adserver().map(|adserver| adserver.implementation.id), + Some("ad-server.example"), + "the plan should carry the external ad server" + ); + } + + #[test] + fn the_selected_ad_server_is_built_from_its_own_table() { + let mut settings = create_test_settings(); + settings.adserver = adserver( + "fixture", + Map::from_iter([( + "endpoint".to_string(), + json!("https://adserver.example/mediate"), + )]), + ); let plan = Arc::new(compile_auction_plan(&settings).expect("should compile plan")); - build_orchestrator_with_plan(plan, &settings) - .expect("should build with enabled exact mediator registration"); + build_orchestrator_with_plan(plan).expect("should build the selected ad server"); + } + + #[test] + fn an_ad_server_table_with_no_endpoint_fails_the_plan() { + let mut settings = create_test_settings(); + settings.adserver = adserver("fixture", Map::new()); + let error = compile_auction_plan(&settings) + .expect_err("should refuse an ad server with no endpoint"); + assert!( + format!("{error:?}").contains("endpoint"), + "should say an endpoint is needed: {error:?}" + ); } #[test] fn cloudflare_and_spin_reject_multi_provider_plans_before_runtime_construction() { let mut settings = create_test_settings(); settings.auction.enabled = true; - settings.auction.providers = - AuctionConfig::legacy_provider_map(&["provider-a", "provider-b"]); + settings.demand = demand_named( + crate::auction::test_support::plain_fixture::MODULE, + &["provider_a", "provider_b"], + ); let plan = compile_auction_plan(&settings).expect("should compile target-independent plan"); for target in [ @@ -208,26 +347,4 @@ mod plan_sharing_tests { ); } } - - #[test] - fn aps_profile_registers_renderer_without_browser_aps_config() { - let mut settings = create_test_settings(); - settings.auction.providers = std::collections::BTreeMap::from([( - "aps-main".parse().expect("should parse APS provider ID"), - ProviderConfig { - protocol: "openrtb-2.6".to_string(), - profile: "aps".to_string(), - endpoint: "https://aps.example/e/pb/bid".to_string(), - timeout_ms: None, - routing: RoutingMode::AllEligible, - notifications: NotificationConfig::default(), - profile_config: serde_json::json!({"account_id":"example-account"}), - }, - )]); - let plan = Arc::new(compile_auction_plan(&settings).expect("should compile APS plan")); - let registry = IntegrationRegistry::with_plan(&settings, plan) - .expect("should build APS renderer registry"); - - assert!(registry.has_route(&http::Method::GET, "/integrations/aps/renderer")); - } } diff --git a/crates/trusted-server-core/src/auction/openrtb.rs b/crates/trusted-server-core/src/auction/openrtb.rs index 74553c9bce..44b8087e92 100644 --- a/crates/trusted-server-core/src/auction/openrtb.rs +++ b/crates/trusted-server-core/src/auction/openrtb.rs @@ -1,41 +1,51 @@ -//! Shared `OpenRTB` 2.6 request/response support for config-first providers. +//! The shared `OpenRTB` 2.6 request and response driver. //! -//! Profiles receive only routed, privacy-approved facts and never the raw -//! downstream request or unrestricted runtime services. +//! Every demand source's request is built here from routed, privacy-approved +//! facts, never from the raw downstream request or unrestricted runtime +//! services. A demand implementation decides only what +//! [`crate::auction::demand`] exposes, and this driver builds the rest the same +//! way for all of them. use std::collections::{BTreeMap, BTreeSet, HashMap}; -use error_stack::Report; +use error_stack::{Report, ResultExt as _}; +use http::StatusCode; use serde_json::{Map, Value, json}; -use url::Url; -use super::plan::{NotificationPolicy, ProviderPlan}; -use super::profile::{ - ApsProfilePlan, CompiledOpenRtbProfile, PrebidProfilePlan, StandardProfilePlan, -}; -use super::routing::{ - PrebidTransportHeaders, ProviderAuctionInput, ProviderSlotInput, RoutedAuction, +use super::demand::{ + CompiledDemand, DemandFieldPolicy, DemandResponse, ImpressionExtension, RegsPolicy, + RequestExtensions, }; +use super::plan::{NotificationPolicy, ProviderPlan}; +use super::routing::{ProviderAuctionInput, ProviderSlotInput, RoutedAuction, TransportHeaders}; use super::types::{AdFormat, AdSlot, AuctionResponse, Bid}; -use crate::consent::ConsentSource; use crate::error::TrustedServerError; use crate::openrtb::{ Banner, ConsentedProvidersSettings, Device, Format, Geo, Imp, OpenRtbRequest, Publisher, Regs, RegsExt, Site, ToExt as _, TrustedServerExt, User, UserExt, to_openrtb_i32, }; +use crate::platform::PlatformResponse; use crate::request_signing::{RequestSigner, SIGNING_VERSION, SigningParams}; const DEFAULT_CURRENCY: &str = "USD"; -const APS_SDK_SOURCE: &str = "prebid"; -const APS_SDK_VERSION: &str = "2.2.0"; -const MAX_CONSERVATIVE_LANGUAGE_BYTES: usize = 8; +/// The request extension the driver writes and no implementation may claim. +const TRUSTED_SERVER_EXT_KEY: &str = "trusted_server"; /// Fixed reasons why an upstream bid failed response admission. +/// +/// Public, with the dimension helpers below, so an auction implementation in +/// a crate of its own admits a bid by the rules core's own response reader +/// applies. #[derive(Debug, Clone, Copy, Eq, Ord, PartialEq, PartialOrd)] -pub(crate) enum BidRejectionReason { +pub enum BidRejectionReason { + /// The bid is malformed. InvalidBid, + /// The bid names an impression the request did not carry. UnrequestedImpression, + /// The bid's dimensions match no format the impression asked for. DimensionMismatch, + /// The bid's dimensions match more than one format the impression asked + /// for. AmbiguousDimensions, } @@ -52,21 +62,21 @@ impl BidRejectionReason { /// Bounded aggregate diagnostics for rejected upstream bids. #[derive(Debug, Default)] -pub(crate) struct ResponseAdmissionDiagnostics { +pub struct ResponseAdmissionDiagnostics { rejected_bid_count: u32, reason_counts: BTreeMap, } impl ResponseAdmissionDiagnostics { /// Record one rejected bid without retaining upstream payload data. - pub(crate) fn record(&mut self, reason: BidRejectionReason) { + pub fn record(&mut self, reason: BidRejectionReason) { self.rejected_bid_count = self.rejected_bid_count.saturating_add(1); let count = self.reason_counts.entry(reason).or_default(); *count = count.saturating_add(1); } /// Attach fixed-cardinality rejection counts to a provider response. - pub(crate) fn attach_to(self, response: &mut AuctionResponse) { + pub fn attach_to(self, response: &mut AuctionResponse) { if self.rejected_bid_count == 0 { return; } @@ -107,7 +117,7 @@ impl DimensionMatch { } /// Precomputed dimension resolutions for one requested impression. -pub(crate) struct SlotBidDimensions { +pub struct SlotBidDimensions { exact: BTreeSet<(u32, u32)>, by_width: BTreeMap, by_height: BTreeMap, @@ -150,17 +160,19 @@ impl SlotBidDimensions { } /// Precomputed requested banner dimensions keyed by impression ID. -pub(crate) type BidDimensionIndex = BTreeMap; +pub type BidDimensionIndex = BTreeMap; /// Build the requested-dimension index once for one provider response. -pub(crate) fn build_bid_dimension_index(input: &ProviderAuctionInput) -> BidDimensionIndex { +#[must_use] +pub fn build_bid_dimension_index(input: &ProviderAuctionInput) -> BidDimensionIndex { build_bid_dimension_index_from_slots(input.slots().iter().map(ProviderSlotInput::slot)) } /// Build the requested-dimension index from plain [`AdSlot`]s. /// /// The first slot wins when several share an ID. -pub(crate) fn build_bid_dimension_index_from_slots<'a>( +#[must_use] +pub fn build_bid_dimension_index_from_slots<'a>( slots: impl IntoIterator, ) -> BidDimensionIndex { let mut index = BidDimensionIndex::new(); @@ -173,7 +185,12 @@ pub(crate) fn build_bid_dimension_index_from_slots<'a>( } /// Parse an optional positive `OpenRTB` bid dimension. -pub(crate) fn parse_optional_bid_dimension( +/// +/// # Errors +/// +/// Returns [`BidRejectionReason::InvalidBid`] when the value is present and is +/// not a positive whole number that fits. +pub fn parse_optional_bid_dimension( value: &Value, key: &str, ) -> Result, BidRejectionReason> { @@ -201,7 +218,12 @@ pub(crate) fn parse_optional_bid_dimension( } /// Validate explicit dimensions or infer them from one matching banner format. -pub(crate) fn resolve_bid_dimensions( +/// +/// # Errors +/// +/// Returns the reason the bid is refused: the impression was not requested, +/// or the dimensions match none or more than one of its formats. +pub fn resolve_bid_dimensions( dimensions_by_slot: &BidDimensionIndex, slot_id: &str, width: Option, @@ -253,12 +275,13 @@ pub(crate) struct RequestFinalization<'a> { pub(crate) signing_params: SigningParams, } -/// Build one provider request from its immutable routed input. +/// Build one demand source's request from its immutable routed input. /// /// # Errors /// -/// Returns an auction error when static/profile extensions cannot be merged or -/// the supplied signing input does not bind the already-fixed request ID. +/// Returns an auction error when an implementation's own extensions cannot be +/// built or the supplied signing input does not bind the already-fixed request +/// ID. pub(crate) fn build_request( input: &ProviderAuctionInput, routed: &RoutedAuction, @@ -266,101 +289,90 @@ pub(crate) fn build_request( effective_timeout_ms: u32, finalization: &RequestFinalization<'_>, ) -> Result> { - let policy = ProfilePolicy::from(&provider.profile); - let mut request = build_common_request(input, routed, policy, effective_timeout_ms); + let demand = provider.demand.as_ref(); + let policy = demand.field_policy(); + let (mut request, slot_indices) = + build_common_request(input, routed, demand, policy, effective_timeout_ms); if request.imp.is_empty() { return Ok(OpenRtbBuildOutcome::NoImpressions); } - policy.augment_request(&mut request, input, routed)?; + let omitted = { + // The implementation sees the extension objects and the slot each + // impression came from, and nothing else of the request. + let OpenRtbRequest { ext, imp, .. } = &mut request; + let impressions = imp + .iter_mut() + .zip(&slot_indices) + .map(|(imp, &index)| ImpressionExtension { + slot: &input.slots()[index], + ext: &mut imp.ext, + omitted: false, + }) + .collect(); + let mut extensions = RequestExtensions { + request: ext, + impressions, + }; + demand.augment_request(&mut extensions, input)?; + extensions + .impressions + .iter() + .map(|impression| impression.omitted) + .collect::>() + }; + // An impression the implementation left without demand goes, and so does + // a request that has none left, so a demand source is never asked for + // nothing. + if omitted.iter().any(|&omitted| omitted) { + let mut omitted = omitted.into_iter(); + request.imp.retain(|_| !omitted.next().unwrap_or(false)); + } if request.imp.is_empty() { return Ok(OpenRtbBuildOutcome::NoImpressions); } + if request + .ext + .as_ref() + .is_some_and(|ext| ext.contains_key(TRUSTED_SERVER_EXT_KEY)) + { + return Err(Report::new(TrustedServerError::Auction { + message: format!( + "Provider {} set ext.{TRUSTED_SERVER_EXT_KEY}, which only the driver writes", + provider.id + ), + })); + } finalize_request(&mut request, policy, finalization)?; Ok(OpenRtbBuildOutcome::Ready(request)) } -#[derive(Clone, Copy)] -enum ProfilePolicy<'a> { - Standard(&'a StandardProfilePlan), - Prebid(&'a PrebidProfilePlan), - Aps(&'a ApsProfilePlan), -} - -impl<'a> From<&'a CompiledOpenRtbProfile> for ProfilePolicy<'a> { - fn from(profile: &'a CompiledOpenRtbProfile) -> Self { - match profile { - CompiledOpenRtbProfile::Standard(plan) => Self::Standard(plan), - CompiledOpenRtbProfile::PrebidServer(plan) => Self::Prebid(plan), - CompiledOpenRtbProfile::Aps(plan) => Self::Aps(plan), - } - } -} - -impl ProfilePolicy<'_> { - fn augment_request( - self, - request: &mut OpenRtbRequest, - input: &ProviderAuctionInput, - routed: &RoutedAuction, - ) -> Result<(), Report> { - match self { - Self::Standard(plan) => apply_standard(request, plan), - Self::Prebid(plan) => apply_prebid(request, input, routed, plan), - Self::Aps(plan) => apply_aps(request, plan), - } - } - - fn keeps_pbs_identity_when_unsigned(self) -> bool { - matches!(self, Self::Prebid(_)) - } -} - +/// Builds every standard field of the request. The second value holds, for +/// each impression built, the index of the routed slot it came from. fn build_common_request( input: &ProviderAuctionInput, routed: &RoutedAuction, - policy: ProfilePolicy<'_>, + demand: &dyn CompiledDemand, + policy: DemandFieldPolicy, effective_timeout_ms: u32, -) -> OpenRtbRequest { +) -> (OpenRtbRequest, Vec) { let common = input.common_request(); - let imps = input + let (slot_indices, imps): (Vec, Vec) = input .slots() .iter() - .filter_map(|slot| build_imp(slot, policy)) - .collect(); - let site_domain = match policy { - ProfilePolicy::Aps(plan) => plan - .inventory_domain - .clone() - .unwrap_or_else(|| common.publisher.domain.clone()), - _ => common.publisher.domain.clone(), - }; - let page = match policy { - ProfilePolicy::Aps(plan) => { - aps_inventory_page(plan, common.publisher.page_url.as_deref(), &site_domain) - } - ProfilePolicy::Prebid(plan) => common.publisher.page_url.as_deref().map(|page| { - plan.debug_query_params.as_deref().map_or_else( - || page.to_string(), - |query| append_query_fragment(page, query), - ) - }), - ProfilePolicy::Standard(_) => common.publisher.page_url.clone(), - }; - let consent = common.user.consent.as_ref(); - let body_consent = match policy { - ProfilePolicy::Prebid(plan) => consent.filter(|value| { - plan.consent_forwarding.includes_body_consent() - || !matches!(value.source, ConsentSource::Cookie) - }), - _ => consent, - }; + .enumerate() + .filter_map(|(index, slot)| build_imp(slot, policy).map(|imp| (index, imp))) + .unzip(); + let site_domain = demand.site_domain(&common.publisher.domain); + let page = demand.site_page(common.publisher.page_url.as_deref(), &site_domain); + let body_consent = demand.body_consent(common.user.consent.as_ref()); let raw_tc = body_consent.and_then(|value| value.raw_tc_string.clone()); let user = Some(User { id: common.user.id.clone(), consent: raw_tc.clone(), ext: UserExt { consent: raw_tc, - consented_providers_settings: matches!(policy, ProfilePolicy::Prebid(_)) + consented_providers_settings: policy + .additional_consent .then(|| { body_consent .and_then(|value| value.raw_ac_string.clone()) @@ -374,7 +386,7 @@ fn build_common_request( .to_ext(), ..Default::default() }); - let language = normalized_language(routed.prebid_transport_headers(), policy); + let language = normalized_language(routed.transport_headers(), policy); let device = common .device .as_ref() @@ -385,8 +397,8 @@ fn build_common_request( country: Some(geo.country.clone()), region: geo.region.clone(), city: Some(geo.city.clone()), - lat: matches!(policy, ProfilePolicy::Prebid(_)).then_some(geo.latitude), - lon: matches!(policy, ProfilePolicy::Prebid(_)).then_some(geo.longitude), + lat: policy.precise_geo.then_some(geo.latitude), + lon: policy.precise_geo.then_some(geo.longitude), metro: (geo.metro_code > 0).then(|| geo.metro_code.to_string()), r#type: Some(2), ..Default::default() @@ -403,14 +415,15 @@ fn build_common_request( }) }); - OpenRtbRequest { + let request = OpenRtbRequest { id: Some(common.id.clone()), imp: imps, site: Some(Site { domain: Some(site_domain.clone()), page, - r#ref: matches!(policy, ProfilePolicy::Prebid(_)) - .then(|| header_string(routed.prebid_transport_headers().referer())) + r#ref: policy + .site_ref + .then(|| header_string(routed.transport_headers().referer())) .flatten(), publisher: Some(Publisher { domain: Some(site_domain), @@ -420,11 +433,8 @@ fn build_common_request( }), user, device, - regs: build_regs(body_consent, policy), - test: match policy { - ProfilePolicy::Prebid(plan) => plan.test_mode.then_some(true), - _ => None, - }, + regs: build_regs(body_consent, policy.regs), + test: policy.test.then_some(true), tmax: to_openrtb_i32( effective_timeout_ms, "tmax", @@ -432,10 +442,11 @@ fn build_common_request( ), cur: vec![DEFAULT_CURRENCY.to_string()], ..Default::default() - } + }; + (request, slot_indices) } -fn build_imp(slot: &ProviderSlotInput, policy: ProfilePolicy<'_>) -> Option { +fn build_imp(slot: &ProviderSlotInput, policy: DemandFieldPolicy) -> Option { let formats = slot .slot() .formats @@ -451,17 +462,17 @@ fn build_imp(slot: &ProviderSlotInput, policy: ProfilePolicy<'_>) -> Option .collect::>(); let first_width = formats.first()?.w; let first_height = formats.first()?.h; - let aps_banner = matches!(policy, ProfilePolicy::Aps(_)); + let primary_size = policy.primary_banner_size; Some(Imp { id: Some(slot.slot().id.clone()), banner: Some(Banner { format: formats, - w: aps_banner.then_some(first_width).flatten(), - h: aps_banner.then_some(first_height).flatten(), - topframe: aps_banner.then_some(false), + w: primary_size.then_some(first_width).flatten(), + h: primary_size.then_some(first_height).flatten(), + topframe: primary_size.then_some(false), ..Default::default() }), - tagid: matches!(policy, ProfilePolicy::Prebid(_)).then(|| slot.slot().id.clone()), + tagid: policy.imp_tagid.then(|| slot.slot().id.clone()), bidfloor: slot.slot().floor_price, bidfloorcur: slot .slot() @@ -472,99 +483,9 @@ fn build_imp(slot: &ProviderSlotInput, policy: ProfilePolicy<'_>) -> Option }) } -fn apply_standard( - request: &mut OpenRtbRequest, - plan: &StandardProfilePlan, -) -> Result<(), Report> { - request.ext = nonempty_map(plan.request_ext.as_object().clone()); - for imp in &mut request.imp { - imp.ext = nonempty_map(plan.imp_ext.as_object().clone()); - } - Ok(()) -} - -fn apply_prebid( - request: &mut OpenRtbRequest, - input: &ProviderAuctionInput, - _routed: &RoutedAuction, - plan: &PrebidProfilePlan, -) -> Result<(), Report> { - // Routing admits only slots with positive banner dimensions that fit in i32, - // so build_imp produces one impression per routed slot, in the same order. - debug_assert_eq!( - request.imp.len(), - input.slots().len(), - "should keep one impression per routed slot" - ); - // Filter paired impressions and slots together so later demand keeps its slot ID. - request.imp = std::mem::take(&mut request.imp) - .into_iter() - .zip(input.slots()) - .filter_map(|(mut imp, slot)| { - let bidder = slot - .bidder_params() - .iter() - .filter_map(|(bidder, params)| { - let mut params = params.clone(); - plan.override_engine.apply_routed( - bidder.as_str(), - slot.prebid_zone(), - &mut params, - ); - params - .as_object() - .is_some_and(|params| !params.is_empty()) - .then(|| (bidder.as_str().to_string(), params)) - }) - .collect::>(); - let mut prebid = Map::new(); - if !bidder.is_empty() { - prebid.insert("bidder".to_string(), Value::Object(bidder)); - } else if slot.allows_stored_fallback() { - prebid.insert("storedrequest".to_string(), json!({"id": slot.slot().id})); - } - if prebid.is_empty() { - return None; - } - imp.ext = Some(Map::from_iter([( - "prebid".to_string(), - Value::Object(prebid), - )])); - Some(imp) - }) - .collect(); - let mut prebid_request = Map::new(); - if plan.debug { - prebid_request.insert("debug".to_string(), Value::Bool(true)); - prebid_request.insert("returnallbidstatus".to_string(), Value::Bool(true)); - } - request.ext = Some(Map::from_iter([( - "prebid".to_string(), - Value::Object(prebid_request), - )])); - Ok(()) -} - -fn apply_aps( - request: &mut OpenRtbRequest, - plan: &ApsProfilePlan, -) -> Result<(), Report> { - request.ext = Some(Map::from_iter([ - ( - "account".to_string(), - Value::String(plan.account_id.clone()), - ), - ( - "sdk".to_string(), - json!({"source": APS_SDK_SOURCE, "version": APS_SDK_VERSION}), - ), - ])); - Ok(()) -} - fn finalize_request( request: &mut OpenRtbRequest, - policy: ProfilePolicy<'_>, + policy: DemandFieldPolicy, finalization: &RequestFinalization<'_>, ) -> Result<(), Report> { let request_id = request.id.as_deref().ok_or_else(|| { @@ -587,7 +508,7 @@ fn finalize_request( request_scheme: Some(finalization.signing_params.request_scheme.clone()), ts: Some(finalization.signing_params.timestamp), }) - } else if policy.keeps_pbs_identity_when_unsigned() { + } else if policy.unsigned_request_identity { Some(TrustedServerExt { version: None, signature: None, @@ -606,19 +527,19 @@ fn finalize_request( message: format!("Failed to serialize Trusted Server extension: {error}"), }) })?; - ext.insert("trusted_server".to_string(), serialized); + ext.insert(TRUSTED_SERVER_EXT_KEY.to_string(), serialized); } Ok(()) } fn build_regs( consent: Option<&crate::consent::ConsentContext>, - policy: ProfilePolicy<'_>, + policy: RegsPolicy, ) -> Option { let consent = consent?; - if matches!(policy, ProfilePolicy::Aps(_)) { - // Preserve APS exactly: any admitted context produces regs and GDPR is - // derived only from the applicability bit, without jurisdiction rules. + if policy == RegsPolicy::ApplicabilityBit { + // Any admitted context produces regs and GDPR comes from the + // applicability bit alone, without jurisdiction rules. let ext = RegsExt { gdpr: Some(u8::from(consent.gdpr_applies)), us_privacy: consent.raw_us_privacy.clone(), @@ -639,8 +560,8 @@ fn build_regs( }); } - // Standard deliberately shares PBS's conservative consent baseline. Keep - // the legacy PBS empty-context and jurisdiction behavior byte-for-byte. + // The jurisdiction policy sends no regs for an empty context and reads + // GDPR from the applicability bit or from a GDPR jurisdiction. let has_data = consent.gdpr_applies || consent.raw_us_privacy.is_some() || consent.raw_gpp_string.is_some() @@ -684,21 +605,16 @@ fn build_regs( }) } -fn normalized_language( - headers: &PrebidTransportHeaders, - policy: ProfilePolicy<'_>, -) -> Option { +fn normalized_language(headers: &TransportHeaders, policy: DemandFieldPolicy) -> Option { let value = header_string(headers.accept_language()) .and_then(|value| value.split(',').next().map(str::to_string)) .and_then(|value| value.split(';').next().map(str::to_string)) .and_then(|value| value.split('-').next().map(str::to_string)) .map(|value| value.trim().to_string()) .filter(|value| !value.is_empty())?; - match policy { - ProfilePolicy::Prebid(_) => Some(value), - ProfilePolicy::Aps(_) | ProfilePolicy::Standard(_) => { - (value.len() <= MAX_CONSERVATIVE_LANGUAGE_BYTES).then_some(value) - } + match policy.language_max_bytes { + None => Some(value), + Some(max_bytes) => (value.len() <= max_bytes).then_some(value), } } @@ -708,54 +624,8 @@ fn header_string(value: Option<&http::HeaderValue>) -> Option { .map(str::to_string) } -fn aps_inventory_page( - plan: &ApsProfilePlan, - publisher_page: Option<&str>, - domain: &str, -) -> Option { - let fallback = publisher_page - .and_then(valid_aps_page_url) - .unwrap_or_else(|| format!("https://{domain}")); - let Some(origin) = plan.inventory_page_origin.as_deref() else { - return Some(fallback); - }; - let (Ok(mut canonical), Ok(current)) = (Url::parse(origin), Url::parse(&fallback)) else { - return Some(fallback); - }; - canonical.set_path(current.path()); - canonical.set_query(current.query()); - canonical.set_fragment(None); - Some(canonical.to_string()) -} - -fn append_query_fragment(url: &str, query: &str) -> String { - if query.is_empty() || url.contains(query) { - return url.to_string(); - } - let separator = if url.contains('?') { '&' } else { '?' }; - format!("{url}{separator}{query}") -} - -fn valid_aps_page_url(value: &str) -> Option { - const MAX_APS_PAGE_URL_BYTES: usize = 8192; - - if value.len() > MAX_APS_PAGE_URL_BYTES { - return None; - } - let parsed = Url::parse(value).ok()?; - (matches!(parsed.scheme(), "http" | "https") - && parsed.host_str().is_some() - && parsed.username().is_empty() - && parsed.password().is_none()) - .then(|| parsed.to_string()) -} - -fn nonempty_map(value: Map) -> Option> { - (!value.is_empty()).then_some(value) -} - /// Suppress notification URLs using exact returned-seat identity. -pub(crate) fn apply_notification_policy(bids: &mut [Bid], policy: &NotificationPolicy) { +pub fn apply_notification_policy(bids: &mut [Bid], policy: &NotificationPolicy) { for bid in bids { let suppress = policy.suppress_all || bid @@ -897,21 +767,24 @@ fn extract_standard_bid( }) } -/// Count bidder parameter objects a profile did not consume. +/// Count bidder parameter objects a demand source did not consume. #[must_use] pub(crate) fn unused_bidder_params_count( - profile: &CompiledOpenRtbProfile, + demand: &dyn CompiledDemand, input: &ProviderAuctionInput, ) -> u32 { - if profile.is_prebid_server() { + if demand.field_policy().consumes_bidder_params { return 0; } ignored_bidder_params_count(input) } -/// Count routed bidder params for a profile known to ignore them. +/// Count routed bidder params for a demand source known to ignore them. +/// +/// Public so an implementation that takes no bidder parameters reports the +/// ones it was routed the way core's own reader does. #[must_use] -pub(crate) fn ignored_bidder_params_count(input: &ProviderAuctionInput) -> u32 { +pub fn ignored_bidder_params_count(input: &ProviderAuctionInput) -> u32 { saturating_bidder_param_counts(input.slots().iter().map(|slot| slot.bidder_params().len())) } @@ -921,20 +794,76 @@ fn saturating_bidder_param_counts(counts: impl IntoIterator) -> u3 }) } +/// The most a demand source's response body may hold. +const MAX_RESPONSE_BYTES: usize = 1024 * 1024; + +/// Reads an ordinary `OpenRTB` response. A `204` is a no-bid, any other +/// non-success status or unreadable JSON is an error response, and a body over +/// the size limit is an `Err`. +/// +/// Public so a demand implementation in a crate of its own, whose exchange +/// answers the ordinary way, has its response read as core reads one. +/// +/// # Errors +/// +/// Returns an error when the response body cannot be read. +pub async fn parse_openrtb_response( + context: DemandResponse<'_>, + response: PlatformResponse, +) -> Result> { + let provider_id = context.provider_id; + let response_time_ms = context.response_time_ms; + let response = response.response; + let status = response.status(); + if status == StatusCode::NO_CONTENT { + return Ok(AuctionResponse::no_bid(provider_id, response_time_ms)); + } + if !status.is_success() { + if status.is_redirection() { + log::warn!( + "Provider '{provider_id}' returned a redirect; generic OpenRTB redirects are refused" + ); + } + return Ok(AuctionResponse::error(provider_id, response_time_ms) + .with_metadata("error_type", json!("http_status")) + .with_metadata("http_status", json!(status.as_u16()))); + } + + let body = response + .into_body() + .into_bytes_bounded(MAX_RESPONSE_BYTES) + .await + .change_context(TrustedServerError::Auction { + message: format!("Provider {provider_id} response body failed"), + })?; + let value: Value = match serde_json::from_slice(&body) { + Ok(value) => value, + Err(error) => { + log::warn!("Provider '{provider_id}' response JSON was invalid: {error}"); + return Ok(AuctionResponse::error(provider_id, response_time_ms) + .with_metadata("error_type", json!("parse_response"))); + } + }; + + Ok(extract_standard_response( + provider_id, + context.input, + &value, + response_time_ms, + )) +} + #[cfg(test)] mod routing_metadata_tests { use std::collections::BTreeMap; use std::str::FromStr as _; - use serde_json::json; - use super::{saturating_bidder_param_counts, unused_bidder_params_count}; - use crate::auction::plan::{ - AuctionPlan, AuctionPlanConfig, BidderId, BidderRouteConfig, NotificationConfig, - ProviderConfig, ProviderId, RoutingMode, - }; + use crate::auction::plan::{AuctionPlan, BidderId, BidderRouteConfig, ProviderId}; use crate::auction::routing::route_auction; - use crate::auction::test_support::canonical_parity_auction_request; + use crate::auction::test_support::{ + canonical_parity_auction_request, demand_table, plan_config, + }; #[test] fn unused_bidder_param_count_saturates_across_slots_and_large_values() { @@ -947,49 +876,32 @@ mod routing_metadata_tests { } #[test] - fn unused_bidder_param_count_is_profile_aware() { - for (profile, profile_config, expected) in [ - ("prebid-server", json!({}), 0), - ("standard", json!({}), 1), - ("aps", json!({"account_id":"example-account"}), 1), - ] { + fn unused_bidder_param_count_follows_the_implementation() { + for (implementation, expected) in [("auction.fixture", 0), ("auction.plain-fixture", 1)] { + let endpoint = "https://provider.example/openrtb"; let provider_id = - ProviderId::from_str("fictional-provider").expect("should parse provider ID"); - let plan = AuctionPlan::compile(AuctionPlanConfig { - timeout_ms: 1_000, - providers: BTreeMap::from([( - provider_id.clone(), - ProviderConfig { - protocol: "openrtb-2.6".to_string(), - profile: profile.to_string(), - endpoint: if profile == "aps" { - "https://aps.example/e/pb/bid".to_string() - } else { - "https://provider.example/openrtb".to_string() - }, - timeout_ms: None, - routing: RoutingMode::Explicit, - notifications: NotificationConfig::default(), - profile_config, - }, - )]), - bidders: BTreeMap::from([( - BidderId::from_str("exampleBidder").expect("should parse bidder ID"), - BidderRouteConfig { - provider: provider_id, - }, - )]), - mediator: None, - request_signing: None, - }) - .expect("should compile profile plan"); + ProviderId::from_str("fictional_provider").expect("should parse provider ID"); + let mut config = plan_config(vec![( + "fictional_provider", + demand_table(implementation, endpoint), + )]); + config.bidders = BTreeMap::from([( + BidderId::from_str("exampleBidder").expect("should parse bidder ID"), + BidderRouteConfig { + module: provider_id, + }, + )]); + let plan = AuctionPlan::compile(config).expect("should compile the plan"); let inbound = http::Request::new(edgezero_core::body::Body::empty()); let routed = route_auction(canonical_parity_auction_request(), &inbound, &plan, None); assert_eq!( - unused_bidder_params_count(&plan.providers()[0].profile, &routed.inputs()[0]), + unused_bidder_params_count( + plan.providers()[0].demand.as_ref(), + &routed.inputs()[0] + ), expected, - "{profile} should report only bidder params it ignores" + "{implementation} should report only bidder params it ignores" ); } } diff --git a/crates/trusted-server-core/src/auction/openrtb/test_executor.rs b/crates/trusted-server-core/src/auction/openrtb/test_executor.rs index d322102e62..2f2fa20f84 100644 --- a/crates/trusted-server-core/src/auction/openrtb/test_executor.rs +++ b/crates/trusted-server-core/src/auction/openrtb/test_executor.rs @@ -66,7 +66,7 @@ pub(super) async fn execute_standard_fixture( return Ok( AuctionResponse::no_bid(provider.id.as_str(), 0).with_metadata( "routing", - json!({"unused_bidder_params_count": unused_bidder_params_count(&provider.profile, input)}), + json!({"unused_bidder_params_count": unused_bidder_params_count(provider.demand.as_ref(), input)}), ), ); } @@ -76,7 +76,7 @@ pub(super) async fn execute_standard_fixture( .with_metadata("http_status", json!(status.as_u16())) .with_metadata( "routing", - json!({"unused_bidder_params_count": unused_bidder_params_count(&provider.profile, input)}), + json!({"unused_bidder_params_count": unused_bidder_params_count(provider.demand.as_ref(), input)}), )); } let body = response @@ -93,7 +93,7 @@ pub(super) async fn execute_standard_fixture( .with_metadata("error_type", json!("parse_response")) .with_metadata( "routing", - json!({"unused_bidder_params_count": unused_bidder_params_count(&provider.profile, input)}), + json!({"unused_bidder_params_count": unused_bidder_params_count(provider.demand.as_ref(), input)}), )); } }; @@ -101,7 +101,7 @@ pub(super) async fn execute_standard_fixture( apply_notification_policy(&mut parsed.bids, &provider.notifications); parsed.metadata.insert( "routing".to_string(), - json!({"unused_bidder_params_count": unused_bidder_params_count(&provider.profile, input)}), + json!({"unused_bidder_params_count": unused_bidder_params_count(provider.demand.as_ref(), input)}), ); Ok(parsed) } diff --git a/crates/trusted-server-core/src/auction/openrtb/tests.rs b/crates/trusted-server-core/src/auction/openrtb/tests.rs index f47efa475e..18eaf5118b 100644 --- a/crates/trusted-server-core/src/auction/openrtb/tests.rs +++ b/crates/trusted-server-core/src/auction/openrtb/tests.rs @@ -9,69 +9,62 @@ use serde_json::{Value, json}; use super::test_executor::execute_standard_fixture; use super::*; +use crate::auction::demand::DemandResponse; use crate::auction::plan::{ - AuctionPlan, AuctionPlanConfig, BidderId, BidderRouteConfig, NotificationConfig, - ProviderConfig, ProviderId, RoutingMode, + AuctionPlan, AuctionPlanConfig, BidderId, BidderRouteConfig, ProviderId, }; use crate::auction::provider::{GenericOpenRtbProvider, ProviderRequestOutcome}; use crate::auction::routing::route_auction; -use crate::auction::test_support::canonical_parity_auction_request; +use crate::auction::test_support::{canonical_parity_auction_request, demand_table, plan_config}; use crate::auction::types::{AdFormat, AdSlot, BidStatus, MediaType}; +use crate::consent::ConsentContext; use crate::consent::jurisdiction::Jurisdiction; -use crate::consent::{ConsentContext, ConsentSource}; -use crate::platform::PlatformHttpClient; use crate::platform::test_support::{ HashMapConfigStore, HashMapSecretStore, NoopHttpClient, StubBackend, StubHttpClient, build_services_with_backend_and_http_client, build_services_with_config_secret_and_http_client, }; +use crate::platform::{PlatformHttpClient, PlatformResponse}; use crate::request_signing::RequestSigner; -fn config(profile: &str, profile_config: Value) -> AuctionPlanConfig { +fn config(implementation: &str, settings: Value) -> AuctionPlanConfig { config_with_endpoint( - profile, - profile_config, + implementation, + settings, "https://exchange.example.test/openrtb", ) } -fn config_with_endpoint(profile: &str, profile_config: Value, endpoint: &str) -> AuctionPlanConfig { - let provider_id = ProviderId::from_str("fictional-provider").expect("should parse provider ID"); - let prebid_server = profile == "prebid-server"; - AuctionPlanConfig { - timeout_ms: 321, - providers: BTreeMap::from([( - provider_id.clone(), - ProviderConfig { - protocol: "openrtb-2.6".to_string(), - profile: profile.to_string(), - endpoint: endpoint.to_string(), - timeout_ms: Some(321), - routing: if prebid_server { - RoutingMode::Explicit - } else { - RoutingMode::AllEligible - }, - notifications: NotificationConfig::default(), - profile_config, +fn config_with_endpoint( + implementation: &str, + settings: Value, + endpoint: &str, +) -> AuctionPlanConfig { + let provider_id = ProviderId::from_str("fictional_provider").expect("should parse provider ID"); + // The stand-in forbids all-eligible routing, so a bidder is routed to it. + let routed_by_bidder = implementation == "auction.fixture"; + let mut table = demand_table(implementation, endpoint); + table.insert("timeout_ms".to_string(), json!(321)); + if !routed_by_bidder { + table.insert("routing".to_string(), json!("all_eligible")); + } + if let Value::Object(settings) = settings { + table.extend(settings); + } + let mut config = plan_config(vec![("fictional_provider", table)]); + config.timeout_ms = 321; + if routed_by_bidder { + config.bidders = BTreeMap::from([( + BidderId::from_str("exampleBidder").expect("should parse bidder ID"), + BidderRouteConfig { + module: provider_id, }, - )]), - bidders: if prebid_server { - BTreeMap::from([( - BidderId::from_str("exampleBidder").expect("should parse bidder ID"), - BidderRouteConfig { - provider: provider_id, - }, - )]) - } else { - BTreeMap::new() - }, - mediator: None, - request_signing: None, + )]); } + config } -fn routed(profile: &str, profile_config: Value) -> (AuctionPlan, RoutedAuction) { - let plan = AuctionPlan::compile(config(profile, profile_config)).expect("should compile plan"); +fn routed(implementation: &str, settings: Value) -> (AuctionPlan, RoutedAuction) { + let plan = AuctionPlan::compile(config(implementation, settings)).expect("should compile plan"); let inbound = Request::builder() .uri("https://publisher.example/auction") .header( @@ -87,12 +80,12 @@ fn routed(profile: &str, profile_config: Value) -> (AuctionPlan, RoutedAuction) } fn routed_with_request( - profile: &str, - profile_config: Value, + implementation: &str, + settings: Value, request: crate::auction::types::AuctionRequest, accept_language: Option<&str>, ) -> (AuctionPlan, RoutedAuction) { - let plan = AuctionPlan::compile(config(profile, profile_config)).expect("should compile plan"); + let plan = AuctionPlan::compile(config(implementation, settings)).expect("should compile plan"); let mut builder = Request::builder().uri("https://publisher.example/auction"); if let Some(language) = accept_language { builder = builder.header(header::ACCEPT_LANGUAGE, language); @@ -105,12 +98,12 @@ fn routed_with_request( } fn build_with_request( - profile: &str, - profile_config: Value, + implementation: &str, + settings: Value, request: crate::auction::types::AuctionRequest, accept_language: Option<&str>, ) -> OpenRtbRequest { - let (plan, routed) = routed_with_request(profile, profile_config, request, accept_language); + let (plan, routed) = routed_with_request(implementation, settings, request, accept_language); match build_request( &routed.inputs()[0], &routed, @@ -137,8 +130,8 @@ fn finalization<'a>(signer: Option<&'a RequestSigner>) -> RequestFinalization<'a } } -fn build(profile: &str, profile_config: Value, signer: Option<&RequestSigner>) -> OpenRtbRequest { - let (plan, routed) = routed(profile, profile_config); +fn build(implementation: &str, settings: Value, signer: Option<&RequestSigner>) -> OpenRtbRequest { + let (plan, routed) = routed(implementation, settings); match build_request( &routed.inputs()[0], &routed, @@ -172,7 +165,7 @@ fn deterministic_signer() -> RequestSigner { } #[test] -fn consent_matrix_preserves_pbs_standard_and_aps_policies() { +fn consent_matrix_holds_under_each_implementation() { let cases = [ ("empty", ConsentContext::default()), ( @@ -212,26 +205,19 @@ fn consent_matrix_preserves_pbs_standard_and_aps_policies() { ), ]; for (name, consent) in cases { - for profile in ["standard", "prebid-server", "aps"] { + for implementation in ["auction.plain-fixture", "auction.fixture"] { let mut canonical = canonical_parity_auction_request(); canonical.user.consent = Some(consent.clone()); - let config = if profile == "aps" { - json!({"account_id": "example-account-id"}) - } else { - json!({}) - }; - let value = serde_json::to_value(build_with_request(profile, config, canonical, None)) - .expect("should serialize request"); + let value = serde_json::to_value(build_with_request( + implementation, + json!({}), + canonical, + None, + )) + .expect("should serialize request"); let regs = value.get("regs"); - if profile == "aps" { - let regs = regs.expect("APS should preserve empty admitted context"); - assert_eq!( - regs["gdpr"], - json!(u8::from(consent.gdpr_applies)), - "{name}" - ); - } else if name == "empty" { - assert!(regs.is_none(), "{profile} should omit empty regs"); + if name == "empty" { + assert!(regs.is_none(), "{implementation} should omit empty regs"); } else { let regs = regs.expect("should emit actionable regs"); let expected_gdpr = match consent.jurisdiction { @@ -242,7 +228,7 @@ fn consent_matrix_preserves_pbs_standard_and_aps_policies() { assert_eq!( regs.get("gdpr"), expected_gdpr.map(|value| json!(u8::from(value))).as_ref(), - "{name} {profile}" + "{name} {implementation}" ); } let serialized = value.to_string(); @@ -263,53 +249,16 @@ fn consent_matrix_preserves_pbs_standard_and_aps_policies() { } } -#[test] -fn pbs_body_consent_respects_source_and_forwarding_mode() { - for (mode, source, expected) in [ - ("cookies_only", ConsentSource::Cookie, false), - ("cookies_only", ConsentSource::PolicyDefault, true), - ("openrtb_only", ConsentSource::Cookie, true), - ("both", ConsentSource::Cookie, true), - ] { - let mut canonical = canonical_parity_auction_request(); - canonical - .user - .consent - .as_mut() - .expect("should have consent context") - .source = source; - let value = serde_json::to_value(build_with_request( - "prebid-server", - json!({"consent_forwarding": mode}), - canonical, - None, - )) - .expect("should serialize request"); - assert_eq!( - value["user"].get("consent").is_some(), - expected, - "{mode:?} {source:?}" - ); - assert_eq!(value.get("regs").is_some(), expected, "{mode:?} {source:?}"); - } -} - #[test] fn language_limits_are_profile_specific() { let language = "abcdefghijk"; - for (profile, expected) in [ - ("prebid-server", Some(language)), - ("aps", None), - ("standard", None), + for (implementation, expected) in [ + ("auction.fixture", Some(language)), + ("auction.plain-fixture", None), ] { - let config = if profile == "aps" { - json!({"account_id": "example-account-id"}) - } else { - json!({}) - }; let request = build_with_request( - profile, - config, + implementation, + json!({}), canonical_parity_auction_request(), Some(language), ); @@ -318,15 +267,10 @@ fn language_limits_are_profile_specific() { expected ); } - for profile in ["prebid-server", "aps", "standard"] { - let config = if profile == "aps" { - json!({"account_id": "example-account-id"}) - } else { - json!({}) - }; + for implementation in ["auction.fixture", "auction.plain-fixture"] { let request = build_with_request( - profile, - config, + implementation, + json!({}), canonical_parity_auction_request(), Some("en-US,en;q=0.9"), ); @@ -338,272 +282,9 @@ fn language_limits_are_profile_specific() { } #[test] -fn pbs_debug_query_fragment_preserves_exact_legacy_configured_semantics() { - for (page, fragment, expected) in [ - ( - "https://publisher.example/article", - "pbjs_debug=true", - "https://publisher.example/article?pbjs_debug=true", - ), - ( - "https://publisher.example/article?existing=1", - "pbjs_debug=true", - "https://publisher.example/article?existing=1&pbjs_debug=true", - ), - ( - "https://publisher.example/article", - "?pbjs_debug=true", - "https://publisher.example/article??pbjs_debug=true", - ), - ( - "https://publisher.example/article?pbjs_debug=true", - "pbjs_debug=true", - "https://publisher.example/article?pbjs_debug=true", - ), - ( - "https://publisher.example/article", - "", - "https://publisher.example/article", - ), - ] { - let mut request = canonical_parity_auction_request(); - request.publisher.page_url = Some(page.to_string()); - let built = build_with_request( - "prebid-server", - json!({"debug_query_params": fragment}), - request, - None, - ); - assert_eq!( - built.site.and_then(|site| site.page).as_deref(), - Some(expected), - "should preserve exact legacy query fragment semantics" - ); - } -} - -#[test] -fn pbs_routed_overrides_are_ordered_and_stored_request_is_trusted_fallback() { - let mut raw = config( - "prebid-server", - json!({ - "debug": true, - "test_mode": true, - "bid_param_overrides": {"exampleBidder": {"generic": 1, "shared": "generic"}}, - "bid_param_zone_overrides": {"exampleBidder": {"zone-a": {"zone": 2, "shared": "zone"}}}, - "bid_param_override_rules": [ - {"when":{"bidder":"exampleBidder"},"set":{"ordered":1,"shared":"rule-one"}}, - {"when":{"bidder":"exampleBidder","zone":"zone-a"},"set":{"ordered":2,"shared":"rule-two"}} - ] - }), - ); - raw.providers - .get_mut(&ProviderId::from_str("fictional-provider").expect("should parse provider")) - .expect("should find provider") - .routing = RoutingMode::Explicit; - raw.bidders.insert( - crate::auction::plan::BidderId::from_str("exampleBidder").expect("should parse bidder"), - BidderRouteConfig { - provider: ProviderId::from_str("fictional-provider").expect("should parse provider"), - }, - ); - let plan = AuctionPlan::compile(raw).expect("should compile PBS override plan"); - let mut request = canonical_parity_auction_request(); - request.slots[0].bidders = HashMap::from([( - "trustedServer".to_string(), - json!({"zone":"zone-a","bidderParams":{"exampleBidder":{"original":true,"shared":"original"}}}), - )]); - let inbound = Request::builder() - .uri("https://publisher.example/auction") - .body(EdgeBody::empty()) - .expect("should build inbound request"); - let routed = route_auction(request, &inbound, &plan, None); - let built = match build_request( - &routed.inputs()[0], - &routed, - &plan.providers()[0], - 321, - &finalization(None), - ) - .expect("should build request") - { - OpenRtbBuildOutcome::Ready(request) => request, - OpenRtbBuildOutcome::NoImpressions => panic!("should retain impression"), - }; - let value = serde_json::to_value(built).expect("should serialize request"); - assert_eq!( - value["imp"][0]["ext"]["prebid"]["bidder"]["exampleBidder"], - json!({"generic":1,"ordered":2,"original":true,"shared":"rule-two","zone":2}) - ); - assert_eq!(value["ext"]["prebid"]["debug"], true); - assert_eq!(value["ext"]["prebid"]["returnallbidstatus"], true); - assert_eq!(value["test"], 1); - - let mut empty_overridden = canonical_parity_auction_request(); - empty_overridden.slots[0].bidders = HashMap::from([( - "trustedServer".to_string(), - json!({"zone":"zone-a","bidderParams":{"exampleBidder":{}}}), - )]); - let routed = route_auction(empty_overridden, &inbound, &plan, None); - let built = match build_request( - &routed.inputs()[0], - &routed, - &plan.providers()[0], - 321, - &finalization(None), - ) - .expect("should build request after populating empty params") - { - OpenRtbBuildOutcome::Ready(request) => request, - OpenRtbBuildOutcome::NoImpressions => panic!("should retain overridden impression"), - }; - let value = serde_json::to_value(built).expect("should serialize overridden request"); - assert_eq!( - value["imp"][0]["ext"]["prebid"]["bidder"]["exampleBidder"], - json!({"generic":1,"ordered":2,"shared":"rule-two","zone":2}), - "should allow profile overrides to populate empty browser params" - ); - - let mut stored = canonical_parity_auction_request(); - stored.slots[0].bidders.clear(); - let routed = route_auction(stored, &inbound, &plan, None); - let built = match build_request( - &routed.inputs()[0], - &routed, - &plan.providers()[0], - 321, - &finalization(None), - ) - .expect("should build stored request") - { - OpenRtbBuildOutcome::Ready(request) => request, - OpenRtbBuildOutcome::NoImpressions => panic!("should retain impression"), - }; - let value = serde_json::to_value(built).expect("should serialize stored request"); - assert_eq!( - value["imp"][0]["ext"]["prebid"]["storedrequest"]["id"], - "fictional-slot" - ); -} - -#[test] -fn pbs_pairs_each_impression_with_its_routed_slot_params() { - let mut raw = config("prebid-server", json!({})); - raw.providers - .get_mut(&ProviderId::from_str("fictional-provider").expect("should parse provider")) - .expect("should find provider") - .routing = RoutingMode::Explicit; - raw.bidders.insert( - crate::auction::plan::BidderId::from_str("exampleBidder").expect("should parse bidder"), - BidderRouteConfig { - provider: ProviderId::from_str("fictional-provider").expect("should parse provider"), - }, - ); - let plan = AuctionPlan::compile(raw).expect("should compile PBS plan"); - let mut request = canonical_parity_auction_request(); - request.slots[0].id = "first-slot".to_string(); - request.slots[0].bidders = HashMap::from([( - "trustedServer".to_string(), - json!({"bidderParams":{"exampleBidder":{"placement":"first"}}}), - )]); - let mut second_slot = request.slots[0].clone(); - second_slot.id = "second-slot".to_string(); - second_slot.bidders = HashMap::from([( - "trustedServer".to_string(), - json!({"bidderParams":{"exampleBidder":{"placement":"second"}}}), - )]); - request.slots.push(second_slot); - let inbound = Request::builder() - .uri("https://publisher.example/auction") - .body(EdgeBody::empty()) - .expect("should build inbound request"); - let routed = route_auction(request, &inbound, &plan, None); - - let built = match build_request( - &routed.inputs()[0], - &routed, - &plan.providers()[0], - 321, - &finalization(None), - ) - .expect("should build request") - { - OpenRtbBuildOutcome::Ready(request) => request, - OpenRtbBuildOutcome::NoImpressions => panic!("should retain impressions"), - }; - let value = serde_json::to_value(built).expect("should serialize request"); - - assert_eq!(value["imp"][0]["id"], "first-slot"); - assert_eq!( - value["imp"][0]["ext"]["prebid"]["bidder"]["exampleBidder"], - json!({"placement":"first"}) - ); - assert_eq!(value["imp"][1]["id"], "second-slot"); - assert_eq!( - value["imp"][1]["ext"]["prebid"]["bidder"]["exampleBidder"], - json!({"placement":"second"}) - ); -} - -#[test] -fn pbs_stored_intent_is_applied_after_overrides_with_inline_first() { - for intent in [None, Some(false), Some(true)] { - for inline in [false, true] { - for fill_override in [false, true] { - let profile = if fill_override { - json!({"bid_param_overrides":{"exampleBidder":{"filled":1}}}) - } else { - json!({}) - }; - let plan = AuctionPlan::compile(config("prebid-server", profile)) - .expect("should compile plan"); - let mut request = canonical_parity_auction_request(); - let mut envelope = json!({"bidderParams":{"exampleBidder":if inline { json!({"original":1}) } else { json!({}) }}}); - if let Some(intent) = intent { - envelope["storedRequest"] = json!(intent); - } - request.slots[0].bidders = HashMap::from([("trustedServer".to_string(), envelope)]); - let inbound = Request::new(EdgeBody::empty()); - let routed = route_auction(request, &inbound, &plan, None); - let result = build_request( - &routed.inputs()[0], - &routed, - &plan.providers()[0], - 321, - &finalization(None), - ) - .expect("should build request"); - if !inline && !fill_override && intent == Some(false) { - assert!(matches!(result, OpenRtbBuildOutcome::NoImpressions)); - continue; - } - let OpenRtbBuildOutcome::Ready(request) = result else { - panic!("should retain demand") - }; - let wire = serde_json::to_value(request).expect("should serialize request"); - let prebid = &wire["imp"][0]["ext"]["prebid"]; - if inline || fill_override { - assert!(prebid.get("storedrequest").is_none()); - assert_eq!( - prebid["bidder"]["exampleBidder"].get("original"), - inline.then_some(&json!(1)) - ); - assert_eq!( - prebid["bidder"]["exampleBidder"].get("filled"), - fill_override.then_some(&json!(1)) - ); - } else { - assert_eq!(prebid["storedrequest"]["id"], "fictional-slot"); - } - } - } - } -} - -#[test] -fn pbs_filtering_keeps_slot_pairs_and_drops_demandless_trusted_routes() { +fn filtering_keeps_slot_pairs_and_drops_demandless_trusted_routes() { let plan = - AuctionPlan::compile(config("prebid-server", json!({}))).expect("should compile plan"); + AuctionPlan::compile(config("auction.fixture", json!({}))).expect("should compile plan"); let mut request = canonical_parity_auction_request(); let template = request.slots[0].clone(); request.slots = [ @@ -667,30 +348,23 @@ fn pbs_filtering_keeps_slot_pairs_and_drops_demandless_trusted_routes() { ); assert_eq!(wire["imp"][0]["id"], "keep-inline"); assert_eq!( - wire["imp"][0]["ext"]["prebid"]["bidder"]["exampleBidder"], + wire["imp"][0]["ext"]["fixture"]["bidder"]["exampleBidder"], json!({"id":2}) ); assert_eq!(wire["imp"][1]["id"], "keep-stored"); - assert_eq!( - wire["imp"][1]["ext"]["prebid"]["storedrequest"]["id"], - "keep-stored" - ); + assert_eq!(wire["imp"][1]["ext"]["fixture"]["stored"], "keep-stored"); } #[test] -fn pbs_disabled_empty_candidate_does_not_become_stored_demand() { - let mut raw = config("prebid-server", json!({})); - raw.providers - .get_mut(&ProviderId::from_str("fictional-provider").expect("should parse provider")) - .expect("should find provider") - .routing = RoutingMode::Explicit; +fn a_disabled_empty_candidate_does_not_become_stored_demand() { + let mut raw = config("auction.fixture", json!({"routing": "explicit"})); raw.bidders.insert( crate::auction::plan::BidderId::from_str("exampleBidder").expect("should parse bidder"), BidderRouteConfig { - provider: ProviderId::from_str("fictional-provider").expect("should parse provider"), + module: ProviderId::from_str("fictional_provider").expect("should parse provider"), }, ); - let plan = AuctionPlan::compile(raw).expect("should compile PBS plan"); + let plan = AuctionPlan::compile(raw).expect("should compile the plan"); let mut request = canonical_parity_auction_request(); request.slots[0].bidders = HashMap::from([( "trustedServer".to_string(), @@ -719,183 +393,33 @@ fn pbs_disabled_empty_candidate_does_not_become_stored_demand() { )); } -#[test] -fn pbs_empty_params_without_matching_override_fall_back_to_stored_request() { - let mut raw = config("prebid-server", json!({})); - raw.providers - .get_mut(&ProviderId::from_str("fictional-provider").expect("should parse provider")) - .expect("should find provider") - .routing = RoutingMode::Explicit; - raw.bidders.insert( - crate::auction::plan::BidderId::from_str("exampleBidder").expect("should parse bidder"), - BidderRouteConfig { - provider: ProviderId::from_str("fictional-provider").expect("should parse provider"), - }, - ); - let plan = AuctionPlan::compile(raw).expect("should compile PBS plan"); - let mut request = canonical_parity_auction_request(); - request.slots[0].bidders = HashMap::from([( - "trustedServer".to_string(), - json!({"bidderParams":{"exampleBidder":{}}}), - )]); - let inbound = Request::builder() - .uri("https://publisher.example/auction") - .body(EdgeBody::empty()) - .expect("should build inbound request"); - let routed = route_auction(request, &inbound, &plan, None); - let built = match build_request( - &routed.inputs()[0], - &routed, - &plan.providers()[0], - 321, - &finalization(None), - ) - .expect("should build stored request") - { - OpenRtbBuildOutcome::Ready(request) => request, - OpenRtbBuildOutcome::NoImpressions => panic!("should retain stored impression"), - }; - let value = serde_json::to_value(built).expect("should serialize stored request"); - assert_eq!( - value["imp"][0]["ext"]["prebid"]["storedrequest"]["id"], - "fictional-slot" - ); - assert!(value["imp"][0]["ext"]["prebid"].get("bidder").is_none()); -} - -#[test] -fn pbs_driver_exact_golden_preserves_profile_policy() { - let mut raw = config("prebid-server", json!({"consent_forwarding": "both"})); - raw.providers - .get_mut(&ProviderId::from_str("fictional-provider").expect("should parse provider")) - .expect("should find provider") - .routing = RoutingMode::Explicit; - raw.bidders.insert( - crate::auction::plan::BidderId::from_str("exampleBidder").expect("should parse bidder"), - BidderRouteConfig { - provider: ProviderId::from_str("fictional-provider").expect("should parse provider"), - }, - ); - let plan = AuctionPlan::compile(raw).expect("should compile PBS plan"); - let mut common = canonical_parity_auction_request(); - common.slots[0].bidders = HashMap::from([( - "exampleBidder".to_string(), - json!({"placement": "fictional-placement"}), - )]); - let inbound = Request::builder() - .uri("https://publisher.example/auction") - .header( - header::REFERER, - "https://referrer.example/story?fictional=1", - ) - .header(header::ACCEPT_LANGUAGE, "en-US,en;q=0.9") - .header("dnt", "1") - .body(EdgeBody::empty()) - .expect("should build inbound request"); - let routed = route_auction(common, &inbound, &plan, None); - let request = match build_request( - &routed.inputs()[0], - &routed, - &plan.providers()[0], - 321, - &finalization(None), - ) - .expect("should build PBS request") - { - OpenRtbBuildOutcome::Ready(request) => request, - OpenRtbBuildOutcome::NoImpressions => panic!("should retain impression"), - }; - assert_eq!( - serde_json::to_string(&request).expect("should serialize PBS driver request"), - r#"{"id":"fictional-auction","imp":[{"id":"fictional-slot","banner":{"format":[{"w":300,"h":250},{"w":728,"h":90}]},"tagid":"fictional-slot","bidfloor":1.0,"bidfloorcur":"USD","secure":1,"ext":{"prebid":{"bidder":{"exampleBidder":{"placement":"fictional-placement"}}}}}],"site":{"domain":"publisher.example","page":"https://publisher.example/article","ref":"https://referrer.example/story?fictional=1","publisher":{"domain":"publisher.example"}},"device":{"geo":{"lat":12.34,"lon":56.78,"type":2,"country":"US","region":"CA","metro":"501","city":"Example City"},"dnt":1,"ua":"Fictional Browser","ip":"192.0.2.10","language":"en"},"user":{"id":"fictional-user","consent":"fictional-tcf","ext":{"ConsentedProvidersSettings":{"consented_providers":"fictional-ac"},"consent":"fictional-tcf","eids":[{"source":"identity.example","uids":[{"atype":1,"id":"fictional-uid"}]}]}},"tmax":321,"cur":["USD"],"regs":{"gdpr":1,"us_privacy":"1YNN","gpp":"fictional-gpp","gpp_sid":[2,6],"ext":{"gdpr":1,"gpp":"fictional-gpp","gpp_sid":[2,6],"us_privacy":"1YNN"}},"ext":{"prebid":{},"trusted_server":{"request_host":"publisher.example","request_scheme":"https"}}}"#, - "should preserve PBS parity differences" - ); -} - -#[test] -fn aps_inventory_identity_and_page_fallback_preserve_legacy_policy() { - let mut request = canonical_parity_auction_request(); - request.publisher.domain = "deployment.example".to_string(); - request.publisher.page_url = - Some("https://deployment.example/news/story?edition=fictional#section".to_string()); - let built = build_with_request( - "aps", - json!({ - "account_id": "example-account-id", - "inventory_domain": "publisher.example", - "inventory_page_origin": "https://www.publisher.example" - }), - request, - None, - ); - let site = built.site.expect("should include APS site"); - assert_eq!(site.domain.as_deref(), Some("publisher.example")); - assert_eq!( - site.page.as_deref(), - Some("https://www.publisher.example/news/story?edition=fictional") - ); - assert_eq!( - site.publisher - .and_then(|publisher| publisher.domain) - .as_deref(), - Some("publisher.example") - ); - - for unsafe_page in [ - "https://user:password@publisher.example/private", - "data:text/html,fictional", - ] { - let mut request = canonical_parity_auction_request(); - request.publisher.page_url = Some(unsafe_page.to_string()); - let built = build_with_request( - "aps", - json!({"account_id":"example-account-id"}), - request, - None, - ); - assert_eq!( - built.site.and_then(|site| site.page).as_deref(), - Some("https://publisher.example"), - "unsafe page should fall back to publisher domain" - ); - } -} - -#[test] -fn aps_driver_exact_golden_preserves_profile_policy() { - let request = build("aps", json!({"account_id": "example-account-id"}), None); - assert_eq!( - serde_json::to_string(&request).expect("should serialize APS driver request"), - r#"{"id":"fictional-auction","imp":[{"id":"fictional-slot","banner":{"format":[{"w":300,"h":250},{"w":728,"h":90}],"w":300,"h":250,"topframe":0},"bidfloor":1.0,"bidfloorcur":"USD","secure":1}],"site":{"domain":"publisher.example","page":"https://publisher.example/article","publisher":{"domain":"publisher.example"}},"device":{"geo":{"type":2,"country":"US","region":"CA","metro":"501","city":"Example City"},"dnt":1,"ua":"Fictional Browser","ip":"192.0.2.10","language":"en"},"user":{"id":"fictional-user","consent":"fictional-tcf","ext":{"consent":"fictional-tcf","eids":[{"source":"identity.example","uids":[{"atype":1,"id":"fictional-uid"}]}]}},"tmax":321,"cur":["USD"],"regs":{"gdpr":1,"us_privacy":"1YNN","gpp":"fictional-gpp","gpp_sid":[2,6],"ext":{"gdpr":1,"gpp":"fictional-gpp","gpp_sid":[2,6],"us_privacy":"1YNN"}},"ext":{"account":"example-account-id","sdk":{"source":"prebid","version":"2.2.0"}}}"#, - "should preserve APS parity differences" - ); -} - #[test] fn signing_finalization_is_after_profiles_and_asserts_every_owned_key() { let signer = deterministic_signer(); - for (profile, config) in [ - ("standard", json!({"request_ext": {"fictional": true}})), - ("prebid-server", json!({})), - ("aps", json!({"account_id": "example-account-id"})), + for (implementation, config) in [ + ( + "auction.plain-fixture", + json!({"request_ext": {"fictional": true}}), + ), + ("auction.fixture", json!({})), ] { - let unsigned = serde_json::to_value(build(profile, config.clone(), None)) + let unsigned = serde_json::to_value(build(implementation, config.clone(), None)) .expect("should serialize unsigned request"); let unsigned_ts = unsigned["ext"].get("trusted_server"); - if profile == "prebid-server" { + if implementation == "auction.fixture" { assert_eq!( unsigned_ts, Some(&json!({"request_host": "publisher.example", "request_scheme": "https"})), - "should retain only PBS host and scheme when unsigned" + "should retain only the host and scheme when unsigned" ); } else { assert!( unsigned_ts.is_none(), - "should omit unsigned non-PBS extension" + "should omit the extension when unsigned" ); } - let signed = serde_json::to_value(build(profile, config, Some(&signer))) + let signed = serde_json::to_value(build(implementation, config, Some(&signer))) .expect("should serialize signed request"); let extension = &signed["ext"]["trusted_server"]; assert_eq!(extension["version"], "1.1", "should set signing version"); @@ -921,35 +445,23 @@ fn signing_finalization_is_after_profiles_and_asserts_every_owned_key() { #[test] fn signed_profiles_and_unsigned_standard_have_exact_full_goldens() { let signer = deterministic_signer(); - let cases = [ - ( - "standard", - json!({"request_ext": {"fictional": true}}), - r#"{"id":"fictional-auction","imp":[{"id":"fictional-slot","banner":{"format":[{"w":300,"h":250},{"w":728,"h":90}]},"bidfloor":1.0,"bidfloorcur":"USD","secure":1}],"site":{"domain":"publisher.example","page":"https://publisher.example/article","publisher":{"domain":"publisher.example"}},"device":{"geo":{"type":2,"country":"US","region":"CA","metro":"501","city":"Example City"},"dnt":1,"ua":"Fictional Browser","ip":"192.0.2.10","language":"en"},"user":{"id":"fictional-user","consent":"fictional-tcf","ext":{"consent":"fictional-tcf","eids":[{"source":"identity.example","uids":[{"atype":1,"id":"fictional-uid"}]}]}},"tmax":321,"cur":["USD"],"regs":{"gdpr":1,"us_privacy":"1YNN","gpp":"fictional-gpp","gpp_sid":[2,6],"ext":{"gdpr":1,"gpp":"fictional-gpp","gpp_sid":[2,6],"us_privacy":"1YNN"}},"ext":{"fictional":true,"trusted_server":{"kid":"fictional-kid","request_host":"publisher.example","request_scheme":"https","signature":"LU_JUIA1BT80ShZNjSa4PIF5T-uMjEeodwKrV_6bXgh0hi1SYVtCKn9g_DTW62krmjCOFgoFYPHsu6L0nAcuDg","ts":1706900000,"version":"1.1"}}}"#, - ), - ( - "prebid-server", - json!({}), - r#"{"id":"fictional-auction","imp":[{"id":"fictional-slot","banner":{"format":[{"w":300,"h":250},{"w":728,"h":90}]},"tagid":"fictional-slot","bidfloor":1.0,"bidfloorcur":"USD","secure":1,"ext":{"prebid":{"bidder":{"exampleBidder":{"placement":"fictional-placement"}}}}}],"site":{"domain":"publisher.example","page":"https://publisher.example/article","ref":"https://referrer.example/story?fictional=1","publisher":{"domain":"publisher.example"}},"device":{"geo":{"lat":12.34,"lon":56.78,"type":2,"country":"US","region":"CA","metro":"501","city":"Example City"},"dnt":1,"ua":"Fictional Browser","ip":"192.0.2.10","language":"en"},"user":{"id":"fictional-user","consent":"fictional-tcf","ext":{"ConsentedProvidersSettings":{"consented_providers":"fictional-ac"},"consent":"fictional-tcf","eids":[{"source":"identity.example","uids":[{"atype":1,"id":"fictional-uid"}]}]}},"tmax":321,"cur":["USD"],"regs":{"gdpr":1,"us_privacy":"1YNN","gpp":"fictional-gpp","gpp_sid":[2,6],"ext":{"gdpr":1,"gpp":"fictional-gpp","gpp_sid":[2,6],"us_privacy":"1YNN"}},"ext":{"prebid":{},"trusted_server":{"kid":"fictional-kid","request_host":"publisher.example","request_scheme":"https","signature":"LU_JUIA1BT80ShZNjSa4PIF5T-uMjEeodwKrV_6bXgh0hi1SYVtCKn9g_DTW62krmjCOFgoFYPHsu6L0nAcuDg","ts":1706900000,"version":"1.1"}}}"#, - ), - ( - "aps", - json!({"account_id": "example-account-id"}), - r#"{"id":"fictional-auction","imp":[{"id":"fictional-slot","banner":{"format":[{"w":300,"h":250},{"w":728,"h":90}],"w":300,"h":250,"topframe":0},"bidfloor":1.0,"bidfloorcur":"USD","secure":1}],"site":{"domain":"publisher.example","page":"https://publisher.example/article","publisher":{"domain":"publisher.example"}},"device":{"geo":{"type":2,"country":"US","region":"CA","metro":"501","city":"Example City"},"dnt":1,"ua":"Fictional Browser","ip":"192.0.2.10","language":"en"},"user":{"id":"fictional-user","consent":"fictional-tcf","ext":{"consent":"fictional-tcf","eids":[{"source":"identity.example","uids":[{"atype":1,"id":"fictional-uid"}]}]}},"tmax":321,"cur":["USD"],"regs":{"gdpr":1,"us_privacy":"1YNN","gpp":"fictional-gpp","gpp_sid":[2,6],"ext":{"gdpr":1,"gpp":"fictional-gpp","gpp_sid":[2,6],"us_privacy":"1YNN"}},"ext":{"account":"example-account-id","sdk":{"source":"prebid","version":"2.2.0"},"trusted_server":{"kid":"fictional-kid","request_host":"publisher.example","request_scheme":"https","signature":"LU_JUIA1BT80ShZNjSa4PIF5T-uMjEeodwKrV_6bXgh0hi1SYVtCKn9g_DTW62krmjCOFgoFYPHsu6L0nAcuDg","ts":1706900000,"version":"1.1"}}}"#, - ), - ]; - for (profile, config, expected) in cases { + let cases = [( + "auction.plain-fixture", + json!({"request_ext": {"fictional": true}}), + r#"{"id":"fictional-auction","imp":[{"id":"fictional-slot","banner":{"format":[{"w":300,"h":250},{"w":728,"h":90}]},"bidfloor":1.0,"bidfloorcur":"USD","secure":1}],"site":{"domain":"publisher.example","page":"https://publisher.example/article","publisher":{"domain":"publisher.example"}},"device":{"geo":{"type":2,"country":"US","region":"CA","metro":"501","city":"Example City"},"dnt":1,"ua":"Fictional Browser","ip":"192.0.2.10","language":"en"},"user":{"id":"fictional-user","consent":"fictional-tcf","ext":{"consent":"fictional-tcf","eids":[{"source":"identity.example","uids":[{"atype":1,"id":"fictional-uid"}]}]}},"tmax":321,"cur":["USD"],"regs":{"gdpr":1,"us_privacy":"1YNN","gpp":"fictional-gpp","gpp_sid":[2,6],"ext":{"gdpr":1,"gpp":"fictional-gpp","gpp_sid":[2,6],"us_privacy":"1YNN"}},"ext":{"fictional":true,"trusted_server":{"kid":"fictional-kid","request_host":"publisher.example","request_scheme":"https","signature":"LU_JUIA1BT80ShZNjSa4PIF5T-uMjEeodwKrV_6bXgh0hi1SYVtCKn9g_DTW62krmjCOFgoFYPHsu6L0nAcuDg","ts":1706900000,"version":"1.1"}}}"#, + )]; + for (implementation, config, expected) in cases { assert_eq!( - serde_json::to_string(&build(profile, config, Some(&signer))) + serde_json::to_string(&build(implementation, config, Some(&signer))) .expect("should serialize signed request"), expected, - "{profile} signed wire fixture should stay exact" + "{implementation} signed wire fixture should stay exact" ); } assert_eq!( serde_json::to_string(&build( - "standard", + "auction.plain-fixture", json!({"request_ext": {"fictional": true}}), None, )) @@ -959,28 +471,9 @@ fn signed_profiles_and_unsigned_standard_have_exact_full_goldens() { ); } -#[test] -fn standard_static_extensions_have_no_invented_bidder_param_location() { - let request = build( - "standard", - json!({ - "request_ext": {"fictional_request": {"enabled": true}}, - "imp_ext": {"fictional_imp": "value"} - }), - None, - ); - let value = serde_json::to_value(request).expect("should serialize request"); - assert_eq!(value["ext"]["fictional_request"]["enabled"], true); - assert_eq!(value["imp"][0]["ext"]["fictional_imp"], "value"); - assert!( - !value.to_string().contains("exampleBidder"), - "standard profile must not invent bidder params placement" - ); -} - #[test] fn defensive_no_impression_outcome_does_not_build_transportable_request() { - let (plan, mut routed) = routed("standard", json!({})); + let (plan, mut routed) = routed("auction.plain-fixture", json!({})); let mut common = routed.inputs()[0].common_request().clone(); common.slots = vec![AdSlot { id: "video-only".to_string(), @@ -1008,17 +501,13 @@ fn standard_fixture_with_formats( formats: Vec, ) -> (AuctionPlan, RoutedAuction, OpenRtbRequest) { let mut raw = config( - "standard", + "auction.plain-fixture", json!({"request_ext": {"fixture": true}, "imp_ext": {"slot_fixture": true}}), ); - raw.providers - .get_mut(&ProviderId::from_str("fictional-provider").expect("should parse provider")) - .expect("should find provider") - .routing = RoutingMode::Explicit; raw.bidders.insert( - crate::auction::plan::BidderId::from_str("exampleBidder").expect("should parse bidder"), + BidderId::from_str("exampleBidder").expect("should parse bidder"), BidderRouteConfig { - provider: ProviderId::from_str("fictional-provider").expect("should parse provider"), + module: ProviderId::from_str("fictional_provider").expect("should parse provider"), }, ); let plan = AuctionPlan::compile(raw).expect("should compile standard fixture plan"); @@ -1056,7 +545,7 @@ fn standard_fixture() -> (AuctionPlan, RoutedAuction, OpenRtbRequest) { fn standard_response_extraction_isolates_malformed_siblings_and_ignores_response_id() { let (_plan, routed, _request) = standard_fixture(); let response = extract_standard_response( - "fictional-provider", + "fictional_provider", &routed.inputs()[0], &json!({ "id": "informational-mismatch", @@ -1088,7 +577,7 @@ fn standard_response_currency_accepts_omitted_and_usd_but_rejects_other_or_malfo value["cur"] = currency; } let response = - extract_standard_response("fictional-provider", &routed.inputs()[0], &value, 0); + extract_standard_response("fictional_provider", &routed.inputs()[0], &value, 0); assert_eq!( response.status, BidStatus::Success, @@ -1099,14 +588,14 @@ fn standard_response_currency_accepts_omitted_and_usd_but_rejects_other_or_malfo let mut eur = bid.clone(); eur["cur"] = json!("EUR"); - let response = extract_standard_response("fictional-provider", &routed.inputs()[0], &eur, 0); + let response = extract_standard_response("fictional_provider", &routed.inputs()[0], &eur, 0); assert_eq!(response.status, BidStatus::NoBid); assert_eq!(response.metadata["unsupported_currency"], "EUR"); let mut malformed = bid; malformed["cur"] = json!(["USD"]); let response = - extract_standard_response("fictional-provider", &routed.inputs()[0], &malformed, 0); + extract_standard_response("fictional_provider", &routed.inputs()[0], &malformed, 0); assert_eq!(response.status, BidStatus::Error); assert_eq!(response.metadata["error_type"], "parse_response"); } @@ -1115,7 +604,7 @@ fn standard_response_currency_accepts_omitted_and_usd_but_rejects_other_or_malfo fn standard_response_rejects_unknown_impressions_and_dimensions_but_keeps_siblings() { let (_plan, routed, _request) = standard_fixture(); let response = extract_standard_response( - "fictional-provider", + "fictional_provider", &routed.inputs()[0], &json!({"seatbid": [{"seat": "seat", "bid": [ {"id":"good","impid":"fictional-slot","price":1.0,"adm":"ok","w":300,"h":250}, @@ -1145,7 +634,7 @@ fn standard_response_rejects_unknown_impressions_and_dimensions_but_keeps_siblin fn standard_response_infers_only_unambiguous_missing_dimensions() { let (_plan, routed, _request) = standard_fixture(); let inferred = extract_standard_response( - "fictional-provider", + "fictional_provider", &routed.inputs()[0], &json!({"seatbid": [{"bid": [ {"id":"inferred","impid":"fictional-slot","price":1.0,"adm":"ok"} @@ -1170,7 +659,7 @@ fn standard_response_infers_only_unambiguous_missing_dimensions() { ]; let (_plan, routed, _request) = standard_fixture_with_formats(formats); let ambiguous = extract_standard_response( - "fictional-provider", + "fictional_provider", &routed.inputs()[0], &json!({"seatbid": [{"bid": [ {"id":"ambiguous","impid":"fictional-slot","price":1.0,"adm":"ok"} @@ -1205,7 +694,7 @@ fn standard_response_infers_dimensions_from_partial_unique_matches() { let (_plan, routed, _request) = standard_fixture_with_formats(formats); let response = extract_standard_response( - "fictional-provider", + "fictional_provider", &routed.inputs()[0], &json!({"seatbid": [{"bid": [ {"id":"width-only","impid":"fictional-slot","price":1.0,"adm":"width","w":300}, @@ -1237,7 +726,7 @@ fn standard_response_accepts_null_and_integral_float_dimensions() { let (_plan, routed, _request) = standard_fixture(); let response = extract_standard_response( - "fictional-provider", + "fictional_provider", &routed.inputs()[0], &json!({"seatbid": [{"bid": [ {"id":"null","impid":"fictional-slot","price":1.0,"adm":"null","w":null,"h":null}, @@ -1287,7 +776,7 @@ fn standard_response_rejects_partial_dimension_mismatch_and_ambiguity() { let (_plan, routed, _request) = standard_fixture_with_formats(formats); let response = extract_standard_response( - "fictional-provider", + "fictional_provider", &routed.inputs()[0], &json!({"seatbid": [{"bid": [ {"id":"ambiguous","impid":"fictional-slot","price":1.0,"adm":"ambiguous","w":300}, @@ -1316,7 +805,7 @@ fn standard_response_rejects_invalid_numeric_dimension_shapes() { let (_plan, routed, _request) = standard_fixture(); let response = extract_standard_response( - "fictional-provider", + "fictional_provider", &routed.inputs()[0], &json!({"seatbid": [{"bid": [ {"id":"fractional","impid":"fictional-slot","price":1.0,"adm":"bad","w":300.5}, @@ -1347,7 +836,7 @@ fn standard_response_rejects_invalid_numeric_dimension_shapes() { fn notification_suppression_matrix_uses_only_exact_valid_returned_seat() { let (_plan, routed, _request) = standard_fixture(); let response = extract_standard_response( - "fictional-provider", + "fictional_provider", &routed.inputs()[0], &json!({"seatbid": [ {"seat": "exact", "bid": [{"id":"exact","impid":"fictional-slot","price":1.0,"adm":"ok","w":300,"h":250,"nurl":"https://n.example","burl":"https://b.example"}]}, @@ -1485,26 +974,29 @@ fn fictional_standard_executor_covers_bid_no_bid_malformed_unused_and_redirect() ); let spec = provider.backend_spec(); assert_eq!(spec.host, "exchange.example.test"); - assert_eq!(spec.discriminator.as_deref(), Some("fictional-provider")); + assert_eq!(spec.discriminator.as_deref(), Some("fictional_provider")); }); } #[test] -fn prebid_endpoint_normalization_reaches_generic_execution_and_preserves_custom_paths() { +fn a_completed_endpoint_reaches_generic_execution_and_a_custom_path_is_kept() { futures::executor::block_on(async { for (configured_endpoint, expected_endpoint) in [ ( - "https://pbs.example", - "https://pbs.example/openrtb2/auction", + "https://exchange.example", + "https://exchange.example/fixture/auction", + ), + ( + "https://exchange.example/bid", + "https://exchange.example/bid", ), - ("https://pbs.example/bid", "https://pbs.example/bid"), ] { let plan = AuctionPlan::compile(config_with_endpoint( - "prebid-server", + "auction.fixture", json!({}), configured_endpoint, )) - .expect("should compile Prebid Server endpoint"); + .expect("should compile the endpoint"); let inbound = Request::builder() .uri("https://publisher.example/auction") .body(EdgeBody::empty()) @@ -1531,26 +1023,24 @@ fn prebid_endpoint_normalization_reaches_generic_execution_and_preserves_custom_ &mut reserved_backend_names, ) .await - .expect("should launch one Prebid Server request"); + .expect("should launch one request"); let ProviderRequestOutcome::Pending { request, .. } = outcome else { - panic!("should launch a pending Prebid Server request"); + panic!("should launch a pending request"); }; let selected = services .http_client() .select(vec![request]) .await - .expect("should select one Prebid Server response"); - let response = selected - .ready - .expect("should receive the Prebid Server response"); + .expect("should select one response"); + let response = selected.ready.expect("should receive the response"); assert_eq!(response.response.status(), http::StatusCode::NO_CONTENT); assert_eq!(client.recorded_backend_names(), vec!["stub-backend"]); assert_eq!(client.recorded_request_methods(), vec!["POST"]); assert_eq!(client.recorded_request_uris(), vec![expected_endpoint]); - assert_eq!(provider_plan.backend_spec().host, "pbs.example"); + assert_eq!(provider_plan.backend_spec().host, "exchange.example"); let body: Value = serde_json::from_slice(&client.recorded_request_bodies()[0]) - .expect("should parse recorded Prebid Server body"); + .expect("should parse the recorded body"); assert!( !body .as_object() @@ -1565,7 +1055,97 @@ fn prebid_endpoint_normalization_reaches_generic_execution_and_preserves_custom_ fn malformed_top_level_standard_response_is_error() { let (_plan, routed, _request) = standard_fixture(); let response = - extract_standard_response("fictional-provider", &routed.inputs()[0], &json!([]), 0); + extract_standard_response("fictional_provider", &routed.inputs()[0], &json!([]), 0); assert_eq!(response.status, BidStatus::Error); assert_eq!(response.metadata["error_type"], "parse_response"); } + +/// A demand implementation that tries to write the extension the driver owns. +struct ForgingDemand; + +#[async_trait::async_trait(?Send)] +impl CompiledDemand for ForgingDemand { + fn field_policy(&self) -> DemandFieldPolicy { + DemandFieldPolicy::default() + } + + fn augment_request( + &self, + extensions: &mut RequestExtensions<'_>, + _input: &ProviderAuctionInput, + ) -> Result<(), Report> { + *extensions.request = Some(serde_json::Map::from_iter([( + "trusted_server".to_string(), + json!({"signature": "forged"}), + )])); + Ok(()) + } + + async fn parse_response( + &self, + context: DemandResponse<'_>, + _response: PlatformResponse, + ) -> Result> { + Ok(AuctionResponse::error( + context.provider_id, + context.response_time_ms, + )) + } + + fn as_any(&self) -> &dyn std::any::Any { + self + } +} + +#[test] +fn the_driver_refuses_an_implementation_that_claims_the_trusted_server_extension() { + let (plan, routed) = routed("auction.plain-fixture", json!({})); + let mut provider = plan.providers()[0].clone(); + provider.demand = Arc::new(ForgingDemand); + + let error = match build_request( + &routed.inputs()[0], + &routed, + &provider, + 321, + &finalization(None), + ) { + Ok(_) => panic!("should refuse a forged trusted_server extension"), + Err(error) => error, + }; + + assert!( + error.to_string().contains("trusted_server"), + "should name the extension the driver owns: {error:?}" + ); +} + +#[test] +fn an_implementation_sees_each_impression_beside_the_slot_it_came_from() { + let (plan, routed) = routed("auction.fixture", json!({})); + let request = match build_request( + &routed.inputs()[0], + &routed, + &plan.providers()[0], + 321, + &finalization(None), + ) + .expect("should build the request") + { + OpenRtbBuildOutcome::Ready(request) => request, + OpenRtbBuildOutcome::NoImpressions => panic!("should keep the routed impression"), + }; + + let slots = routed.inputs()[0].slots(); + assert_eq!(request.imp.len(), slots.len()); + for (imp, slot) in request.imp.iter().zip(slots) { + assert_eq!(imp.id.as_deref(), Some(slot.slot().id.as_str())); + assert!( + imp.ext + .as_ref() + .and_then(|ext| ext.get("fixture")) + .is_some(), + "each impression should carry the extension built from its own slot" + ); + } +} diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 204201e605..77a0266749 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -40,7 +40,7 @@ pub struct DispatchedAuction { timeout_ms: u32, floor_prices: HashMap, provider_request_context: Box>, - /// Carried so the mediator call in collect can pass it as the auction request. + /// Carried so the ad server call in collect can pass it as the auction request. request: AuctionRequest, planned_unused_bidder_params: HashMap, planned_unroutable_bidder_count: u32, @@ -148,9 +148,9 @@ pub(crate) const ERROR_TYPE_TRANSPORT: &str = "transport"; pub(crate) const ERROR_TYPE_TIMEOUT: &str = "timeout"; /// A non-2xx HTTP status from an upstream SSP (e.g. a PBS 4xx/5xx). Distinct /// from [`ERROR_TYPE_TRANSPORT`] (a connection-level failure) so telemetry can -/// bucket it separately. `pub(crate)` so producers such as the prebid provider -/// tag errors with the exact value the telemetry layer recognises. -pub(crate) const ERROR_TYPE_HTTP_STATUS: &str = "http_status"; +/// bucket it separately. Public so a producer in a crate of its own tags +/// errors with the exact value the telemetry layer recognises. +pub const ERROR_TYPE_HTTP_STATUS: &str = "http_status"; /// Every server-owned `error_type` classification. /// @@ -309,7 +309,7 @@ pub struct AuctionOrchestrator { plan_backed: bool, plan: Arc, planned_providers: Vec>, - mediator: Option>, + adserver: Option>, #[cfg(test)] config: AuctionConfig, #[cfg(test)] @@ -321,7 +321,7 @@ pub struct AuctionOrchestrator { pub(crate) struct AuctionOrchestratorHarness { plan: Arc, providers: Vec>, - mediator: Option>, + adserver: Option>, } struct PlannedLaunchState { @@ -338,7 +338,7 @@ struct PlannedLaunchState { impl AuctionOrchestratorHarness { pub(crate) fn new( plan: impl Into>, - mediator: Option>, + adserver: Option>, ) -> Self { let plan = plan.into(); let providers = plan @@ -351,7 +351,7 @@ impl AuctionOrchestratorHarness { Self { plan, providers, - mediator, + adserver, } } @@ -359,8 +359,8 @@ impl AuctionOrchestratorHarness { self.providers.len() } - pub(crate) fn mediator(&self) -> Option<&Arc> { - self.mediator.as_ref() + pub(crate) fn adserver(&self) -> Option<&Arc> { + self.adserver.as_ref() } /// Route and execute config-first bidder providers in deterministic order. @@ -423,11 +423,11 @@ impl AuctionOrchestratorHarness { ( input.provider_id().as_str().to_string(), unused_bidder_params_count( - &self - .plan + self.plan .provider(input.provider_id()) .expect("should find routed provider in compiled plan") - .profile, + .demand + .as_ref(), input, ), ) @@ -642,33 +642,33 @@ impl AuctionOrchestratorHarness { .collect::>(); let helper = AuctionOrchestrator::new(AuctionConfig::default()); let local_winners = || helper.select_winning_bids(&responses, &floor_prices); - let (mediator_response, winning_bids) = if let Some(mediator) = &self.mediator { + let (adserver_response, winning_bids) = if let Some(adserver) = &self.adserver { let remaining_ms = remaining_budget_ms(auction_start, context.timeout_ms); - let logical_budget_ms = remaining_ms.min(mediator.timeout_ms()); + let logical_budget_ms = remaining_ms.min(adserver.timeout_ms()); if logical_budget_ms == 0 { log::warn!( - "Auction deadline exhausted before planned mediator; using local ranking" + "Auction deadline exhausted before planned adserver; using local ranking" ); (None, local_winners()) } else { let transport_timeout_ms = context .services .backend() - .canonicalize_transport_timeout_ms(logical_budget_ms, mediator.timeout_ms()); + .canonicalize_transport_timeout_ms(logical_budget_ms, adserver.timeout_ms()); if transport_timeout_ms == 0 { log::warn!( - "Planned mediator transport budget canonicalized to zero; using local ranking" + "Planned adserver transport budget canonicalized to zero; using local ranking" ); let winning_bids = local_winners(); return Ok(OrchestrationResult { provider_responses: responses, - mediator_response: None, + adserver_response: None, winning_bids, total_time_ms: auction_start.elapsed().as_millis() as u64, metadata: routing_metadata(routed.diagnostics().unroutable_bidder_count()), }); } - let mediator_context = AuctionContext { + let adserver_context = AuctionContext { settings: context.settings, request: context.request, timeout_ms: logical_budget_ms, @@ -676,9 +676,9 @@ impl AuctionOrchestratorHarness { provider_responses: Some(&responses), services: context.services, }; - let mediator_start = Instant::now(); - let mediated = match mediator - .request_bids(original_request, &mediator_context) + let adserver_start = Instant::now(); + let decided = match adserver + .request_bids(original_request, &adserver_context) .await { Ok(ProviderRequestOutcome::Immediate(response)) => Some(response), @@ -687,30 +687,30 @@ impl AuctionOrchestratorHarness { parse_state, }) => match context.services.http_client().wait(pending).await { Ok(platform_response) => { - let response_time_ms = mediator_start.elapsed().as_millis() as u64; + let response_time_ms = adserver_start.elapsed().as_millis() as u64; if AuctionDeadlinePolicy::for_runtime(context.services) .rejects_late_completion(auction_start, context.timeout_ms) { log::warn!( - "Planned mediator '{}' completed after the hard auction deadline; using local ranking ({}ms)", - mediator.provider_name(), + "Planned adserver '{}' completed after the hard auction deadline; using local ranking ({}ms)", + adserver.provider_name(), response_time_ms ); None } else { - mediator + adserver .parse_response_with_context_and_state( platform_response, response_time_ms, original_request, - &mediator_context, + &adserver_context, parse_state.as_deref(), ) .await .map_err(|error| { log::warn!( - "Planned mediator '{}' parse failed: {:?}", - mediator.provider_name(), + "Planned adserver '{}' parse failed: {:?}", + adserver.provider_name(), error ); }) @@ -719,8 +719,8 @@ impl AuctionOrchestratorHarness { } Err(error) => { log::warn!( - "Planned mediator '{}' request failed: {:?}", - mediator.provider_name(), + "Planned adserver '{}' request failed: {:?}", + adserver.provider_name(), error ); None @@ -728,21 +728,21 @@ impl AuctionOrchestratorHarness { }, Err(error) => { log::warn!( - "Planned mediator '{}' failed to launch: {:?}", - mediator.provider_name(), + "Planned adserver '{}' failed to launch: {:?}", + adserver.provider_name(), error ); None } }; - if let Some(mediated) = mediated { - let winners = mediated + if let Some(decided) = decided { + let winners = decided .bids .iter() .filter_map(|bid| { if bid.price.is_none() { log::warn!( - "Planned mediator returned a bid without a decoded price" + "Planned adserver returned a bid without a decoded price" ); None } else { @@ -751,7 +751,7 @@ impl AuctionOrchestratorHarness { }) .collect(); ( - Some(mediated), + Some(decided), helper.apply_floor_prices(winners, &floor_prices), ) } else { @@ -769,7 +769,7 @@ impl AuctionOrchestratorHarness { ); Ok(OrchestrationResult { provider_responses: responses, - mediator_response, + adserver_response, winning_bids, total_time_ms: auction_start.elapsed().as_millis() as u64, metadata: routing_metadata(unroutable_bidder_count), @@ -785,10 +785,7 @@ impl AuctionOrchestrator { let plan = Arc::new( AuctionPlan::compile(super::plan::AuctionPlanConfig { timeout_ms: config.timeout_ms, - providers: std::collections::BTreeMap::new(), - bidders: std::collections::BTreeMap::new(), - mediator: None, - request_signing: None, + ..super::plan::AuctionPlanConfig::default() }) .expect("should compile empty legacy test plan") .with_enabled(config.enabled), @@ -799,14 +796,14 @@ impl AuctionOrchestrator { config, plan, planned_providers: Vec::new(), - mediator: None, + adserver: None, providers: HashMap::new(), } } /// Create the live orchestrator from one shared compiled auction plan. #[must_use] - pub fn from_plan(plan: Arc, mediator: Option>) -> Self { + pub fn from_plan(plan: Arc, adserver: Option>) -> Self { let planned_providers = plan .providers() .iter() @@ -819,7 +816,7 @@ impl AuctionOrchestrator { plan_backed: true, plan, planned_providers, - mediator, + adserver, #[cfg(test)] config: AuctionConfig::default(), #[cfg(test)] @@ -833,6 +830,21 @@ impl AuctionOrchestrator { Arc::ptr_eq(&self.plan, plan) } + /// Return whether each auction publishes a token with its winning bids. + #[must_use] + pub fn publishes_auction_token(&self) -> bool { + self.plan.publishes_auction_token() + } + + /// The same orchestrator in a deployment where a module reads the + /// auction token. + #[cfg(test)] + #[must_use] + pub(crate) fn publishing_auction_token(mut self) -> Self { + self.plan = Arc::new((*self.plan).clone().with_auction_token(true)); + self + } + /// Register an auction provider in the legacy parity harness. #[cfg(test)] pub(crate) fn register_provider(&mut self, provider: Arc) { @@ -886,7 +898,7 @@ impl AuctionOrchestrator { /// # Errors /// /// Returns an error if the auction execution fails due to provider errors or - /// mediation errors. + /// ad server decision errors. pub async fn run_auction( &self, request: &AuctionRequest, @@ -904,12 +916,12 @@ impl AuctionOrchestrator { #[cfg(test)] let start_time = Instant::now(); - // Auto-detect strategy based on mediator configuration. + // Auto-detect strategy based on ad server configuration. #[cfg(test)] - let (strategy_name, result) = if self.config.has_mediator() { + let (strategy_name, result) = if self.config.has_adserver() { ( - "parallel_mediation", - self.run_parallel_mediation(request, context).await?, + "parallel_adserver", + self.run_parallel_adserver(request, context).await?, ) } else { ( @@ -920,7 +932,7 @@ impl AuctionOrchestrator { #[cfg(test)] log::info!( - "Running auction with strategy: {} (auto-detected from mediator config)", + "Running auction with strategy: {} (auto-detected from adserver config)", strategy_name ); @@ -931,128 +943,130 @@ impl AuctionOrchestrator { }) } - /// Run auction with parallel bidding + mediation. + /// Run auction with parallel bidding + ad server decision. #[cfg(test)] /// /// Flow: /// 1. Run all bidders in parallel /// 2. Collect bids from all bidders - /// 3. Send combined bids to mediator for final decision - async fn run_parallel_mediation( + /// 3. Send combined bids to ad server for final decision + async fn run_parallel_adserver( &self, request: &AuctionRequest, context: &AuctionContext<'_>, ) -> Result> { - let mediation_start = Instant::now(); + let adserver_start = Instant::now(); let provider_responses = self.run_providers_parallel(request, context).await?; let floor_prices = self.floor_prices_by_slot(request); - let (mediator_response, winning_bids) = if let Some(mediator_name) = &self.config.mediator { - let mediator = self.get_provider(mediator_name)?; + let (adserver_response, winning_bids) = if let Some(adserver_name) = + &self.config.adserver_name + { + let adserver = self.get_provider(adserver_name)?; log::info!( - "Sending {} provider responses to mediator: {}", + "Sending {} provider responses to adserver: {}", provider_responses.len(), - mediator.provider_name() + adserver.provider_name() ); - // Give the mediator only the remaining time from the auction + // Give the ad server only the remaining time from the auction // deadline, not the full timeout — the bidding phase already // consumed part of it. Canonicalize the transport timeout so the // backend name remains stable across equivalent budget values. - let remaining_ms = remaining_budget_ms(mediation_start, context.timeout_ms); - let mediator_timeout = context + let remaining_ms = remaining_budget_ms(adserver_start, context.timeout_ms); + let adserver_timeout = context .services .backend() - .canonicalize_transport_timeout_ms(remaining_ms, mediator.timeout_ms()); + .canonicalize_transport_timeout_ms(remaining_ms, adserver.timeout_ms()); - if mediator_timeout == 0 { - log::warn!("Auction timeout exhausted during bidding phase; skipping mediator"); + if adserver_timeout == 0 { + log::warn!("Auction timeout exhausted during bidding phase; skipping adserver"); let winning = self.select_winning_bids(&provider_responses, &floor_prices); return Ok(OrchestrationResult { provider_responses, - mediator_response: None, + adserver_response: None, winning_bids: winning, total_time_ms: 0, metadata: HashMap::new(), }); } - let mediator_context = AuctionContext { + let adserver_context = AuctionContext { settings: context.settings, request: context.request, - timeout_ms: mediator_timeout, - transport_timeout_ms: mediator_timeout, + timeout_ms: adserver_timeout, + transport_timeout_ms: adserver_timeout, provider_responses: Some(&provider_responses), services: context.services, }; let start_time = Instant::now(); - let mediator_resp = match mediator - .request_bids(request, &mediator_context) + let adserver_resp = match adserver + .request_bids(request, &adserver_context) .await .change_context(TrustedServerError::Auction { - message: format!("Mediator {} failed to launch", mediator.provider_name()), + message: format!("AdServer {} failed to launch", adserver.provider_name()), })? { ProviderRequestOutcome::Immediate(response) => response, ProviderRequestOutcome::Pending { request: pending, parse_state, } => { - let platform_resp = mediator_context + let platform_resp = adserver_context .services .http_client() .wait(pending) .await .change_context(TrustedServerError::Auction { message: format!( - "Mediator {} request failed", - mediator.provider_name() + "AdServer {} request failed", + adserver.provider_name() ), })?; let response_time_ms = start_time.elapsed().as_millis() as u64; if AuctionDeadlinePolicy::for_runtime(context.services) - .rejects_late_completion(mediation_start, context.timeout_ms) + .rejects_late_completion(adserver_start, context.timeout_ms) { log::warn!( - "Mediator '{}' completed after the hard auction deadline; using local ranking ({}ms)", - mediator.provider_name(), + "AdServer '{}' completed after the hard auction deadline; using local ranking ({}ms)", + adserver.provider_name(), response_time_ms ); let winning = self.select_winning_bids(&provider_responses, &floor_prices); return Ok(OrchestrationResult { provider_responses, - mediator_response: None, + adserver_response: None, winning_bids: winning, total_time_ms: 0, metadata: HashMap::new(), }); } - mediator + adserver .parse_response_with_context_and_state( platform_resp, response_time_ms, request, - &mediator_context, + &adserver_context, parse_state.as_deref(), ) .await .change_context(TrustedServerError::Auction { - message: format!("Mediator {} parse failed", mediator.provider_name()), + message: format!("AdServer {} parse failed", adserver.provider_name()), })? } }; - // Extract only mediator bids with comparable numeric prices. - let winning = mediator_resp + // Extract only ad server bids with comparable numeric prices. + let winning = adserver_resp .bids .iter() .filter_map(|bid| { if bid.price.is_none() { log::warn!( - "Mediator '{}' returned bid for slot '{}' without a price - skipping", - mediator.provider_name(), + "AdServer '{}' returned bid for slot '{}' without a price - skipping", + adserver.provider_name(), bid.slot_id ); None @@ -1063,25 +1077,25 @@ impl AuctionOrchestrator { .collect(); ( - Some(mediator_resp), + Some(adserver_resp), self.apply_floor_prices(winning, &floor_prices), ) } else { - // No mediator - select best bid per slot from bidder responses + // No ad server - select best bid per slot from bidder responses let winning = self.select_winning_bids(&provider_responses, &floor_prices); (None, winning) }; Ok(OrchestrationResult { provider_responses, - mediator_response, + adserver_response, winning_bids, total_time_ms: 0, // Will be set by caller metadata: HashMap::new(), }) } - /// Run auction with only parallel bidding (no mediation). + /// Run auction with only parallel bidding (no ad server decision). #[cfg(test)] async fn run_parallel_only( &self, @@ -1094,7 +1108,7 @@ impl AuctionOrchestrator { Ok(OrchestrationResult { provider_responses, - mediator_response: None, + adserver_response: None, winning_bids, total_time_ms: 0, metadata: HashMap::new(), @@ -1113,9 +1127,9 @@ impl AuctionOrchestrator { ) -> Result, Report> { let provider_names = self .config - .providers - .keys() - .map(super::plan::ProviderId::as_str) + .provider_names + .iter() + .map(String::as_str) .collect::>(); if provider_names.is_empty() { @@ -1616,10 +1630,10 @@ impl AuctionOrchestrator { ( input.provider_id().as_str().to_string(), unused_bidder_params_count( - &plan - .provider(input.provider_id()) + plan.provider(input.provider_id()) .expect("should find routed provider in compiled plan") - .profile, + .demand + .as_ref(), input, ), ) @@ -1648,10 +1662,10 @@ impl AuctionOrchestrator { materialize_planned_response( provider_launch_failed_response(input.provider_id().as_str(), 0), unused_bidder_params_count( - &plan - .provider(input.provider_id()) + plan.provider(input.provider_id()) .expect("should find routed provider in compiled plan") - .profile, + .demand + .as_ref(), input, ), ) @@ -1859,9 +1873,9 @@ impl AuctionOrchestrator { #[cfg(test)] let provider_names = self .config - .providers - .keys() - .map(super::plan::ProviderId::as_str) + .provider_names + .iter() + .map(String::as_str) .collect::>(); #[cfg(test)] if provider_names.is_empty() { @@ -2085,7 +2099,7 @@ impl AuctionOrchestrator { /// Collect bid responses from a previously-dispatched auction. /// /// Runs the select-loop phase (equivalent to Phase 2 of - /// `run_providers_parallel`) and, if the orchestrator has a mediator + /// `run_providers_parallel`) and, if the orchestrator has an ad server /// configured, forwards collected bids to it. The overall auction deadline /// is enforced from `dispatched.auction_start`. /// @@ -2292,7 +2306,7 @@ impl AuctionOrchestrator { // collect phase the remaining handles may already be ready even if // wall-clock time elapsed while the origin was slow. Dropping them // here would discard SSP responses that already arrived. The - // mediator launch below still observes A_deadline via + // ad server launch below still observes A_deadline via // `remaining_budget_ms`. } @@ -2329,35 +2343,35 @@ impl AuctionOrchestrator { } #[cfg(not(test))] - let mediator = self.mediator.as_ref(); + let adserver = self.adserver.as_ref(); #[cfg(test)] - let mediator = self.mediator.as_ref().or_else(|| { + let adserver = self.adserver.as_ref().or_else(|| { self.config - .mediator + .adserver_name .as_ref() .and_then(|name| self.providers.get(name)) }); - let (mediator_response, winning_bids) = if let Some(mediator) = mediator { + let (adserver_response, winning_bids) = if let Some(adserver) = adserver { { - // Cap the mediator at whichever is tighter: its own configured + // Cap the ad server at whichever is tighter, being its own configured // timeout or the remaining auction budget (A_deadline). Backend // first-byte and between-bytes timeouts bound normal collection, but // they are transport timers rather than absolute wall-clock limits: // connection setup and byte-trickling can still consume more of the // auction budget. Recomputing the remaining budget here prevents the - // mediator from extending that bounded response hold. + // ad server from extending that bounded response hold. let remaining = remaining_budget_ms(auction_start, timeout_ms); - let logical_budget_ms = remaining.min(mediator.timeout_ms()); + let logical_budget_ms = remaining.min(adserver.timeout_ms()); if logical_budget_ms == 0 { log::warn!( - "A_deadline exhausted before mediator '{}' — returning {} SSP bids without mediation", - mediator.provider_name(), + "A_deadline exhausted before adserver '{}', returning {} SSP bids without an ad server decision", + adserver.provider_name(), responses.len(), ); let winning = self.select_winning_bids(&responses, &floor_prices); return OrchestrationResult { provider_responses: responses, - mediator_response: None, + adserver_response: None, winning_bids: winning, total_time_ms: auction_start.elapsed().as_millis() as u64, metadata: routing_metadata(planned_unroutable_bidder_count), @@ -2365,43 +2379,43 @@ impl AuctionOrchestrator { } let transport_timeout_ms = services .backend() - .canonicalize_transport_timeout_ms(logical_budget_ms, mediator.timeout_ms()); + .canonicalize_transport_timeout_ms(logical_budget_ms, adserver.timeout_ms()); if transport_timeout_ms == 0 { log::warn!( - "Mediator '{}' transport budget canonicalized to zero — returning {} SSP bids without mediation", - mediator.provider_name(), + "AdServer '{}' transport budget canonicalized to zero, returning {} SSP bids without an ad server decision", + adserver.provider_name(), responses.len(), ); let winning = self.select_winning_bids(&responses, &floor_prices); return OrchestrationResult { provider_responses: responses, - mediator_response: None, + adserver_response: None, winning_bids: winning, total_time_ms: auction_start.elapsed().as_millis() as u64, metadata: routing_metadata(planned_unroutable_bidder_count), }; } - let mediator_start = Instant::now(); - // This logs a configured mediator identifier and timeout values, not request data or secrets. + let adserver_start = Instant::now(); + // This logs a configured ad server identifier and timeout values, not request data or secrets. log::info!( - "Running mediator '{}' with {}ms logical budget and {}ms transport timeout (A_deadline remaining: {}ms, configured: {}ms)", - mediator.provider_name(), + "Running adserver '{}' with {}ms logical budget and {}ms transport timeout (A_deadline remaining: {}ms, configured: {}ms)", + adserver.provider_name(), logical_budget_ms, transport_timeout_ms, remaining, - mediator.timeout_ms(), + adserver.timeout_ms(), ); - // The mediator runs on the collect path. See the doc-comment on + // The ad server runs on the collect path. See the doc-comment on // `AuctionContext::request`: the real client request was already // consumed by `send_async` during dispatch, so we substitute a - // canonical placeholder URL. Any future mediator that needs real + // canonical placeholder URL. Any future ad server that needs real // client headers must snapshot them at dispatch time onto // `DispatchedAuction` rather than reading `context.request` here. let placeholder = http::Request::builder() .uri(crate::auction::types::MEDIATOR_PLACEHOLDER_URL) .body(edgezero_core::body::Body::empty()) .unwrap_or_else(|_| http::Request::new(edgezero_core::body::Body::empty())); - let mediator_context = AuctionContext { + let adserver_context = AuctionContext { settings: context.settings, request: &placeholder, timeout_ms: logical_budget_ms, @@ -2409,8 +2423,8 @@ impl AuctionOrchestrator { provider_responses: Some(&responses), services: context.services, }; - let mediator_response = match mediator - .request_bids(&request, &mediator_context) + let adserver_response = match adserver + .request_bids(&request, &adserver_context) .await { Ok(ProviderRequestOutcome::Immediate(response)) => Some(response), @@ -2420,27 +2434,27 @@ impl AuctionOrchestrator { }) => match services.http_client().wait(pending).await.change_context( TrustedServerError::Auction { message: format!( - "Mediator {} request failed", - mediator.provider_name() + "AdServer {} request failed", + adserver.provider_name() ), }, ) { Ok(platform_resp) => { - let response_time_ms = mediator_start.elapsed().as_millis() as u64; + let response_time_ms = adserver_start.elapsed().as_millis() as u64; if deadline_policy.rejects_late_completion(auction_start, timeout_ms) { log::warn!( - "Mediator '{}' completed after the hard auction deadline; using local ranking ({}ms)", - mediator.provider_name(), + "AdServer '{}' completed after the hard auction deadline; using local ranking ({}ms)", + adserver.provider_name(), response_time_ms ); None } else { - match mediator + match adserver .parse_response_with_context_and_state( platform_resp, response_time_ms, &request, - &mediator_context, + &adserver_context, parse_state.as_deref(), ) .await @@ -2448,8 +2462,8 @@ impl AuctionOrchestrator { Ok(response) => Some(response), Err(error) => { log::warn!( - "Mediator '{}' parse failed: {:?}", - mediator.provider_name(), + "AdServer '{}' parse failed: {:?}", + adserver.provider_name(), error ); None @@ -2458,29 +2472,29 @@ impl AuctionOrchestrator { } } Err(error) => { - log::warn!("Mediator request failed: {:?}", error); + log::warn!("AdServer request failed: {:?}", error); None } }, Err(error) => { log::warn!( - "Mediator '{}' failed to dispatch: {:?}", - mediator.provider_name(), + "AdServer '{}' failed to dispatch: {:?}", + adserver.provider_name(), error ); None } }; - if let Some(mediator_response) = mediator_response { - let winning = mediator_response + if let Some(adserver_response) = adserver_response { + let winning = adserver_response .bids .iter() .filter_map(|bid| { if bid.price.is_none() { log::warn!( - "Mediator '{}' returned bid for slot '{}' without decoded price - skipping", - mediator.provider_name(), + "AdServer '{}' returned bid for slot '{}' without decoded price - skipping", + adserver.provider_name(), bid.slot_id ); None @@ -2490,7 +2504,7 @@ impl AuctionOrchestrator { }) .collect(); let winning = self.apply_floor_prices(winning, &floor_prices); - (Some(mediator_response), winning) + (Some(adserver_response), winning) } else { (None, self.select_winning_bids(&responses, &floor_prices)) } @@ -2501,7 +2515,7 @@ impl AuctionOrchestrator { OrchestrationResult { provider_responses: responses, - mediator_response, + adserver_response, winning_bids, total_time_ms: auction_start.elapsed().as_millis() as u64, metadata: routing_metadata(planned_unroutable_bidder_count), @@ -2520,8 +2534,8 @@ impl AuctionOrchestrator { pub struct OrchestrationResult { /// All responses from providers pub provider_responses: Vec, - /// Final response from mediator (if used) - pub mediator_response: Option, + /// Final response from ad server (if used) + pub adserver_response: Option, /// Winning bids per slot pub winning_bids: HashMap, /// Total orchestration time in milliseconds @@ -2534,7 +2548,7 @@ impl OrchestrationResult { fn no_bid() -> Self { Self { provider_responses: Vec::new(), - mediator_response: None, + adserver_response: None, winning_bids: HashMap::new(), total_time_ms: 0, metadata: HashMap::new(), @@ -2566,30 +2580,25 @@ impl OrchestrationResult { #[cfg(test)] mod tests { - use std::str::FromStr as _; use std::time::Duration; - use base64::Engine as _; use web_time::Instant; use crate::auction::config::AuctionConfig; use crate::auction::orchestrator::DispatchAuctionOutcome; - use crate::auction::plan::{ - AuctionPlan, AuctionPlanConfig, NotificationConfig, ProviderConfig, ProviderId, RoutingMode, - }; + use crate::auction::plan::{AuctionPlan, AuctionPlanConfig, NotificationConfig, RoutingMode}; use crate::auction::provider::{ AuctionProvider, GenericOpenRtbProvider, ProviderRequestOutcome, }; use crate::auction::routing::{RoutingDiagnostics, route_auction}; - use crate::auction::test_support::create_test_auction_context; + use crate::auction::test_support::{create_test_auction_context, demand_table, plan_config}; use crate::auction::types::{ - AdFormat, AdSlot, ApsRendererV1, ApsTagType, AuctionContext, AuctionRequest, - AuctionResponse, Bid, BidRenderer, BidStatus, MediaType, PublisherInfo, UserInfo, + AdFormat, AdSlot, AuctionContext, AuctionRequest, AuctionResponse, Bid, BidRenderer, + BidStatus, MediaType, PublisherInfo, UserInfo, }; use crate::error::TrustedServerError; - use crate::integrations::adserver_mock::{AdServerMockConfig, AdServerMockProvider}; use crate::platform::test_support::{ - StubHttpClient, build_services_with_backend_and_http_client, + NamingBackend, StubHttpClient, build_services_with_backend_and_http_client, build_services_with_http_client, noop_services, }; use crate::platform::{ @@ -2600,7 +2609,7 @@ mod tests { }; use crate::test_support::tests::crate_test_settings_str; use error_stack::{Report, ResultExt}; - use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; + use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; @@ -2609,98 +2618,72 @@ mod tests { ERROR_TYPE_LAUNCH_FAILED, ERROR_TYPE_TIMEOUT, ERROR_TYPE_TRANSPORT, OrchestrationResult, }; - fn planned_config(providers: &[(&str, RoutingMode)], signing: bool) -> AuctionPlanConfig { - AuctionPlanConfig { - timeout_ms: 777, - providers: providers - .iter() - .map(|(id, routing)| { - ( - ProviderId::from_str(id).expect("should parse fictional provider ID"), - ProviderConfig { - protocol: "openrtb-2.6".to_string(), - profile: "standard".to_string(), - endpoint: "https://example.test/openrtb".to_string(), - timeout_ms: Some(1_000), - routing: *routing, - notifications: Default::default(), - profile_config: serde_json::json!({}), - }, - ) - }) - .collect(), - bidders: BTreeMap::new(), - mediator: None, - request_signing: signing.then(|| crate::settings::RequestSigning { - enabled: true, - config_store_id: "fictional-config-store".to_string(), - secret_store_id: "fictional-secret-store".to_string(), - }), + /// One `[demand.]` table for a planned test source. + fn planned_table( + implementation: &str, + endpoint: &str, + routing: RoutingMode, + settings: &serde_json::Value, + notifications: &NotificationConfig, + ) -> serde_json::Map { + let mut entry = demand_table(implementation, endpoint); + entry.insert("timeout_ms".to_string(), serde_json::json!(1_000)); + if routing == RoutingMode::AllEligible { + entry.insert("routing".to_string(), serde_json::json!("all_eligible")); } - } - - fn planned_prebid_config( - providers: &[(&str, serde_json::Value, NotificationConfig)], - ) -> AuctionPlanConfig { - AuctionPlanConfig { - timeout_ms: 777, - providers: providers - .iter() - .map(|(id, profile_config, notifications)| { - ( - ProviderId::from_str(id).expect("should parse fictional provider ID"), - ProviderConfig { - protocol: "openrtb-2.6".to_string(), - profile: "prebid-server".to_string(), - endpoint: format!("https://{id}.example.test/openrtb"), - timeout_ms: Some(1_000), - routing: RoutingMode::Explicit, - notifications: notifications.clone(), - profile_config: profile_config.clone(), - }, - ) - }) - .collect(), - bidders: BTreeMap::new(), - mediator: None, - request_signing: None, + entry.insert( + "notifications".to_string(), + serde_json::to_value(notifications).expect("should serialize notifications"), + ); + if let serde_json::Value::Object(settings) = settings { + entry.extend(settings.clone()); } + entry } - fn planned_aps_config() -> AuctionPlanConfig { - planned_aps_instances_config(&[( - "aps-instance", - serde_json::json!({"account_id": "example-account"}), - NotificationConfig::default(), - )]) + fn planned_config(providers: &[(&str, RoutingMode)], signing: bool) -> AuctionPlanConfig { + let tables = providers + .iter() + .map(|(id, routing)| { + ( + *id, + planned_table( + "auction.plain-fixture", + "https://example.test/openrtb", + *routing, + &serde_json::json!({}), + &NotificationConfig::default(), + ), + ) + }) + .collect::>(); + let mut config = plan_config(tables); + config.timeout_ms = 777; + config.request_signing = signing.then(|| crate::settings::RequestSigning::new(true)); + config } - fn planned_aps_instances_config( + fn planned_fixture_config( providers: &[(&str, serde_json::Value, NotificationConfig)], ) -> AuctionPlanConfig { - AuctionPlanConfig { - timeout_ms: 777, - providers: providers - .iter() - .map(|(id, profile_config, notifications)| { - ( - ProviderId::from_str(id).expect("should parse fictional provider ID"), - ProviderConfig { - protocol: "openrtb-2.6".to_string(), - profile: "aps".to_string(), - endpoint: "https://aps.example/e/pb/bid".to_string(), - timeout_ms: Some(1_000), - routing: RoutingMode::AllEligible, - notifications: notifications.clone(), - profile_config: profile_config.clone(), - }, - ) - }) - .collect(), - bidders: BTreeMap::new(), - mediator: None, - request_signing: None, - } + let tables = providers + .iter() + .map(|(id, settings, notifications)| { + ( + *id, + planned_table( + "auction.fixture", + &format!("https://{id}.example.test/openrtb"), + RoutingMode::Explicit, + settings, + notifications, + ), + ) + }) + .collect::>(); + let mut config = plan_config(tables); + config.timeout_ms = 777; + config } fn planned_request() -> AuctionRequest { @@ -2732,7 +2715,7 @@ mod tests { } } - fn planned_prebid_request() -> AuctionRequest { + fn planned_envelope_request() -> AuctionRequest { let mut request = planned_request(); request.slots[0] .bidders @@ -2744,7 +2727,7 @@ mod tests { async fn disabled_from_plan_is_a_no_work_kill_switch_for_sync_and_split_paths() { let plan = Arc::new( AuctionPlan::compile(planned_config( - &[("provider-a", RoutingMode::AllEligible)], + &[("provider_a", RoutingMode::AllEligible)], false, )) .expect("should compile plan") @@ -2815,14 +2798,14 @@ mod tests { async fn all_planned_launch_failures_error_direct_and_surface_split_failure() { let plan = Arc::new( AuctionPlan::compile(planned_config( - &[("launch-fail", RoutingMode::AllEligible)], + &[("launch_fail", RoutingMode::AllEligible)], false, )) .expect("should compile launch-failure plan"), ); let orchestrator = AuctionOrchestrator::from_plan(plan, None); let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); - backend.fail_ensure_for("launch-fail"); + backend.fail_ensure_for("launch_fail"); let http = Arc::new(StubHttpClient::new()); let services = build_services_with_backend_and_http_client( Arc::clone(&backend) as Arc<_>, @@ -2869,7 +2852,7 @@ mod tests { "should keep one launch-failure response" ); assert_eq!( - provider_responses[0].provider, "launch-fail", + provider_responses[0].provider, "launch_fail", "should attribute the response to the failed provider" ); assert_eq!( @@ -2884,7 +2867,7 @@ mod tests { let plan = Arc::new( AuctionPlan::compile(planned_config( &[ - ("launch-fail", RoutingMode::AllEligible), + ("launch_fail", RoutingMode::AllEligible), ("timeout", RoutingMode::AllEligible), ], false, @@ -2921,7 +2904,7 @@ mod tests { ); assert!( result.provider_responses.iter().any(|response| { - response.provider == "launch-fail" + response.provider == "launch_fail" && response.metadata["error_type"] == ERROR_TYPE_LAUNCH_FAILED }), "should retain the launch-failure outcome" @@ -2992,71 +2975,6 @@ mod tests { } } - struct NamingBackend { - policy: BackendNamingPolicy, - predicted: AtomicUsize, - ensured: AtomicUsize, - specs: Mutex>, - fail_ensure_for: Mutex>, - } - - impl NamingBackend { - fn new(policy: BackendNamingPolicy) -> Self { - Self { - policy, - predicted: AtomicUsize::new(0), - ensured: AtomicUsize::new(0), - specs: Mutex::new(Vec::new()), - fail_ensure_for: Mutex::new(HashSet::new()), - } - } - - fn fail_ensure_for(&self, provider_id: &str) { - self.fail_ensure_for - .lock() - .expect("should lock failing provider IDs") - .insert(provider_id.to_string()); - } - - fn name(&self, spec: &PlatformBackendSpec) -> Result> { - self.policy - .predict(spec) - .map(|prediction| prediction.name) - .change_context(PlatformError::Backend) - } - } - - impl PlatformBackend for NamingBackend { - fn naming_policy(&self) -> BackendNamingPolicy { - self.policy - } - - fn predict_name( - &self, - spec: &PlatformBackendSpec, - ) -> Result> { - self.predicted.fetch_add(1, Ordering::Relaxed); - self.name(spec) - } - - fn ensure(&self, spec: &PlatformBackendSpec) -> Result> { - self.ensured.fetch_add(1, Ordering::Relaxed); - if spec.discriminator.as_deref().is_some_and(|provider_id| { - self.fail_ensure_for - .lock() - .expect("should lock failing provider IDs") - .contains(provider_id) - }) { - return Err(Report::new(PlatformError::Backend)); - } - self.specs - .lock() - .expect("should lock planned backend specs") - .push(spec.clone()); - self.name(spec) - } - } - struct ZeroCanonicalBackend { predicted: AtomicUsize, ensured: AtomicUsize, @@ -3452,19 +3370,19 @@ mod tests { } } - type RecordedMediatorBudgets = Arc>>; + type RecordedAdServerBudgets = Arc>>; - struct DeadlineRecordingMediator { + struct DeadlineRecordingAdServer { launches: Arc, - budgets: Option, + budgets: Option, } - struct PendingDeadlineMediator; + struct PendingDeadlineAdServer; #[async_trait::async_trait(?Send)] - impl AuctionProvider for PendingDeadlineMediator { + impl AuctionProvider for PendingDeadlineAdServer { fn provider_name(&self) -> &str { - "pending-deadline-mediator" + "pending_deadline_adserver" } async fn request_bids( @@ -3477,8 +3395,8 @@ mod tests { .method("POST") .uri("https://example.com/mediate") .body(edgezero_core::body::Body::empty()) - .expect("should build pending mediator request"), - "pending-mediator-backend", + .expect("should build pending adserver request"), + "pending-adserver-backend", ); context .services @@ -3486,7 +3404,7 @@ mod tests { .send_async(request) .await .change_context(TrustedServerError::Auction { - message: "pending mediator launch failed".to_string(), + message: "pending adserver launch failed".to_string(), }) .map(ProviderRequestOutcome::pending) } @@ -3498,7 +3416,7 @@ mod tests { ) -> Result> { Ok(AuctionResponse::success( self.provider_name(), - vec![auction_bid("mediated", 9.0)], + vec![auction_bid("adserver", 9.0)], response_time_ms, )) } @@ -3508,14 +3426,14 @@ mod tests { } fn backend_name(&self, _services: &RuntimeServices, _timeout_ms: u32) -> Option { - Some("pending-mediator-backend".to_string()) + Some("pending-adserver-backend".to_string()) } } #[async_trait::async_trait(?Send)] - impl AuctionProvider for DeadlineRecordingMediator { + impl AuctionProvider for DeadlineRecordingAdServer { fn provider_name(&self) -> &str { - "deadline-mediator" + "deadline_adserver" } async fn request_bids( @@ -3527,7 +3445,7 @@ mod tests { if let Some(budgets) = &self.budgets { budgets .lock() - .expect("should lock mediator budgets") + .expect("should lock adserver budgets") .push((context.timeout_ms, context.transport_timeout_ms)); } Ok(ProviderRequestOutcome::Immediate(AuctionResponse::no_bid( @@ -3541,7 +3459,7 @@ mod tests { _response: PlatformResponse, _response_time_ms: u64, ) -> Result> { - panic!("immediate mediator response should not be parsed"); + panic!("immediate adserver response should not be parsed"); } fn timeout_ms(&self) -> u32 { @@ -3719,24 +3637,23 @@ mod tests { } } - /// Mediator whose context-aware parse restores `nurl`/`ad_id` (mirroring - /// `adserver_mock`), while its context-free parse does not. Lets a test prove - /// the synchronous mediation path calls `parse_response_with_context`. - struct CacheRestoringMediator; + /// Ad server whose context-aware parse restores `nurl`/`ad_id`, as one that + /// keeps them out of its request does, while its context-free parse does not. Lets a test prove + /// the synchronous ad server decision path calls `parse_response_with_context`. + struct CacheRestoringAdServer; fn auction_bid(bidder: &str, price: f64) -> Bid { - let renderer = (bidder == "aps").then(|| { - BidRenderer::Aps(ApsRendererV1 { - version: 1, - account_id: "example-account".to_string(), - bid_id: "aps-selected-bid".to_string(), - creative_id: None, - tag_type: ApsTagType::Iframe, - creative_url: "https://creative.example/render".to_string(), - aax_response: "fictional-base64".to_string(), - width: 300, - height: 250, - }) + let renderer = (bidder == "typed").then(|| { + BidRenderer::new( + "example", + serde_json::json!({ + "version": 1, + "bidId": "typed-selected-bid", + "creativeUrl": "https://creative.example/render", + "envelope": "fictional-base64", + }), + ) + .expect("should build the example renderer descriptor") }); Bid { slot_id: "slot-1".to_string(), @@ -3752,7 +3669,7 @@ mod tests { height: 250, nurl: None, burl: None, - bid_id: (bidder == "aps").then(|| "aps-selected-bid".to_string()), + bid_id: (bidder == "typed").then(|| "typed-selected-bid".to_string()), ad_id: None, creative_id: None, renderer, @@ -3763,14 +3680,14 @@ mod tests { } } - fn mediated_bid(nurl: Option) -> Bid { + fn adserver_bid(nurl: Option) -> Bid { Bid { slot_id: "header-banner".to_string(), price: Some(2.5), currency: "USD".to_string(), creative: Some("
ad
".to_string()), adomain: None, - bidder: "mediator".to_string(), + bidder: "adserver".to_string(), returned_seat: None, width: 728, height: 90, @@ -3788,9 +3705,9 @@ mod tests { } #[async_trait::async_trait(?Send)] - impl AuctionProvider for CacheRestoringMediator { + impl AuctionProvider for CacheRestoringAdServer { fn provider_name(&self) -> &str { - "mediator" + "adserver" } async fn request_bids( @@ -3803,8 +3720,8 @@ mod tests { .method("POST") .uri("https://example.com/mediate") .body(edgezero_core::body::Body::empty()) - .expect("should build mediator request"), - "mediator-backend", + .expect("should build adserver request"), + "adserver-backend", ); context .services @@ -3812,7 +3729,7 @@ mod tests { .send_async(req) .await .change_context(TrustedServerError::Auction { - message: "mediator launch failed".to_string(), + message: "adserver launch failed".to_string(), }) .map(ProviderRequestOutcome::pending) } @@ -3824,8 +3741,8 @@ mod tests { ) -> Result> { // Context-free path: cannot restore SSP-only render/accounting fields. Ok(AuctionResponse::success( - "mediator", - vec![mediated_bid(None)], + "adserver", + vec![adserver_bid(None)], response_time_ms, )) } @@ -3839,8 +3756,8 @@ mod tests { ) -> Result> { // Context-aware path: restores nurl/ad_id from the collected SSP bids. Ok(AuctionResponse::success( - "mediator", - vec![mediated_bid(Some("https://nurl.example/win".to_string()))], + "adserver", + vec![adserver_bid(Some("https://nurl.example/win".to_string()))], response_time_ms, )) } @@ -3850,16 +3767,16 @@ mod tests { } fn backend_name(&self, _services: &RuntimeServices, _timeout_ms: u32) -> Option { - Some("mediator-backend".to_string()) + Some("adserver-backend".to_string()) } } - struct ImmediateMediator; + struct ImmediateAdServer; #[async_trait::async_trait(?Send)] - impl AuctionProvider for ImmediateMediator { + impl AuctionProvider for ImmediateAdServer { fn provider_name(&self) -> &str { - "immediate-mediator" + "immediate_adserver" } async fn request_bids( @@ -3869,7 +3786,7 @@ mod tests { ) -> Result> { Ok(ProviderRequestOutcome::Immediate(AuctionResponse::success( self.provider_name(), - vec![mediated_bid(Some( + vec![adserver_bid(Some( "https://nurl.example/immediate".to_string(), ))], 0, @@ -3881,7 +3798,7 @@ mod tests { _response: PlatformResponse, _response_time_ms: u64, ) -> Result> { - panic!("immediate mediator response should not be parsed"); + panic!("immediate adserver response should not be parsed"); } fn timeout_ms(&self) -> u32 { @@ -3890,22 +3807,22 @@ mod tests { } #[tokio::test] - async fn mediated_bid_preserves_restored_fields_through_run_auction() { - // run_parallel_mediation must parse the mediator response via + async fn adserver_bid_preserves_restored_fields_through_run_auction() { + // run_parallel_adserver must parse the ad server response via // parse_response_with_context so cache/nurl fields restored from SSP - // responses survive the synchronous mediation path (POST /auction, + // responses survive the synchronous ad server decision path (POST /auction, // /_ts/page-bids), matching the dispatched collect path. let stub = Arc::new(StubHttpClient::new()); stub.push_response(200, b"{}".to_vec()); // bidder send_async - stub.push_response(200, b"{}".to_vec()); // mediator send_async + stub.push_response(200, b"{}".to_vec()); // adserver send_async let services = build_services_with_http_client(stub); // SAFETY: `Box::leak` creates a `'static` reference for test use only. let services: &'static RuntimeServices = Box::leak(Box::new(services)); let config = AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["bidder"]), - mediator: Some("mediator".to_string()), + provider_names: vec!["bidder".to_string()], + adserver_name: Some("adserver".to_string()), timeout_ms: 2000, ..Default::default() }; @@ -3914,7 +3831,7 @@ mod tests { name: "bidder", backend: "bidder-backend", })); - orchestrator.register_provider(Arc::new(CacheRestoringMediator)); + orchestrator.register_provider(Arc::new(CacheRestoringAdServer)); let request = create_test_auction_request(); let settings = create_test_settings(); @@ -3935,34 +3852,34 @@ mod tests { let result = orchestrator .run_auction(&request, &context) .await - .expect("mediated auction should complete"); + .expect("ad server auction should complete"); let bid = result .winning_bids .get("header-banner") - .expect("mediator should produce a winning bid for the slot"); + .expect("adserver should produce a winning bid for the slot"); assert_eq!( bid.nurl.as_deref(), Some("https://nurl.example/win"), - "synchronous mediation must restore nurl via parse_response_with_context" + "the synchronous ad server must restore nurl via parse_response_with_context" ); assert_eq!( bid.ad_id.as_deref(), Some("creative-123"), - "mediated bid must keep its restored ad_id" + "decided bid must keep its restored ad_id" ); } #[tokio::test] - async fn immediate_mediator_completes_in_sync_and_split_paths() { + async fn immediate_adserver_completes_in_sync_and_split_paths() { for split in [false, true] { let stub = Arc::new(StubHttpClient::new()); stub.push_response(200, b"{}".to_vec()); let services = build_services_with_http_client(stub); let config = AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["bidder"]), - mediator: Some("immediate-mediator".to_string()), + provider_names: vec!["bidder".to_string()], + adserver_name: Some("immediate_adserver".to_string()), timeout_ms: 2000, ..Default::default() }; @@ -3971,7 +3888,7 @@ mod tests { name: "bidder", backend: "bidder-backend", })); - orchestrator.register_provider(Arc::new(ImmediateMediator)); + orchestrator.register_provider(Arc::new(ImmediateAdServer)); let request = create_test_auction_request(); let settings = create_test_settings(); let downstream = http::Request::new(edgezero_core::body::Body::empty()); @@ -3997,15 +3914,15 @@ mod tests { orchestrator .run_auction(&request, &context) .await - .expect("auction with immediate mediator should complete") + .expect("auction with immediate adserver should complete") }; assert_eq!( result - .mediator_response + .adserver_response .as_ref() .map(|response| response.provider.as_str()), - Some("immediate-mediator") + Some("immediate_adserver") ); assert_eq!( result @@ -4029,17 +3946,17 @@ mod tests { let services = build_services_with_http_client(Arc::clone(&stub) as Arc<_>); let config = AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["late-one", "late-two"]), + provider_names: vec!["late_one".to_string(), "late_two".to_string()], timeout_ms: 10, ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); orchestrator.register_provider(Arc::new(DeadlineBidProvider { - name: "late-one", + name: "late_one", backend: "late-one-backend", })); orchestrator.register_provider(Arc::new(DeadlineBidProvider { - name: "late-two", + name: "late_two", backend: "late-two-backend", })); let request = create_test_auction_request(); @@ -4076,9 +3993,9 @@ mod tests { for split in [false, true] { let result = collect_deadline_test_result(split, false).await; assert_eq!(result.provider_responses.len(), 2); - assert_eq!(result.provider_responses[0].provider, "late-one"); + assert_eq!(result.provider_responses[0].provider, "late_one"); assert_eq!(result.provider_responses[0].status, BidStatus::Success); - assert_eq!(result.provider_responses[1].provider, "late-two"); + assert_eq!(result.provider_responses[1].provider, "late_two"); assert_eq!(result.provider_responses[1].status, BidStatus::Success); assert!( result @@ -4088,7 +4005,7 @@ mod tests { "late response times should retain actual elapsed duration" ); assert_eq!( - result.winning_bids["slot-1"].bidder, "late-one", + result.winning_bids["slot-1"].bidder, "late_one", "a completed response remains eligible after the logical deadline" ); } @@ -4108,7 +4025,7 @@ mod tests { } } - async fn pending_mediator_deadline_test_result( + async fn pending_adserver_deadline_test_result( split: bool, enforceable_total_request_deadline: bool, ) -> OrchestrationResult { @@ -4121,8 +4038,8 @@ mod tests { let services = build_services_with_http_client(Arc::clone(&stub) as Arc<_>); let config = AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["local"]), - mediator: Some("pending-deadline-mediator".to_string()), + provider_names: vec!["local".to_string()], + adserver_name: Some("pending_deadline_adserver".to_string()), timeout_ms: 20, ..Default::default() }; @@ -4131,7 +4048,7 @@ mod tests { name: "local", backend: "local-backend", })); - orchestrator.register_provider(Arc::new(PendingDeadlineMediator)); + orchestrator.register_provider(Arc::new(PendingDeadlineAdServer)); let request = create_test_auction_request(); let settings = create_test_settings(); let downstream = http::Request::new(edgezero_core::body::Body::empty()); @@ -4162,31 +4079,31 @@ mod tests { } #[tokio::test] - async fn pending_mediator_late_completion_policy_is_equivalent_in_sync_and_split_paths() { + async fn pending_adserver_late_completion_policy_is_equivalent_in_sync_and_split_paths() { for split in [false, true] { - let current = pending_mediator_deadline_test_result(split, false).await; - let current_mediator = current - .mediator_response + let current = pending_adserver_deadline_test_result(split, false).await; + let current_adserver = current + .adserver_response .as_ref() - .expect("current adapters should accept completed late mediator responses"); + .expect("current adapters should accept completed late adserver responses"); assert!( - current_mediator.response_time_ms >= 50, - "mediator timing should preserve actual elapsed duration" + current_adserver.response_time_ms >= 50, + "adserver timing should preserve actual elapsed duration" ); - assert_eq!(current.winning_bids["slot-1"].bidder, "mediated"); + assert_eq!(current.winning_bids["slot-1"].bidder, "adserver"); - let hard = pending_mediator_deadline_test_result(split, true).await; - assert!(hard.mediator_response.is_none()); + let hard = pending_adserver_deadline_test_result(split, true).await; + assert!(hard.adserver_response.is_none()); assert_eq!(hard.winning_bids["slot-1"].bidder, "local"); assert!( hard.total_time_ms >= 50, - "discarding a late mediator must retain actual total elapsed time" + "discarding a late adserver must retain actual total elapsed time" ); } } #[tokio::test] - async fn split_deadline_skips_mediator_and_falls_back_to_provider_winner() { + async fn split_deadline_skips_adserver_and_falls_back_to_provider_winner() { let stub = Arc::new(StubHttpClient::new()); stub.push_response(200, b"{}".to_vec()); stub.push_select_delay(Duration::from_millis(50)); @@ -4194,17 +4111,17 @@ mod tests { let launches = Arc::new(AtomicUsize::new(0)); let config = AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["late-one"]), - mediator: Some("deadline-mediator".to_string()), + provider_names: vec!["late_one".to_string()], + adserver_name: Some("deadline_adserver".to_string()), timeout_ms: 10, ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); orchestrator.register_provider(Arc::new(DeadlineBidProvider { - name: "late-one", + name: "late_one", backend: "late-one-backend", })); - orchestrator.register_provider(Arc::new(DeadlineRecordingMediator { + orchestrator.register_provider(Arc::new(DeadlineRecordingAdServer { launches: Arc::clone(&launches), budgets: None, })); @@ -4229,12 +4146,12 @@ mod tests { .await; assert_eq!(launches.load(Ordering::Relaxed), 0); - assert!(result.mediator_response.is_none()); - assert_eq!(result.winning_bids["slot-1"].bidder, "late-one"); + assert!(result.adserver_response.is_none()); + assert_eq!(result.winning_bids["slot-1"].bidder, "late_one"); } #[tokio::test] - async fn synchronous_deadline_skips_mediator_and_falls_back_to_provider_winner() { + async fn synchronous_deadline_skips_adserver_and_falls_back_to_provider_winner() { let stub = Arc::new(StubHttpClient::new()); stub.push_response(200, b"{}".to_vec()); stub.push_select_delay(Duration::from_millis(50)); @@ -4242,17 +4159,17 @@ mod tests { let launches = Arc::new(AtomicUsize::new(0)); let config = AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["late-one"]), - mediator: Some("deadline-mediator".to_string()), + provider_names: vec!["late_one".to_string()], + adserver_name: Some("deadline_adserver".to_string()), timeout_ms: 10, ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); orchestrator.register_provider(Arc::new(DeadlineBidProvider { - name: "late-one", + name: "late_one", backend: "late-one-backend", })); - orchestrator.register_provider(Arc::new(DeadlineRecordingMediator { + orchestrator.register_provider(Arc::new(DeadlineRecordingAdServer { launches: Arc::clone(&launches), budgets: None, })); @@ -4273,8 +4190,8 @@ mod tests { .expect("synchronous deadline test should complete"); assert_eq!(launches.load(Ordering::Relaxed), 0); - assert!(result.mediator_response.is_none()); - assert_eq!(result.winning_bids["slot-1"].bidder, "late-one"); + assert!(result.adserver_response.is_none()); + assert_eq!(result.winning_bids["slot-1"].bidder, "late_one"); } fn create_test_auction_request() -> AuctionRequest { @@ -4412,7 +4329,7 @@ mod tests { async fn synchronous_auction_accepts_an_all_immediate_no_bid_result() { let config = AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["immediate"]), + provider_names: vec!["immediate".to_string()], timeout_ms: 2000, ..Default::default() }; @@ -4437,7 +4354,7 @@ mod tests { async fn split_auction_accepts_an_all_immediate_no_bid_result() { let config = AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["immediate"]), + provider_names: vec!["immediate".to_string()], timeout_ms: 2000, ..Default::default() }; @@ -4468,7 +4385,7 @@ mod tests { for split in [false, true] { let config = AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["immediate", "pending"]), + provider_names: vec!["immediate".to_string(), "pending".to_string()], timeout_ms: 2000, ..Default::default() }; @@ -4682,7 +4599,7 @@ mod tests { // Timeout paths still need a controllable delayed pending response: // - Deadline check in select() loop (drops remaining requests) - // - Mediator skip when remaining_ms == 0 (bidding exhausts budget) + // - Ad server skip when remaining_ms == 0 (bidding exhausts budget) // - Provider skip when effective_timeout == 0 (budget exhausted before launch) // - Provider context receives reduced timeout_ms per remaining budget // @@ -4696,12 +4613,13 @@ mod tests { enabled: true, sanitize_creatives: true, rewrite_creatives: true, - providers: AuctionConfig::legacy_provider_map(&[]), + provider_names: vec![], bidders: Default::default(), - mediator: None, + adserver_name: None, timeout_ms: 2000, creative_store: "creative_store".to_string(), allowed_context_keys: BTreeSet::from(["permutive_segments".to_string()]), + ..Default::default() }; let orchestrator = AuctionOrchestrator::new(config); @@ -4728,7 +4646,7 @@ mod tests { futures::executor::block_on(async { let config = AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["launch-failing"]), + provider_names: vec!["launch_failing".to_string()], timeout_ms: 2000, ..Default::default() }; @@ -4763,17 +4681,17 @@ mod tests { for split in [false, true] { let config = AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["provider-a", "provider-b"]), + provider_names: vec!["provider_a".to_string(), "provider_b".to_string()], timeout_ms: 2000, ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-a", + name: "provider_a", backend: "shared-backend", })); orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-b", + name: "provider_b", backend: "shared-backend", })); let stub = Arc::new(StubHttpClient::new()); @@ -4801,10 +4719,10 @@ mod tests { }; assert!(result.provider_responses.iter().any(|response| { - response.provider == "provider-a" && response.status == BidStatus::Success + response.provider == "provider_a" && response.status == BidStatus::Success })); assert!(result.provider_responses.iter().any(|response| { - response.provider == "provider-b" && response.status == BidStatus::Error + response.provider == "provider_b" && response.status == BidStatus::Error })); } } @@ -4878,7 +4796,7 @@ mod tests { let requested = Arc::new(Mutex::new(Vec::new())); let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["bidder"]), + provider_names: vec!["bidder".to_string()], timeout_ms: 2000, ..Default::default() }); @@ -4922,7 +4840,7 @@ mod tests { let requested = Arc::new(Mutex::new(Vec::new())); let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["bidder"]), + provider_names: vec!["bidder".to_string()], timeout_ms: 2000, ..Default::default() }); @@ -4948,7 +4866,7 @@ mod tests { } #[test] - fn synchronous_mediation_applies_canonical_timeout_to_mediator() { + fn synchronous_adserver_applies_canonical_timeout_to_adserver() { futures::executor::block_on(async { let stub = Arc::new(StubHttpClient::new()); stub.push_response(200, b"{}".to_vec()); @@ -4965,8 +4883,8 @@ mod tests { let requested = Arc::new(Mutex::new(Vec::new())); let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["bidder"]), - mediator: Some("mediator".to_string()), + provider_names: vec!["bidder".to_string()], + adserver_name: Some("adserver".to_string()), timeout_ms: 2000, ..Default::default() }); @@ -4975,8 +4893,8 @@ mod tests { backend: "bidder-backend", })); orchestrator.register_provider(Arc::new(recording_provider( - "mediator", - "mediator-backend", + "adserver", + "adserver-backend", 2000, &predicted, &requested, @@ -4988,7 +4906,7 @@ mod tests { orchestrator .run_auction(&create_test_auction_request(), &context) .await - .expect("should complete mediated auction"); + .expect("should complete ad server auction"); assert!(predicted.lock().expect("should lock predicted").is_empty()); assert_eq!(*requested.lock().expect("should lock requested"), vec![500]); @@ -5011,12 +4929,12 @@ mod tests { ); let bidder_predicted = Arc::new(Mutex::new(Vec::new())); let bidder_requested = Arc::new(Mutex::new(Vec::new())); - let mediator_predicted = Arc::new(Mutex::new(Vec::new())); - let mediator_requested = Arc::new(Mutex::new(Vec::new())); + let adserver_predicted = Arc::new(Mutex::new(Vec::new())); + let adserver_requested = Arc::new(Mutex::new(Vec::new())); let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["bidder"]), - mediator: Some("mediator".to_string()), + provider_names: vec!["bidder".to_string()], + adserver_name: Some("adserver".to_string()), timeout_ms: 2000, ..Default::default() }); @@ -5028,11 +4946,11 @@ mod tests { &bidder_requested, ))); orchestrator.register_provider(Arc::new(recording_provider( - "mediator", - "mediator-backend", + "adserver", + "adserver-backend", 2000, - &mediator_predicted, - &mediator_requested, + &adserver_predicted, + &adserver_requested, ))); let settings = create_test_settings(); let downstream = http::Request::new(edgezero_core::body::Body::empty()); @@ -5057,20 +4975,20 @@ mod tests { vec![500] ); assert!( - mediator_predicted + adserver_predicted .lock() .expect("should lock predicted") .is_empty() ); assert_eq!( - *mediator_requested.lock().expect("should lock requested"), + *adserver_requested.lock().expect("should lock requested"), vec![500] ); }); } #[test] - fn planned_collect_skips_mediator_with_zero_canonical_transport_budget() { + fn planned_collect_skips_adserver_with_zero_canonical_transport_budget() { futures::executor::block_on(async { let calls = Arc::new(Mutex::new(Vec::new())); let services = build_services_with_backend_and_http_client( @@ -5082,18 +5000,24 @@ mod tests { ); let launches = Arc::new(AtomicUsize::new(0)); let budgets = Arc::new(Mutex::new(Vec::new())); - let plan = AuctionPlan::compile(AuctionPlanConfig { - timeout_ms: 49, - providers: BTreeMap::new(), - bidders: BTreeMap::new(), - mediator: Some("adserver_mock".to_string()), - request_signing: None, - }) - .expect("should compile mediator-only plan") - .with_enabled(true); + let mut adserver_only = plan_config(Vec::new()); + adserver_only.timeout_ms = 49; + adserver_only.adserver = crate::provider_table::ProviderChoice::new( + Some("fixture".to_string()), + BTreeMap::from([( + "fixture".to_string(), + serde_json::Map::from_iter([( + "endpoint".to_string(), + serde_json::json!("https://adserver.example/mediate"), + )]), + )]), + ); + let plan = AuctionPlan::compile(adserver_only) + .expect("should compile adserver-only plan") + .with_enabled(true); let orchestrator = AuctionOrchestrator::from_plan( Arc::new(plan), - Some(Arc::new(DeadlineRecordingMediator { + Some(Arc::new(DeadlineRecordingAdServer { launches: Arc::clone(&launches), budgets: Some(Arc::clone(&budgets)), })), @@ -5119,11 +5043,11 @@ mod tests { assert!( budgets .lock() - .expect("should lock mediator budgets") + .expect("should lock adserver budgets") .is_empty() ); assert_eq!(calls.lock().expect("should lock calls").len(), 1); - assert!(result.mediator_response.is_none()); + assert!(result.adserver_response.is_none()); }); } @@ -5135,12 +5059,12 @@ mod tests { let services = build_services_with_http_client(stub); let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["provider-a"]), + provider_names: vec!["provider_a".to_string()], timeout_ms: 2000, ..Default::default() }); orchestrator.register_provider(Arc::new(DivergentBackendProvider { - name: "provider-a", + name: "provider_a", predicted: "predicted-backend", resolved: "resolved-backend", })); @@ -5159,7 +5083,7 @@ mod tests { .await; assert!(result.provider_responses.iter().any(|response| { - response.provider == "provider-a" && response.status == BidStatus::Success + response.provider == "provider_a" && response.status == BidStatus::Success })); }); } @@ -5173,17 +5097,17 @@ mod tests { let services = build_services_with_http_client(stub); let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["provider-a", "provider-b"]), + provider_names: vec!["provider_a".to_string(), "provider_b".to_string()], timeout_ms: 2000, ..Default::default() }); orchestrator.register_provider(Arc::new(DivergentBackendProvider { - name: "provider-a", + name: "provider_a", predicted: "predicted-a", resolved: "shared-resolved", })); orchestrator.register_provider(Arc::new(DivergentBackendProvider { - name: "provider-b", + name: "provider_b", predicted: "predicted-b", resolved: "shared-resolved", })); @@ -5202,10 +5126,10 @@ mod tests { .await; assert!(result.provider_responses.iter().any(|response| { - response.provider == "provider-a" && response.status == BidStatus::Success + response.provider == "provider_a" && response.status == BidStatus::Success })); assert!(result.provider_responses.iter().any(|response| { - response.provider == "provider-b" && response.status == BidStatus::Error + response.provider == "provider_b" && response.status == BidStatus::Error })); }); } @@ -5228,18 +5152,18 @@ mod tests { let config = AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["provider-a", "provider-b"]), + provider_names: vec!["provider_a".to_string(), "provider_b".to_string()], timeout_ms: 2000, - mediator: None, + adserver_name: None, ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-a", + name: "provider_a", backend: "backend-a", })); orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-b", + name: "provider_b", backend: "backend-b", })); @@ -5275,12 +5199,12 @@ mod tests { let provider_a = result .provider_responses .iter() - .find(|r| r.provider == "provider-a") + .find(|r| r.provider == "provider_a") .expect("should have provider-a response"); let provider_b = result .provider_responses .iter() - .find(|r| r.provider == "provider-b") + .find(|r| r.provider == "provider_b") .expect("should have provider-b response"); assert_eq!( @@ -5307,7 +5231,7 @@ mod tests { ); let plan = Arc::new( AuctionPlan::compile(planned_config( - &[("provider-a", RoutingMode::AllEligible)], + &[("provider_a", RoutingMode::AllEligible)], false, )) .expect("should compile planned auction"), @@ -5357,7 +5281,7 @@ mod tests { ); let plan = Arc::new( AuctionPlan::compile(planned_config( - &[("provider-a", RoutingMode::AllEligible)], + &[("provider_a", RoutingMode::AllEligible)], false, )) .expect("should compile planned auction"), @@ -5404,8 +5328,8 @@ mod tests { let plan = Arc::new( AuctionPlan::compile(planned_config( &[ - ("provider-b", RoutingMode::AllEligible), - ("provider-a", RoutingMode::AllEligible), + ("provider_b", RoutingMode::AllEligible), + ("provider_a", RoutingMode::AllEligible), ], false, )) @@ -5441,8 +5365,8 @@ mod tests { .iter() .map(|response| response.provider.as_str()) .collect::>(), - vec!["provider-a", "provider-b"], - "outer select errors should retain deterministic plan order" + vec!["provider_b", "provider_a"], + "outer select errors should keep the order the deployment selected" ); for response in &result.provider_responses { assert_eq!(response.status, BidStatus::Error); @@ -5462,14 +5386,14 @@ mod tests { let services = build_services_with_http_client(stub); let config = AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["provider-a"]), + provider_names: vec!["provider_a".to_string()], timeout_ms: 750, - mediator: None, + adserver_name: None, ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-a", + name: "provider_a", backend: "backend-a", })); let request = create_test_auction_request(); @@ -5542,18 +5466,18 @@ mod tests { let config = AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["provider-a", "provider-b"]), + provider_names: vec!["provider_a".to_string(), "provider_b".to_string()], timeout_ms: 2000, - mediator: None, + adserver_name: None, ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-a", + name: "provider_a", backend: "backend-a", })); orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-b", + name: "provider_b", backend: "backend-b", })); @@ -5608,18 +5532,18 @@ mod tests { let config = AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["provider-a", "provider-b"]), + provider_names: vec!["provider_a".to_string(), "provider_b".to_string()], timeout_ms: 2000, - mediator: None, + adserver_name: None, ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-a", + name: "provider_a", backend: "backend-a", })); orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-b", + name: "provider_b", backend: "backend-b", })); @@ -5673,13 +5597,13 @@ mod tests { Arc::clone(&backend) as Arc<_>, Arc::clone(&http) as Arc<_>, ); - let mut config = planned_config(&[("provider-a", RoutingMode::AllEligible)], false); + let mut config = planned_config(&[("provider_a", RoutingMode::AllEligible)], false); config.bidders.insert( "routed-bidder" .parse() .expect("should parse fictional bidder ID"), crate::auction::plan::BidderRouteConfig { - provider: "provider-a" + module: "provider_a" .parse() .expect("should parse fictional provider ID"), }, @@ -5725,7 +5649,7 @@ mod tests { assert_eq!(http.recorded_backend_names().len(), 1); assert_eq!(backend.ensured.load(Ordering::Relaxed), 1); assert_eq!(result.provider_responses.len(), 1); - assert_eq!(result.provider_responses[0].provider, "provider-a"); + assert_eq!(result.provider_responses[0].provider, "provider_a"); assert_eq!(result.provider_responses[0].status, BidStatus::Success); assert_eq!( result.provider_responses[0].metadata["routing"]["unused_bidder_params_count"], @@ -5743,7 +5667,7 @@ mod tests { } #[tokio::test] - async fn planned_executor_invokes_immediate_mediator_and_applies_floor() { + async fn planned_executor_invokes_immediate_adserver_and_applies_floor() { let http = Arc::new(StubHttpClient::new()); http.push_response( 200, @@ -5761,11 +5685,11 @@ mod tests { Arc::clone(&http) as Arc<_>, ); let plan = AuctionPlan::compile(planned_config( - &[("provider-a", RoutingMode::AllEligible)], + &[("provider_a", RoutingMode::AllEligible)], false, )) .expect("should compile planned auction"); - let orchestrator = AuctionOrchestratorHarness::new(plan, Some(Arc::new(ImmediateMediator))); + let orchestrator = AuctionOrchestratorHarness::new(plan, Some(Arc::new(ImmediateAdServer))); let request = planned_request(); let settings = create_test_settings(); let inbound = http::Request::new(edgezero_core::body::Body::empty()); @@ -5781,14 +5705,14 @@ mod tests { let result = orchestrator .run_auction(&request, &context) .await - .expect("should execute planned mediation"); + .expect("should execute the planned ad server"); assert_eq!( result - .mediator_response + .adserver_response .as_ref() .map(|response| response.provider.as_str()), - Some("immediate-mediator") + Some("immediate_adserver") ); assert_eq!( result.winning_bids["header-banner"].nurl.as_deref(), @@ -5796,11 +5720,11 @@ mod tests { ); assert!( !result.winning_bids.contains_key("fictional-slot"), - "mediator output owns final selection" + "adserver output owns final selection" ); } - async fn planned_pending_mediator_deadline_result( + async fn planned_pending_adserver_deadline_result( enforceable_total_request_deadline: bool, ) -> OrchestrationResult { let http = Arc::new(StubHttpClient::new()); @@ -5824,12 +5748,12 @@ mod tests { Arc::clone(&http) as Arc<_>, ); let plan = AuctionPlan::compile(planned_config( - &[("provider-a", RoutingMode::AllEligible)], + &[("provider_a", RoutingMode::AllEligible)], false, )) .expect("should compile planned auction"); let orchestrator = - AuctionOrchestratorHarness::new(plan, Some(Arc::new(PendingDeadlineMediator))); + AuctionOrchestratorHarness::new(plan, Some(Arc::new(PendingDeadlineAdServer))); let request = planned_request(); let settings = create_test_settings(); let inbound = http::Request::new(edgezero_core::body::Body::empty()); @@ -5845,21 +5769,21 @@ mod tests { orchestrator .run_auction(&request, &context) .await - .expect("should execute planned pending mediator") + .expect("should execute planned pending adserver") } #[tokio::test] - async fn planned_pending_mediator_applies_explicit_hard_deadline_policy() { - let current = planned_pending_mediator_deadline_result(false).await; - let current_mediator = current - .mediator_response + async fn planned_pending_adserver_applies_explicit_hard_deadline_policy() { + let current = planned_pending_adserver_deadline_result(false).await; + let current_adserver = current + .adserver_response .as_ref() - .expect("current adapters should accept completed late mediator responses"); - assert!(current_mediator.response_time_ms >= 50); - assert_eq!(current.winning_bids["slot-1"].bidder, "mediated"); + .expect("current adapters should accept completed late adserver responses"); + assert!(current_adserver.response_time_ms >= 50); + assert_eq!(current.winning_bids["slot-1"].bidder, "adserver"); - let hard = planned_pending_mediator_deadline_result(true).await; - assert!(hard.mediator_response.is_none()); + let hard = planned_pending_adserver_deadline_result(true).await; + assert!(hard.adserver_response.is_none()); assert_eq!( hard.winning_bids["fictional-slot"].bid_id.as_deref(), Some("provider") @@ -5868,7 +5792,7 @@ mod tests { } #[tokio::test] - async fn planned_executor_mediator_transport_failure_falls_back_locally() { + async fn planned_executor_adserver_transport_failure_falls_back_locally() { let http = Arc::new(StubHttpClient::new()); http.push_response( 200, @@ -5889,12 +5813,12 @@ mod tests { Arc::clone(&http) as Arc<_>, ); let plan = AuctionPlan::compile(planned_config( - &[("provider-a", RoutingMode::AllEligible)], + &[("provider_a", RoutingMode::AllEligible)], false, )) .expect("should compile planned auction"); let orchestrator = - AuctionOrchestratorHarness::new(plan, Some(Arc::new(CacheRestoringMediator))); + AuctionOrchestratorHarness::new(plan, Some(Arc::new(CacheRestoringAdServer))); let request = planned_request(); let settings = create_test_settings(); let inbound = http::Request::new(edgezero_core::body::Body::empty()); @@ -5910,9 +5834,9 @@ mod tests { let result = orchestrator .run_auction(&request, &context) .await - .expect("should fall back from mediator transport failure"); + .expect("should fall back from adserver transport failure"); - assert!(result.mediator_response.is_none()); + assert!(result.adserver_response.is_none()); assert_eq!( result.winning_bids["fictional-slot"].bid_id.as_deref(), Some("provider") @@ -5920,10 +5844,11 @@ mod tests { } #[tokio::test] - async fn planned_prebid_stored_intent_filters_wire_demand_and_skips_empty_transports() { + async fn planned_stored_intent_filters_wire_demand_and_skips_empty_transports() { for inline_providers in 0..=2 { let http = Arc::new(StubHttpClient::new()); - // Providers launch in ID order: APS, then only PBS instances with usable demand. + // Providers launch in ID order: the source that takes every slot, then + // only the stored-request sources with usable demand. http.push_response(204, Vec::new()); for index in 0..inline_providers { http.push_response( @@ -5940,7 +5865,7 @@ mod tests { } ]}] })) - .expect("should serialize PBS response"), + .expect("should serialize the upstream response"), ); } let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Fastly)); @@ -5948,24 +5873,45 @@ mod tests { Arc::clone(&backend) as Arc<_>, Arc::clone(&http) as Arc<_>, ); - let mut config = planned_prebid_config(&[ + let mut config = planned_fixture_config(&[ ( - "pbs-a", + "stored_a", serde_json::json!({}), NotificationConfig::default(), ), ( - "pbs-b", + "stored_b", serde_json::json!({}), NotificationConfig::default(), ), ]); - config.providers.extend(planned_aps_config().providers); - for (bidder, provider) in [("alpha", "pbs-a"), ("beta", "pbs-b")] { + let all_eligible = + planned_config(&[("all_eligible_source", RoutingMode::AllEligible)], false); + let mut tables = config.demand.tables().clone(); + tables.extend(all_eligible.demand.tables().clone()); + let mut selected = config + .demand + .selected() + .into_iter() + .map(str::to_owned) + .collect::>(); + selected.extend( + all_eligible + .demand + .selected() + .into_iter() + .map(str::to_owned), + ); + // Sources launch in the order they are selected, and this test reads + // the all-eligible source's request first, so the selection is in + // name order. + selected.sort(); + config.demand = crate::provider_table::ProviderList::new(selected, tables); + for (bidder, provider) in [("alpha", "stored_a"), ("beta", "stored_b")] { config.bidders.insert( bidder.parse().expect("should parse bidder"), crate::auction::plan::BidderRouteConfig { - provider: provider.parse().expect("should parse provider"), + module: provider.parse().expect("should parse provider"), }, ); } @@ -5973,7 +5919,7 @@ mod tests { let orchestrator = AuctionOrchestratorHarness::new(plan, None); let mut request = planned_request(); let template = request.slots[0].clone(); - // Both PBS instances must evaluate candidates but omit them after overrides. + // Both stored-request sources must evaluate candidates and omit them. request.slots[0].bidders.insert( "trustedServer".to_string(), serde_json::json!({ @@ -5981,7 +5927,7 @@ mod tests { }), ); request.slots.push(AdSlot { - id: "synthetic-no-pbs".to_string(), + id: "synthetic-no-demand".to_string(), bidders: HashMap::from([( "trustedServer".to_string(), serde_json::json!({"storedRequest":false,"bidderParams":{}}), @@ -6017,12 +5963,12 @@ mod tests { assert_eq!( bodies.len(), 1 + inline_providers, - "should never transport an empty PBS request" + "should never transport an empty request" ); - let aps: serde_json::Value = - serde_json::from_slice(&bodies[0]).expect("should parse APS wire request"); + let all_eligible_wire: serde_json::Value = serde_json::from_slice(&bodies[0]) + .expect("should parse the all-eligible source's wire request"); assert_eq!( - aps["imp"] + all_eligible_wire["imp"] .as_array() .expect("should have impressions") .len(), @@ -6034,12 +5980,12 @@ mod tests { "should reject a bid for an omitted impression" ); for index in inline_providers..2 { - let provider_id = if index == 0 { "pbs-a" } else { "pbs-b" }; + let provider_id = if index == 0 { "stored_a" } else { "stored_b" }; let response = result .provider_responses .iter() .find(|response| response.provider == provider_id) - .expect("should retain skipped PBS provider response"); + .expect("should retain the skipped source's response"); assert_eq!( response.metadata["routing"]["skipped_no_usable_demand"], true @@ -6047,7 +5993,7 @@ mod tests { } for index in 0..inline_providers { let wire: serde_json::Value = serde_json::from_slice(&bodies[index + 1]) - .expect("should parse PBS wire request"); + .expect("should parse the source's wire request"); assert_eq!( wire["imp"] .as_array() @@ -6056,11 +6002,11 @@ mod tests { 1 ); assert_eq!(wire["imp"][0]["id"], format!("inline-{index}")); - let prebid = &wire["imp"][0]["ext"]["prebid"]; - assert!(prebid.get("storedrequest").is_none()); + let fixture = &wire["imp"][0]["ext"]["fixture"]; + assert!(fixture.get("stored").is_none()); let bidder = if index == 0 { "alpha" } else { "beta" }; assert_eq!( - prebid["bidder"], + fixture["bidder"], serde_json::json!({bidder:{"placement":index}}) ); assert_eq!( @@ -6069,12 +6015,12 @@ mod tests { .as_deref(), Some(format!("bid-{index}").as_str()) ); - let provider_id = if index == 0 { "pbs-a" } else { "pbs-b" }; + let provider_id = if index == 0 { "stored_a" } else { "stored_b" }; let response = result .provider_responses .iter() .find(|response| response.provider == provider_id) - .expect("should retain PBS provider response"); + .expect("should retain the source's response"); assert_eq!( response.metadata["response_admission"]["rejected_bid_count"], 1 @@ -6088,59 +6034,119 @@ mod tests { } #[tokio::test] - async fn planned_prebid_instances_preserve_headers_metadata_suppression_and_identity() { + async fn planned_provider_outcome_matrix_has_fixed_count_only_routing_metadata() { + let standard_plan = AuctionPlan::compile(planned_config( + &[("standard", RoutingMode::AllEligible)], + false, + )) + .expect("should compile standard plan"); + let fixture_plan = AuctionPlan::compile(planned_fixture_config(&[( + "stored", + serde_json::json!({}), + NotificationConfig::default(), + )])) + .expect("should compile the stand-in plan"); + let cases = [ + (&standard_plan, 204, Vec::new(), BidStatus::NoBid), + (&standard_plan, 502, Vec::new(), BidStatus::Error), + (&standard_plan, 200, b"not-json".to_vec(), BidStatus::Error), + ( + &standard_plan, + 200, + br#"{"seatbid":[]}"#.to_vec(), + BidStatus::NoBid, + ), + (&fixture_plan, 204, b"{}".to_vec(), BidStatus::NoBid), + (&fixture_plan, 502, Vec::new(), BidStatus::Error), + (&fixture_plan, 200, b"not-json".to_vec(), BidStatus::Error), + ( + &fixture_plan, + 200, + br#"{"seatbid":[]}"#.to_vec(), + BidStatus::NoBid, + ), + ]; + + for (plan, status, body, expected) in cases { + let routed = route_auction( + planned_envelope_request(), + &http::Request::new(edgezero_core::body::Body::empty()), + plan, + None, + ); + let provider = GenericOpenRtbProvider::new(plan.providers()[0].clone()); + let state = provider.parse_state_for_test(routed.inputs()[0].clone()); + let response = PlatformResponse::new( + edgezero_core::http::response_builder() + .status(status) + .body(edgezero_core::body::Body::from(body)) + .expect("should build provider matrix response"), + ); + let parsed = provider + .parse_response_with_state(response, 4, Some(state.as_ref())) + .await + .expect("should classify provider matrix response"); + assert_eq!(parsed.status, expected, "status {status}"); + if expected == BidStatus::Error { + let expected_error_type = if (200..300).contains(&status) { + "parse_response" + } else { + "http_status" + }; + assert_eq!(parsed.metadata["error_type"], expected_error_type); + } + assert_eq!( + parsed.metadata["routing"], + serde_json::json!({"unused_bidder_params_count": 0}) + ); + let serialized = serde_json::to_string(&parsed.metadata["routing"]) + .expect("should serialize routing metadata"); + assert!(!serialized.contains("fictional_provider")); + assert!(!serialized.contains("fictional-slot")); + } + } + + #[tokio::test] + async fn planned_standard_instances_have_distinct_backends_and_independent_results() { let http = Arc::new(StubHttpClient::new()); http.push_response( 200, serde_json::to_vec(&serde_json::json!({ - "seatbid": [{"seat": "suppress-exact", "bid": [ - {"id":"good-a","impid":"fictional-slot","price":1.25,"adm":"
a
","w":300,"h":250,"nurl":"https://notify.example/win","burl":"https://notify.example/bill","ext":{"prebid":{"cache":{"bids":{"cacheId":"cache-a","url":"https://cache-a.example/cache/path"}}}}}, - {"id":"bad-a","price":2.0} - ]}], - "ext": {"responsetimemillis":{"suppress-exact":4},"errors":{"other":["fictional"]},"warnings":{"other":["warning"]},"debug":{"httpcalls":[]},"prebid":{"bidstatus":{"suppress-exact":[{"bidid":"good-a"}]}}} + "seatbid": [{"seat": "fictional-seat-a", "bid": [{ + "id": "bid-a", "impid": "fictional-slot", "price": 1.25, + "adm": "
a
", "w": 300, "h": 250 + }]}] })) - .expect("should serialize PBS response a"), + .expect("should serialize response a"), ); http.push_response( 200, serde_json::to_vec(&serde_json::json!({ - "seatbid": [{"seat": "keep-seat", "bid": [{ - "id":"good-b","impid":"fictional-slot","price":2.5,"adm":"
b
","w":300,"h":250,"nurl":"https://notify.example/win","burl":"https://notify.example/bill" + "seatbid": [{"seat": "fictional-seat-b", "bid": [{ + "id": "bid-b", "impid": "fictional-slot", "price": 2.5, + "adm": "
b
", "w": 300, "h": 250 }]}] })) - .expect("should serialize PBS response b"), + .expect("should serialize response b"), ); let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Fastly)); let services = build_services_with_backend_and_http_client( Arc::clone(&backend) as Arc<_>, Arc::clone(&http) as Arc<_>, ); - let notifications = NotificationConfig { - suppress_all: false, - suppress_seats: vec!["suppress-exact".to_string()], - }; - let plan = AuctionPlan::compile(planned_prebid_config(&[ - ( - "pbs-a", - serde_json::json!({"debug":true,"test_mode":true,"consent_forwarding":"openrtb_only"}), - notifications, - ), - ("pbs-b", serde_json::json!({}), NotificationConfig::default()), - ])) - .expect("should compile planned PBS auction"); + let plan = AuctionPlan::compile(planned_config( + &[ + ("provider_a", RoutingMode::AllEligible), + ("provider_b", RoutingMode::AllEligible), + ], + false, + )) + .expect("should compile planned auction"); let orchestrator = AuctionOrchestratorHarness::new(plan, None); - let request = planned_prebid_request(); + let request = planned_request(); let settings = create_test_settings(); let inbound = http::Request::builder() .uri("https://publisher.example/auction") - .header( - http::header::COOKIE, - "consent=keep; euconsent-v2=drop; other=value", - ) - .header(http::header::USER_AGENT, "Fictional Browser/7") - .header(http::header::REFERER, "https://referrer.example/story") - .header(http::header::ACCEPT_LANGUAGE, "en-US,en;q=0.9") - .header("x-forwarded-for", "203.0.113.250") .body(edgezero_core::body::Body::empty()) .expect("should build inbound request"); let context = AuctionContext { @@ -6155,830 +6161,75 @@ mod tests { let result = orchestrator .run_auction(&request, &context) .await - .expect("should execute planned PBS auction"); + .expect("should execute planned auction"); + assert_eq!(orchestrator.provider_count(), 2); + assert!(orchestrator.adserver().is_none()); assert_eq!(result.provider_responses.len(), 2); - let first = &result.provider_responses[0]; - assert_eq!(first.provider, "pbs-a"); - assert_eq!(first.bids.len(), 1, "should isolate malformed sibling"); + assert_eq!(result.provider_responses[0].provider, "provider_a"); + assert_eq!( + result.provider_responses[0].bids[0].bid_id.as_deref(), + Some("bid-a") + ); assert_eq!( - first.bids[0].returned_seat.as_deref(), - Some("suppress-exact") + result.provider_responses[0].bids[0] + .returned_seat + .as_deref(), + Some("fictional-seat-a") ); - assert_eq!(first.bids[0].bidder, "suppress-exact"); - assert!( - first.bids[0].nurl.is_none(), - "should suppress after normalization" + assert_eq!( + result.provider_responses[0].metadata["routing"]["unused_bidder_params_count"], + 0 ); - assert!( - first.bids[0].burl.is_none(), - "should suppress billing notification" - ); - assert_eq!(first.bids[0].cache_id.as_deref(), Some("cache-a")); - assert_eq!(first.bids[0].cache_host.as_deref(), Some("cache-a.example")); - assert_eq!(first.bids[0].cache_path.as_deref(), Some("/cache/path")); - assert_eq!(first.metadata["responsetimemillis"]["suppress-exact"], 4); - assert!(first.metadata.contains_key("errors")); - assert!(first.metadata.contains_key("warnings")); - assert!(first.metadata.contains_key("debug")); - assert!(first.metadata.contains_key("bidstatus")); - let second = &result.provider_responses[1]; - assert_eq!(second.provider, "pbs-b"); - assert_eq!(second.bids[0].returned_seat.as_deref(), Some("keep-seat")); - assert!(second.bids[0].nurl.is_some()); - assert!(!second.metadata.contains_key("debug")); - assert!(!second.metadata.contains_key("bidstatus")); - - let headers = http.recorded_request_headers(); - assert_eq!(headers.len(), 2); - for request_headers in &headers { - assert!( - request_headers - .iter() - .any(|(name, value)| name == "user-agent" && value == "Fictional Browser/7") - ); - assert!(request_headers.iter().any( - |(name, value)| name == "referer" && value == "https://referrer.example/story" - )); - assert!( - request_headers - .iter() - .any(|(name, value)| name == "accept-language" && value == "en-US,en;q=0.9") - ); + assert_eq!(result.provider_responses[1].provider, "provider_b"); + assert_eq!( + result.provider_responses[1].bids[0].bid_id.as_deref(), + Some("bid-b") + ); + assert_eq!( + result.provider_responses[1].bids[0] + .returned_seat + .as_deref(), + Some("fictional-seat-b") + ); + assert_eq!( + result.provider_responses[1].metadata["routing"]["unused_bidder_params_count"], + 0 + ); + assert_eq!( + result.winning_bids["fictional-slot"].bid_id.as_deref(), + Some("bid-b") + ); + let request_headers = http.recorded_request_headers(); + assert_eq!(request_headers.len(), 2); + for headers in request_headers { assert!( - request_headers + headers .iter() - .all(|(name, _)| name != "x-forwarded-for"), - "must ignore inbound XFF without attestation" + .any(|(name, value)| name == "accept" && value == "application/json"), + "standard planned transport should retain its JSON Accept header" ); + } + let backend_names = http.recorded_backend_names(); + assert_eq!(backend_names.len(), 2); + assert_ne!(backend_names[0], backend_names[1]); + let request_bodies = http.recorded_request_bodies(); + assert_eq!(request_bodies.len(), 2); + for body in request_bodies { + let value: serde_json::Value = + serde_json::from_slice(&body).expect("should parse planned request"); + let tmax = value["tmax"].as_u64().expect("should include logical tmax"); assert!( - request_headers.iter().all(|(name, _)| name != "accept"), - "planned PBS transport must not add Accept beyond legacy headers" + (750..=777).contains(&tmax), + "logical budget should remain near the auction budget, got {tmax}" ); + assert_eq!(value["imp"].as_array().map(Vec::len), Some(1)); } - let first_cookie = headers[0] - .iter() - .find(|(name, _)| name == "cookie") - .map(|(_, value)| value.as_str()); - assert_eq!(first_cookie, Some("consent=keep; other=value")); - let second_cookie = headers[1] - .iter() - .find(|(name, _)| name == "cookie") - .map(|(_, value)| value.as_str()); - assert_eq!( - second_cookie, - Some("consent=keep; euconsent-v2=drop; other=value") - ); - } - - #[tokio::test] - async fn planned_aps_mock_mediation_preserves_three_identities_and_renderer() { - let http = Arc::new(StubHttpClient::new()); - http.push_response( - 200, - serde_json::to_vec(&serde_json::json!({ - "seatbid": [{"seat": "upstream-seat", "bid": [{ - "id": "aps-bid", "impid": "fictional-slot", "price": 2.0, - "w": 300, "h": 250, - "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"} - }]}] - })) - .expect("should serialize APS response"), - ); - http.push_response( - 200, - serde_json::to_vec(&serde_json::json!({ - "seatbid": [{"seat": "aps-instance", "bid": [{ - "id": "mediated-aps", "impid": "fictional-slot", "price": 2.0, - "adm": "ignored", "w": 300, "h": 250, "crid": "aps-creative" - }]}] - })) - .expect("should serialize mediator response"), - ); - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, - ); - let plan = - AuctionPlan::compile(planned_aps_config()).expect("should compile planned APS auction"); - let mediator = AdServerMockProvider::new(AdServerMockConfig { - enabled: true, - endpoint: "https://mediator.example/mediate".to_string(), - timeout_ms: 500, - ..AdServerMockConfig::default() - }); - let orchestrator = AuctionOrchestratorHarness::new(plan, Some(Arc::new(mediator))); - let request = planned_request(); - let settings = create_test_settings(); - let inbound = http::Request::new(edgezero_core::body::Body::empty()); - let context = AuctionContext { - settings: &settings, - request: &inbound, - timeout_ms: 777, - transport_timeout_ms: 777, - provider_responses: None, - services: &services, - }; - - let result = orchestrator - .run_auction(&request, &context) - .await - .expect("should mediate planned APS bid"); - - let provider_bid = &result.provider_responses[0].bids[0]; - assert_eq!(result.provider_responses[0].provider, "aps-instance"); - assert_eq!(provider_bid.returned_seat.as_deref(), Some("upstream-seat")); - assert_eq!(provider_bid.bidder, "aps"); - let winner = &result.winning_bids["fictional-slot"]; - assert_eq!(winner.returned_seat.as_deref(), Some("upstream-seat")); - assert_eq!(winner.bidder, "aps"); - assert!(winner.renderer.is_some()); - assert!(winner.creative.is_none()); - assert_eq!( - result - .mediator_response - .as_ref() - .map(|response| response.provider.as_str()), - Some("adserver_mock") - ); - } - - #[tokio::test] - async fn planned_aps_transport_omits_accept_header() { - let http = Arc::new(StubHttpClient::new()); - http.push_response(400, Vec::new()); - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Fastly)); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, - ); - let plan = - AuctionPlan::compile(planned_aps_config()).expect("should compile planned APS auction"); - let orchestrator = AuctionOrchestratorHarness::new(plan, None); - let request = planned_request(); - let settings = create_test_settings(); - let inbound = http::Request::builder() - .uri("https://publisher.example/auction") - .body(edgezero_core::body::Body::empty()) - .expect("should build inbound request"); - let context = AuctionContext { - settings: &settings, - request: &inbound, - timeout_ms: 777, - transport_timeout_ms: 777, - provider_responses: None, - services: &services, - }; - - orchestrator - .run_auction(&request, &context) - .await - .expect("should execute planned APS auction"); - - let headers = http.recorded_request_headers(); - assert_eq!(headers.len(), 1); - assert!( - headers[0].iter().all(|(name, _)| name != "accept"), - "planned APS transport must not add Accept beyond legacy headers" - ); - } - - #[tokio::test] - async fn planned_aps_profile_normalizes_renderer_reduction_and_metadata() { - let http = Arc::new(StubHttpClient::new()); - http.push_response_with_headers( - 200, - serde_json::to_vec(&serde_json::json!({ - "cur": "USD", - "seatbid": [ - {"seat": "returned-seat", "bid": [ - {"id": "z-high", "impid": "fictional-slot", "price": 2.0, "w": 300, "h": 250, - "nurl": "https://notice.example/win", "burl": "https://notice.example/bill", - "crid": "fictional-creative", "adomain": ["advertiser.example"], - "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, - {"id": "a-high", "impid": "fictional-slot", "price": 2.0, "w": 300, "h": 250, - "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, - {"id": "bad-script", "impid": "fictional-slot", "price": 9.0, "w": 300, "h": 250, - "ext": {"creativeurl": "https://creative.example/render", "tagtype": "script"}}, - {"id": "bad-domain", "impid": "fictional-slot", "price": 8.0, "w": 300, "h": 250, - "ext": {"creativeurl": "https://publisher.example/render", "tagtype": "iframe"}}, - {"id": "bad-credentials", "impid": "fictional-slot", "price": 8.0, "w": 300, "h": 250, - "ext": {"creativeurl": "https://user:password@creative.example/render", "tagtype": "iframe"}}, - {"id": "bad-imp", "impid": "unknown-slot", "price": 8.0, "w": 300, "h": 250, - "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, - {"id": "bad-dimensions", "impid": "fictional-slot", "price": 8.0, "w": 320, "h": 50, - "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, - {"id": "bad-price", "impid": "fictional-slot", "price": "high", "w": 300, "h": 250, - "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, - {"id": "bad-mtype", "impid": "fictional-slot", "price": 8.0, "mtype": 2, "w": 300, "h": 250, - "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, - {"id": "bad-tag", "impid": "fictional-slot", "price": 8.0, "w": 300, "h": 250, - "ext": {"creativeurl": "https://creative.example/render", "tagtype": "native"}}, - {"id": "bad-crid", "impid": "fictional-slot", "price": 8.0, "w": 300, "h": 250, - "crid": "x".repeat(1025), - "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, - {"impid": "fictional-slot", "price": 8.0, "w": 300, "h": 250, - "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}} - ]}, - {"seat": 7, "bid": "bad-shape"} - ] - })) - .expect("should serialize APS profile response"), - vec![ - ("content-type", "application/json"), - ("authorization", "fictional-secret"), - ], - ); - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Fastly)); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, - ); - let plan = AuctionPlan::compile(planned_aps_instances_config(&[( - "aps-instance", - serde_json::json!({"account_id": "example-account", "debug": true}), - NotificationConfig { - suppress_all: false, - suppress_seats: vec!["returned-seat".to_string()], - }, - )])) - .expect("should compile planned APS profile"); - let orchestrator = AuctionOrchestratorHarness::new(plan, None); - let request = planned_request(); - let settings = create_test_settings(); - let inbound = http::Request::new(edgezero_core::body::Body::empty()); - let context = AuctionContext { - settings: &settings, - request: &inbound, - timeout_ms: 777, - transport_timeout_ms: 777, - provider_responses: None, - services: &services, - }; - - let result = orchestrator - .run_auction(&request, &context) - .await - .expect("should execute planned APS profile"); - - let response = &result.provider_responses[0]; - assert_eq!(response.provider, "aps-instance"); - assert_eq!(response.status, BidStatus::Success); - assert_eq!( - response.bids.len(), - 1, - "should retain one bid per impression" - ); - let bid = &response.bids[0]; - assert_eq!(bid.bidder, "aps"); - assert_eq!(bid.returned_seat.as_deref(), Some("returned-seat")); - assert_eq!( - bid.bid_id.as_deref(), - Some("a-high"), - "lexical ID should break equal-price tie" - ); - assert!(bid.creative.is_none()); - assert!( - bid.nurl.is_none() && bid.burl.is_none(), - "APS must discard notification URLs" - ); - let renderer = bid - .renderer - .as_ref() - .and_then(BidRenderer::as_aps) - .expect("should construct typed APS renderer"); - assert_eq!(renderer.account_id, "example-account"); - let decoded = base64::engine::general_purpose::STANDARD - .decode(&renderer.aax_response) - .expect("should decode minimized APS response"); - assert_eq!( - serde_json::from_slice::(&decoded) - .expect("should parse minimized APS response"), - serde_json::json!({"seatbid":[{"bid":[{ - "id":"a-high","price":2.0,"w":300,"h":250, - "ext":{"creativeurl":"https://creative.example/render","tagtype":"iframe"} - }]}]}) - ); - assert_eq!(response.metadata["seatbid_count"], 2); - assert_eq!(response.metadata["accepted_bid_count"], 1); - assert_eq!(response.metadata["dropped_bid_count"], 12); - for reason in [ - "lost_to_higher_bid", - "script_rendering_disabled", - "unknown_impid", - "invalid_dimensions", - "invalid_price", - "unsupported_media_type", - "unsupported_tagtype", - "creative_id_too_large", - "missing_render_source", - "empty_seatbid_bids", - ] { - assert_eq!(response.metadata["drop_reasons"][reason], 1, "{reason}"); - } - assert_eq!( - response.metadata["drop_reasons"]["invalid_creative_url"], 2, - "same-publisher and credentialed URLs should both be rejected" - ); - assert_eq!( - response.metadata["routing"]["unused_bidder_params_count"], - 0 - ); - let debug = &response.metadata["debug"]["httpcalls"]["aps"][0]; - assert_eq!(debug["uri"], "https://aps.example/e/pb/bid"); - assert_eq!( - debug["responseheaders"], - serde_json::json!({"content-type": ["application/json"]}), - "async stub should preserve allowlisted response headers" - ); - assert!( - debug["requestbody"] - .as_str() - .is_some_and(|body| body.contains("example-account")) - ); - assert!(debug["requestheaders"].get("authorization").is_none()); - assert!(debug["responseheaders"].get("authorization").is_none()); - } - - #[tokio::test] - async fn two_planned_aps_instances_correlate_independently() { - let http = Arc::new(StubHttpClient::new()); - for (seat, id, price) in [("seat-a", "bid-a", 1.0), ("seat-b", "bid-b", 2.0)] { - http.push_response( - 200, - serde_json::to_vec(&serde_json::json!({"seatbid":[{"seat":seat,"bid":[{ - "id":id,"impid":"fictional-slot","price":price,"w":300,"h":250, - "ext":{"creativeurl":"https://creative.example/render","tagtype":"iframe"} - }]}]})) - .expect("should serialize APS instance response"), - ); - } - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Fastly)); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, - ); - let plan = AuctionPlan::compile(planned_aps_instances_config(&[ - ( - "aps-a", - serde_json::json!({"account_id":"account-a"}), - NotificationConfig::default(), - ), - ( - "aps-b", - serde_json::json!({"account_id":"account-b"}), - NotificationConfig::default(), - ), - ])) - .expect("should compile two APS instances"); - let orchestrator = AuctionOrchestratorHarness::new(plan, None); - let request = planned_request(); - let settings = create_test_settings(); - let inbound = http::Request::new(edgezero_core::body::Body::empty()); - let context = AuctionContext { - settings: &settings, - request: &inbound, - timeout_ms: 777, - transport_timeout_ms: 777, - provider_responses: None, - services: &services, - }; - - let result = orchestrator - .run_auction(&request, &context) - .await - .expect("should execute two APS instances"); - - assert_eq!(result.provider_responses.len(), 2); - assert_eq!(result.provider_responses[0].provider, "aps-a"); - assert_eq!( - result.provider_responses[0].bids[0].bid_id.as_deref(), - Some("bid-a") - ); - assert_eq!(result.provider_responses[1].provider, "aps-b"); - assert_eq!( - result.provider_responses[1].bids[0].bid_id.as_deref(), - Some("bid-b") - ); - assert_eq!(http.recorded_request_bodies().len(), 2); - assert_eq!( - result.winning_bids["fictional-slot"].bid_id.as_deref(), - Some("bid-b"), - "global ranking should remain orchestrator-owned" - ); - let specs = backend.specs.lock().expect("should lock specs"); - assert_eq!(specs.len(), 2); - assert_ne!(specs[0].discriminator, specs[1].discriminator); - } - - #[tokio::test] - async fn planned_aps_returned_seat_accepts_only_valid_nonempty_strings() { - let plan = AuctionPlan::compile(planned_aps_config()).expect("should compile APS plan"); - let routed = route_auction( - planned_request(), - &http::Request::new(edgezero_core::body::Body::empty()), - &plan, - None, - ); - let provider = GenericOpenRtbProvider::new(plan.providers()[0].clone()); - for (seat, expected) in [ - (serde_json::Value::Null, None), - (serde_json::json!(7), None), - (serde_json::json!(""), None), - (serde_json::json!("exact-seat"), Some("exact-seat")), - ] { - let state = provider.parse_state_for_test(routed.inputs()[0].clone()); - let response = PlatformResponse::new( - edgezero_core::http::response_builder() - .status(200) - .body(edgezero_core::body::Body::from( - serde_json::to_vec(&serde_json::json!({"seatbid":[{"seat":seat,"bid":[{ - "id":"bid","impid":"fictional-slot","price":1.0,"w":300,"h":250, - "nurl":"https://notice.example/win","burl":"https://notice.example/bill", - "ext":{"creativeurl":"https://creative.example/render","tagtype":"iframe"} - }]}]})) - .expect("should serialize seat identity response"), - )) - .expect("should build seat identity response"), - ); - let parsed = provider - .parse_response_with_state(response, 4, Some(state.as_ref())) - .await - .expect("should parse seat identity response"); - assert_eq!(parsed.bids[0].returned_seat.as_deref(), expected); - assert!(parsed.bids[0].nurl.is_none() && parsed.bids[0].burl.is_none()); - } - } - - #[tokio::test] - async fn planned_aps_response_status_shape_and_currency_matrix() { - let plan = AuctionPlan::compile(planned_aps_config()).expect("should compile APS plan"); - let routed = route_auction( - planned_request(), - &http::Request::new(edgezero_core::body::Body::empty()), - &plan, - None, - ); - let provider = GenericOpenRtbProvider::new(plan.providers()[0].clone()); - let cases = [ - (204, Vec::new(), BidStatus::NoBid, None, None), - (400, Vec::new(), BidStatus::Error, None, Some("http_status")), - ( - 200, - b"not-json".to_vec(), - BidStatus::Error, - Some("unexpected_response_shape"), - Some("parse_response"), - ), - ( - 200, - b"[]".to_vec(), - BidStatus::Error, - Some("unexpected_response_shape"), - Some("parse_response"), - ), - ( - 200, - br#"{"contextual":true}"#.to_vec(), - BidStatus::Error, - Some("unexpected_response_shape"), - Some("parse_response"), - ), - ( - 200, - br#"{"cur":"EUR","seatbid":[]}"#.to_vec(), - BidStatus::NoBid, - Some("unsupported_currency"), - None, - ), - ]; - for (status, body, expected, reason, error_type) in cases { - let state = provider.parse_state_for_test(routed.inputs()[0].clone()); - let response = PlatformResponse::new( - edgezero_core::http::response_builder() - .status(status) - .body(edgezero_core::body::Body::from(body)) - .expect("should build APS matrix response"), - ); - let parsed = provider - .parse_response_with_state(response, 4, Some(state.as_ref())) - .await - .expect("should classify APS matrix response"); - assert_eq!(parsed.status, expected, "status {status}"); - if let Some(reason) = reason { - assert_eq!( - parsed.metadata["drop_reasons"][reason], 1, - "status {status}" - ); - } - if let Some(error_type) = error_type { - assert_eq!(parsed.metadata["error_type"], error_type, "status {status}"); - } - } - } - - #[tokio::test] - async fn planned_provider_outcome_matrix_has_fixed_count_only_routing_metadata() { - let standard_plan = AuctionPlan::compile(planned_config( - &[("standard", RoutingMode::AllEligible)], - false, - )) - .expect("should compile standard plan"); - let prebid_plan = AuctionPlan::compile(planned_prebid_config(&[( - "pbs", - serde_json::json!({}), - NotificationConfig::default(), - )])) - .expect("should compile PBS plan"); - let cases = [ - (&standard_plan, 204, Vec::new(), BidStatus::NoBid), - (&standard_plan, 502, Vec::new(), BidStatus::Error), - (&standard_plan, 200, b"not-json".to_vec(), BidStatus::Error), - ( - &standard_plan, - 200, - br#"{"seatbid":[]}"#.to_vec(), - BidStatus::NoBid, - ), - (&prebid_plan, 204, b"{}".to_vec(), BidStatus::NoBid), - (&prebid_plan, 502, Vec::new(), BidStatus::Error), - (&prebid_plan, 200, b"not-json".to_vec(), BidStatus::Error), - ( - &prebid_plan, - 200, - br#"{"seatbid":[]}"#.to_vec(), - BidStatus::NoBid, - ), - ]; - - for (plan, status, body, expected) in cases { - let routed = route_auction( - planned_prebid_request(), - &http::Request::new(edgezero_core::body::Body::empty()), - plan, - None, - ); - let provider = GenericOpenRtbProvider::new(plan.providers()[0].clone()); - let state = provider.parse_state_for_test(routed.inputs()[0].clone()); - let response = PlatformResponse::new( - edgezero_core::http::response_builder() - .status(status) - .body(edgezero_core::body::Body::from(body)) - .expect("should build provider matrix response"), - ); - let parsed = provider - .parse_response_with_state(response, 4, Some(state.as_ref())) - .await - .expect("should classify provider matrix response"); - assert_eq!(parsed.status, expected, "status {status}"); - if expected == BidStatus::Error { - let expected_error_type = if (200..300).contains(&status) { - "parse_response" - } else { - "http_status" - }; - assert_eq!(parsed.metadata["error_type"], expected_error_type); - } - assert_eq!( - parsed.metadata["routing"], - serde_json::json!({"unused_bidder_params_count": 0}) - ); - let serialized = serde_json::to_string(&parsed.metadata["routing"]) - .expect("should serialize routing metadata"); - assert!(!serialized.contains("fictional-provider")); - assert!(!serialized.contains("fictional-slot")); - } - } - - #[tokio::test] - async fn planned_aps_script_opt_in_matches_shared_renderer_fixture() { - let plan = AuctionPlan::compile(planned_aps_instances_config(&[( - "aps-instance", - serde_json::json!({ - "account_id":"example-account-id", - "allow_script_creatives":true - }), - NotificationConfig::default(), - )])) - .expect("should compile script-enabled APS plan"); - let routed = route_auction( - planned_request(), - &http::Request::new(edgezero_core::body::Body::empty()), - &plan, - None, - ); - let provider = GenericOpenRtbProvider::new(plan.providers()[0].clone()); - let state = provider.parse_state_for_test(routed.inputs()[0].clone()); - let response = PlatformResponse::new( - edgezero_core::http::response_builder() - .status(200) - .body(edgezero_core::body::Body::from( - serde_json::to_vec(&serde_json::json!({"seatbid":[{"bid":[{ - "id":"fictional-selected-bid-id","impid":"fictional-slot","price":1.23, - "w":300,"h":250,"crid":"fictional-creative", - "ext":{"creativeurl":"https://creative.example/render","tagtype":"iframe"} - },{ - "id":"script-bid","impid":"fictional-slot","price":1.0, - "w":300,"h":250, - "ext":{"creativeurl":"https://creative.example/script","tagtype":"script"} - }]}]})) - .expect("should serialize APS renderer fixture response"), - )) - .expect("should build APS renderer fixture response"), - ); - - let parsed = provider - .parse_response_with_state(response, 3, Some(state.as_ref())) - .await - .expect("should parse APS renderer fixture response"); - - assert_eq!(parsed.status, BidStatus::Success); - assert_eq!( - parsed.metadata["drop_reasons"]["lost_to_higher_bid"], 1, - "enabled script creative should be eligible before reduction" - ); - let renderer = parsed.bids[0] - .renderer - .as_ref() - .and_then(BidRenderer::as_aps) - .expect("should construct APS renderer"); - let decoded = base64::engine::general_purpose::STANDARD - .decode(&renderer.aax_response) - .expect("should decode APS fixture envelope"); - let fixture: serde_json::Value = serde_json::from_str(include_str!( - "../../../trusted-server-js/lib/test/fixtures/aps-renderer-v1.json" - )) - .expect("should parse shared APS renderer fixture"); - assert_eq!( - serde_json::from_slice::(&decoded) - .expect("should parse decoded APS renderer"), - fixture - ); - } - - #[tokio::test] - async fn planned_aps_debug_response_headers_are_allowlisted() { - let plan = AuctionPlan::compile(planned_aps_instances_config(&[( - "aps-instance", - serde_json::json!({"account_id":"example-account","debug":true}), - NotificationConfig::default(), - )])) - .expect("should compile debug APS plan"); - let routed = route_auction( - planned_request(), - &http::Request::new(edgezero_core::body::Body::empty()), - &plan, - None, - ); - let provider = GenericOpenRtbProvider::new(plan.providers()[0].clone()); - let state = provider.parse_state_for_test(routed.inputs()[0].clone()); - let response = PlatformResponse::new( - edgezero_core::http::response_builder() - .status(200) - .header("content-type", "application/json") - .header("authorization", "fictional-secret") - .body(edgezero_core::body::Body::from("{}")) - .expect("should build debug APS response"), - ); - - let parsed = provider - .parse_response_with_state(response, 3, Some(state.as_ref())) - .await - .expect("should parse debug APS response"); - - let headers = &parsed.metadata["debug"]["httpcalls"]["aps"][0]["responseheaders"]; - assert_eq!( - headers, - &serde_json::json!({"content-type":["application/json"]}) - ); - assert!(headers.get("authorization").is_none()); - } - - #[tokio::test] - async fn planned_standard_instances_have_distinct_backends_and_independent_results() { - let http = Arc::new(StubHttpClient::new()); - http.push_response( - 200, - serde_json::to_vec(&serde_json::json!({ - "seatbid": [{"seat": "fictional-seat-a", "bid": [{ - "id": "bid-a", "impid": "fictional-slot", "price": 1.25, - "adm": "
a
", "w": 300, "h": 250 - }]}] - })) - .expect("should serialize response a"), - ); - http.push_response( - 200, - serde_json::to_vec(&serde_json::json!({ - "seatbid": [{"seat": "fictional-seat-b", "bid": [{ - "id": "bid-b", "impid": "fictional-slot", "price": 2.5, - "adm": "
b
", "w": 300, "h": 250 - }]}] - })) - .expect("should serialize response b"), - ); - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Fastly)); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, - ); - let plan = AuctionPlan::compile(planned_config( - &[ - ("provider-a", RoutingMode::AllEligible), - ("provider-b", RoutingMode::AllEligible), - ], - false, - )) - .expect("should compile planned auction"); - let orchestrator = AuctionOrchestratorHarness::new(plan, None); - let request = planned_request(); - let settings = create_test_settings(); - let inbound = http::Request::builder() - .uri("https://publisher.example/auction") - .body(edgezero_core::body::Body::empty()) - .expect("should build inbound request"); - let context = AuctionContext { - settings: &settings, - request: &inbound, - timeout_ms: 777, - transport_timeout_ms: 777, - provider_responses: None, - services: &services, - }; - - let result = orchestrator - .run_auction(&request, &context) - .await - .expect("should execute planned auction"); - - assert_eq!(orchestrator.provider_count(), 2); - assert!(orchestrator.mediator().is_none()); - assert_eq!(result.provider_responses.len(), 2); - assert_eq!(result.provider_responses[0].provider, "provider-a"); - assert_eq!( - result.provider_responses[0].bids[0].bid_id.as_deref(), - Some("bid-a") - ); - assert_eq!( - result.provider_responses[0].bids[0] - .returned_seat - .as_deref(), - Some("fictional-seat-a") - ); - assert_eq!( - result.provider_responses[0].metadata["routing"]["unused_bidder_params_count"], - 0 - ); - assert_eq!(result.provider_responses[1].provider, "provider-b"); - assert_eq!( - result.provider_responses[1].bids[0].bid_id.as_deref(), - Some("bid-b") - ); - assert_eq!( - result.provider_responses[1].bids[0] - .returned_seat - .as_deref(), - Some("fictional-seat-b") - ); - assert_eq!( - result.provider_responses[1].metadata["routing"]["unused_bidder_params_count"], - 0 - ); - assert_eq!( - result.winning_bids["fictional-slot"].bid_id.as_deref(), - Some("bid-b") - ); - let request_headers = http.recorded_request_headers(); - assert_eq!(request_headers.len(), 2); - for headers in request_headers { - assert!( - headers - .iter() - .any(|(name, value)| name == "accept" && value == "application/json"), - "standard planned transport should retain its JSON Accept header" - ); - } - let backend_names = http.recorded_backend_names(); - assert_eq!(backend_names.len(), 2); - assert_ne!(backend_names[0], backend_names[1]); - let request_bodies = http.recorded_request_bodies(); - assert_eq!(request_bodies.len(), 2); - for body in request_bodies { - let value: serde_json::Value = - serde_json::from_slice(&body).expect("should parse planned request"); - let tmax = value["tmax"].as_u64().expect("should include logical tmax"); - assert!( - (750..=777).contains(&tmax), - "logical budget should remain near the auction budget, got {tmax}" - ); - assert_eq!(value["imp"].as_array().map(Vec::len), Some(1)); - } - let specs = backend.specs.lock().expect("should lock specs"); - assert_eq!(specs.len(), 2); - assert_eq!(specs[0].first_byte_timeout, Duration::from_millis(750)); - assert_eq!(specs[1].first_byte_timeout, Duration::from_millis(750)); - assert_ne!(specs[0].discriminator, specs[1].discriminator); + let specs = backend.specs.lock().expect("should lock specs"); + assert_eq!(specs.len(), 2); + assert_eq!(specs[0].first_byte_timeout, Duration::from_millis(750)); + assert_eq!(specs[1].first_byte_timeout, Duration::from_millis(750)); + assert_ne!(specs[0].discriminator, specs[1].discriminator); } #[tokio::test] @@ -7001,8 +6252,8 @@ mod tests { ); let plan = AuctionPlan::compile(planned_config( &[ - ("provider-a", RoutingMode::AllEligible), - ("provider-b", RoutingMode::AllEligible), + ("provider_a", RoutingMode::AllEligible), + ("provider_b", RoutingMode::AllEligible), ], false, )) @@ -7026,12 +6277,12 @@ mod tests { .expect("should isolate backend collision"); assert_eq!(http.recorded_backend_names().len(), 1); - assert_eq!(result.provider_responses[0].provider, "provider-a"); + assert_eq!(result.provider_responses[0].provider, "provider_a"); assert_eq!( result.provider_responses[0].bids[0].bid_id.as_deref(), Some("first-bid") ); - assert_eq!(result.provider_responses[1].provider, "provider-b"); + assert_eq!(result.provider_responses[1].provider, "provider_b"); assert_eq!( result.provider_responses[1].metadata["error_type"], "launch_failed" @@ -7051,8 +6302,8 @@ mod tests { ); let plan = AuctionPlan::compile(planned_config( &[ - ("provider-a", RoutingMode::AllEligible), - ("provider-b", RoutingMode::AllEligible), + ("provider_a", RoutingMode::AllEligible), + ("provider_b", RoutingMode::AllEligible), ], false, )) @@ -7076,12 +6327,12 @@ mod tests { .expect("should isolate divergent pending backend"); assert_eq!(result.provider_responses.len(), 2); - assert_eq!(result.provider_responses[0].provider, "provider-a"); + assert_eq!(result.provider_responses[0].provider, "provider_a"); assert_eq!( result.provider_responses[0].metadata["error_type"], "launch_failed" ); - assert_eq!(result.provider_responses[1].provider, "provider-b"); + assert_eq!(result.provider_responses[1].provider, "provider_b"); assert_eq!(result.provider_responses[1].status, BidStatus::NoBid); } @@ -7098,8 +6349,8 @@ mod tests { ); let plan = AuctionPlan::compile(planned_config( &[ - ("provider-a", RoutingMode::AllEligible), - ("provider-b", RoutingMode::AllEligible), + ("provider_a", RoutingMode::AllEligible), + ("provider_b", RoutingMode::AllEligible), ], false, )) @@ -7123,12 +6374,12 @@ mod tests { .expect("should isolate missing pending backend"); assert_eq!(result.provider_responses.len(), 2); - assert_eq!(result.provider_responses[0].provider, "provider-a"); + assert_eq!(result.provider_responses[0].provider, "provider_a"); assert_eq!( result.provider_responses[0].metadata["error_type"], "launch_failed" ); - assert_eq!(result.provider_responses[1].provider, "provider-b"); + assert_eq!(result.provider_responses[1].provider, "provider_b"); assert_eq!(result.provider_responses[1].status, BidStatus::NoBid); } @@ -7136,8 +6387,8 @@ mod tests { async fn planned_same_profile_rejects_cross_provider_parse_state() { let plan = AuctionPlan::compile(planned_config( &[ - ("provider-a", RoutingMode::AllEligible), - ("provider-b", RoutingMode::AllEligible), + ("provider_a", RoutingMode::AllEligible), + ("provider_b", RoutingMode::AllEligible), ], false, )) @@ -7164,28 +6415,28 @@ mod tests { .expect_err("should reject another provider's parse state"); assert!( - error.to_string().contains("owned by provider provider-a"), + error.to_string().contains("owned by provider provider_a"), "should identify cross-provider state ownership" ); } #[tokio::test] - async fn planned_prebid_rejects_cross_provider_parse_state() { - let plan = AuctionPlan::compile(planned_prebid_config(&[ + async fn planned_source_rejects_cross_provider_parse_state() { + let plan = AuctionPlan::compile(planned_fixture_config(&[ ( - "pbs-a", + "stored_a", serde_json::json!({}), NotificationConfig::default(), ), ( - "pbs-b", + "stored_b", serde_json::json!({}), NotificationConfig::default(), ), ])) - .expect("should compile planned PBS auction"); + .expect("should compile the planned auction"); let routed = route_auction( - planned_prebid_request(), + planned_envelope_request(), &http::Request::new(edgezero_core::body::Body::empty()), &plan, None, @@ -7197,17 +6448,17 @@ mod tests { edgezero_core::http::response_builder() .status(200) .body(edgezero_core::body::Body::from_bytes(b"{}".as_slice())) - .expect("should build PBS response"), + .expect("should build the upstream response"), ); let error = provider_b .parse_response_with_state(response, 1, Some(parse_state.as_ref())) .await - .expect_err("should reject another PBS provider's parse state"); + .expect_err("should reject another source's parse state"); assert!( - error.to_string().contains("owned by provider pbs-a"), - "should identify cross-provider PBS state ownership" + error.to_string().contains("owned by provider stored_a"), + "should identify cross-provider state ownership" ); } @@ -7274,18 +6525,27 @@ mod tests { Arc::clone(&http) as Arc<_>, ); let mut config = planned_config(&[("provider", RoutingMode::AllEligible)], false); - config.mediator = Some("adserver_mock".to_string()); + config.adserver = crate::provider_table::ProviderChoice::new( + Some("fixture".to_string()), + BTreeMap::from([( + "fixture".to_string(), + serde_json::Map::from_iter([( + "endpoint".to_string(), + serde_json::json!("https://adserver.example/mediate"), + )]), + )]), + ); let plan = AuctionPlan::compile(config).expect("should compile planned auction"); - let mediator_predicted = Arc::new(Mutex::new(Vec::new())); - let mediator_requested = Arc::new(Mutex::new(Vec::new())); - let mediator = Arc::new(recording_provider( + let adserver_predicted = Arc::new(Mutex::new(Vec::new())); + let adserver_requested = Arc::new(Mutex::new(Vec::new())); + let adserver = Arc::new(recording_provider( "adserver_mock", - "mediator-backend", + "adserver-backend", 777, - &mediator_predicted, - &mediator_requested, + &adserver_predicted, + &adserver_requested, )); - let orchestrator = AuctionOrchestrator::from_plan(Arc::new(plan), Some(mediator)); + let orchestrator = AuctionOrchestrator::from_plan(Arc::new(plan), Some(adserver)); let request = planned_request(); let settings = create_test_settings(); let inbound = http::Request::new(edgezero_core::body::Body::empty()); @@ -7323,15 +6583,15 @@ mod tests { assert_eq!(backend.ensured.load(Ordering::Relaxed), 0); assert!(http.recorded_backend_names().is_empty()); assert!( - mediator_predicted + adserver_predicted .lock() - .expect("should lock mediator predictions") + .expect("should lock adserver predictions") .is_empty() ); assert!( - mediator_requested + adserver_requested .lock() - .expect("should lock mediator requests") + .expect("should lock adserver requests") .is_empty() ); } @@ -7340,12 +6600,14 @@ mod tests { #[tokio::test] async fn planned_launch_transport_parse_failures_are_isolated_from_valid_winner_and_floor() { let http = Arc::new(StubHttpClient::new()); - // BTreeMap plan order is alphabetical: below-floor, parse-fail, - // transport-fail, valid-winner. Queue responses in that exact order. + // The plan runs its sources in the order the deployment selected them, + // so queue the responses in that same order: below_floor, parse_fail, + // transport_fail, valid_winner, with launch_fail never reaching + // transport. http.push_response( 200, serde_json::to_vec(&serde_json::json!({ - "seatbid": [{"seat": "below-floor", "bid": [{ + "seatbid": [{"seat": "below_floor", "bid": [{ "id": "below", "impid": "fictional-slot", "price": 0.5, "adm": "
below
", "w": 300, "h": 250 }, { @@ -7371,18 +6633,18 @@ mod tests { http.push_select_success(); http.push_select_error(); let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); - backend.fail_ensure_for("launch-fail"); + backend.fail_ensure_for("launch_fail"); let services = build_services_with_backend_and_http_client( Arc::clone(&backend) as Arc<_>, Arc::clone(&http) as Arc<_>, ); let plan = AuctionPlan::compile(planned_config( &[ - ("launch-fail", RoutingMode::AllEligible), - ("transport-fail", RoutingMode::AllEligible), - ("parse-fail", RoutingMode::AllEligible), - ("below-floor", RoutingMode::AllEligible), - ("valid-winner", RoutingMode::AllEligible), + ("below_floor", RoutingMode::AllEligible), + ("launch_fail", RoutingMode::AllEligible), + ("parse_fail", RoutingMode::AllEligible), + ("transport_fail", RoutingMode::AllEligible), + ("valid_winner", RoutingMode::AllEligible), ], false, )) @@ -7415,7 +6677,7 @@ mod tests { .collect::>(); assert_eq!( by_provider - .get("launch-fail") + .get("launch_fail") .unwrap_or_else(|| panic!( "should include launch-fail response; got {:?}", by_provider.keys().collect::>() @@ -7423,7 +6685,7 @@ mod tests { .metadata["error_type"], "launch_failed" ); - let below_floor = by_provider.get("below-floor").unwrap_or_else(|| { + let below_floor = by_provider.get("below_floor").unwrap_or_else(|| { panic!( "should include below-floor response; got {:?}", by_provider.keys().collect::>() @@ -7434,10 +6696,10 @@ mod tests { assert_eq!(below_floor.bids[0].price, Some(0.5)); assert_eq!(below_floor.bids[1].bid_id.as_deref(), Some("below-only")); assert_eq!( - by_provider["transport-fail"].metadata["error_type"], + by_provider["transport_fail"].metadata["error_type"], "transport" ); - let parse_failure = by_provider.get("parse-fail").unwrap_or_else(|| { + let parse_failure = by_provider.get("parse_fail").unwrap_or_else(|| { panic!( "should include parse-fail response; got {:?}", by_provider.keys().collect::>() @@ -7455,7 +6717,7 @@ mod tests { "every materialized planned provider response should have routing count" ); } - assert_eq!(by_provider["valid-winner"].status, BidStatus::Success); + assert_eq!(by_provider["valid_winner"].status, BidStatus::Success); assert_eq!( result.winning_bids["fictional-slot"].bid_id.as_deref(), Some("winner") @@ -7467,62 +6729,57 @@ mod tests { } #[tokio::test] - async fn planned_routing_count_survives_standard_and_aps_bounded_body_failures() { - for (profile, provider_id) in [("standard", "standard"), ("aps", "aps-instance")] { - let http = Arc::new(StubHttpClient::new()); - http.push_response(200, vec![b'x'; 1024 * 1024 + 1]); - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, - ); - let mut config = if profile == "standard" { - planned_config(&[(provider_id, RoutingMode::AllEligible)], false) - } else { - planned_aps_config() - }; - config.bidders.insert( - "example-bidder" + async fn planned_routing_count_survives_a_bounded_body_failure() { + let (profile, provider_id) = ("standard", "standard"); + let http = Arc::new(StubHttpClient::new()); + http.push_response(200, vec![b'x'; 1024 * 1024 + 1]); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let mut config = planned_config(&[(provider_id, RoutingMode::AllEligible)], false); + config.bidders.insert( + "example-bidder" + .parse() + .expect("should parse fictional bidder ID"), + crate::auction::plan::BidderRouteConfig { + module: provider_id .parse() - .expect("should parse fictional bidder ID"), - crate::auction::plan::BidderRouteConfig { - provider: provider_id - .parse() - .expect("should parse fictional provider ID"), - }, - ); - let plan = AuctionPlan::compile(config).expect("should compile bounded-body plan"); - let orchestrator = AuctionOrchestratorHarness::new(plan, None); - let mut request = planned_request(); - request.slots[0].bidders.insert( - "example-bidder".to_string(), - serde_json::json!({"private": "value"}), - ); - let settings = create_test_settings(); - let inbound = http::Request::new(edgezero_core::body::Body::empty()); - let context = AuctionContext { - settings: &settings, - request: &inbound, - timeout_ms: 777, - transport_timeout_ms: 777, - provider_responses: None, - services: &services, - }; + .expect("should parse fictional provider ID"), + }, + ); + let plan = AuctionPlan::compile(config).expect("should compile bounded-body plan"); + let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let mut request = planned_request(); + request.slots[0].bidders.insert( + "example-bidder".to_string(), + serde_json::json!({"private": "value"}), + ); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; - let result = orchestrator - .run_auction(&request, &context) - .await - .expect("should materialize bounded-body failure"); - let response = &result.provider_responses[0]; - assert_eq!(response.status, BidStatus::Error, "{profile}"); - assert_eq!( - response.metadata["routing"]["unused_bidder_params_count"], 1, - "{profile} bounded-body failure should retain the input-derived count" - ); - let routing = serde_json::to_string(&response.metadata["routing"]) - .expect("should serialize routing metadata"); - assert!(!routing.contains("example-bidder") && !routing.contains("private")); - } + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should materialize bounded-body failure"); + let response = &result.provider_responses[0]; + assert_eq!(response.status, BidStatus::Error, "{profile}"); + assert_eq!( + response.metadata["routing"]["unused_bidder_params_count"], 1, + "{profile} bounded-body failure should retain the input-derived count" + ); + let routing = serde_json::to_string(&response.metadata["routing"]) + .expect("should serialize routing metadata"); + assert!(!routing.contains("example-bidder") && !routing.contains("private")); } #[tokio::test] @@ -7620,8 +6877,8 @@ mod tests { .build(); let plan = AuctionPlan::compile(planned_config( &[ - ("provider-a", RoutingMode::AllEligible), - ("provider-b", RoutingMode::AllEligible), + ("provider_a", RoutingMode::AllEligible), + ("provider_b", RoutingMode::AllEligible), ], true, )) @@ -7684,7 +6941,7 @@ mod tests { .parse() .expect("should parse fictional bidder ID"), crate::auction::plan::BidderRouteConfig { - provider: "signed" + module: "signed" .parse() .expect("should parse fictional provider ID"), }, @@ -7828,11 +7085,11 @@ mod tests { )) .expect("should compile signed plan"), ); - let mediator_launches = Arc::new(AtomicUsize::new(0)); + let adserver_launches = Arc::new(AtomicUsize::new(0)); let orchestrator = AuctionOrchestrator::from_plan( plan, - Some(Arc::new(DeadlineRecordingMediator { - launches: Arc::clone(&mediator_launches), + Some(Arc::new(DeadlineRecordingAdServer { + launches: Arc::clone(&adserver_launches), budgets: None, })), ); @@ -7868,9 +7125,9 @@ mod tests { assert_eq!(backend.ensured.load(Ordering::Relaxed), 0); assert!(http.recorded_backend_names().is_empty()); assert_eq!( - mediator_launches.load(Ordering::Relaxed), + adserver_launches.load(Ordering::Relaxed), 0, - "zero budget must not invoke even an immediate mediator" + "zero budget must not invoke even an immediate adserver" ); } @@ -7885,8 +7142,8 @@ mod tests { ); let plan = AuctionPlan::compile(planned_config( &[ - ("provider-a", RoutingMode::AllEligible), - ("provider-b", RoutingMode::AllEligible), + ("provider_a", RoutingMode::AllEligible), + ("provider_b", RoutingMode::AllEligible), ], false, )) @@ -7935,26 +7192,28 @@ mod tests { } #[test] - fn decoded_aps_bid_competes_directly_by_cpm() { + fn a_bid_with_a_typed_renderer_competes_directly_by_cpm() { let orchestrator = AuctionOrchestrator::new(AuctionConfig::default()); let floor_prices = HashMap::new(); let response = |provider: &str, bid: Bid| AuctionResponse::success(provider, vec![bid], 1); - let aps_wins = orchestrator.select_winning_bids( + let typed_wins = orchestrator.select_winning_bids( &[ - response("aps", auction_bid("aps", 2.0)), + response("typed", auction_bid("typed", 2.0)), response("ordinary", auction_bid("ordinary", 1.0)), ], &floor_prices, ); - let winner = aps_wins.get("slot-1").expect("should select APS bid"); - assert_eq!(winner.bidder, "aps"); + let winner = typed_wins + .get("slot-1") + .expect("should select the typed renderer bid"); + assert_eq!(winner.bidder, "typed"); assert!(winner.renderer.is_some()); assert!(winner.creative.is_none()); let ordinary_wins = orchestrator.select_winning_bids( &[ - response("aps", auction_bid("aps", 2.0)), + response("typed", auction_bid("typed", 2.0)), response("ordinary", auction_bid("ordinary", 3.0)), ], &floor_prices, diff --git a/crates/trusted-server-core/src/auction/plan.rs b/crates/trusted-server-core/src/auction/plan.rs index df36fc09b7..49fff990f0 100644 --- a/crates/trusted-server-core/src/auction/plan.rs +++ b/crates/trusted-server-core/src/auction/plan.rs @@ -1,37 +1,57 @@ -//! Target-independent config-first auction plan compiler. +//! Target-independent auction plan compiler. +//! +//! The plan is compiled once at startup from `[demand]`, `[ad-server]` and +//! `[auction.bidders]`. Every selected name resolves to an implementation an +//! integration registered, so the compiler names no vendor, and request +//! handling reads only the compiled plan. use std::collections::{BTreeMap, BTreeSet}; use std::str::FromStr; +use std::sync::Arc; use std::time::Duration; use error_stack::{Report, ResultExt as _}; use serde::{Deserialize, Serialize}; -use serde_json::Value; -use url::Url; +use serde_json::{Map, Value}; +use url::{Host, Url}; -use super::profile::{CompiledOpenRtbProfile, ProfileTimeoutDefault, find_profile}; +use super::demand::{ + AdServerImplementation, CompiledDemand, DemandImplementation, DemandTimeoutDefault, +}; use crate::error::TrustedServerError; use crate::platform::{AuctionTargetId, PlatformBackendSpec}; +use crate::provider_table::{ProviderChoice, ProviderList}; use crate::settings::RequestSigning; const MAX_ID_BYTES: usize = 128; const MAX_SUPPRESS_SEATS: usize = 128; const MAX_SUPPRESS_SEAT_BYTES: usize = 128; -const MOCK_MEDIATOR_ID: &str = "adserver_mock"; const RESERVED_BROWSER_ENVELOPE_BIDDER_ID: &str = "trustedServer"; -/// Validated operator-defined provider identifier. +/// The keys of a `[demand.]` table that the common driver reads itself, +/// so an implementation receives the rest. +const ENDPOINT_KEY: &str = "endpoint"; +const TIMEOUT_KEY: &str = "timeout_ms"; +const ROUTING_KEY: &str = "routing"; +const NOTIFICATIONS_KEY: &str = "notifications"; + +/// A validated provider name, as written in `[demand] modules` or +/// `[ad-server] module`. #[derive(Debug, Clone, Eq, Hash, Ord, PartialEq, PartialOrd, derive_more::Display)] pub struct ProviderId(String); impl ProviderId { - /// Borrow the validated identifier. + /// Borrow the validated name. #[must_use] pub fn as_str(&self) -> &str { &self.0 } #[cfg(test)] + #[allow( + dead_code, + reason = "kept for a legacy test path the provider types will retire" + )] pub(crate) fn unchecked_for_legacy_test(value: &str) -> Self { Self(value.to_string()) } @@ -47,10 +67,10 @@ impl FromStr for ProviderId { && value .as_bytes() .iter() - .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || *byte == b'-'); + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || *byte == b'_'); if !valid { return Err(configuration_error(format!( - "provider ID `{value}` must match ^[a-z][a-z0-9-]{{0,62}}$" + "provider name `{value}` must be snake_case, matching ^[a-z][a-z0-9_]{{0,62}}$" ))); } Ok(Self(value.to_string())) @@ -124,80 +144,55 @@ impl Serialize for BidderId { } } -/// Raw config-first provider declaration. -#[derive(Debug, Clone, Deserialize, Serialize)] -#[serde(deny_unknown_fields)] -pub struct ProviderConfig { - /// Protocol identifier. Version one accepts only `openrtb-2.6`. - pub protocol: String, - /// Registered profile identifier. - #[serde(default = "default_profile")] - pub profile: String, - /// Fixed provider endpoint. - pub endpoint: String, - /// Optional profile-default timeout override. - #[serde(default)] - pub timeout_ms: Option, - /// Slot routing mode. - #[serde(default)] - pub routing: RoutingMode, - /// Common `OpenRTB` notification policy. - #[serde(default)] - pub notifications: NotificationConfig, - /// Selected profile's typed configuration object. - #[serde(default = "empty_object")] - pub profile_config: Value, -} - -/// Raw central bidder route. +/// A bidder code's route to one selected demand source. #[derive(Debug, Clone, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct BidderRouteConfig { - /// Referenced provider identifier. - pub provider: ProviderId, + /// The `[demand]` module the bidder is sent to. + pub module: ProviderId, } -/// Provider slot routing behavior. +/// Which slots a demand source receives. #[derive(Debug, Clone, Copy, Default, Deserialize, Eq, PartialEq, Serialize)] #[serde(rename_all = "snake_case")] pub enum RoutingMode { - /// Route only centrally assigned or trusted demand. + /// Only slots carrying a bidder routed to the source, or demand the server + /// routed to it. #[default] Explicit, - /// Route every banner-compatible slot. + /// Every banner-compatible slot. AllEligible, } -/// Common normalized-notification suppression configuration. +/// Notification URL suppression for one demand source. #[derive(Debug, Clone, Default, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct NotificationConfig { - /// Suppress notification URLs for every normalized bid. + /// Suppress notification URLs for every bid. #[serde(default)] pub suppress_all: bool, - /// Suppress notification URLs for exact returned-seat matches. + /// Suppress notification URLs for bids from these returned seats. #[serde(default)] pub suppress_seats: Vec, } -/// Raw internal input for target-independent plan compilation. -#[derive(Debug, Clone, Default, Deserialize, Serialize)] -#[serde(deny_unknown_fields)] +/// Input for target-independent plan compilation. +#[derive(Debug, Clone, Default)] pub struct AuctionPlanConfig { /// Auction-wide logical timeout. pub timeout_ms: u32, - /// Operator-defined provider instances. - #[serde(default)] - pub providers: BTreeMap, - /// Client bidder-to-provider routes. - #[serde(default)] + /// The `[demand]` table. + pub demand: ProviderList, + /// The `[ad-server]` table. + pub adserver: ProviderChoice, + /// Client bidder routes from `[auction.bidders]`. pub bidders: BTreeMap, - /// Existing separately registered mock mediator. - #[serde(default)] - pub mediator: Option, - /// Existing global Trusted Server signing configuration. - #[serde(default)] + /// Trusted Server's request signing configuration. pub request_signing: Option, + /// The demand implementations the integration builders registered. + pub demand_implementations: Vec<&'static DemandImplementation>, + /// The ad server implementations the integration builders registered. + pub adserver_implementations: Vec<&'static AdServerImplementation>, } /// Canonical absolute provider endpoint. @@ -216,13 +211,6 @@ impl CanonicalProviderEndpoint { } } -/// Closed first-version protocol plan. -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -pub enum ProtocolPlan { - /// `OpenRTB` version 2.6 subset. - OpenRtb26, -} - /// Immutable common notification policy. #[derive(Debug, Clone, Default, Eq, PartialEq)] pub struct NotificationPolicy { @@ -232,23 +220,58 @@ pub struct NotificationPolicy { pub suppress_seats: BTreeSet, } -/// Immutable compiled provider instance. -#[derive(Debug, Clone)] +/// One compiled demand source. +#[derive(Clone)] pub struct ProviderPlan { - /// Provider identity. + /// The configured name. pub id: ProviderId, + /// The implementation the name resolved to. + pub implementation: &'static DemandImplementation, /// Canonical endpoint. pub endpoint: CanonicalProviderEndpoint, - /// Resolved profile-default or explicit timeout. + /// The table's timeout, or the implementation's default. pub timeout_ms: u32, /// Slot routing mode. pub routing: RoutingMode, /// Common notification policy. pub notifications: NotificationPolicy, - /// Compiled protocol behavior. - pub protocol: ProtocolPlan, - /// Compiled typed profile behavior. - pub profile: CompiledOpenRtbProfile, + /// The compiled request and response behavior. + pub demand: Arc, +} + +impl core::fmt::Debug for ProviderPlan { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + formatter + .debug_struct("ProviderPlan") + .field("id", &self.id) + .field("implementation", &self.implementation.id) + .field("endpoint", &self.endpoint) + .field("timeout_ms", &self.timeout_ms) + .field("routing", &self.routing) + .field("notifications", &self.notifications) + .finish_non_exhaustive() + } +} + +/// The compiled ad server selection. +#[derive(Clone)] +pub struct AdServerPlan { + /// The configured name. + pub id: ProviderId, + /// The implementation the name resolved to. + pub implementation: &'static AdServerImplementation, + /// The table's own settings, without `implementation`. + pub settings: Map, +} + +impl core::fmt::Debug for AdServerPlan { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + formatter + .debug_struct("AdServerPlan") + .field("id", &self.id) + .field("implementation", &self.implementation.id) + .finish_non_exhaustive() + } } /// Immutable target-independent auction plan. @@ -259,7 +282,8 @@ pub struct AuctionPlan { providers: Vec, bidder_routes: BTreeMap, signing_enabled: bool, - mediator: Option, + adserver: Option, + publishes_auction_token: bool, } impl AuctionPlan { @@ -269,77 +293,118 @@ impl AuctionPlan { self.enabled } - pub(crate) fn with_enabled(mut self, enabled: bool) -> Self { + /// Return the plan with auction execution switched on or off, which is how + /// `[auction] enabled` is applied to a compiled plan. + #[must_use] + pub fn with_enabled(mut self, enabled: bool) -> Self { self.enabled = enabled; self } + /// Return whether each auction publishes a token with its winning bids, + /// which it does when a module that runs reads one. + #[must_use] + pub fn publishes_auction_token(&self) -> bool { + self.publishes_auction_token + } + + pub(crate) fn with_auction_token(mut self, publishes: bool) -> Self { + self.publishes_auction_token = publishes; + self + } + /// Compile a deterministic plan without adapter-specific validation. /// /// # Errors /// - /// Returns a configuration error for invalid identifiers, protocol/profile - /// declarations, endpoints, profile configuration, routes, notifications, - /// signing structure, or mediator selection. + /// Returns a configuration error for a table no selector names, a label + /// that is not `snake_case`, an implementation no builder registered, an + /// invalid endpoint, timeout, route or notification setting, settings an + /// implementation rejects, or invalid signing structure. pub fn compile(config: AuctionPlanConfig) -> Result> { if config.timeout_ms == 0 { return Err(configuration_error( "auction timeout_ms must be greater than zero", )); } - validate_mediator(config.mediator.as_deref())?; - let signing_enabled = compile_signing_enabled(config.request_signing.as_ref())?; - let mut providers = Vec::with_capacity(config.providers.len()); + config + .demand + .validate("demand") + .map_err(configuration_error)?; + config + .adserver + .validate("ad-server") + .map_err(configuration_error)?; + let signing_enabled = config + .request_signing + .as_ref() + .is_some_and(|signing| signing.enabled); + + let mut providers = Vec::new(); let mut provider_indices = BTreeMap::new(); - for (id, raw) in config.providers { - if raw.protocol != "openrtb-2.6" { + for name in config.demand.selected() { + let id = ProviderId::from_str(name)?; + let implementation = resolve_implementation( + "demand", + name, + config.demand.implementation_of(name), + &config.demand_implementations, + |implementation| implementation.id, + )?; + let mut settings = config.demand.settings_of(name); + let endpoint = take_setting::(&mut settings, "demand", name, ENDPOINT_KEY)? + .ok_or_else(|| configuration_error(format!("[demand.{name}] needs an endpoint")))?; + let timeout_ms = take_setting::(&mut settings, "demand", name, TIMEOUT_KEY)?; + let routing = take_setting::(&mut settings, "demand", name, ROUTING_KEY)? + .unwrap_or_default(); + let notifications = take_setting::( + &mut settings, + "demand", + name, + NOTIFICATIONS_KEY, + )? + .unwrap_or_default(); + + if routing == RoutingMode::AllEligible && !implementation.allows_all_eligible { return Err(configuration_error(format!( - "provider `{id}` uses unsupported protocol `{}`", - raw.protocol + "[demand.{name}] cannot use routing `all_eligible` with implementation `{}`; configure explicit bidder routes", + implementation.id ))); } - let registration = find_profile(&raw.profile).ok_or_else(|| { - configuration_error(format!( - "provider `{id}` uses unknown OpenRTB profile `{}`", - raw.profile - )) + let mut endpoint_url = check_endpoint("demand", name, &endpoint)?; + (implementation.canonicalize_endpoint)(&mut endpoint_url).map_err(|reason| { + configuration_error(format!("[demand.{name}] endpoint {reason}")) })?; - if registration.id == "prebid-server" && raw.routing == RoutingMode::AllEligible { - return Err(configuration_error(format!( - "provider `{id}` cannot use routing `all_eligible` with profile `prebid-server`; configure explicit bidder routes" - ))); - } - if !raw.profile_config.is_object() { - return Err(configuration_error(format!( - "provider `{id}` profile_config must be an object" - ))); - } - let endpoint = canonicalize_endpoint(&id, registration.id, &raw.endpoint)?; - let timeout_ms = raw - .timeout_ms - .unwrap_or(match registration.default_timeout { - ProfileTimeoutDefault::Auction => config.timeout_ms, - ProfileTimeoutDefault::Fixed(value) => value, - }); + let timeout_ms = timeout_ms.unwrap_or(match implementation.default_timeout { + DemandTimeoutDefault::Auction => config.timeout_ms, + DemandTimeoutDefault::Fixed(value) => value, + }); if timeout_ms == 0 { return Err(configuration_error(format!( - "provider `{id}` timeout_ms must be greater than zero" + "[demand.{name}] timeout_ms must be greater than zero" ))); } - let notifications = compile_notifications(&id, raw.notifications)?; - let profile = registration.compile(&raw.profile_config)?; - let index = providers.len(); - provider_indices.insert(id.clone(), index); + let notifications = compile_notifications(&id, notifications)?; + let demand = (implementation.compile)(&settings).change_context( + TrustedServerError::Configuration { + message: format!( + "[demand.{name}] settings are not valid for implementation `{}`", + implementation.id + ), + }, + )?; + provider_indices.insert(id.clone(), providers.len()); providers.push(ProviderPlan { id, - endpoint, + implementation, + endpoint: CanonicalProviderEndpoint(endpoint_url), timeout_ms, - routing: raw.routing, + routing, notifications, - protocol: ProtocolPlan::OpenRtb26, - profile, + demand, }); } + let mut bidder_routes = BTreeMap::new(); for (bidder, route) in config.bidders { if bidder.as_str() == RESERVED_BROWSER_ENVELOPE_BIDDER_ID { @@ -349,23 +414,59 @@ impl AuctionPlan { } let provider_index = provider_indices - .get(&route.provider) + .get(&route.module) .copied() .ok_or_else(|| { configuration_error(format!( - "bidder `{bidder}` references unknown provider `{}`", - route.provider + "[auction.bidders.{bidder}] sends the bidder to `{}`, which [demand] modules does not select", + route.module )) })?; bidder_routes.insert(bidder, provider_index); } + + let adserver = match config.adserver.selected().first().copied() { + None => None, + Some(name) => { + let id = ProviderId::from_str(name)?; + let implementation = resolve_implementation( + "ad-server", + name, + config.adserver.implementation_of(name), + &config.adserver_implementations, + |implementation| implementation.id, + )?; + let settings = config.adserver.settings_of(name); + if let Some(endpoint) = settings.get(ENDPOINT_KEY) { + let endpoint = endpoint.as_str().ok_or_else(|| { + configuration_error(format!("[ad-server.{name}] endpoint must be a URL")) + })?; + check_endpoint("ad-server", name, endpoint)?; + } + (implementation.build)(name, &settings).change_context( + TrustedServerError::Configuration { + message: format!( + "[ad-server.{name}] settings are not valid for implementation `{}`", + implementation.id + ), + }, + )?; + Some(AdServerPlan { + id, + implementation, + settings, + }) + } + }; + Ok(Self { enabled: true, timeout_ms: config.timeout_ms, providers, bidder_routes, signing_enabled, - mediator: config.mediator, + adserver, + publishes_auction_token: false, }) } } @@ -470,38 +571,39 @@ impl AuctionPlan { Ok(()) } - /// Borrow compiled providers in deterministic provider-ID order. + /// Borrow compiled demand sources in the order `[demand] modules` lists + /// them. #[must_use] pub fn providers(&self) -> &[ProviderPlan] { &self.providers } - /// Borrow a compiled provider by its validated identity. + /// Borrow a compiled demand source by its name. #[must_use] pub(crate) fn provider(&self, id: &ProviderId) -> Option<&ProviderPlan> { self.providers.iter().find(|provider| provider.id == *id) } - /// Return whether any compiled provider uses the named profile. + /// Return whether any compiled demand source uses the named implementation. /// - /// This narrow query allows capability activation to follow the validated - /// plan without exposing profile configuration. + /// This narrow query lets an implementation's integration activate its own + /// capabilities from the validated plan. #[must_use] - pub fn has_profile(&self, profile_id: &str) -> bool { + pub fn has_implementation(&self, implementation_id: &str) -> bool { self.providers .iter() - .any(|provider| provider.profile.id() == profile_id) + .any(|provider| provider.implementation.id == implementation_id) } - /// Borrow validated client-visible bidder route codes in deterministic order. + /// Borrow the validated bidder route codes in deterministic order. /// - /// This intentionally exposes route keys rather than provider identities or - /// profile configuration for the browser Prebid injection boundary. - pub(crate) fn browser_bidder_codes(&self) -> impl Iterator { + /// These are the route keys alone, with no source name and no + /// implementation setting, so a page module may show them to a browser. + pub fn browser_bidder_codes(&self) -> impl Iterator { self.bidder_routes.keys().map(BidderId::as_str) } - /// Resolve a bidder route to a compiled provider. + /// Resolve a bidder route to a compiled demand source. #[must_use] pub fn provider_for_bidder(&self, bidder: &BidderId) -> Option<&ProviderPlan> { self.bidder_routes @@ -515,95 +617,106 @@ impl AuctionPlan { self.signing_enabled } - /// Borrow the separately validated static mediator identifier. + /// Borrow the selected ad server, when `[ad-server] module` names one. #[must_use] - pub fn mediator(&self) -> Option<&str> { - self.mediator.as_deref() + pub fn adserver(&self) -> Option<&AdServerPlan> { + self.adserver.as_ref() } } -fn default_profile() -> String { - "standard".to_string() -} - -fn empty_object() -> Value { - Value::Object(serde_json::Map::new()) -} - fn configuration_error(message: impl Into) -> Report { Report::new(TrustedServerError::Configuration { message: message.into(), }) } -fn compile_signing_enabled( - request_signing: Option<&RequestSigning>, -) -> Result> { - let Some(request_signing) = request_signing else { - return Ok(false); - }; - if request_signing.enabled - && (request_signing.config_store_id.trim().is_empty() - || request_signing.secret_store_id.trim().is_empty()) - { - return Err(configuration_error( - "enabled request_signing requires nonblank config_store_id and secret_store_id", - )); - } - Ok(request_signing.enabled) +/// The registered implementation `written` names in `[section]`. +/// +/// An implementation is named by its module path, which a section named by +/// the implementation's type folder may shorten, so `mock` in `[ad-server]` +/// is `ad-server.mock`. The refusal lists what the build has, as the section +/// would write it. +fn resolve_implementation( + section: &str, + name: &str, + written: &str, + registered: &[&'static T], + id: fn(&T) -> &'static str, +) -> Result<&'static T, Report> { + let ids = registered + .iter() + .map(|implementation| id(implementation)) + .collect::>(); + crate::module_name::resolve(section, written, &ids) + .and_then(|resolved| { + registered + .iter() + .copied() + .find(|implementation| id(implementation) == resolved) + }) + .ok_or_else(|| { + let mut known = ids + .iter() + .map(|id| crate::module_name::short_form(section, id)) + .collect::>(); + known.sort_unstable(); + configuration_error(format!( + "[{section}] `{name}` uses implementation `{written}`, which this build does not have. The implementations it has are: {}", + known.join(", ") + )) + }) } -fn validate_mediator(mediator: Option<&str>) -> Result<(), Report> { - if mediator.is_some_and(|value| value != MOCK_MEDIATOR_ID) { - return Err(configuration_error(format!( - "auction mediator must be `{MOCK_MEDIATOR_ID}` when configured" - ))); - } - Ok(()) +/// Takes one common setting out of a table's settings, parsing it as `T`. +fn take_setting( + settings: &mut Map, + type_name: &str, + name: &str, + key: &str, +) -> Result, Report> +where + T: for<'de> Deserialize<'de>, +{ + settings + .remove(key) + .map(|value| { + T::deserialize(value).map_err(|error| { + configuration_error(format!("[{type_name}.{name}] {key} is not valid: {error}")) + }) + }) + .transpose() } -fn canonicalize_endpoint( - provider_id: &ProviderId, - profile_id: &str, +/// Checks an endpoint the common way. It must be HTTPS, or plain HTTP to a +/// loopback address, with a host and no credentials or fragment. +fn check_endpoint( + type_name: &str, + name: &str, value: &str, -) -> Result> { - let mut endpoint = Url::parse(value).map_err(|error| { +) -> Result> { + let endpoint = Url::parse(value).map_err(|error| { configuration_error(format!( - "provider `{provider_id}` endpoint must be an absolute HTTPS URL: {error}" + "[{type_name}.{name}] endpoint must be an absolute URL: {error}" )) })?; - if endpoint.scheme() != "https" - || endpoint.host_str().is_none() + let loopback = match endpoint.host() { + Some(Host::Domain(domain)) => domain.eq_ignore_ascii_case("localhost"), + Some(Host::Ipv4(address)) => address.is_loopback(), + Some(Host::Ipv6(address)) => address.is_loopback(), + None => false, + }; + let scheme_allowed = endpoint.scheme() == "https" || (endpoint.scheme() == "http" && loopback); + if !scheme_allowed + || endpoint.host().is_none() || !endpoint.username().is_empty() || endpoint.password().is_some() || endpoint.fragment().is_some() { return Err(configuration_error(format!( - "provider `{provider_id}` endpoint must be absolute HTTPS with a host and no credentials or fragment" - ))); - } - if profile_id == "aps" - && endpoint - .path() - .trim_end_matches('/') - .ends_with("/e/dtb/bid") - { - return Err(configuration_error(format!( - "provider `{provider_id}` uses unsupported legacy APS endpoint `/e/dtb/bid`" + "[{type_name}.{name}] endpoint must be HTTPS, or HTTP to 127.0.0.1, ::1 or localhost, with a host and no credentials or fragment" ))); } - if profile_id == "prebid-server" { - normalize_prebid_server_endpoint(&mut endpoint); - } - Ok(CanonicalProviderEndpoint(endpoint)) -} - -fn normalize_prebid_server_endpoint(endpoint: &mut Url) { - match endpoint.path() { - "" | "/" => endpoint.set_path("/openrtb2/auction"), - "/openrtb2/auction/" => endpoint.set_path("/openrtb2/auction"), - _ => {} - } + Ok(endpoint) } fn compile_notifications( @@ -640,28 +753,63 @@ fn compile_notifications( #[cfg(test)] mod tests { use super::*; - use crate::auction::profile::CompiledOpenRtbProfile; - - fn provider(profile: &str) -> ProviderConfig { - ProviderConfig { - protocol: "openrtb-2.6".to_string(), - profile: profile.to_string(), - endpoint: "https://bid.example/openrtb2/auction".to_string(), - timeout_ms: None, - routing: RoutingMode::Explicit, - notifications: NotificationConfig::default(), - profile_config: empty_object(), - } + use crate::auction::test_support::plain_fixture::PlainDemand; + use crate::provider_table::IMPLEMENTATION_KEY; + use serde_json::json; + + /// The demand implementations the built-in builders register. + fn demand_implementations() -> Vec<&'static DemandImplementation> { + crate::integrations::all_builders(&[]) + .filter_map(|builder| builder.demand()) + .collect() + } + + /// The ad server implementations the built-in builders register. + fn adserver_implementations() -> Vec<&'static AdServerImplementation> { + crate::integrations::all_builders(&[]) + .filter_map(|builder| builder.adserver()) + .collect() + } + + /// One `[demand.]` table naming an implementation. + fn table(implementation: &str) -> Map { + Map::from_iter([ + (IMPLEMENTATION_KEY.to_string(), json!(implementation)), + ( + ENDPOINT_KEY.to_string(), + json!("https://bid.example/openrtb2/auction"), + ), + ]) } - fn config(providers: BTreeMap) -> AuctionPlanConfig { + /// A `[demand]` table selecting every name given, in order. + fn demand(tables: Vec<(&str, Map)>) -> ProviderList { + let selected = tables + .iter() + .map(|(name, _)| (*name).to_string()) + .collect::>(); + let tables = tables + .into_iter() + .map(|(name, table)| (name.to_string(), table)) + .collect::>(); + ProviderList::new(selected, tables) + } + + fn config(tables: Vec<(&str, Map)>) -> AuctionPlanConfig { AuctionPlanConfig { timeout_ms: 1500, - providers, + demand: demand(tables), + demand_implementations: demand_implementations(), + adserver_implementations: adserver_implementations(), ..AuctionPlanConfig::default() } } + /// A plan with one ordinary `OpenRTB` source called `one`. + fn one_source() -> AuctionPlanConfig { + config(vec![("one", table("auction.plain-fixture"))]) + } + fn id(value: &str) -> ProviderId { ProviderId::from_str(value).expect("should parse provider ID") } @@ -670,32 +818,13 @@ mod tests { BidderId::from_str(value).expect("should parse bidder ID") } - fn nested_object(levels: usize) -> Value { - let mut value = Value::String("leaf".to_string()); - for level in 0..levels { - value = Value::Object(serde_json::Map::from_iter([( - format!("level-{level}"), - value, - )])); - } - value - } - - fn nested_array(levels: usize) -> Value { - let mut value = Value::String("leaf".to_string()); - for _ in 0..levels { - value = Value::Array(vec![value]); - } - value - } - #[test] fn target_validation_accepts_fanout_and_rejects_unsupported_targets() { - let providers = BTreeMap::from([ - (id("provider-one"), provider("standard")), - (id("provider-two"), provider("standard")), - ]); - let plan = AuctionPlan::compile(config(providers)).expect("should compile plan"); + let plan = AuctionPlan::compile(config(vec![ + ("provider_one", table("auction.plain-fixture")), + ("provider_two", table("auction.plain-fixture")), + ])) + .expect("should compile plan"); assert!( plan.validate_for_target(crate::platform::AuctionTargetId::Fastly) @@ -708,253 +837,171 @@ mod tests { "Axum should accept provider fanout" ); for target in [ - crate::platform::AuctionTargetId::Cloudflare, crate::platform::AuctionTargetId::Spin, + crate::platform::AuctionTargetId::Cloudflare, ] { - let error = plan - .validate_for_target(target) - .expect_err("should reject unsupported provider fanout"); assert!( - error.to_string().contains("fanout"), - "should explain fanout rejection: {error:?}" + plan.validate_for_target(target).is_err(), + "{target:?} runs one demand source at a time and should refuse fanout" ); } } #[test] - fn fastly_target_validation_canonicalizes_url_derived_ipv6_prediction() { - let mut ipv6_provider = provider("standard"); - ipv6_provider.endpoint = "https://[2001:db8::5]:8443/openrtb".to_string(); - ipv6_provider.timeout_ms = Some(750); - let plan = AuctionPlan::compile(config(BTreeMap::from([( - id("ipv6-provider"), - ipv6_provider, - )]))) - .expect("should compile IPv6 provider plan"); - - plan.validate_for_target(crate::platform::AuctionTargetId::Fastly) - .expect("Fastly target validation should accept canonical IPv6 prediction"); - let bracketed_spec = plan.providers()[0].backend_spec(); - assert_eq!(bracketed_spec.host, "[2001:db8::5]"); - let mut bare_spec = bracketed_spec.clone(); - bare_spec.host = "2001:db8::5".to_string(); - let policy = crate::platform::BackendNamingPolicy::Fastly; - assert_eq!( - policy - .predict(&bracketed_spec) - .expect("should predict URL-derived bracketed IPv6 backend"), - policy - .predict(&bare_spec) - .expect("should predict runtime-normalized bare IPv6 backend"), - "startup target validation and Fastly runtime must hash the same backend spec" + fn a_table_no_selector_names_is_refused() { + let mut raw = one_source(); + raw.demand = ProviderList::new( + vec!["one".to_string()], + BTreeMap::from([ + ("one".to_string(), table("auction.plain-fixture")), + ("left_behind".to_string(), table("auction.plain-fixture")), + ]), ); - } - - #[test] - fn fastly_target_validation_reserves_dynamic_backends_outside_auction() { - let plan_with = |count: usize| { - let providers = (0..count) - .map(|index| { - let mut provider = provider("standard"); - provider.timeout_ms = Some(1000); - (id(&format!("provider-{index}")), provider) - }) - .collect(); - AuctionPlan::compile(config(providers)).expect("should compile provider plan") - }; - - plan_with(19) - .validate_for_target(crate::platform::AuctionTargetId::Fastly) - .expect("below-budget provider plan should validate"); - plan_with(20) - .validate_for_target(crate::platform::AuctionTargetId::Fastly) - .expect("at-budget provider plan should validate"); - let error = plan_with(21) - .validate_for_target(crate::platform::AuctionTargetId::Fastly) - .expect_err("over-budget provider plan should fail"); - assert!( - error - .to_string() - .contains("exceeding its auction budget of 160") - ); - } - - #[test] - fn fastly_backend_quota_uses_auction_timeout_as_reachable_bucket_ceiling() { - let providers = (0..21) - .map(|index| { - let mut provider = provider("standard"); - provider.timeout_ms = Some(1000); - (id(&format!("provider-{index}")), provider) - }) - .collect(); - let mut bounded = config(providers); - bounded.timeout_ms = 100; - let plan = AuctionPlan::compile(bounded).expect("should compile bounded provider plan"); - + let error = AuctionPlan::compile(raw).expect_err("should refuse a table nothing selects"); + let message = error.to_string(); assert!( - plan.providers() - .iter() - .all(|provider| provider.timeout_ms == 1000), - "quota validation must not rewrite configured provider timeouts" + message.contains("left_behind") && message.contains("modules"), + "should name the table and the selector: {error:?}" ); - plan.validate_for_target(crate::platform::AuctionTargetId::Fastly) - .expect("21 providers reach only the 50ms and 100ms Fastly buckets"); } #[test] - fn disabled_target_validation_skips_fanout_and_collision_checks() { - let providers = BTreeMap::from([ - (id("provider-one"), provider("standard")), - (id("provider-two"), provider("standard")), - ]); - let disabled = AuctionPlan::compile(config(providers)) - .expect("should compile plan") - .with_enabled(false); - - for target in [ - crate::platform::AuctionTargetId::Cloudflare, - crate::platform::AuctionTargetId::Spin, + fn an_unknown_implementation_names_the_ones_this_build_has() { + let mut absent = table("auction.plain-fixture"); + absent.insert(IMPLEMENTATION_KEY.to_string(), json!("fictional_exchange")); + let error = AuctionPlan::compile(config(vec![("one", absent)])) + .expect_err("should refuse an implementation this build does not have"); + let message = error.to_string(); + for expected in [ + "fictional_exchange", + "auction.plain-fixture", + "auction.fixture", ] { - disabled - .validate_for_target(target) - .expect("disabled dormant providers should skip target validation"); + assert!( + message.contains(expected), + "should name the unknown implementation and the known ones: {error:?}" + ); } - - let provider = disabled.providers[0].clone(); - let disabled_collision = AuctionPlan { - enabled: false, - timeout_ms: disabled.timeout_ms, - providers: vec![provider.clone(), provider], - bidder_routes: BTreeMap::new(), - signing_enabled: false, - mediator: None, - }; - disabled_collision - .validate_for_target(crate::platform::AuctionTargetId::Axum) - .expect("disabled dormant providers should skip collision validation"); } #[test] - fn target_validation_keeps_same_origin_timeout_profile_instances_distinct() { - let shared = ProviderConfig { - timeout_ms: Some(777), - ..provider("standard") - }; - let plan = AuctionPlan::compile(config(BTreeMap::from([ - (id("provider-one"), shared.clone()), - (id("provider-two"), shared), - ]))) - .expect("should compile same-origin provider instances"); - - for target in [ - crate::platform::AuctionTargetId::Fastly, - crate::platform::AuctionTargetId::Axum, - ] { - plan.validate_for_target(target) - .expect("provider ID discriminators should prevent predicted collisions"); - } + fn a_demand_source_names_its_implementation_in_full() { + // `[demand]` is no implementation's type folder, so a source's name + // never stands for its implementation. + let unnamed = Map::from_iter([( + ENDPOINT_KEY.to_string(), + json!("https://bid.example/openrtb2/auction"), + )]); + let error = AuctionPlan::compile(config(vec![("exchange", unnamed)])) + .expect_err("should refuse a source with no implementation line"); + let message = error.to_string(); + assert!( + message.contains("uses implementation `exchange`") + && message.contains("auction.plain-fixture"), + "should list the implementations by module path: {error:?}" + ); } #[test] - fn target_validation_rejects_predicted_name_collisions() { - let compiled = AuctionPlan::compile(config(BTreeMap::from([( - id("provider-a"), - provider("standard"), - )]))) - .expect("should compile plan"); - let provider = compiled.providers[0].clone(); - // The compiler prevents duplicate provider IDs. Construct the otherwise - // impossible duplicate internally to pin validation's defense-in-depth - // collision rejection independently of compiler invariants. - let collision_plan = AuctionPlan { - enabled: true, - timeout_ms: compiled.timeout_ms, - providers: vec![provider.clone(), provider], - bidder_routes: BTreeMap::new(), - signing_enabled: false, - mediator: None, - }; + fn an_ad_server_named_by_its_short_form_needs_no_implementation_line() { + let mut raw = one_source(); + raw.adserver = ProviderChoice::new( + Some("fixture".to_string()), + BTreeMap::from([( + "fixture".to_string(), + Map::from_iter([( + ENDPOINT_KEY.to_string(), + json!("https://adserver.example/mediate"), + )]), + )]), + ); + let plan = AuctionPlan::compile(raw).expect("should resolve `fixture` within [ad-server]"); + assert_eq!( + plan.adserver().map(|adserver| adserver.implementation.id), + Some("ad-server.fixture") + ); + } - let error = collision_plan - .validate_for_target(crate::platform::AuctionTargetId::Axum) - .expect_err("should reject predicted backend collision"); - assert!(error.to_string().contains("same backend name")); + #[test] + fn a_demand_table_needs_an_endpoint() { + let mut without = table("auction.plain-fixture"); + without.remove(ENDPOINT_KEY); + let error = AuctionPlan::compile(config(vec![("one", without)])) + .expect_err("should refuse a source with no endpoint"); + assert!( + error.to_string().contains("endpoint"), + "should say an endpoint is needed: {error:?}" + ); } #[test] fn provider_id_enforces_exact_grammar_and_bounds() { - for valid in ["a", "pbs-primary", &format!("a{}", "0".repeat(62))] { - assert!(ProviderId::from_str(valid).is_ok(), "should accept {valid}"); + for value in ["a", "provider_one", "p0", &format!("a{}", "b".repeat(62))] { + assert!( + ProviderId::from_str(value).is_ok(), + "should accept `{value}`" + ); } - for invalid in [ + for value in [ "", - "A", + "Provider", + "provider-one", "1provider", - "provider_name", - "provider.name", - "provider/one", - &format!("a{}", "0".repeat(63)), + "_provider", + "provider.one", + &format!("a{}", "b".repeat(63)), ] { assert!( - ProviderId::from_str(invalid).is_err(), - "should reject {invalid}" + ProviderId::from_str(value).is_err(), + "should reject `{value}`" ); } } #[test] fn bidder_id_enforces_admission_bounds() { - assert!(BidderId::from_str("exampleBidder").is_ok()); - for invalid in ["", " bidder", "bidder\n", &"a".repeat(129)] { - let error = BidderId::from_str(invalid).expect_err("should reject invalid bidder ID"); - assert!( - error.to_string().contains(&format!("{invalid:?}")), - "should identify invalid bidder ID {invalid:?}: {error:?}" - ); - } + assert!(BidderId::from_str("example").is_ok()); + assert!(BidderId::from_str("Example").is_ok()); + assert!(BidderId::from_str("").is_err()); + assert!(BidderId::from_str(&"a".repeat(MAX_ID_BYTES + 1)).is_err()); } #[test] - fn compiler_rejects_all_eligible_for_prebid_server_only() { - let mut prebid = provider("prebid-server"); - prebid.routing = RoutingMode::AllEligible; - let error = AuctionPlan::compile(config(BTreeMap::from([(id("pbs-main"), prebid)]))) - .expect_err("should reject all_eligible Prebid Server routing"); + fn all_eligible_is_refused_for_an_implementation_that_forbids_it() { + let mut explicit_only = table("auction.fixture"); + explicit_only.insert(ROUTING_KEY.to_string(), json!("all_eligible")); + let error = AuctionPlan::compile(config(vec![("explicit_main", explicit_only)])) + .expect_err("should refuse all_eligible routing for the implementation"); let message = error.to_string(); - for expected in ["pbs-main", "all_eligible", "prebid-server"] { + for expected in ["explicit_main", "all_eligible", "auction.fixture"] { assert!( message.contains(expected), - "should identify provider, routing, and profile: {error:?}" + "should name the source, the routing and the implementation: {error:?}" ); } - let mut standard = provider("standard"); - standard.routing = RoutingMode::AllEligible; - AuctionPlan::compile(config(BTreeMap::from([(id("standard-main"), standard)]))) - .expect("should retain all_eligible for non-Prebid profiles"); + let mut plain = table("auction.plain-fixture"); + plain.insert(ROUTING_KEY.to_string(), json!("all_eligible")); + AuctionPlan::compile(config(vec![("plain_main", plain)])) + .expect("should keep all_eligible for an implementation that allows it"); } #[test] fn compiler_rejects_exact_reserved_browser_envelope_bidder_id() { - let mut raw = config(BTreeMap::from([(id("one"), provider("standard"))])); + let mut raw = one_source(); raw.bidders.insert( bidder("trustedServer"), - BidderRouteConfig { - provider: id("one"), - }, + BidderRouteConfig { module: id("one") }, ); assert!( AuctionPlan::compile(raw).is_err(), "exact reserved bidder ID should be rejected" ); - let mut case_distinct = config(BTreeMap::from([(id("one"), provider("standard"))])); + let mut case_distinct = one_source(); case_distinct.bidders.insert( bidder("TrustedServer"), - BidderRouteConfig { - provider: id("one"), - }, + BidderRouteConfig { module: id("one") }, ); assert!( AuctionPlan::compile(case_distinct).is_ok(), @@ -963,26 +1010,30 @@ mod tests { } #[test] - fn compiler_orders_providers_and_routes_deterministically() { - let mut providers = BTreeMap::new(); - providers.insert(id("z-provider"), provider("standard")); - providers.insert(id("a-provider"), provider("standard")); - let mut raw = config(providers); + fn sources_keep_the_order_they_were_selected_in_and_routes_are_deterministic() { + let mut raw = config(vec![ + ("z_provider", table("auction.plain-fixture")), + ("a_provider", table("auction.plain-fixture")), + ]); raw.bidders.insert( bidder("z-bidder"), BidderRouteConfig { - provider: id("z-provider"), + module: id("z_provider"), }, ); raw.bidders.insert( bidder("a-bidder"), BidderRouteConfig { - provider: id("a-provider"), + module: id("a_provider"), }, ); let plan = AuctionPlan::compile(raw).expect("should compile deterministic plan"); - assert_eq!(plan.providers()[0].id.as_str(), "a-provider"); - assert_eq!(plan.providers()[1].id.as_str(), "z-provider"); + assert_eq!( + plan.providers()[0].id.as_str(), + "z_provider", + "the selector's order is the deployment's order" + ); + assert_eq!(plan.providers()[1].id.as_str(), "a_provider"); assert_eq!( plan.browser_bidder_codes().collect::>(), vec!["a-bidder", "z-bidder"], @@ -991,113 +1042,95 @@ mod tests { assert_eq!( plan.provider_for_bidder(&bidder("a-bidder")) .map(|provider| provider.id.as_str()), - Some("a-provider") + Some("a_provider") ); } #[test] - fn compiler_supports_two_instances_of_the_same_profile() { - let mut providers = BTreeMap::new(); - providers.insert(id("pbs-a"), provider("prebid-server")); - providers.insert(id("pbs-b"), provider("prebid-server")); - let plan = AuctionPlan::compile(config(providers)).expect("should compile two PBS plans"); + fn two_sources_can_run_one_implementation_under_their_own_names() { + let plan = AuctionPlan::compile(config(vec![ + ("source_a", table("auction.plain-fixture")), + ("source_b", table("auction.plain-fixture")), + ])) + .expect("should compile two sources of one implementation"); assert_eq!(plan.providers().len(), 2); assert!( - plan.providers().iter().all(|provider| matches!( - provider.profile, - CompiledOpenRtbProfile::PrebidServer(_) - )) + plan.providers() + .iter() + .all(|provider| provider.implementation.id == "auction.plain-fixture"), + "both names should resolve to the same implementation" + ); + assert!( + plan.providers() + .iter() + .all(|provider| provider.demand.as_any().is::()), + "each source should compile its own settings" ); } #[test] - fn profile_defaults_and_explicit_timeout_override_are_resolved() { - let mut providers = BTreeMap::new(); - providers.insert(id("standard-one"), provider("standard")); - providers.insert(id("pbs-one"), provider("prebid-server")); - providers.insert( - id("aps-one"), - ProviderConfig { - endpoint: "https://aps.example/e/pb/bid".to_string(), - profile_config: serde_json::json!({"account_id": "example-account"}), - ..provider("aps") - }, - ); - providers.insert( - id("pbs-override"), - ProviderConfig { - timeout_ms: Some(321), - ..provider("prebid-server") - }, - ); - let plan = AuctionPlan::compile(config(providers)).expect("should resolve timeouts"); + fn implementation_defaults_and_an_explicit_timeout_are_resolved() { + let mut override_table = table("auction.fixture"); + override_table.insert(TIMEOUT_KEY.to_string(), json!(321)); + let plan = AuctionPlan::compile(config(vec![ + ("plain_one", table("auction.plain-fixture")), + ("fixed_one", table("auction.fixture")), + ("fixed_override", override_table), + ])) + .expect("should resolve timeouts"); let timeouts = plan .providers() .iter() .map(|provider| (provider.id.as_str(), provider.timeout_ms)) .collect::>(); - assert_eq!(timeouts["standard-one"], 1500); - assert_eq!(timeouts["pbs-one"], 1000); - assert_eq!(timeouts["aps-one"], 800); - assert_eq!(timeouts["pbs-override"], 321); + assert_eq!(timeouts["plain_one"], 1500); + assert_eq!(timeouts["fixed_one"], 1000); + assert_eq!(timeouts["fixed_override"], 321); } #[test] - fn profile_registry_is_independent_of_browser_configuration() { - let ids = crate::auction::profile::profile_registrations() - .iter() - .map(|registration| registration.id) - .collect::>(); - assert_eq!(ids, vec!["standard", "prebid-server", "aps"]); - let mut providers = BTreeMap::new(); - providers.insert(id("pbs"), provider("prebid-server")); - let plan = AuctionPlan::compile(config(providers)) + fn the_plan_reports_which_implementations_it_selected() { + let plan = AuctionPlan::compile(config(vec![("source", table("auction.fixture"))])) .expect("should compile without Settings or browser integration state"); - assert!(!plan.has_profile("aps")); - - let mut providers = BTreeMap::new(); - providers.insert( - id("aps-instance"), - ProviderConfig { - endpoint: "https://aps.example/e/pb/bid".to_string(), - profile_config: serde_json::json!({"account_id": "example-account"}), - ..provider("aps") - }, + assert!(!plan.has_implementation("auction.plain-fixture")); + assert!(plan.has_implementation("auction.fixture")); + + let plan = AuctionPlan::compile(one_source()).expect("should compile a plain source"); + assert!( + plan.has_implementation("auction.plain-fixture"), + "a validated plan should say which implementation it selected" ); - let plan = AuctionPlan::compile(config(providers)).expect("should compile APS plan"); + assert!(!plan.has_implementation("auction.fixture")); assert!( - plan.has_profile("aps"), - "validated plan should expose APS renderer capability" + plan.providers()[0].demand.as_any().is::(), + "the source should compile its own settings" ); - assert!(!plan.has_profile("prebid-server")); } #[test] - fn compiler_rejects_unknown_protocol_profile_and_route() { - let mut unknown_protocol = provider("standard"); - unknown_protocol.protocol = "openrtb-2.5".to_string(); - assert!( - AuctionPlan::compile(config(BTreeMap::from([(id("one"), unknown_protocol)]))).is_err() - ); - assert!( - AuctionPlan::compile(config(BTreeMap::from([(id("one"), provider("unknown"))]))) - .is_err() - ); - let mut raw = config(BTreeMap::from([(id("one"), provider("standard"))])); + fn a_bidder_route_must_name_a_selected_source() { + let mut raw = one_source(); raw.bidders.insert( bidder("example"), BidderRouteConfig { - provider: id("missing"), + module: id("missing"), }, ); - assert!(AuctionPlan::compile(raw).is_err()); + let error = AuctionPlan::compile(raw).expect_err("should refuse an unrouted bidder"); + assert!( + error.to_string().contains("missing"), + "should name the source the route wanted: {error:?}" + ); } #[test] fn compiler_canonicalizes_https_endpoints_and_rejects_unsafe_forms() { - let mut canonical = provider("standard"); - canonical.endpoint = "https://BID.EXAMPLE:443/path".to_string(); - let plan = AuctionPlan::compile(config(BTreeMap::from([(id("one"), canonical)]))) + let mut canonical = table("auction.plain-fixture"); + canonical.insert( + ENDPOINT_KEY.to_string(), + json!("https://BID.EXAMPLE:443/path"), + ); + let plan = AuctionPlan::compile(config(vec![("one", canonical)])) .expect("should canonicalize endpoint"); assert_eq!( plan.providers()[0].endpoint.as_str(), @@ -1110,52 +1143,68 @@ mod tests { "https://bid.example/path#fragment", "/relative", ] { - let mut raw_provider = provider("standard"); - raw_provider.endpoint = endpoint.to_string(); + let mut raw = table("auction.plain-fixture"); + raw.insert(ENDPOINT_KEY.to_string(), json!(endpoint)); assert!( - AuctionPlan::compile(config(BTreeMap::from([(id("one"), raw_provider)]))).is_err(), + AuctionPlan::compile(config(vec![("one", raw)])).is_err(), "should reject {endpoint}" ); } - let mut aps = provider("aps"); - aps.endpoint = "https://aps.example/e/dtb/bid".to_string(); - aps.profile_config = serde_json::json!({"account_id": "example-account"}); - assert!(AuctionPlan::compile(config(BTreeMap::from([(id("aps"), aps)]))).is_err()); } #[test] - fn compiler_normalizes_only_prebid_server_origin_and_canonical_paths() { + fn plain_http_is_allowed_only_to_a_loopback_host() { + for endpoint in [ + "http://127.0.0.1:8000/openrtb2/auction", + "http://[::1]:8000/openrtb2/auction", + "http://localhost:8000/openrtb2/auction", + "http://LOCALHOST:8000/openrtb2/auction", + ] { + let mut raw = table("auction.plain-fixture"); + raw.insert(ENDPOINT_KEY.to_string(), json!(endpoint)); + AuctionPlan::compile(config(vec![("local", raw)])) + .unwrap_or_else(|error| panic!("should accept {endpoint}: {error:?}")); + } + for endpoint in [ + "http://192.0.2.10:8000/openrtb2/auction", + "http://bid.example/openrtb2/auction", + "http://localhost.example/openrtb2/auction", + ] { + let mut raw = table("auction.plain-fixture"); + raw.insert(ENDPOINT_KEY.to_string(), json!(endpoint)); + let error = AuctionPlan::compile(config(vec![("remote", raw)])) + .expect_err("should reject plain HTTP off the loopback"); + assert!( + error.to_string().contains("127.0.0.1"), + "should say which hosts plain HTTP may reach: {error:?}" + ); + } + } + + #[test] + fn an_implementation_completes_an_endpoint_and_the_plain_one_leaves_it_as_written() { for (configured, expected) in [ ( - "https://pbs.example", - "https://pbs.example/openrtb2/auction", + "https://exchange.example", + "https://exchange.example/fixture/auction", ), ( - "https://pbs.example/", - "https://pbs.example/openrtb2/auction", + "https://exchange.example/", + "https://exchange.example/fixture/auction", ), ( - "https://pbs.example/openrtb2/auction", - "https://pbs.example/openrtb2/auction", + "https://exchange.example?region=example", + "https://exchange.example/fixture/auction?region=example", ), ( - "https://pbs.example/openrtb2/auction/", - "https://pbs.example/openrtb2/auction", - ), - ( - "https://pbs.example?region=example", - "https://pbs.example/openrtb2/auction?region=example", - ), - ("https://pbs.example/bid", "https://pbs.example/bid"), - ( - "https://pbs.example/custom/pbs", - "https://pbs.example/custom/pbs", + "https://exchange.example/bid", + "https://exchange.example/bid", ), ] { - let mut pbs = provider("prebid-server"); - pbs.endpoint = configured.to_string(); - let plan = AuctionPlan::compile(config(BTreeMap::from([(id("pbs"), pbs)]))) - .expect("should compile Prebid Server endpoint"); + let mut source = table("auction.fixture"); + source.insert(ENDPOINT_KEY.to_string(), json!(configured)); + let plan = AuctionPlan::compile(config(vec![("source", source)])) + .expect("should compile the endpoint"); assert_eq!( plan.providers()[0].endpoint.as_str(), expected, @@ -1163,124 +1212,26 @@ mod tests { ); } - let mut standard = provider("standard"); - standard.endpoint = "https://bid.example/".to_string(); - let plan = AuctionPlan::compile(config(BTreeMap::from([(id("standard"), standard)]))) - .expect("should compile standard root endpoint"); + let mut plain = table("auction.plain-fixture"); + plain.insert(ENDPOINT_KEY.to_string(), json!("https://bid.example/")); + let plan = AuctionPlan::compile(config(vec![("plain", plain)])) + .expect("should compile a root endpoint unchanged"); assert_eq!( plan.providers()[0].endpoint.as_str(), "https://bid.example/" ); - - let mut aps = provider("aps"); - aps.endpoint = "https://aps.example/e/pb/bid".to_string(); - aps.profile_config = serde_json::json!({"account_id": "example-account"}); - let plan = AuctionPlan::compile(config(BTreeMap::from([(id("aps"), aps)]))) - .expect("should compile APS endpoint"); - assert_eq!( - plan.providers()[0].endpoint.as_str(), - "https://aps.example/e/pb/bid" - ); - } - - #[test] - fn standard_extensions_are_typed_bounded_and_cannot_claim_reserved_fields() { - let mut valid = provider("standard"); - valid.profile_config = serde_json::json!({ - "request_ext": {"fictional_account": "example"}, - "imp_ext": {"placement_group": "display"} - }); - let plan = AuctionPlan::compile(config(BTreeMap::from([(id("one"), valid)]))) - .expect("should compile static extensions"); - let CompiledOpenRtbProfile::Standard(standard) = &plan.providers()[0].profile else { - panic!("should compile standard profile") - }; - assert_eq!( - standard.request_ext.as_object()["fictional_account"], - "example" - ); - - let mut standard_owned_fields = provider("standard"); - standard_owned_fields.profile_config = serde_json::json!({ - "request_ext": { - "account": "example-account", - "sdk": {"source": "example"}, - "prebid": {"example": true} - }, - "imp_ext": {"prebid": {"example": true}} - }); - AuctionPlan::compile(config(BTreeMap::from([( - id("standard-owned-fields"), - standard_owned_fields, - )]))) - .expect("should allow standard static extensions outside common-owned fields"); - - for profile_config in [ - serde_json::json!({"request_ext": "bad"}), - serde_json::json!({"request_ext": {"trusted_server": {}}}), - ] { - let mut invalid = provider("standard"); - invalid.profile_config = profile_config; - assert!(AuctionPlan::compile(config(BTreeMap::from([(id("one"), invalid)]))).is_err()); - } - let oversized = "x".repeat(16 * 1024); - let mut invalid = provider("standard"); - invalid.profile_config = serde_json::json!({"request_ext": {"value": oversized}}); - assert!(AuctionPlan::compile(config(BTreeMap::from([(id("one"), invalid)]))).is_err()); - - let too_many_keys = (0..257) - .map(|index| (format!("key-{index}"), Value::Bool(true))) - .collect::>(); - let mut invalid = provider("standard"); - invalid.profile_config = serde_json::json!({"imp_ext": too_many_keys}); - assert!(AuctionPlan::compile(config(BTreeMap::from([(id("one"), invalid)]))).is_err()); - } - - #[test] - fn standard_extension_depth_counts_container_levels() { - let mut object_valid = provider("standard"); - object_valid.profile_config = serde_json::json!({"request_ext": nested_object(8)}); - AuctionPlan::compile(config(BTreeMap::from([(id("object-valid"), object_valid)]))) - .expect("should accept eight nested object levels"); - - let mut object_invalid = provider("standard"); - object_invalid.profile_config = serde_json::json!({"request_ext": nested_object(9)}); - assert!( - AuctionPlan::compile(config(BTreeMap::from([( - id("object-invalid"), - object_invalid, - )]))) - .is_err(), - "should reject nine nested object levels" - ); - - let mut array_valid = provider("standard"); - array_valid.profile_config = serde_json::json!({"request_ext": {"value": nested_array(7)}}); - AuctionPlan::compile(config(BTreeMap::from([(id("array-valid"), array_valid)]))) - .expect("should accept one object plus seven nested array levels"); - - let mut array_invalid = provider("standard"); - array_invalid.profile_config = - serde_json::json!({"request_ext": {"value": nested_array(8)}}); - assert!( - AuctionPlan::compile(config(BTreeMap::from([( - id("array-invalid"), - array_invalid, - )]))) - .is_err(), - "should reject one object plus eight nested array levels" - ); } #[test] fn notification_policy_rejects_duplicates_and_bounds() { - let mut valid = provider("standard"); - valid.notifications = NotificationConfig { - suppress_all: true, - suppress_seats: vec!["seat-b".to_string(), "seat-a".to_string()], - }; - let plan = AuctionPlan::compile(config(BTreeMap::from([(id("one"), valid)]))) + let mut valid = table("auction.plain-fixture"); + valid.insert( + NOTIFICATIONS_KEY.to_string(), + json!({"suppress_all": true, "suppress_seats": ["seat-b", "seat-a"]}), + ); + let plan = AuctionPlan::compile(config(vec![("one", valid)])) .expect("should compile notifications"); + assert!(plan.providers()[0].notifications.suppress_all); assert_eq!( plan.providers()[0] .notifications @@ -1291,82 +1242,116 @@ mod tests { vec!["seat-a", "seat-b"] ); for seats in [ - vec!["same".to_string(), "same".to_string()], - vec![String::new()], - vec!["bad\nseat".to_string()], - vec!["x".repeat(129)], - (0..129).map(|index| format!("seat-{index}")).collect(), + json!(["same", "same"]), + json!([""]), + json!(["bad\nseat"]), + json!(["x".repeat(129)]), + Value::Array( + (0..129) + .map(|index| json!(format!("seat-{index}"))) + .collect(), + ), ] { - let mut invalid = provider("standard"); - invalid.notifications.suppress_seats = seats; - assert!(AuctionPlan::compile(config(BTreeMap::from([(id("one"), invalid)]))).is_err()); + let mut invalid = table("auction.plain-fixture"); + invalid.insert( + NOTIFICATIONS_KEY.to_string(), + json!({"suppress_seats": seats}), + ); + assert!( + AuctionPlan::compile(config(vec![("one", invalid)])).is_err(), + "should refuse {seats}" + ); } } #[test] - fn typed_profile_config_rejects_unknown_fields_and_validates_aps_pairing() { - let mut pbs = provider("prebid-server"); - pbs.profile_config = serde_json::json!({"browser_only": true}); - assert!(AuctionPlan::compile(config(BTreeMap::from([(id("pbs"), pbs)]))).is_err()); - - let mut non_object = provider("standard"); - non_object.profile_config = Value::Null; + fn an_implementation_rejects_settings_it_does_not_know() { + let mut source = table("auction.fixture"); + source.insert("browser_only".to_string(), json!(true)); + let error = AuctionPlan::compile(config(vec![("source", source)])) + .expect_err("should refuse a setting the implementation does not know"); assert!( - AuctionPlan::compile(config(BTreeMap::from([(id("standard"), non_object,)]))).is_err() + format!("{error:?}").contains("browser_only"), + "should name the setting: {error:?}" ); - - let mut aps = provider("aps"); - aps.endpoint = "https://aps.example/e/pb/bid".to_string(); - aps.profile_config = serde_json::json!({ - "account_id": "example-account", - "inventory_domain": "publisher.example" - }); - assert!(AuctionPlan::compile(config(BTreeMap::from([(id("aps"), aps)]))).is_err()); } #[test] - fn enabled_signing_requires_nonblank_existing_global_store_ids() { - let mut raw = config(BTreeMap::from([(id("one"), provider("standard"))])); - raw.request_signing = Some(RequestSigning { - enabled: true, - config_store_id: " ".to_string(), - secret_store_id: "example-secret-store".to_string(), - }); - assert!( - AuctionPlan::compile(raw).is_err(), - "should reject enabled signing without a config store ID" + fn routing_signing_and_the_selected_ad_server_are_preserved_in_the_plan() { + let mut plain = table("auction.plain-fixture"); + plain.insert(ROUTING_KEY.to_string(), json!("all_eligible")); + let mut raw = config(vec![("one", plain)]); + raw.request_signing = Some(RequestSigning::new(true)); + raw.adserver = ProviderChoice::new( + Some("fixture".to_string()), + BTreeMap::from([( + "fixture".to_string(), + Map::from_iter([( + ENDPOINT_KEY.to_string(), + json!("http://127.0.0.1:6767/adserver/mediate"), + )]), + )]), + ); + let plan = AuctionPlan::compile(raw).expect("should compile common policies"); + assert_eq!(plan.providers()[0].routing, RoutingMode::AllEligible); + assert!(plan.signing_enabled()); + assert_eq!( + plan.adserver().map(|adserver| adserver.id.as_str()), + Some("fixture") ); - let mut raw = config(BTreeMap::from([(id("one"), provider("standard"))])); - raw.request_signing = Some(RequestSigning { - enabled: true, - config_store_id: "example-config-store".to_string(), - secret_store_id: "\t".to_string(), - }); + let mut invalid = config(Vec::new()); + invalid.adserver = + ProviderChoice::new(Some("fictional_adserver".to_string()), BTreeMap::new()); + let error = AuctionPlan::compile(invalid) + .expect_err("should refuse an ad server this build does not have"); assert!( - AuctionPlan::compile(raw).is_err(), - "should reject enabled signing without a secret store ID" + error.to_string().contains("fictional_adserver"), + "should name the ad server: {error:?}" ); } #[test] - fn routing_signing_and_static_mediator_are_preserved_in_plan() { - let mut provider = provider("standard"); - provider.routing = RoutingMode::AllEligible; - let mut raw = config(BTreeMap::from([(id("one"), provider)])); - raw.request_signing = Some(RequestSigning { - enabled: true, - config_store_id: "example-config-store".to_string(), - secret_store_id: "example-secret-store".to_string(), - }); - raw.mediator = Some(MOCK_MEDIATOR_ID.to_string()); - let plan = AuctionPlan::compile(raw).expect("should compile common policies"); - assert_eq!(plan.providers()[0].routing, RoutingMode::AllEligible); - assert!(plan.signing_enabled()); - assert_eq!(plan.mediator(), Some(MOCK_MEDIATOR_ID)); - - let mut invalid = config(BTreeMap::new()); - invalid.mediator = Some("generic-mediator".to_string()); - assert!(AuctionPlan::compile(invalid).is_err()); + fn an_ad_server_is_built_from_its_own_table() { + let mut raw = one_source(); + raw.adserver = ProviderChoice::new( + Some("house".to_string()), + BTreeMap::from([( + "house".to_string(), + Map::from_iter([ + (IMPLEMENTATION_KEY.to_string(), json!("fixture")), + ( + ENDPOINT_KEY.to_string(), + json!("https://adserver.example/mediate"), + ), + ("timeout_ms".to_string(), json!(750)), + ]), + )]), + ); + let plan = AuctionPlan::compile(raw).expect("should compile a named ad server"); + let adserver = plan.adserver().expect("should select an ad server"); + assert_eq!(adserver.id.as_str(), "house"); + assert_eq!(adserver.implementation.id, "ad-server.fixture"); + + let mut unknown_setting = one_source(); + unknown_setting.adserver = ProviderChoice::new( + Some("fixture".to_string()), + BTreeMap::from([( + "fixture".to_string(), + Map::from_iter([ + ( + ENDPOINT_KEY.to_string(), + json!("https://adserver.example/mediate"), + ), + ("enabled".to_string(), json!(true)), + ]), + )]), + ); + let error = AuctionPlan::compile(unknown_setting) + .expect_err("should refuse a setting the ad server does not know"); + assert!( + format!("{error:?}").contains("enabled"), + "should name the setting: {error:?}" + ); } } diff --git a/crates/trusted-server-core/src/auction/profile.rs b/crates/trusted-server-core/src/auction/profile.rs deleted file mode 100644 index 51a91e69c6..0000000000 --- a/crates/trusted-server-core/src/auction/profile.rs +++ /dev/null @@ -1,325 +0,0 @@ -//! Compile-time `OpenRTB` profile registry and typed profile plans. - -use std::collections::BTreeMap; - -use error_stack::Report; -use serde::Deserialize; -use serde_json::{Map, Value}; - -use crate::consent_config::ConsentForwardingMode; -use crate::error::TrustedServerError; -use crate::integrations::aps::compile_profile_config as compile_aps_profile_config; -use crate::integrations::prebid::{ - BidParamOverrideEngine, BidParamOverrideRule, compile_profile_override_rules, -}; - -const STANDARD_PROFILE_ID: &str = "standard"; -const PREBID_PROFILE_ID: &str = "prebid-server"; -const APS_PROFILE_ID: &str = "aps"; -const STATIC_EXTENSION_MAX_BYTES: usize = 16 * 1024; -const STATIC_EXTENSION_MAX_DEPTH: usize = 8; -const STATIC_EXTENSION_MAX_KEYS: usize = 256; - -/// A registered profile's provider-timeout default. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ProfileTimeoutDefault { - /// Inherit the configured auction timeout. - Auction, - /// Use this fixed profile timeout. - Fixed(u32), -} - -/// Compile-time profile registration. -#[derive(Clone, Copy)] -pub struct OpenRtbProfileRegistration { - /// Stable profile identifier used by configuration. - pub id: &'static str, - /// Profile timeout used when a provider has no explicit override. - pub default_timeout: ProfileTimeoutDefault, - compile: fn(&Value) -> Result>, -} - -impl core::fmt::Debug for OpenRtbProfileRegistration { - fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - formatter - .debug_struct("OpenRtbProfileRegistration") - .field("id", &self.id) - .field("default_timeout", &self.default_timeout) - .finish_non_exhaustive() - } -} - -impl OpenRtbProfileRegistration { - pub(crate) fn compile( - self, - config: &Value, - ) -> Result> { - (self.compile)(config) - } -} - -/// Immutable, typed profile behavior selected during plan compilation. -#[derive(Debug, Clone)] -pub enum CompiledOpenRtbProfile { - /// Generic `OpenRTB` 2.6 profile. - Standard(StandardProfilePlan), - /// Prebid Server compatibility profile. - PrebidServer(PrebidProfilePlan), - /// APS `OpenRTB` compatibility profile. - Aps(ApsProfilePlan), -} - -impl CompiledOpenRtbProfile { - /// Return the stable profile identifier. - #[must_use] - pub fn id(&self) -> &'static str { - match self { - Self::Standard(_) => STANDARD_PROFILE_ID, - Self::PrebidServer(_) => PREBID_PROFILE_ID, - Self::Aps(_) => APS_PROFILE_ID, - } - } - - /// Return whether this plan uses the Prebid Server profile. - #[must_use] - pub(crate) fn is_prebid_server(&self) -> bool { - matches!(self, Self::PrebidServer(_)) - } -} - -/// Validated static extension object. -#[derive(Debug, Clone, Default, PartialEq)] -pub struct StaticExtension(Map); - -impl StaticExtension { - /// Borrow the validated extension object. - #[must_use] - pub fn as_object(&self) -> &Map { - &self.0 - } -} - -/// Compiled generic `OpenRTB` profile configuration. -#[derive(Debug, Clone, Default)] -pub struct StandardProfilePlan { - /// Static request-level extension fields. - pub request_ext: StaticExtension, - /// Static impression-level extension fields. - pub imp_ext: StaticExtension, -} - -/// Compiled Prebid profile configuration. -#[derive(Debug, Clone)] -pub struct PrebidProfilePlan { - /// Include Prebid HTTP exchange diagnostics. - pub debug: bool, - /// Set `OpenRTB` test mode. - pub test_mode: bool, - /// Optional query fragment appended to the page URL under legacy rules. - pub debug_query_params: Option, - /// Compiled override matching and merge index. - pub(crate) override_engine: BidParamOverrideEngine, - /// Consent transport policy. - pub consent_forwarding: ConsentForwardingMode, -} - -/// Compiled APS profile configuration. -#[derive(Debug, Clone)] -pub struct ApsProfilePlan { - /// APS account identifier. - pub account_id: String, - /// Include APS request/response diagnostics. - pub debug: bool, - /// Permit APS script creatives. - pub allow_script_creatives: bool, - /// Optional authorized inventory domain. - pub inventory_domain: Option, - /// Optional canonical inventory page origin. - pub inventory_page_origin: Option, -} - -#[derive(Debug, Deserialize, Default)] -#[serde(deny_unknown_fields)] -struct StandardProfileConfig { - #[serde(default)] - request_ext: Option, - #[serde(default)] - imp_ext: Option, -} - -/// Typed operator configuration compiled into a [`PrebidProfilePlan`]. -#[derive(Debug, Deserialize, Default)] -#[serde(deny_unknown_fields)] -pub(crate) struct PrebidProfileConfig { - #[serde(default)] - debug: bool, - #[serde(default)] - test_mode: bool, - #[serde(default)] - debug_query_params: Option, - #[serde(default)] - bid_param_zone_overrides: BTreeMap>>, - #[serde(default)] - bid_param_overrides: BTreeMap>, - #[serde(default)] - bid_param_override_rules: Vec, - #[serde(default)] - consent_forwarding: ConsentForwardingMode, -} - -const PROFILE_REGISTRATIONS: [OpenRtbProfileRegistration; 3] = [ - OpenRtbProfileRegistration { - id: STANDARD_PROFILE_ID, - default_timeout: ProfileTimeoutDefault::Auction, - compile: compile_standard, - }, - OpenRtbProfileRegistration { - id: PREBID_PROFILE_ID, - default_timeout: ProfileTimeoutDefault::Fixed(1000), - compile: compile_prebid, - }, - OpenRtbProfileRegistration { - id: APS_PROFILE_ID, - default_timeout: ProfileTimeoutDefault::Fixed(800), - compile: compile_aps, - }, -]; - -/// Return the compile-time profile registry. -#[must_use] -pub fn profile_registrations() -> &'static [OpenRtbProfileRegistration] { - &PROFILE_REGISTRATIONS -} - -pub(crate) fn find_profile(id: &str) -> Option { - profile_registrations() - .iter() - .copied() - .find(|registration| registration.id == id) -} - -fn configuration_error(message: impl Into) -> Report { - Report::new(TrustedServerError::Configuration { - message: message.into(), - }) -} - -fn deserialize_profile(id: &str, value: &Value) -> Result> -where - T: for<'de> Deserialize<'de>, -{ - T::deserialize(value) - .map_err(|error| configuration_error(format!("invalid `{id}` profile_config: {error}"))) -} - -fn compile_standard(value: &Value) -> Result> { - let config: StandardProfileConfig = deserialize_profile(STANDARD_PROFILE_ID, value)?; - Ok(CompiledOpenRtbProfile::Standard(StandardProfilePlan { - request_ext: validate_static_extension("request_ext", config.request_ext)?, - imp_ext: validate_static_extension("imp_ext", config.imp_ext)?, - })) -} - -fn compile_prebid(value: &Value) -> Result> { - let config: PrebidProfileConfig = deserialize_profile(PREBID_PROFILE_ID, value)?; - let override_engine = compile_profile_override_rules( - &config.bid_param_zone_overrides, - &config.bid_param_overrides, - &config.bid_param_override_rules, - )?; - Ok(CompiledOpenRtbProfile::PrebidServer(PrebidProfilePlan { - debug: config.debug, - test_mode: config.test_mode, - debug_query_params: config.debug_query_params, - override_engine, - consent_forwarding: config.consent_forwarding, - })) -} - -fn compile_aps(value: &Value) -> Result> { - let config = compile_aps_profile_config(value.clone())?; - Ok(CompiledOpenRtbProfile::Aps(ApsProfilePlan { - account_id: config.account_id, - debug: config.debug, - allow_script_creatives: config.allow_script_creatives, - inventory_domain: config.inventory_domain, - inventory_page_origin: config.inventory_page_origin, - })) -} - -fn validate_static_extension( - field: &str, - value: Option, -) -> Result> { - let Some(value) = value else { - return Ok(StaticExtension::default()); - }; - let object = value.as_object().ok_or_else(|| { - configuration_error(format!("standard profile {field} must be an object")) - })?; - let size = serde_json::to_vec(&value) - .map_err(|error| configuration_error(format!("cannot serialize {field}: {error}")))? - .len(); - if size > STATIC_EXTENSION_MAX_BYTES { - return Err(configuration_error(format!( - "standard profile {field} exceeds {STATIC_EXTENSION_MAX_BYTES} bytes" - ))); - } - validate_extension_value(field, &value, 0)?; - reject_reserved_fields(field, object)?; - Ok(StaticExtension(object.clone())) -} - -fn validate_extension_value( - field: &str, - value: &Value, - container_depth: usize, -) -> Result<(), Report> { - match value { - Value::Object(object) => { - let container_depth = container_depth + 1; - if container_depth > STATIC_EXTENSION_MAX_DEPTH { - return Err(configuration_error(format!( - "standard profile {field} exceeds nesting depth {STATIC_EXTENSION_MAX_DEPTH}" - ))); - } - if object.len() > STATIC_EXTENSION_MAX_KEYS { - return Err(configuration_error(format!( - "standard profile {field} object exceeds {STATIC_EXTENSION_MAX_KEYS} keys" - ))); - } - for nested in object.values() { - validate_extension_value(field, nested, container_depth)?; - } - } - Value::Array(array) => { - let container_depth = container_depth + 1; - if container_depth > STATIC_EXTENSION_MAX_DEPTH { - return Err(configuration_error(format!( - "standard profile {field} exceeds nesting depth {STATIC_EXTENSION_MAX_DEPTH}" - ))); - } - for nested in array { - validate_extension_value(field, nested, container_depth)?; - } - } - _ => {} - } - Ok(()) -} - -fn reject_reserved_fields( - field: &str, - object: &Map, -) -> Result<(), Report> { - let reserved: &[&str] = match field { - "request_ext" => &["trusted_server"], - _ => &[], - }; - if let Some(key) = reserved.iter().find(|key| object.contains_key(**key)) { - return Err(configuration_error(format!( - "standard profile {field} cannot claim reserved field `{key}`" - ))); - } - Ok(()) -} diff --git a/crates/trusted-server-core/src/auction/provider.rs b/crates/trusted-server-core/src/auction/provider.rs index 62b2ddc8fc..d727629862 100644 --- a/crates/trusted-server-core/src/auction/provider.rs +++ b/crates/trusted-server-core/src/auction/provider.rs @@ -6,11 +6,8 @@ use std::collections::HashSet; use async_trait::async_trait; use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt as _}; -use http::{Method, Request, StatusCode, header}; -use serde_json::{Value, json}; - -use crate::integrations::aps::{ApsDebugRequest, parse_planned_aps_response}; -use crate::integrations::prebid::{apply_prebid_transport_headers, parse_planned_prebid_response}; +use http::{Method, Request, header}; +use serde_json::json; use crate::error::TrustedServerError; use crate::platform::{ @@ -18,25 +15,23 @@ use crate::platform::{ }; use crate::request_signing::{RequestSigner, SigningParams}; +use super::demand::{CompiledDemand, DemandResponse, DemandTransport}; use super::openrtb::{ OpenRtbBuildOutcome, RequestFinalization, apply_notification_policy, build_request, - extract_standard_response, unused_bidder_params_count, + unused_bidder_params_count, }; use super::plan::ProviderPlan; -use super::profile::CompiledOpenRtbProfile; use super::routing::{ProviderAuctionInput, RoutedAuction}; use super::types::{AuctionContext, AuctionRequest, AuctionResponse}; -const MAX_PLANNED_RESPONSE_BYTES: usize = 1024 * 1024; - fn attach_provider_routing_metadata( response: &mut AuctionResponse, - profile: &CompiledOpenRtbProfile, + demand: &dyn CompiledDemand, input: &ProviderAuctionInput, ) { response.metadata.insert( "routing".to_string(), - json!({"unused_bidder_params_count": unused_bidder_params_count(profile, input)}), + json!({"unused_bidder_params_count": unused_bidder_params_count(demand, input)}), ); } @@ -197,27 +192,14 @@ pub(crate) struct GenericOpenRtbProvider { plan: ProviderPlan, } -/// Typed state created by and returned to one [`GenericOpenRtbProvider`]. -#[allow( - dead_code, - clippy::large_enum_variant, - reason = "typed Stage 6 state avoids provider-state confusion; Stage 7/8 replace profile variants" -)] -pub(crate) enum GenericOpenRtbParseState { - Standard { - provider_id: String, - input: ProviderAuctionInput, - }, - Prebid { - provider_id: String, - auction_id: String, - input: ProviderAuctionInput, - }, - Aps { - provider_id: String, - input: ProviderAuctionInput, - debug_request: Option, - }, +/// State created by and returned to one [`GenericOpenRtbProvider`]. +/// +/// `captured` holds whatever the demand implementation kept from its own +/// request, and only that implementation reads it back. +pub(crate) struct GenericOpenRtbParseState { + provider_id: String, + input: ProviderAuctionInput, + captured: Option>, } impl GenericOpenRtbProvider { @@ -233,25 +215,13 @@ impl GenericOpenRtbProvider { self.plan.timeout_ms } - #[cfg(test)] + #[cfg(any(test, feature = "test-utils"))] pub(crate) fn parse_state_for_test(&self, input: ProviderAuctionInput) -> ProviderParseState { - let state = match &self.plan.profile { - CompiledOpenRtbProfile::Standard(_) => GenericOpenRtbParseState::Standard { - provider_id: self.provider_name().to_string(), - input, - }, - CompiledOpenRtbProfile::PrebidServer(_) => GenericOpenRtbParseState::Prebid { - provider_id: self.provider_name().to_string(), - auction_id: input.common_request().id.clone(), - input, - }, - CompiledOpenRtbProfile::Aps(_) => GenericOpenRtbParseState::Aps { - provider_id: self.provider_name().to_string(), - input, - debug_request: None, - }, - }; - Box::new(state) + Box::new(GenericOpenRtbParseState { + provider_id: self.provider_name().to_string(), + input, + captured: None, + }) } /// Build, register, and start exactly one routed provider request. @@ -341,26 +311,39 @@ impl GenericOpenRtbProvider { })); } + let sent_impression_ids = request + .imp + .iter() + .filter_map(|impression| impression.id.as_deref()) + .collect::>(); let body = serde_json::to_vec(&request).change_context(TrustedServerError::Auction { message: format!( "Provider {} request serialization failed", self.provider_name() ), })?; - let mut outbound = Request::builder() + let demand = self.plan.demand.as_ref(); + let mut builder = Request::builder() .method(Method::POST) .uri(self.plan.endpoint.as_str()) .header(header::CONTENT_TYPE, "application/json"); - if matches!(&self.plan.profile, CompiledOpenRtbProfile::Standard(_)) { - outbound = outbound.header(header::ACCEPT, "application/json"); + if demand.field_policy().accept_json { + builder = builder.header(header::ACCEPT, "application/json"); } - let aps_debug_body = matches!( - &self.plan.profile, - CompiledOpenRtbProfile::Aps(profile) if profile.debug - ) - .then(|| body.clone()); - let mut outbound = - outbound + if let Some(headers) = builder.headers_mut() { + demand.prepare_outbound( + headers, + DemandTransport { + headers: routed.transport_headers(), + attested_client_ip: routed.attested_client_ip(), + }, + ); + } + let captured = builder + .headers_ref() + .and_then(|headers| demand.capture_request(&body, headers)); + let outbound = + builder .body(EdgeBody::from(body)) .change_context(TrustedServerError::Auction { message: format!( @@ -368,17 +351,6 @@ impl GenericOpenRtbProvider { self.provider_name() ), })?; - let aps_debug_request = aps_debug_body - .as_deref() - .map(|body| ApsDebugRequest::capture(body, outbound.headers())); - if let CompiledOpenRtbProfile::PrebidServer(profile) = &self.plan.profile { - apply_prebid_transport_headers( - routed.prebid_transport_headers(), - &mut outbound, - profile.consent_forwarding, - routed.attested_client_ip(), - ); - } let pending = services .http_client() .send_async(PlatformHttpRequest::new(outbound, backend_name.clone())) @@ -394,42 +366,23 @@ impl GenericOpenRtbProvider { ), })); } - let parse_state = match &self.plan.profile { - CompiledOpenRtbProfile::Standard(_) => GenericOpenRtbParseState::Standard { - provider_id: self.provider_name().to_string(), - input: input.clone(), - }, - CompiledOpenRtbProfile::PrebidServer(_) => { - debug_assert!( - request.imp.iter().all(|impression| impression.id.is_some()), - "should populate every PBS impression ID" - ); - let sent_impression_ids = request - .imp - .iter() - .filter_map(|impression| impression.id.as_deref()) - .collect::>(); - GenericOpenRtbParseState::Prebid { - provider_id: self.provider_name().to_string(), - auction_id: input.common_request().id.clone(), - input: input.filtered_slots(|slot| { - sent_impression_ids.contains(slot.slot().id.as_str()) - }), - } - } - CompiledOpenRtbProfile::Aps(_) => GenericOpenRtbParseState::Aps { - provider_id: self.provider_name().to_string(), - input: input.clone(), - debug_request: aps_debug_request, - }, - }; Ok(ProviderRequestOutcome::pending_with_state( pending, - Box::new(parse_state), + Box::new(GenericOpenRtbParseState { + provider_id: self.provider_name().to_string(), + input: input + .filtered_slots(|slot| sent_impression_ids.contains(slot.slot().id.as_str())), + captured, + }), )) } /// Parse a response using state created by this exact provider instance. + /// + /// The implementation reads its own response and reports an unusable one as + /// an error response. The driver then applies the common notification + /// policy and routing diagnostics, so every demand source reports them the + /// same way. pub(crate) async fn parse_response_with_state( &self, response: PlatformResponse, @@ -446,153 +399,31 @@ impl GenericOpenRtbProvider { ), }) })?; - let state_provider_id = match parse_state { - GenericOpenRtbParseState::Standard { provider_id, .. } - | GenericOpenRtbParseState::Prebid { provider_id, .. } - | GenericOpenRtbParseState::Aps { provider_id, .. } => provider_id, - }; - if state_provider_id != self.provider_name() { + if parse_state.provider_id != self.provider_name() { return Err(Report::new(TrustedServerError::Auction { message: format!( "Provider {} received response state owned by provider {}", self.provider_name(), - state_provider_id + parse_state.provider_id ), })); } - if let GenericOpenRtbParseState::Prebid { - auction_id, input, .. - } = parse_state - { - let CompiledOpenRtbProfile::PrebidServer(profile) = &self.plan.profile else { - return Err(Report::new(TrustedServerError::Auction { - message: format!( - "Provider {} received PBS response state for profile {}", - self.provider_name(), - self.plan.profile.id() - ), - })); - }; - let mut parsed = match parse_planned_prebid_response( - self.provider_name(), - profile, - input, + let demand = self.plan.demand.as_ref(); + let mut parsed = demand + .parse_response( + DemandResponse { + provider_id: self.provider_name(), + endpoint: self.plan.endpoint.as_str(), + input: &parse_state.input, + response_time_ms, + captured: parse_state.captured.as_deref(), + }, response, - response_time_ms, - auction_id, ) - .await - { - Ok(parsed) => parsed, - Err(error) => { - log::warn!( - "Provider '{}' PBS response parse failed: {:?}", - self.provider_name(), - error - ); - AuctionResponse::error(self.provider_name(), response_time_ms) - .with_metadata("error_type", json!("parse_response")) - } - }; - apply_notification_policy(&mut parsed.bids, &self.plan.notifications); - attach_provider_routing_metadata(&mut parsed, &self.plan.profile, input); - return Ok(parsed); - } - - if let GenericOpenRtbParseState::Aps { - input, - debug_request, - .. - } = parse_state - { - let CompiledOpenRtbProfile::Aps(profile) = &self.plan.profile else { - return Err(Report::new(TrustedServerError::Auction { - message: format!( - "Provider {} received APS response state for profile {}", - self.provider_name(), - self.plan.profile.id() - ), - })); - }; - let mut parsed = match parse_planned_aps_response( - self.provider_name(), - profile, - self.plan.endpoint.as_str(), - input, - response, - response_time_ms, - debug_request.clone(), - ) - .await - { - Ok(parsed) => parsed, - Err(error) => { - log::warn!( - "Provider '{}' APS response parse failed: {:?}", - self.provider_name(), - error - ); - let mut parsed = AuctionResponse::error(self.provider_name(), response_time_ms) - .with_metadata("error_type", json!("parse_response")); - attach_provider_routing_metadata(&mut parsed, &self.plan.profile, input); - parsed - } - }; - apply_notification_policy(&mut parsed.bids, &self.plan.notifications); - return Ok(parsed); - } - - let response = response.response; - let status = response.status(); - let GenericOpenRtbParseState::Standard { input, .. } = parse_state else { - unreachable!("profile-specific states are handled before standard parsing"); - }; - if status == StatusCode::NO_CONTENT { - let mut parsed = AuctionResponse::no_bid(self.provider_name(), response_time_ms); - attach_provider_routing_metadata(&mut parsed, &self.plan.profile, input); - return Ok(parsed); - } - if !status.is_success() { - if status.is_redirection() { - log::warn!( - "Provider '{}' returned a redirect; generic OpenRTB redirects are refused", - self.provider_name() - ); - } - let mut parsed = AuctionResponse::error(self.provider_name(), response_time_ms) - .with_metadata("error_type", json!("http_status")) - .with_metadata("http_status", json!(status.as_u16())); - attach_provider_routing_metadata(&mut parsed, &self.plan.profile, input); - return Ok(parsed); - } - - let body = response - .into_body() - .into_bytes_bounded(MAX_PLANNED_RESPONSE_BYTES) - .await - .change_context(TrustedServerError::Auction { - message: format!("Provider {} response body failed", self.provider_name()), - })?; - let value: Value = match serde_json::from_slice(&body) { - Ok(value) => value, - Err(error) => { - log::warn!( - "Provider '{}' response JSON was invalid: {}", - self.provider_name(), - error - ); - let mut parsed = AuctionResponse::error(self.provider_name(), response_time_ms) - .with_metadata("error_type", json!("parse_response")); - attach_provider_routing_metadata(&mut parsed, &self.plan.profile, input); - return Ok(parsed); - } - }; - - let mut parsed = - extract_standard_response(self.provider_name(), input, &value, response_time_ms); + .await?; apply_notification_policy(&mut parsed.bids, &self.plan.notifications); - attach_provider_routing_metadata(&mut parsed, &self.plan.profile, input); + attach_provider_routing_metadata(&mut parsed, demand, &parse_state.input); Ok(parsed) } } diff --git a/crates/trusted-server-core/src/auction/routing.rs b/crates/trusted-server-core/src/auction/routing.rs index 2b8135da3c..a8df9c1a29 100644 --- a/crates/trusted-server-core/src/auction/routing.rs +++ b/crates/trusted-server-core/src/auction/routing.rs @@ -17,8 +17,8 @@ const STORED_REQUEST_FIELD: &str = "storedRequest"; /// Maximum bidder entries admitted from one browser `bidderParams` envelope. pub(crate) const MAX_BIDDER_ENTRIES: usize = 128; -/// Maximum UTF-8 byte length of the optional Prebid zone fact. -pub(crate) const MAX_PREBID_ZONE_BYTES: usize = 256; +/// Maximum UTF-8 byte length of the zone a browser may name for a slot. +pub(crate) const MAX_ZONE_BYTES: usize = 256; /// Immutable provider-local routing output in deterministic provider-ID order. #[derive(Debug, Clone)] @@ -26,7 +26,7 @@ pub(crate) struct RoutedAuction { inputs: Vec, skipped_no_eligible_provider_ids: Vec, diagnostics: RoutingDiagnostics, - transport_headers: PrebidTransportHeaders, + transport_headers: TransportHeaders, attested_client_ip: Option, dnt: Option, } @@ -47,8 +47,8 @@ impl RoutedAuction { self.diagnostics } - /// Request headers approved for later Prebid transport forwarding. - pub(crate) fn prebid_transport_headers(&self) -> &PrebidTransportHeaders { + /// Request headers approved for forwarding to a demand source. + pub(crate) fn transport_headers(&self) -> &TransportHeaders { &self.transport_headers } @@ -122,7 +122,7 @@ impl RoutingDiagnostics { /// Provider-local immutable auction input. #[derive(Debug, Clone)] -pub(crate) struct ProviderAuctionInput { +pub struct ProviderAuctionInput { provider_id: ProviderId, #[cfg_attr( not(test), @@ -137,7 +137,9 @@ pub(crate) struct ProviderAuctionInput { } impl ProviderAuctionInput { - pub(crate) fn provider_id(&self) -> &ProviderId { + /// The configured name of the demand source this input is routed to. + #[must_use] + pub fn provider_id(&self) -> &ProviderId { &self.provider_id } @@ -147,11 +149,14 @@ impl ProviderAuctionInput { } /// Common privacy-approved request data. Its slot list is always empty. - pub(crate) fn common_request(&self) -> &AuctionRequest { + #[must_use] + pub fn common_request(&self) -> &AuctionRequest { &self.common_request } - pub(crate) fn slots(&self) -> &[ProviderSlotInput] { + /// The eligible slots routed to this demand source. + #[must_use] + pub fn slots(&self) -> &[ProviderSlotInput] { &self.slots } @@ -165,25 +170,30 @@ impl ProviderAuctionInput { /// One eligible slot with only the demand assigned to this provider. #[derive(Debug, Clone)] -pub(crate) struct ProviderSlotInput { +pub struct ProviderSlotInput { slot: AdSlot, bidder_params: BTreeMap, - prebid_zone: Option, + zone: Option, stored_request: StoredRequestIntent, } impl ProviderSlotInput { /// Common slot facts. The legacy `bidders` map is always empty. - pub(crate) fn slot(&self) -> &AdSlot { + #[must_use] + pub fn slot(&self) -> &AdSlot { &self.slot } - pub(crate) fn bidder_params(&self) -> &BTreeMap { + /// The bidder parameters routed to this demand source for this slot. + #[must_use] + pub fn bidder_params(&self) -> &BTreeMap { &self.bidder_params } - pub(crate) fn prebid_zone(&self) -> Option<&str> { - self.prebid_zone.as_deref() + /// The zone the browser named for this slot, where one was admitted. + #[must_use] + pub fn zone(&self) -> Option<&str> { + self.zone.as_deref() } #[cfg(test)] @@ -191,38 +201,49 @@ impl ProviderSlotInput { self.bidder_params.is_empty() && self.allows_stored_fallback() } - pub(crate) fn allows_stored_fallback(&self) -> bool { + /// Whether a stored request may stand in for this slot when it is left + /// with no usable inline bidder parameters. + #[must_use] + pub fn allows_stored_fallback(&self) -> bool { self.stored_request .allows_fallback(!self.bidder_params.is_empty()) } } -/// Request headers approved for later Prebid transport forwarding. +/// Request headers approved for forwarding to a demand source. /// /// Values remain as raw [`HeaderValue`] instances so non-ASCII bytes retain /// the same legacy handling. Client-supplied `X-Forwarded-For` is never read. #[derive(Debug, Clone, Default)] -pub(crate) struct PrebidTransportHeaders { +pub struct TransportHeaders { cookie: Option, user_agent: Option, referer: Option, accept_language: Option, } -impl PrebidTransportHeaders { - pub(crate) fn cookie(&self) -> Option<&HeaderValue> { +impl TransportHeaders { + /// The admitted `Cookie` header. + #[must_use] + pub fn cookie(&self) -> Option<&HeaderValue> { self.cookie.as_ref() } - pub(crate) fn user_agent(&self) -> Option<&HeaderValue> { + /// The admitted `User-Agent` header. + #[must_use] + pub fn user_agent(&self) -> Option<&HeaderValue> { self.user_agent.as_ref() } - pub(crate) fn referer(&self) -> Option<&HeaderValue> { + /// The admitted `Referer` header. + #[must_use] + pub fn referer(&self) -> Option<&HeaderValue> { self.referer.as_ref() } - pub(crate) fn accept_language(&self) -> Option<&HeaderValue> { + /// The admitted `Accept-Language` header. + #[must_use] + pub fn accept_language(&self) -> Option<&HeaderValue> { self.accept_language.as_ref() } @@ -286,14 +307,14 @@ impl StoredRequestIntent { struct NormalizedSlotDemand { bidder_params: BTreeMap, stored_request: StoredRequestIntent, - prebid_zone: Option, + zone: Option, } #[derive(Debug)] struct ProviderInputBuilder { provider_id: ProviderId, timeout_ms: u32, - is_prebid: bool, + serves_stored_requests: bool, routing: RoutingMode, slots: Vec, } @@ -330,7 +351,7 @@ pub(crate) fn route_auction_with_trusted_routes( ) -> RoutedAuction { let slots = std::mem::take(&mut request.slots); let common_request = request; - let transport_headers = PrebidTransportHeaders::snapshot(inbound_request); + let transport_headers = TransportHeaders::snapshot(inbound_request); let dnt = inbound_request .headers() .get("dnt") @@ -344,7 +365,7 @@ pub(crate) fn route_auction_with_trusted_routes( .map(|provider| ProviderInputBuilder { provider_id: provider.id.clone(), timeout_ms: provider.timeout_ms, - is_prebid: provider.profile.is_prebid_server(), + serves_stored_requests: provider.implementation.serves_stored_requests, routing: provider.routing, slots: Vec::new(), }) @@ -385,7 +406,7 @@ pub(crate) fn route_auction_with_trusted_routes( let trusted_route = trusted_provider_indices.contains(&provider_index); let include = !bidder_params.is_empty() || trusted_route - || if builder.is_prebid { + || if builder.serves_stored_requests { demand.stored_request.allows_fallback(false) } else { builder.routing == RoutingMode::AllEligible @@ -396,9 +417,9 @@ pub(crate) fn route_auction_with_trusted_routes( builder.slots.push(ProviderSlotInput { slot: common_slot.clone(), bidder_params, - prebid_zone: builder - .is_prebid - .then(|| demand.prebid_zone.clone()) + zone: builder + .serves_stored_requests + .then(|| demand.zone.clone()) .flatten(), stored_request: demand.stored_request, }); @@ -513,9 +534,9 @@ fn normalize_envelope(envelope: &Value) -> Option { }) { return None; } - let prebid_zone = match object.get(ZONE_FIELD) { + let zone = match object.get(ZONE_FIELD) { None => None, - Some(Value::String(zone)) if zone.len() <= MAX_PREBID_ZONE_BYTES => Some(zone.clone()), + Some(Value::String(zone)) if zone.len() <= MAX_ZONE_BYTES => Some(zone.clone()), Some(_) => return None, }; let params = match object.get(BIDDER_PARAMS_FIELD) { @@ -533,7 +554,7 @@ fn normalize_envelope(envelope: &Value) -> Option { let Some(params) = params else { return Some(NormalizedSlotDemand { stored_request, - prebid_zone, + zone, ..Default::default() }); }; @@ -552,7 +573,7 @@ fn normalize_envelope(envelope: &Value) -> Option { Some(NormalizedSlotDemand { bidder_params, stored_request, - prebid_zone, + zone, }) } @@ -565,97 +586,68 @@ mod tests { use std::str::FromStr as _; use super::*; - use crate::auction::plan::{ - AuctionPlanConfig, BidderRouteConfig, NotificationConfig, ProviderConfig, - }; + use crate::auction::plan::BidderRouteConfig; + use crate::auction::test_support::{demand_table, plan_config}; use crate::auction::types::{AdFormat, DeviceInfo, PublisherInfo, SiteInfo, UserInfo}; use http::HeaderName; use serde_json::{Map, json}; - fn provider(profile: &str, routing: RoutingMode) -> ProviderConfig { - ProviderConfig { - protocol: "openrtb-2.6".to_string(), - profile: profile.to_string(), - endpoint: format!("https://{profile}.example.test/openrtb"), - timeout_ms: None, - routing, - notifications: NotificationConfig::default(), - profile_config: if profile == "aps" { - json!({"account_id": "example-account"}) - } else { - json!({}) - }, + fn provider(implementation: &str, routing: RoutingMode) -> Map { + let mut table = demand_table( + implementation, + &format!("https://{implementation}.example.test/openrtb"), + ); + if routing == RoutingMode::AllEligible { + table.insert("routing".to_string(), json!("all_eligible")); } + table + } + + fn routing_plan_config(open_routing: RoutingMode) -> crate::auction::plan::AuctionPlanConfig { + let mut config = plan_config(vec![ + ( + "open_primary", + provider("auction.plain-fixture", open_routing), + ), + ( + "stored_a", + provider("auction.fixture", RoutingMode::Explicit), + ), + ( + "stored_b", + provider("auction.fixture", RoutingMode::Explicit), + ), + ( + "openrtb_direct", + provider("auction.plain-fixture", RoutingMode::Explicit), + ), + ]); + config.timeout_ms = 900; + config } fn plan() -> AuctionPlan { - AuctionPlan::compile(AuctionPlanConfig { - timeout_ms: 900, - providers: BTreeMap::from([ - ( - ProviderId::from_str("aps-primary").expect("should parse provider"), - provider("aps", RoutingMode::AllEligible), - ), - ( - ProviderId::from_str("pbs-a").expect("should parse provider"), - provider("prebid-server", RoutingMode::Explicit), - ), - ( - ProviderId::from_str("pbs-b").expect("should parse provider"), - provider("prebid-server", RoutingMode::Explicit), - ), - ( - ProviderId::from_str("standard-direct").expect("should parse provider"), - provider("standard", RoutingMode::Explicit), - ), - ]), - bidders: BTreeMap::from([ - ( - BidderId::from_str("alpha").expect("should parse bidder"), - BidderRouteConfig { - provider: ProviderId::from_str("pbs-a").expect("should parse provider"), - }, - ), - ( - BidderId::from_str("beta").expect("should parse bidder"), - BidderRouteConfig { - provider: ProviderId::from_str("standard-direct") - .expect("should parse provider"), - }, - ), - ]), - mediator: None, - request_signing: None, - }) - .expect("should compile plan") + let mut config = routing_plan_config(RoutingMode::AllEligible); + config.bidders = BTreeMap::from([ + ( + BidderId::from_str("alpha").expect("should parse bidder"), + BidderRouteConfig { + module: ProviderId::from_str("stored_a").expect("should parse provider"), + }, + ), + ( + BidderId::from_str("beta").expect("should parse bidder"), + BidderRouteConfig { + module: ProviderId::from_str("openrtb_direct").expect("should parse provider"), + }, + ), + ]); + AuctionPlan::compile(config).expect("should compile plan") } fn explicit_plan() -> AuctionPlan { - AuctionPlan::compile(AuctionPlanConfig { - timeout_ms: 900, - providers: BTreeMap::from([ - ( - ProviderId::from_str("aps-primary").expect("should parse provider"), - provider("aps", RoutingMode::Explicit), - ), - ( - ProviderId::from_str("pbs-a").expect("should parse provider"), - provider("prebid-server", RoutingMode::Explicit), - ), - ( - ProviderId::from_str("pbs-b").expect("should parse provider"), - provider("prebid-server", RoutingMode::Explicit), - ), - ( - ProviderId::from_str("standard-direct").expect("should parse provider"), - provider("standard", RoutingMode::Explicit), - ), - ]), - bidders: BTreeMap::new(), - mediator: None, - request_signing: None, - }) - .expect("should compile explicit plan") + AuctionPlan::compile(routing_plan_config(RoutingMode::Explicit)) + .expect("should compile explicit plan") } fn slot(bidders: HashMap) -> AdSlot { @@ -722,8 +714,8 @@ mod tests { } #[test] - fn pbs_admission_respects_intent_without_changing_aps_eligibility() { - for (params, pbs_ids) in [ + fn stored_admission_respects_intent_without_changing_all_eligible_routing() { + for (params, stored_ids) in [ (json!({"bidderParams":{}, "storedRequest":false}), vec![]), ( json!({"bidderParams":{"unknown":{"id":1}}, "storedRequest":false}), @@ -732,18 +724,18 @@ mod tests { (json!({"bidderParams":{"unknown":{"id":1}}}), vec![]), ( json!({"bidderParams":{"alpha":{}}, "storedRequest":false}), - vec!["pbs-a"], + vec!["stored_a"], ), - (json!({"bidderParams":{"alpha":{}}}), vec!["pbs-a"]), + (json!({"bidderParams":{"alpha":{}}}), vec!["stored_a"]), ( json!({"bidderParams":{"alpha":{"id":1}}, "storedRequest":true}), - vec!["pbs-a", "pbs-b"], + vec!["stored_a", "stored_b"], ), ( json!({"bidderParams":{}, "storedRequest":true}), - vec!["pbs-a", "pbs-b"], + vec!["stored_a", "stored_b"], ), - (json!({"bidderParams":{}}), vec!["pbs-a", "pbs-b"]), + (json!({"bidderParams":{}}), vec!["stored_a", "stored_b"]), ] { let routed = route_auction( request(vec![slot(HashMap::from([( @@ -754,8 +746,8 @@ mod tests { &plan(), None, ); - let mut expected = vec!["aps-primary"]; - expected.extend(pbs_ids); + let mut expected = vec!["open_primary"]; + expected.extend(stored_ids); assert_eq!( routed .inputs() @@ -790,11 +782,11 @@ mod tests { route_auction(request(vec![slot(bidders)]), &inbound(), &plan(), None); assert_eq!(routed.diagnostics().malformed_envelope_count(), 1); assert_eq!(routed.inputs().len(), if direct { 2 } else { 1 }); - assert_eq!(routed.inputs()[0].provider_id().as_str(), "aps-primary"); + assert_eq!(routed.inputs()[0].provider_id().as_str(), "open_primary"); if direct { - let slot = &input(&routed, "pbs-a").slots()[0]; + let slot = &input(&routed, "stored_a").slots()[0]; assert!(!slot.allows_stored_fallback()); - assert_eq!(slot.prebid_zone(), None); + assert_eq!(slot.zone(), None); assert_eq!( slot.bidder_params().values().collect::>(), vec![&json!({"direct":1})] @@ -810,11 +802,11 @@ mod tests { for (params, expected) in [ ( json!({"bidderParams": {}, "storedRequest": true}), - vec!["aps-primary", "pbs-a", "pbs-b"], + vec!["open_primary", "stored_a", "stored_b"], ), ( json!({"bidderParams": {"alpha": {"placement": 1}}, "storedRequest": false}), - vec!["aps-primary", "pbs-a"], + vec!["open_primary", "stored_a"], ), ] { let routed = route_auction( @@ -862,16 +854,16 @@ mod tests { .collect::>(); assert_eq!( ids, - vec!["aps-primary", "pbs-a", "pbs-b"], - "{name} should fan out to both PBS providers while APS remains all-eligible" + vec!["open_primary", "stored_a", "stored_b"], + "{name} should fan out to both stored-request sources while the all-eligible source keeps every slot" ); assert!( - input(&routed, "pbs-a").slots()[0].allows_stored_fallback_without_candidates(), + input(&routed, "stored_a").slots()[0].allows_stored_fallback_without_candidates(), "{name} should create stored intent" ); assert!( - input(&routed, "pbs-b").slots()[0].allows_stored_fallback_without_candidates(), - "{name} should create stored intent for every PBS provider" + input(&routed, "stored_b").slots()[0].allows_stored_fallback_without_candidates(), + "{name} should create stored intent for every stored-request source" ); } } @@ -885,12 +877,12 @@ mod tests { None, ); assert!( - input(&routed, "pbs-a").slots()[0].allows_stored_fallback_without_candidates(), + input(&routed, "stored_a").slots()[0].allows_stored_fallback_without_candidates(), "empty canonical demand should preserve stored-request behavior" ); assert!( - input(&routed, "pbs-b").slots()[0].allows_stored_fallback_without_candidates(), - "empty canonical demand should fan out to same-profile PBS plans" + input(&routed, "stored_b").slots()[0].allows_stored_fallback_without_candidates(), + "empty canonical demand should fan out to the sources of one implementation" ); } @@ -921,7 +913,7 @@ mod tests { ("too many bidders", envelope(Some(too_many))), ( "oversized zone", - json!({"bidderParams": {}, "zone": "z".repeat(MAX_PREBID_ZONE_BYTES + 1)}), + json!({"bidderParams": {}, "zone": "z".repeat(MAX_ZONE_BYTES + 1)}), ), ("nonstring zone", json!({"bidderParams": {}, "zone": 1})), ]; @@ -938,13 +930,13 @@ mod tests { ); assert!( routed.inputs().iter().all(|provider| { - provider.provider_id().as_str() != "pbs-a" - && provider.provider_id().as_str() != "pbs-b" + provider.provider_id().as_str() != "stored_a" + && provider.provider_id().as_str() != "stored_b" }), - "{name} should not produce stored or inline PBS demand" + "{name} should not produce stored or inline demand" ); assert_eq!( - input(&routed, "standard-direct").slots()[0] + input(&routed, "openrtb_direct").slots()[0] .bidder_params() .len(), 1, @@ -954,7 +946,7 @@ mod tests { routed .inputs() .iter() - .any(|provider| provider.provider_id().as_str() == "aps-primary"), + .any(|provider| provider.provider_id().as_str() == "open_primary"), "{name} should preserve independent all-eligible participation" ); } @@ -1025,12 +1017,12 @@ mod tests { assert_eq!( routed.inputs().len(), 1, - "only all-eligible APS should remain" + "only the all-eligible source should remain" ); assert_eq!( routed.inputs()[0].provider_id().as_str(), - "aps-primary", - "unknown demand should not cause PBS fallback" + "open_primary", + "unknown demand should not cause a stored fallback" ); } @@ -1055,7 +1047,7 @@ mod tests { None, ); assert_eq!( - input(&routed, "pbs-a").slots()[0].bidder_params() + input(&routed, "stored_a").slots()[0].bidder_params() [&BidderId::from_str("alpha").expect("should parse bidder")]["source"], expected_source, "{name} should follow deterministic collision semantics" @@ -1110,7 +1102,7 @@ mod tests { TRUSTED_SERVER_ENVELOPE.to_string(), json!({"zone": "home", "bidderParams": null}), ), - ("alpha".to_string(), json!({"placement": "pbs"})), + ("alpha".to_string(), json!({"placement": "stored"})), ("beta".to_string(), json!({"placement": "direct"})), ]))]), &inbound(), @@ -1124,31 +1116,35 @@ mod tests { .collect::>(); assert_eq!( ids, - vec!["aps-primary", "pbs-a", "pbs-b", "standard-direct"], + vec!["open_primary", "stored_a", "stored_b", "openrtb_direct"], "inputs should follow deterministic provider-ID order" ); - let aps = input(&routed, "aps-primary") + let open = input(&routed, "open_primary") .slots() .first() .expect("should have slot"); assert!( - aps.bidder_params().is_empty(), - "APS must receive no foreign params" + open.bidder_params().is_empty(), + "a source that takes no bidder params must receive none" + ); + assert_eq!( + open.zone(), + None, + "a source that takes no bidder params must receive no zone" ); - assert_eq!(aps.prebid_zone(), None, "APS must receive no Prebid zone"); - let pbs_a = &input(&routed, "pbs-a").slots()[0]; - assert_eq!(pbs_a.bidder_params().len(), 1); + let stored_a = &input(&routed, "stored_a").slots()[0]; + assert_eq!(stored_a.bidder_params().len(), 1); assert!( - !pbs_a.allows_stored_fallback_without_candidates(), - "inline params should win for this PBS provider" + !stored_a.allows_stored_fallback_without_candidates(), + "inline params should win for this source" ); - assert_eq!(pbs_a.prebid_zone(), Some("home")); - let pbs_b = &input(&routed, "pbs-b").slots()[0]; - assert!(pbs_b.bidder_params().is_empty()); - assert!(pbs_b.allows_stored_fallback_without_candidates()); - let direct = &input(&routed, "standard-direct").slots()[0]; + assert_eq!(stored_a.zone(), Some("home")); + let stored_b = &input(&routed, "stored_b").slots()[0]; + assert!(stored_b.bidder_params().is_empty()); + assert!(stored_b.allows_stored_fallback_without_candidates()); + let direct = &input(&routed, "openrtb_direct").slots()[0]; assert_eq!(direct.bidder_params().len(), 1); - assert!(direct.prebid_zone().is_none()); + assert!(direct.zone().is_none()); for provider in routed.inputs() { assert!(provider.common_request().slots.is_empty()); assert!(provider.slots()[0].slot().bidders.is_empty()); @@ -1198,7 +1194,7 @@ mod tests { assert_eq!( routed.inputs().len(), 3, - "APS and two PBS providers should receive the eligible slot" + "the all-eligible source and two stored-request sources should receive the eligible slot" ); for provider in routed.inputs() { assert_eq!(provider.slots().len(), 1); @@ -1225,7 +1221,7 @@ mod tests { .iter() .map(ProviderId::as_str) .collect::>(), - vec!["aps-primary", "pbs-a", "pbs-b", "standard-direct"], + vec!["open_primary", "stored_a", "stored_b", "openrtb_direct"], "no-banner auction should retain every provider's deterministic skip outcome" ); } @@ -1247,17 +1243,17 @@ mod tests { .iter() .map(ProviderId::as_str) .collect::>(), - vec!["pbs-a", "pbs-b", "standard-direct"], + vec!["stored_a", "stored_b", "openrtb_direct"], "explicit providers with no routed demand should be retained as skipped" ); } #[test] - fn trusted_routes_admit_explicit_aps_and_standard_and_ignore_unknown_provider() { + fn trusted_routes_admit_explicit_sources_and_ignore_unknown_provider() { let trusted_routes = TrustedProviderRoutes::new(vec![vec![ - ProviderId::from_str("aps-primary").expect("should parse provider"), - ProviderId::from_str("standard-direct").expect("should parse provider"), - ProviderId::from_str("unknown-provider").expect("should parse provider"), + ProviderId::from_str("open_primary").expect("should parse provider"), + ProviderId::from_str("openrtb_direct").expect("should parse provider"), + ProviderId::from_str("unknown_provider").expect("should parse provider"), ]]); let routed = route_auction_with_trusted_routes( request(vec![slot(HashMap::from([( @@ -1275,7 +1271,7 @@ mod tests { .iter() .map(|input| input.provider_id().as_str()) .collect::>(), - vec!["aps-primary", "standard-direct"], + vec!["open_primary", "openrtb_direct"], "only known server-owned provider routes should admit explicit providers" ); assert_eq!( @@ -1289,7 +1285,7 @@ mod tests { .iter() .map(ProviderId::as_str) .collect::>(), - vec!["pbs-a", "pbs-b"], + vec!["stored_a", "stored_b"], "unrouted explicit providers should retain skip outcomes" ); } @@ -1335,7 +1331,7 @@ mod tests { &plan(), Some(attested), ); - let headers = routed.prebid_transport_headers(); + let headers = routed.transport_headers(); assert_eq!(headers.cookie(), Some(&HeaderValue::from_static("first=1"))); assert_eq!( headers.user_agent().expect("should retain UA").as_bytes(), @@ -1355,8 +1351,8 @@ mod tests { assert_eq!(routed.dnt(), Some(true)); for provider in routed.inputs() { let expected_timeout = match provider.provider_id().as_str() { - "aps-primary" => 800, - id if id.starts_with("pbs-") => 1000, + "open_primary" => 900, + id if id.starts_with("stored_") => 1000, _ => 900, }; assert_eq!(provider.timeout_ms(), expected_timeout); diff --git a/crates/trusted-server-core/src/auction/telemetry.rs b/crates/trusted-server-core/src/auction/telemetry.rs index e0a819ba38..0beb1508e4 100644 --- a/crates/trusted-server-core/src/auction/telemetry.rs +++ b/crates/trusted-server-core/src/auction/telemetry.rs @@ -76,7 +76,7 @@ impl AuctionTerminalStatus { pub struct AbandonedProviderCall { /// Provider name. pub provider: String, - /// Provider role, usually `bidder` or `mediator`. + /// Provider role, usually `bidder` or `adserver`. pub provider_role: &'static str, /// Optional elapsed time for this provider at abandonment. pub response_time_ms: Option, @@ -338,7 +338,7 @@ pub struct AuctionEventRow { pub winning_bid_count: Option, /// Provider name. pub provider: Option, - /// `bidder` or `mediator`. + /// `bidder` or `adserver`. pub provider_role: Option, /// Provider-call status. pub status: Option, @@ -576,13 +576,13 @@ pub fn build_auction_events( &result.provider_responses, "bidder", ); - if let Some(mediator_response) = &result.mediator_response { + if let Some(adserver_response) = &result.adserver_response { push_provider_row( &mut rows, &observation, &event_ts, - mediator_response, - "mediator", + adserver_response, + "adserver", ); } push_bid_rows( @@ -775,14 +775,14 @@ fn push_bid_rows( } } - if let Some(mediator_response) = &result.mediator_response { + if let Some(adserver_response) = &result.adserver_response { for (slot_id, winning) in &result.winning_bids { if delivered_winner_slots.is_some_and(|slots| !slots.contains(slot_id)) || matched_wins.contains(slot_id) { continue; } - if mediator_response + if adserver_response .bids .iter() .any(|bid| bid_matches_winning_bid(bid, winning)) @@ -791,7 +791,7 @@ fn push_bid_rows( observation, event_ts, request, - &mediator_response.provider, + &adserver_response.provider, winning, 1, winning.price, @@ -1208,7 +1208,7 @@ mod tests { let winning = provider_success.bids[0].clone(); let result = OrchestrationResult { provider_responses: vec![provider_success, provider_no_bid, provider_error], - mediator_response: None, + adserver_response: None, winning_bids: HashMap::from([("slot-1".to_owned(), winning)]), total_time_ms: 99, metadata: HashMap::new(), @@ -1274,7 +1274,7 @@ mod tests { let provider = AuctionResponse::success("aps-primary", vec![aps_bid.clone()], 12); let result = OrchestrationResult { provider_responses: vec![provider], - mediator_response: None, + adserver_response: None, winning_bids: HashMap::from([("slot-1".to_owned(), aps_bid.clone())]), total_time_ms: 12, metadata: HashMap::new(), @@ -1310,7 +1310,7 @@ mod tests { vec![fallback_bid.clone()], 12, )], - mediator_response: None, + adserver_response: None, winning_bids: HashMap::from([("slot-1".to_owned(), fallback_bid)]), total_time_ms: 12, metadata: HashMap::new(), @@ -1334,15 +1334,15 @@ mod tests { } #[test] - fn mediated_aps_telemetry_retains_provider_upstream_seat_and_delivery_identity() { + fn adserver_aps_telemetry_retains_provider_upstream_seat_and_delivery_identity() { let request = test_request("ts-ec-derived-id"); let mut aps_bid = bid("slot-1", "aps", Some("ad-1"), Some(1.25)); aps_bid.returned_seat = Some("upstream-seat".to_string()); let provider = AuctionResponse::success("aps-primary", vec![aps_bid.clone()], 12); - let mediator = AuctionResponse::success("adserver_mock", vec![aps_bid.clone()], 3); + let adserver = AuctionResponse::success("adserver_mock", vec![aps_bid.clone()], 3); let result = OrchestrationResult { provider_responses: vec![provider], - mediator_response: Some(mediator), + adserver_response: Some(adserver), winning_bids: HashMap::from([("slot-1".to_owned(), aps_bid.clone())]), total_time_ms: 15, metadata: HashMap::new(), @@ -1382,7 +1382,7 @@ mod tests { ); let result = OrchestrationResult { provider_responses: vec![provider_success.clone()], - mediator_response: None, + adserver_response: None, winning_bids: HashMap::from([("slot-1".to_owned(), provider_success.bids[0].clone())]), total_time_ms: 42, metadata: HashMap::new(), @@ -1423,7 +1423,7 @@ mod tests { .with_metadata("status", json!(403)); let result = OrchestrationResult { provider_responses: vec![provider_http_error], - mediator_response: None, + adserver_response: None, winning_bids: HashMap::new(), total_time_ms: 12, metadata: HashMap::new(), @@ -1452,17 +1452,17 @@ mod tests { } #[test] - fn mediated_win_marks_original_bid_once() { + fn adserver_win_marks_original_bid_once() { let request = test_request("req"); let original_bid = bid("slot-1", "kargo", Some("ad-1"), None); let provider_success = AuctionResponse::success("prebid", vec![original_bid], 42); - let mediator_bid = bid("slot-1", "kargo", Some("ad-1"), Some(2.0)); - let mediator_response = - AuctionResponse::success("adserver_mock", vec![mediator_bid.clone()], 15); + let adserver_bid = bid("slot-1", "kargo", Some("ad-1"), Some(2.0)); + let adserver_response = + AuctionResponse::success("adserver_mock", vec![adserver_bid.clone()], 15); let result = OrchestrationResult { provider_responses: vec![provider_success], - mediator_response: Some(mediator_response), - winning_bids: HashMap::from([("slot-1".to_owned(), mediator_bid)]), + adserver_response: Some(adserver_response), + winning_bids: HashMap::from([("slot-1".to_owned(), adserver_bid)]), total_time_ms: 80, metadata: HashMap::new(), }; @@ -1487,12 +1487,12 @@ mod tests { assert_eq!( winning_rows[0].provider.as_deref(), Some("prebid"), - "should mark original provider row when mediator winner matches" + "should mark original provider row when adserver winner matches" ); assert_eq!( winning_rows[0].price_cpm, Some(2.0), - "should copy mediator decoded price onto original null-price bid" + "should copy adserver decoded price onto original null-price bid" ); } @@ -1501,7 +1501,7 @@ mod tests { let request = test_request("ts-ec-derived-id"); let result = OrchestrationResult { provider_responses: Vec::new(), - mediator_response: None, + adserver_response: None, winning_bids: HashMap::new(), total_time_ms: 1, metadata: HashMap::new(), diff --git a/crates/trusted-server-core/src/auction/test_support.rs b/crates/trusted-server-core/src/auction/test_support.rs index a83f8899a8..411e1d268f 100644 --- a/crates/trusted-server-core/src/auction/test_support.rs +++ b/crates/trusted-server-core/src/auction/test_support.rs @@ -1,3 +1,6 @@ +//! Helpers for tests of the auction, in core and in an implementation's own +//! crate. + use std::collections::HashMap; use std::sync::LazyLock; @@ -17,7 +20,8 @@ use crate::settings::Settings; static TEST_SERVICES: LazyLock = LazyLock::new(noop_services); -pub(crate) fn create_test_auction_context<'a>( +/// An auction context over services that do nothing. +pub fn create_test_auction_context<'a>( settings: &'a Settings, request: &'a Request, timeout_ms: u32, @@ -33,13 +37,13 @@ pub(crate) fn create_test_auction_context<'a>( } } -/// Build canonical request facts shared by the PBS and APS Stage 1 wire goldens. +/// Build canonical request facts shared by the implementations' wire goldens. /// -/// The supported and unsupported formats deliberately exercise each profile's -/// existing filtering and field-ownership policy. `trustedServer` bidder -/// parameters are included to pin that PBS consumes them while APS ignores -/// them. -pub(crate) fn canonical_parity_auction_request() -> AuctionRequest { +/// The supported and unsupported formats deliberately exercise each +/// implementation's filtering and field-ownership policy. `trustedServer` +/// bidder parameters are included to pin which implementation consumes them +/// and which ignores them. +pub fn canonical_parity_auction_request() -> AuctionRequest { AuctionRequest { id: "fictional-auction".to_string(), slots: vec![AdSlot { @@ -119,3 +123,642 @@ pub(crate) fn canonical_parity_auction_request() -> AuctionRequest { context: HashMap::new(), } } + +/// The signing input the request goldens were captured with. +pub fn golden_signing_params() -> crate::request_signing::SigningParams { + crate::request_signing::SigningParams { + request_id: "fictional-auction".to_string(), + request_host: "publisher.example".to_string(), + request_scheme: "https".to_string(), + timestamp: 1_706_900_000, + } +} + +/// The one-source plan the request goldens were captured with: a source +/// called `fictional_provider`, running `implementation` with `settings`. +/// +/// The source takes every eligible slot when `all_eligible` is set. Otherwise +/// the bidder `exampleBidder` is routed to it, which is how a source that +/// takes only routed bidders is reached. +pub fn golden_plan_config( + implementation: &str, + settings: serde_json::Value, + all_eligible: bool, + extra: &[crate::integrations::IntegrationBuilder], +) -> crate::auction::plan::AuctionPlanConfig { + let mut table = demand_table(implementation, "https://exchange.example.test/openrtb"); + table.insert("timeout_ms".to_string(), json!(321)); + if all_eligible { + table.insert("routing".to_string(), json!("all_eligible")); + } + if let serde_json::Value::Object(settings) = settings { + table.extend(settings); + } + let mut config = plan_config_with(vec![("fictional_provider", table)], extra); + config.timeout_ms = 321; + if !all_eligible { + config.bidders.insert( + "exampleBidder".parse().expect("should parse bidder ID"), + crate::auction::plan::BidderRouteConfig { + module: "fictional_provider" + .parse() + .expect("should parse provider ID"), + }, + ); + } + config +} + +/// The inbound request the request goldens were captured with. +pub fn golden_inbound_request() -> Request { + Request::builder() + .uri("https://publisher.example/auction") + .header( + http::header::REFERER, + "https://referrer.example/story?fictional=1", + ) + .header(http::header::ACCEPT_LANGUAGE, "en-US,en;q=0.9") + .header("dnt", "1") + .body(EdgeBody::empty()) + .expect("should build inbound request") +} + +/// A signer over one fixed key, so a signed golden is the same on every run. +pub fn deterministic_signer() -> crate::request_signing::RequestSigner { + use base64::Engine as _; + + use crate::platform::test_support::{ + HashMapConfigStore, HashMapSecretStore, NoopHttpClient, + build_services_with_config_secret_and_http_client, + }; + + let config_data = HashMap::from([("current-kid".to_string(), "fictional-kid".to_string())]); + let secret_data = HashMap::from([( + "fictional-kid".to_string(), + base64::engine::general_purpose::STANDARD + .encode([7_u8; 32]) + .into_bytes(), + )]); + let services = build_services_with_config_secret_and_http_client( + HashMapConfigStore::new(config_data), + HashMapSecretStore::new(secret_data), + std::sync::Arc::new(NoopHttpClient), + ); + crate::request_signing::RequestSigner::from_services(&services) + .expect("should load deterministic signer") +} + +/// Builds the request the driver would send to the first demand source of +/// `plan` once `request` is routed to it, or `None` when the source keeps no +/// impression. +/// +/// A test in an implementation's own crate checks what its field policy and +/// its extensions do to a request through this. The signature, when `signer` +/// is given, is made over [`golden_signing_params`]. +/// +/// # Errors +/// +/// Returns the error the request builder returns. +pub fn build_for_first_source( + plan: &crate::auction::AuctionPlan, + request: AuctionRequest, + inbound: &Request, + transport_timeout_ms: u32, + signer: Option<&crate::request_signing::RequestSigner>, +) -> Result< + Option, + error_stack::Report, +> { + use crate::auction::openrtb::{OpenRtbBuildOutcome, RequestFinalization, build_request}; + + let routed = crate::auction::routing::route_auction(request, inbound, plan, None); + let outcome = build_request( + &routed.inputs()[0], + &routed, + &plan.providers()[0], + transport_timeout_ms, + &RequestFinalization { + signer, + signing_params: golden_signing_params(), + }, + )?; + Ok(match outcome { + OpenRtbBuildOutcome::Ready(request) => Some(request), + OpenRtbBuildOutcome::NoImpressions => None, + }) +} + +/// Routes `request` through `plan` and returns the input its first demand +/// source receives, or `None` when no source takes a slot. +/// +/// A test in an implementation's own crate hands this input to its own +/// response reader. +#[must_use] +pub fn route_to_first_source( + plan: &crate::auction::AuctionPlan, + request: AuctionRequest, + inbound: &Request, +) -> Option { + crate::auction::routing::route_auction(request, inbound, plan, None) + .inputs() + .first() + .cloned() +} + +/// The transport headers a routed auction admits from `inbound`, with the +/// attested client address it carries to every source's outbound request. +/// +/// A test in an implementation's own crate checks what it forwards upstream +/// through this. +#[must_use] +pub fn routed_transport( + plan: &crate::auction::AuctionPlan, + request: AuctionRequest, + inbound: &Request, + attested_client_ip: Option, +) -> ( + crate::auction::demand::TransportHeaders, + Option, +) { + let routed = crate::auction::routing::route_auction(request, inbound, plan, attested_client_ip); + ( + routed.transport_headers().clone(), + routed.attested_client_ip(), + ) +} + +/// Reads `response` as the first demand source of `plan` would, once +/// `request` is routed to it. +/// +/// A test in an implementation's own crate checks how its implementation +/// answers through this, with no upstream. +/// +/// # Errors +/// +/// Returns the error the source's own reading returns. +pub async fn parse_as_first_source( + plan: &crate::auction::AuctionPlan, + request: AuctionRequest, + response: crate::platform::PlatformResponse, + response_time_ms: u64, +) -> Result< + crate::auction::types::AuctionResponse, + error_stack::Report, +> { + let inbound = Request::new(EdgeBody::empty()); + let routed = crate::auction::routing::route_auction(request, &inbound, plan, None); + let provider = + crate::auction::provider::GenericOpenRtbProvider::new(plan.providers()[0].clone()); + let state = provider.parse_state_for_test(routed.inputs()[0].clone()); + provider + .parse_response_with_state(response, response_time_ms, Some(state.as_ref())) + .await +} + +/// One `[demand.]` table naming an implementation, for tests that need a +/// compiled plan. +pub fn demand_table( + implementation: &str, + endpoint: &str, +) -> serde_json::Map { + serde_json::Map::from_iter([ + ("implementation".to_string(), json!(implementation)), + ("endpoint".to_string(), json!(endpoint)), + ]) +} + +/// An `[demand]` table selecting every name given, in the order given. +pub fn demand_selection( + tables: Vec<(&str, serde_json::Map)>, +) -> crate::provider_table::ProviderList { + let selected = tables + .iter() + .map(|(name, _)| (*name).to_string()) + .collect::>(); + let tables = tables + .into_iter() + .map(|(name, table)| (name.to_string(), table)) + .collect::>(); + crate::provider_table::ProviderList::new(selected, tables) +} + +/// A plan configuration selecting the named demand tables, with every +/// implementation the built-in builders register. +pub fn plan_config( + tables: Vec<(&str, serde_json::Map)>, +) -> crate::auction::plan::AuctionPlanConfig { + plan_config_with(tables, &[]) +} + +/// A plan configuration selecting the named demand tables, with every +/// implementation the built-in builders register followed by the ones +/// `extra` registers, which is how a crate's own tests reach its +/// implementation. +pub fn plan_config_with( + tables: Vec<(&str, serde_json::Map)>, + extra: &[crate::integrations::IntegrationBuilder], +) -> crate::auction::plan::AuctionPlanConfig { + let builders = crate::integrations::all_builders(extra).collect::>(); + crate::auction::plan::AuctionPlanConfig { + timeout_ms: 1000, + demand: demand_selection(tables), + demand_implementations: builders + .iter() + .filter_map(crate::integrations::IntegrationBuilder::demand) + .collect(), + adserver_implementations: builders + .iter() + .filter_map(crate::integrations::IntegrationBuilder::adserver) + .collect(), + ..crate::auction::plan::AuctionPlanConfig::default() + } +} + +/// A `[demand]` selection of sources under the names given, each running +/// `implementation` at an endpoint of its own and taking every eligible slot. +pub fn demand_named(implementation: &str, names: &[&str]) -> crate::provider_table::ProviderList { + demand_selection( + names + .iter() + .map(|name| { + let mut table = demand_table( + implementation, + &format!("https://{name}.example/openrtb2/auction"), + ); + table.insert("routing".to_string(), json!("all_eligible")); + (*name, table) + }) + .collect(), + ) +} + +/// The legacy test orchestrator's configuration for these settings, carrying +/// the demand source names `[demand] modules` selects. +/// +/// Production compiles its sources from the plan instead, so this exists only +/// so the parity tests can drive the pre-plan orchestrator. +#[cfg(test)] +pub(crate) fn legacy_auction_config(settings: &Settings) -> crate::auction::AuctionConfig { + let mut config = settings.auction.clone(); + config.provider_names = settings + .demand + .selected() + .into_iter() + .map(str::to_string) + .collect(); + config +} + +/// A stand-in ad server implementation for core's own tests of the ad server +/// seam, selected with `[ad-server] module = "fixture"`. +/// +/// It reads an `endpoint`, which has to be a URL, and an optional +/// `timeout_ms` from its table and refuses any other setting, as an +/// implementation of a vendor's does. Its provider answers every request +/// with no bid and calls nothing. +#[cfg(test)] +pub(crate) mod adserver_fixture { + use std::sync::Arc; + + use async_trait::async_trait; + use error_stack::Report; + use serde::Deserialize; + + use crate::auction::demand::AdServerImplementation; + use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; + use crate::auction::types::{AuctionContext, AuctionRequest, AuctionResponse}; + use crate::error::TrustedServerError; + use crate::platform::PlatformResponse; + + /// The name the stand-in is selected by, which `[ad-server]` shortens to + /// `fixture`. + pub(crate) const MODULE: &str = "ad-server.fixture"; + + /// The stand-in ad server implementation. + pub(crate) static ADSERVER: AdServerImplementation = + AdServerImplementation { id: MODULE, build }; + + #[derive(Debug, Deserialize)] + #[serde(deny_unknown_fields)] + struct FixtureSettings { + endpoint: String, + #[serde(default = "default_timeout_ms")] + timeout_ms: u32, + } + + fn default_timeout_ms() -> u32 { + 500 + } + + fn build( + name: &str, + settings: &serde_json::Map, + ) -> Result, Report> { + let settings: FixtureSettings = serde_json::from_value(serde_json::Value::Object( + settings.clone(), + )) + .map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: format!("invalid `{MODULE}` settings: {error}"), + }) + })?; + url::Url::parse(&settings.endpoint).map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: format!("invalid `{MODULE}` endpoint: {error}"), + }) + })?; + Ok(Arc::new(FixtureAdServer::new(name, settings.timeout_ms))) + } + + /// The stand-in's provider. + pub(crate) struct FixtureAdServer { + name: String, + timeout_ms: u32, + } + + impl FixtureAdServer { + /// A provider called `name`. + pub(crate) fn new(name: &str, timeout_ms: u32) -> Self { + Self { + name: name.to_owned(), + timeout_ms, + } + } + } + + #[async_trait(?Send)] + impl AuctionProvider for FixtureAdServer { + fn provider_name(&self) -> &str { + &self.name + } + + async fn request_bids( + &self, + _request: &AuctionRequest, + _context: &AuctionContext<'_>, + ) -> Result> { + Ok(ProviderRequestOutcome::Immediate(AuctionResponse::no_bid( + self.name.clone(), + 0, + ))) + } + + async fn parse_response( + &self, + _response: PlatformResponse, + response_time_ms: u64, + ) -> Result> { + Ok(AuctionResponse::no_bid(self.name.clone(), response_time_ms)) + } + + fn timeout_ms(&self) -> u32 { + self.timeout_ms + } + } +} + +/// A stand-in demand implementation for core's own tests of what the driver +/// does for an implementation that departs from the plain one. It serves +/// stored requests, forbids all-eligible routing, takes bidder parameters, +/// keeps the unsigned request identity and completes an endpoint that names +/// a host alone. +/// +/// It writes each impression's usable bidder parameters under +/// `ext.fixture.bidder`, or the slot id under `ext.fixture.stored` where the +/// slot allows a stored request, and leaves out an impression that has +/// neither. +#[cfg(test)] +pub(crate) mod demand_fixture { + use core::any::Any; + use std::sync::Arc; + + use async_trait::async_trait; + use error_stack::Report; + use serde::Deserialize; + use serde_json::{Map, Value, json}; + use url::Url; + + use crate::auction::demand::{ + CompiledDemand, DemandFieldPolicy, DemandImplementation, DemandResponse, + DemandTimeoutDefault, ProviderAuctionInput, RegsPolicy, RequestExtensions, + }; + use crate::auction::types::AuctionResponse; + use crate::error::TrustedServerError; + use crate::platform::PlatformResponse; + + /// The name an `implementation` line gives the stand-in. + pub(crate) const MODULE: &str = "auction.fixture"; + + /// The path the stand-in completes a bare host with. + pub(crate) const AUCTION_PATH: &str = "/fixture/auction"; + + /// The stand-in demand implementation. + pub(crate) static DEMAND: DemandImplementation = DemandImplementation { + id: MODULE, + default_timeout: DemandTimeoutDefault::Fixed(1000), + allows_all_eligible: false, + serves_stored_requests: true, + canonicalize_endpoint, + compile, + }; + + /// One compiled stand-in source. + #[derive(Debug, Clone, Default)] + pub(crate) struct FixtureDemand; + + #[derive(Debug, Deserialize, Default)] + #[serde(deny_unknown_fields)] + struct FixtureSettings {} + + fn compile( + settings: &Map, + ) -> Result, Report> { + FixtureSettings::deserialize(Value::Object(settings.clone())).map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: format!("invalid `{MODULE}` settings: {error}"), + }) + })?; + Ok(Arc::new(FixtureDemand)) + } + + fn canonicalize_endpoint(endpoint: &mut Url) -> Result<(), String> { + if matches!(endpoint.path(), "" | "/") { + endpoint.set_path(AUCTION_PATH); + } + Ok(()) + } + + #[async_trait(?Send)] + impl CompiledDemand for FixtureDemand { + fn field_policy(&self) -> DemandFieldPolicy { + DemandFieldPolicy { + imp_tagid: true, + site_ref: true, + precise_geo: true, + additional_consent: true, + unsigned_request_identity: true, + consumes_bidder_params: true, + regs: RegsPolicy::Jurisdiction, + ..DemandFieldPolicy::default() + } + } + + fn augment_request( + &self, + extensions: &mut RequestExtensions<'_>, + _input: &ProviderAuctionInput, + ) -> Result<(), Report> { + for impression in &mut extensions.impressions { + let slot = impression.slot; + let bidder = slot + .bidder_params() + .iter() + .filter(|(_, params)| { + params.as_object().is_some_and(|params| !params.is_empty()) + }) + .map(|(bidder, params)| (bidder.as_str().to_string(), params.clone())) + .collect::>(); + let mut fixture = Map::new(); + if !bidder.is_empty() { + fixture.insert("bidder".to_string(), Value::Object(bidder)); + } else if slot.allows_stored_fallback() { + fixture.insert("stored".to_string(), json!(slot.slot().id)); + } + if fixture.is_empty() { + impression.omitted = true; + continue; + } + *impression.ext = Some(Map::from_iter([( + "fixture".to_string(), + Value::Object(fixture), + )])); + } + *extensions.request = Some(Map::from_iter([( + "fixture".to_string(), + Value::Object(Map::new()), + )])); + Ok(()) + } + + async fn parse_response( + &self, + context: DemandResponse<'_>, + response: PlatformResponse, + ) -> Result> { + crate::auction::openrtb::parse_openrtb_response(context, response).await + } + + fn as_any(&self) -> &dyn Any { + self + } + } +} + +/// A stand-in for the plainest demand implementation there can be, which +/// core's own tests of the plan compiler and the request driver name where +/// they need a source. +/// +/// It takes every eligible slot, inherits the auction's timeout, leaves its +/// endpoint as written and has its response read the ordinary way. Its table +/// may set `request_ext` and `imp_ext`, which it writes into the request and +/// into every impression as given, so the driver's tests have an +/// implementation that adds extensions. +#[cfg(test)] +pub(crate) mod plain_fixture { + use core::any::Any; + use std::sync::Arc; + + use async_trait::async_trait; + use error_stack::Report; + use serde::Deserialize; + use serde_json::{Map, Value}; + + use crate::auction::demand::{ + CONSERVATIVE_LANGUAGE_MAX_BYTES, CompiledDemand, DemandFieldPolicy, DemandImplementation, + DemandResponse, DemandTimeoutDefault, ProviderAuctionInput, RegsPolicy, RequestExtensions, + accept_endpoint, + }; + use crate::auction::types::AuctionResponse; + use crate::error::TrustedServerError; + use crate::platform::PlatformResponse; + + /// The name an `implementation` line gives the stand-in. + pub(crate) const MODULE: &str = "auction.plain-fixture"; + + /// The stand-in demand implementation. + pub(crate) static DEMAND: DemandImplementation = DemandImplementation { + id: MODULE, + default_timeout: DemandTimeoutDefault::Auction, + allows_all_eligible: true, + serves_stored_requests: false, + canonicalize_endpoint: accept_endpoint, + compile, + }; + + /// One compiled stand-in source. + #[derive(Debug, Clone, Default)] + pub(crate) struct PlainDemand { + request_ext: Map, + imp_ext: Map, + } + + #[derive(Debug, Deserialize, Default)] + #[serde(deny_unknown_fields)] + struct PlainSettings { + #[serde(default)] + request_ext: Map, + #[serde(default)] + imp_ext: Map, + } + + fn compile( + settings: &Map, + ) -> Result, Report> { + let settings = + PlainSettings::deserialize(Value::Object(settings.clone())).map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: format!("invalid `{MODULE}` settings: {error}"), + }) + })?; + Ok(Arc::new(PlainDemand { + request_ext: settings.request_ext, + imp_ext: settings.imp_ext, + })) + } + + #[async_trait(?Send)] + impl CompiledDemand for PlainDemand { + fn field_policy(&self) -> DemandFieldPolicy { + DemandFieldPolicy { + language_max_bytes: Some(CONSERVATIVE_LANGUAGE_MAX_BYTES), + regs: RegsPolicy::Jurisdiction, + accept_json: true, + ..DemandFieldPolicy::default() + } + } + + fn augment_request( + &self, + extensions: &mut RequestExtensions<'_>, + _input: &ProviderAuctionInput, + ) -> Result<(), Report> { + *extensions.request = (!self.request_ext.is_empty()).then(|| self.request_ext.clone()); + for impression in &mut extensions.impressions { + *impression.ext = (!self.imp_ext.is_empty()).then(|| self.imp_ext.clone()); + } + Ok(()) + } + + async fn parse_response( + &self, + context: DemandResponse<'_>, + response: PlatformResponse, + ) -> Result> { + crate::auction::openrtb::parse_openrtb_response(context, response).await + } + + fn as_any(&self) -> &dyn Any { + self + } + } +} diff --git a/crates/trusted-server-core/src/auction/types.rs b/crates/trusted-server-core/src/auction/types.rs index 4069157068..7e500f263e 100644 --- a/crates/trusted-server-core/src/auction/types.rs +++ b/crates/trusted-server-core/src/auction/types.rs @@ -1,11 +1,14 @@ //! Core types for auction requests and responses. use edgezero_core::body::Body as EdgeBody; +use error_stack::{Report, ResultExt as _, bail, ensure}; use http::Request; +use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, HashMap}; use crate::auction::context::ContextValue; +use crate::error::TrustedServerError; use crate::geo::GeoInfo; use crate::platform::RuntimeServices; use crate::settings::Settings; @@ -93,7 +96,7 @@ pub struct UserInfo { /// cookies/headers, not from stored data. #[serde(skip)] pub consent: Option, - /// Extended User IDs parsed from the [`crate::constants::COOKIE_TS_EIDS`] cookie. + /// Extended User IDs parsed from the [`crate::constants::COOKIE_EIDS`] cookie. /// /// Raw (un-gated) values from the browser; consent gating via /// [`crate::consent::gate_eids_by_consent`] is applied centrally in the @@ -129,14 +132,14 @@ pub struct SiteInfo { /// (DNT, User-Agent, cookies, X-* customs) directly off it. /// /// In the **collect path** ([`collect_dispatched_auction`][collect]) the -/// mediator is invoked with a synthetic placeholder request +/// ad server is invoked with a synthetic placeholder request /// (`https://placeholder.invalid/`), because the real client request has /// already been consumed by `send_async` during dispatch and the host pipeline -/// can't lend it across the `.await`. **Mediators must not depend on reading +/// can't lend it across the `.await`. **Ad servers must not depend on reading /// client state from `context.request`** — the placeholder has none of the -/// real headers. If a future mediator needs that data, snapshot it into a new +/// real headers. If a future ad server needs that data, snapshot it into a new /// field on this struct at dispatch time and stash it on the -/// [`DispatchedAuction`] token so collect can attach it to the mediator's +/// [`DispatchedAuction`] token so collect can attach it to the ad server's /// context. See /// (P2-1) for the open follow-up. /// @@ -154,15 +157,15 @@ pub struct AuctionContext<'a> { /// encode timers. Providers that register a backend should use this value /// for transport timers while retaining `timeout_ms` for logical policy. pub transport_timeout_ms: u32, - /// Provider responses from the bidding phase, used by mediators. - /// This is `None` for regular bidders and `Some` when calling a mediator. + /// Provider responses from the bidding phase, used by ad servers. + /// This is `None` for regular bidders and `Some` when calling an ad server. pub provider_responses: Option<&'a [AuctionResponse]>, /// Platform services (config store, secret store, etc.) for use by providers. pub services: &'a RuntimeServices, } -/// URL used by the orchestrator when invoking a mediator from the collect -/// path. Providers can `debug_assert` against this value to catch a mediator +/// URL used by the orchestrator when invoking an ad server from the collect +/// path. Providers can `debug_assert` against this value to catch an ad server /// that has accidentally started depending on `context.request` carrying real /// client headers. pub const MEDIATOR_PLACEHOLDER_URL: &str = "https://placeholder.invalid/"; @@ -182,56 +185,203 @@ pub struct AuctionResponse { pub metadata: HashMap, } -/// APS creative tag type accepted by the Trusted Server renderer. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum ApsTagType { - /// APS loads the creative URL in a nested iframe. - Iframe, - /// APS fetches creative HTML and executes it in its nested renderer frame. - Script, -} - -/// Version 1 APS renderer descriptor shared with browser clients. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ApsRendererV1 { - /// Renderer contract version. - pub version: u8, - /// APS account identifier used to initialize the fixed runner. - pub account_id: String, - /// Selected `OpenRTB` bid identifier. - pub bid_id: String, - /// Optional `OpenRTB` creative identifier. - #[serde(skip_serializing_if = "Option::is_none")] - pub creative_id: Option, - /// APS creative delivery mode. - pub tag_type: ApsTagType, - /// HTTPS creative URL consumed by the fixed APS runner. - pub creative_url: String, - /// Base64-encoded exact one-bid APS response envelope. - pub aax_response: String, - /// Creative width. - pub width: u32, - /// Creative height. - pub height: u32, -} +/// Wire key carrying the renderer type tag. +/// +/// A payload may not use this key, since it would collide with the tag when +/// the descriptor is serialized flat. +const RENDERER_TYPE_KEY: &str = "type"; -/// Typed browser renderer capability carried by a bid. +/// Browser renderer capability carried by a bid: a type tag and the payload +/// the auction provider that produced the bid defines. +/// +/// Serialized flat, as `{"type": "", ...payload}`, so a page receives +/// the same bytes whether the provider lives in core or in its own crate. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "lowercase")] -pub enum BidRenderer { - /// APS renderer version 1. - Aps(ApsRendererV1), +pub struct BidRenderer { + #[serde(rename = "type")] + renderer_type: String, + #[serde(flatten)] + payload: serde_json::Map, + /// The payload key the code that built the descriptor said holds the + /// identifier the renderer picks its bid by. A statement about the + /// payload and no part of what is sent. + #[serde(skip)] + bid_id_key: Option<&'static str>, } impl BidRenderer { - /// Return the APS renderer descriptor when this is an APS renderer. + /// Build a descriptor from a type tag and the provider's JSON payload. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::Auction`] when `payload` is not a JSON + /// object, or when it carries its own `type` key, which would collide with + /// the tag once the descriptor is serialized flat. + /// + /// # Examples + /// + /// ``` + /// use serde_json::json; + /// use trusted_server_core::auction::types::BidRenderer; + /// + /// let renderer = BidRenderer::new("example", json!({ "version": 1 })) + /// .expect("should accept an object payload"); + /// assert_eq!(renderer.renderer_type(), "example"); + /// ``` + pub fn new( + renderer_type: &str, + payload: serde_json::Value, + ) -> Result> { + let serde_json::Value::Object(payload) = payload else { + bail!(TrustedServerError::Auction { + message: format!("Renderer '{renderer_type}' payload must be a JSON object"), + }); + }; + ensure!( + !payload.contains_key(RENDERER_TYPE_KEY), + TrustedServerError::Auction { + message: format!( + "Renderer '{renderer_type}' payload must not carry a '{RENDERER_TYPE_KEY}' key" + ), + } + ); + Ok(Self { + renderer_type: renderer_type.to_string(), + payload, + bid_id_key: None, + }) + } + + /// Build a descriptor by serializing a provider's own payload type. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::Auction`] when `payload` cannot be + /// serialized, and when the serialized form is rejected by + /// [`new`](Self::new). + /// + /// # Examples + /// + /// ``` + /// use serde::Serialize; + /// use trusted_server_core::auction::types::BidRenderer; + /// + /// #[derive(Serialize)] + /// struct ExampleRendererV1 { + /// version: u8, + /// } + /// + /// let renderer = BidRenderer::from_typed("example", &ExampleRendererV1 { version: 1 }) + /// .expect("should accept a struct payload"); + /// assert_eq!(renderer.renderer_type(), "example"); + /// ``` + pub fn from_typed( + renderer_type: &str, + payload: &T, + ) -> Result> { + let payload = + serde_json::to_value(payload).change_context(TrustedServerError::Auction { + message: format!("Failed to serialize renderer '{renderer_type}' payload"), + })?; + Self::new(renderer_type, payload) + } + + /// Return the renderer type tag a page reads to select its renderer. #[must_use] - pub fn as_aps(&self) -> Option<&ApsRendererV1> { - match self { - Self::Aps(renderer) => Some(renderer), + pub fn renderer_type(&self) -> &str { + &self.renderer_type + } + + /// Deserialize the payload into the provider's own descriptor type. + /// + /// Returns `None` when the descriptor carries a different tag, and when the + /// payload does not match `T`. + /// + /// Clones the whole payload map and deserializes all of it, so use + /// [`payload_field`](Self::payload_field) when the caller wants one field. + /// An APS payload carries a base64 encoding of a creative envelope of up + /// to 256 KB. + #[must_use] + pub fn payload_as(&self, renderer_type: &str) -> Option { + if self.renderer_type != renderer_type { + return None; } + serde_json::from_value(serde_json::Value::Object(self.payload.clone())).ok() + } + + /// Borrow one field of the payload, copying nothing. + /// + /// Returns `None` when the descriptor carries a different tag, and when + /// the payload has no such key. `key` is the wire key, so a payload type + /// that renames its fields for serialization must be asked for the + /// renamed form. + /// + /// Unlike [`payload_as`](Self::payload_as) this reads the one field + /// asked for and does not check that the rest of the payload matches the + /// provider's descriptor type. + /// + /// # Examples + /// + /// ``` + /// use serde_json::json; + /// use trusted_server_core::auction::types::BidRenderer; + /// + /// let renderer = BidRenderer::new("example", json!({ "bidId": "fictional-bid-id" })) + /// .expect("should accept an object payload"); + /// + /// assert_eq!( + /// renderer + /// .payload_field("example", "bidId") + /// .and_then(serde_json::Value::as_str), + /// Some("fictional-bid-id"), + /// ); + /// assert!(renderer.payload_field("other", "bidId").is_none()); + /// ``` + #[must_use] + pub fn payload_field(&self, renderer_type: &str, key: &str) -> Option<&serde_json::Value> { + if self.renderer_type != renderer_type { + return None; + } + self.payload.get(key) + } + + /// States that the payload key `key` holds the identifier this renderer + /// picks its bid by. + /// + /// The statement is the builder's own about its payload. A descriptor + /// built without it has no such identifier, whatever its payload holds, + /// so a key of the same name under another renderer's type tag is not + /// read. + #[must_use] + pub fn picking_bid_by(mut self, key: &'static str) -> Self { + self.bid_id_key = Some(key); + self + } + + /// The identifier the renderer picks its bid by, when the code that built + /// the descriptor said which payload key holds it. + /// + /// # Examples + /// + /// ``` + /// use serde_json::json; + /// use trusted_server_core::auction::types::BidRenderer; + /// + /// let payload = json!({ "bidId": "fictional-bid-id" }); + /// let unstated = BidRenderer::new("example", payload.clone()) + /// .expect("should accept an object payload"); + /// assert_eq!(unstated.bid_id(), None); + /// + /// let stated = BidRenderer::new("example", payload) + /// .expect("should accept an object payload") + /// .picking_bid_by("bidId"); + /// assert_eq!(stated.bid_id(), Some("fictional-bid-id")); + /// ``` + #[must_use] + pub fn bid_id(&self) -> Option<&str> { + self.payload + .get(self.bid_id_key?) + .and_then(serde_json::Value::as_str) } } @@ -599,59 +749,177 @@ mod tests { ); } - #[test] - fn aps_renderer_serializes_to_versioned_camel_case_contract() { - let renderer = BidRenderer::Aps(ApsRendererV1 { + /// A provider's own payload type, as an implementation in a crate of its + /// own defines one. + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] + #[serde(rename_all = "camelCase")] + struct ExampleRenderer { + version: u8, + bid_id: String, + creative_url: String, + } + + const EXAMPLE_RENDERER_TYPE: &str = "example"; + + fn example_descriptor() -> ExampleRenderer { + ExampleRenderer { version: 1, - account_id: "example-account-id".to_string(), bid_id: "fictional-bid-id".to_string(), - creative_id: Some("fictional-creative-id".to_string()), - tag_type: ApsTagType::Iframe, creative_url: "https://creative.example/render".to_string(), - aax_response: "base64-data".to_string(), - width: 300, - height: 250, - }); + } + } - let serialized = serde_json::to_value(&renderer).expect("should serialize renderer"); + #[test] + fn renderer_serializes_flat_under_its_type_tag() { + let renderer = BidRenderer::from_typed(EXAMPLE_RENDERER_TYPE, &example_descriptor()) + .expect("should build renderer descriptor"); assert_eq!( - serialized, + serde_json::to_value(&renderer).expect("should serialize renderer"), json!({ - "type": "aps", + "type": "example", "version": 1, - "accountId": "example-account-id", "bidId": "fictional-bid-id", - "creativeId": "fictional-creative-id", - "tagType": "iframe", - "creativeUrl": "https://creative.example/render", - "aaxResponse": "base64-data", - "width": 300, - "height": 250 + "creativeUrl": "https://creative.example/render" }), - "should match renderer wire contract" + "should carry the tag beside the payload's own keys" ); } #[test] - fn aps_renderer_omits_absent_creative_id() { - let renderer = BidRenderer::Aps(ApsRendererV1 { - version: 1, - account_id: "example-account-id".to_string(), - bid_id: "fictional-bid-id".to_string(), - creative_id: None, - tag_type: ApsTagType::Iframe, - creative_url: "https://creative.example/render".to_string(), - aax_response: "base64-data".to_string(), - width: 300, - height: 250, - }); + fn renderer_round_trips_through_its_wire_form() { + let descriptor = example_descriptor(); + let renderer = BidRenderer::from_typed(EXAMPLE_RENDERER_TYPE, &descriptor) + .expect("should build renderer descriptor"); + + let serialized = serde_json::to_string(&renderer).expect("should serialize renderer"); + let restored: BidRenderer = + serde_json::from_str(&serialized).expect("should deserialize renderer"); + + assert_eq!( + restored.renderer_type(), + EXAMPLE_RENDERER_TYPE, + "should round-trip the renderer type tag" + ); + assert_eq!( + restored + .payload_as::(EXAMPLE_RENDERER_TYPE) + .expect("should deserialize the payload"), + descriptor, + "should round-trip the provider payload" + ); + } + + #[test] + fn renderer_payload_is_hidden_from_a_different_type_tag() { + let renderer = BidRenderer::new(EXAMPLE_RENDERER_TYPE, json!({ "version": 1 })) + .expect("should build renderer descriptor"); + + assert!( + renderer.payload_as::("other").is_none(), + "should refuse a payload requested under a different tag" + ); + } + + #[test] + fn renderer_payload_field_borrows_the_same_value_the_whole_descriptor_carries() { + let descriptor = example_descriptor(); + let renderer = BidRenderer::from_typed(EXAMPLE_RENDERER_TYPE, &descriptor) + .expect("should build renderer descriptor"); + + assert_eq!( + renderer + .payload_field(EXAMPLE_RENDERER_TYPE, "bidId") + .and_then(serde_json::Value::as_str), + Some(descriptor.bid_id.as_str()), + "should read the same bid id the whole descriptor carries" + ); + assert_eq!( + renderer + .payload_field(EXAMPLE_RENDERER_TYPE, "bidId") + .and_then(serde_json::Value::as_str), + renderer + .payload_as::(EXAMPLE_RENDERER_TYPE) + .as_ref() + .map(|full| full.bid_id.as_str()), + "should agree with the field read through the whole descriptor" + ); + assert!( + renderer + .payload_field(EXAMPLE_RENDERER_TYPE, "notAKey") + .is_none(), + "should return nothing for a key the payload does not carry" + ); + } + + #[test] + fn renderer_payload_field_is_hidden_from_a_different_type_tag() { + let renderer = BidRenderer::new( + EXAMPLE_RENDERER_TYPE, + json!({ "bidId": "fictional-bid-id" }), + ) + .expect("should build renderer descriptor"); + + assert!( + renderer.payload_field("other", "bidId").is_none(), + "should refuse a field requested under a different tag" + ); + } + + #[test] + fn renderer_names_the_bid_it_picks_only_when_its_builder_said_which_key_holds_it() { + let unstated = BidRenderer::from_typed(EXAMPLE_RENDERER_TYPE, &example_descriptor()) + .expect("should build renderer descriptor"); + assert_eq!( + unstated.bid_id(), + None, + "should name no bid for a descriptor whose builder made no statement" + ); + + let stated = unstated.clone().picking_bid_by("bidId"); + assert_eq!( + stated.bid_id(), + Some("fictional-bid-id"), + "should read the key the builder stated" + ); + assert_eq!( + stated.clone().bid_id(), + Some("fictional-bid-id"), + "should keep the statement when the descriptor is cloned" + ); + assert_eq!( + serde_json::to_value(&stated).expect("should serialize renderer"), + serde_json::to_value(&unstated).expect("should serialize renderer"), + "should send the same bytes with or without the statement" + ); - let serialized = serde_json::to_value(&renderer).expect("should serialize renderer"); + let absent = BidRenderer::new(EXAMPLE_RENDERER_TYPE, json!({ "version": 1 })) + .expect("should build renderer descriptor") + .picking_bid_by("bidId"); + assert_eq!( + absent.bid_id(), + None, + "should name no bid when the payload does not carry the stated key" + ); + } + #[test] + fn renderer_rejects_a_payload_that_is_not_an_object() { + assert!( + BidRenderer::new(EXAMPLE_RENDERER_TYPE, json!("not-an-object")).is_err(), + "should reject a payload that is not a JSON object" + ); + } + + #[test] + fn renderer_rejects_a_payload_carrying_its_own_type_key() { assert!( - serialized.get("creativeId").is_none(), - "should omit absent creative ID" + BidRenderer::new( + EXAMPLE_RENDERER_TYPE, + json!({ "type": "other", "version": 1 }) + ) + .is_err(), + "should reject a payload that would collide with the type tag" ); } diff --git a/crates/trusted-server-core/src/auction_config_types.rs b/crates/trusted-server-core/src/auction_config_types.rs index 56171e1ea5..9a7f5ab019 100644 --- a/crates/trusted-server-core/src/auction_config_types.rs +++ b/crates/trusted-server-core/src/auction_config_types.rs @@ -1,20 +1,22 @@ //! Auction configuration types shared by settings and auction planning. -use serde::de::{Error as _, MapAccess, SeqAccess, Visitor, value::MapAccessDeserializer}; +use serde::de::Error as _; use serde::{Deserialize, Deserializer, Serialize}; use std::collections::{BTreeMap, BTreeSet}; -use std::fmt; use validator::Validate; -const LEGACY_PROVIDER_LIST_MESSAGE: &str = "Configuration field `auction.providers` uses the removed list schema; migrate to `[auction.providers.]` map entries as described in the CHANGELOG.md breaking migration"; +const MOVED_PROVIDERS_MESSAGE: &str = "`[auction.providers]` has moved. Select demand sources with `[demand] modules = [...]` and give each its settings in `[demand.]`, as the configuration rules describe"; + +const MOVED_MEDIATOR_MESSAGE: &str = "`[auction] mediator` has moved. Select the ad server with `[ad-server] module = \"\"` and give it its settings in `[ad-server.]`, as the configuration rules describe"; + +use crate::provider_table::SectionModules; pub use crate::auction::plan::{ - BidderId, BidderRouteConfig, NotificationConfig, ProviderConfig, ProviderId, RoutingMode, + BidderId, BidderRouteConfig, NotificationConfig, ProviderId, RoutingMode, }; /// Auction orchestration configuration. #[derive(Debug, Clone, Deserialize, Serialize, Validate)] -#[serde(deny_unknown_fields)] pub struct AuctionConfig { /// Enable the auction orchestrator #[serde(default)] @@ -49,18 +51,45 @@ pub struct AuctionConfig { )] pub rewrite_creatives: bool, - /// Operator-defined bidder-provider instances, keyed by provider ID. - #[serde(default, deserialize_with = "deserialize_provider_map")] - pub providers: BTreeMap, + /// Refuses the removed `providers` table with a message naming its new + /// home, so an old configuration fails with the fix rather than as an + /// unknown field. + #[serde(default, skip_serializing, deserialize_with = "reject_moved_providers")] + #[allow( + dead_code, + reason = "the field exists so deserialization refuses the removed table" + )] + pub(crate) providers: MovedSetting, - /// Client-visible bidder routes, keyed by bidder code. + /// Client-visible bidder routes, keyed by bidder code, each naming the + /// `[demand]` provider the bidder is sent to. #[serde(default)] pub bidders: BTreeMap, - /// Optional separately registered mediator provider name. - /// When set, runs parallel mediation strategy (bidders in parallel, then mediator decides) - /// When omitted, runs parallel only strategy (bidders in parallel, highest CPM wins) - pub mediator: Option, + /// The ad server name the legacy test orchestrator runs. + /// + /// Production selects its ad server in `[ad-server]` instead, so this never + /// appears in a configuration file. + #[cfg(test)] + #[serde(skip)] + pub(crate) adserver_name: Option, + + /// The demand source names the legacy test orchestrator runs. + /// + /// Production compiles its sources from `[demand]` instead, so this never + /// appears in a configuration file. + #[cfg(test)] + #[serde(skip)] + pub(crate) provider_names: Vec, + + /// Refuses the removed `mediator` setting with a message naming its new + /// home. + #[serde(default, skip_serializing, deserialize_with = "reject_moved_mediator")] + #[allow( + dead_code, + reason = "the field exists so deserialization refuses the removed setting" + )] + pub(crate) mediator: MovedSetting, /// Timeout in milliseconds #[serde(default = "default_timeout")] @@ -68,7 +97,10 @@ pub struct AuctionConfig { pub timeout_ms: u32, /// KV store name for creative storage (deprecated: creatives are now delivered inline) - #[serde(default = "default_creative_store")] + #[serde( + default = "default_creative_store", + serialize_with = "crate::redacted::sensitive" + )] pub creative_store: String, /// Keys allowed in the auction request context map. @@ -79,6 +111,11 @@ pub struct AuctionConfig { /// deduplicated and serialized in sorted order so config hashes are stable. #[serde(default = "default_allowed_context_keys")] pub allowed_context_keys: BTreeSet, + + /// The modules this section selects, with each one's settings in the + /// table at its name. + #[serde(flatten)] + pub modules: SectionModules, } impl Default for AuctionConfig { @@ -87,47 +124,47 @@ impl Default for AuctionConfig { enabled: false, sanitize_creatives: default_sanitize_creatives(), rewrite_creatives: default_rewrite_creatives(), - providers: BTreeMap::new(), + providers: MovedSetting, + #[cfg(test)] + adserver_name: None, + #[cfg(test)] + provider_names: Vec::new(), bidders: BTreeMap::new(), - mediator: None, + mediator: MovedSetting, timeout_ms: default_timeout(), creative_store: default_creative_store(), allowed_context_keys: BTreeSet::new(), + modules: SectionModules::default(), } } } -fn deserialize_provider_map<'de, D>( - deserializer: D, -) -> Result, D::Error> +#[cfg(test)] +impl AuctionConfig { + /// Whether the legacy test orchestrator runs an ad server. + pub(crate) fn has_adserver(&self) -> bool { + self.adserver_name.is_some() + } +} + +/// A setting that has moved elsewhere. It deserializes only by failing. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) struct MovedSetting; + +fn reject_moved_providers<'de, D>(deserializer: D) -> Result where D: Deserializer<'de>, { - struct ProviderMapVisitor; - - impl<'de> Visitor<'de> for ProviderMapVisitor { - type Value = BTreeMap; - - fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str("a map of auction provider IDs to provider configurations") - } - - fn visit_map(self, map: A) -> Result - where - A: MapAccess<'de>, - { - Self::Value::deserialize(MapAccessDeserializer::new(map)) - } - - fn visit_seq(self, _sequence: A) -> Result - where - A: SeqAccess<'de>, - { - Err(A::Error::custom(LEGACY_PROVIDER_LIST_MESSAGE)) - } - } + let _ = serde::de::IgnoredAny::deserialize(deserializer)?; + Err(D::Error::custom(MOVED_PROVIDERS_MESSAGE)) +} - deserializer.deserialize_any(ProviderMapVisitor) +fn reject_moved_mediator<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let _ = serde::de::IgnoredAny::deserialize(deserializer)?; + Err(D::Error::custom(MOVED_MEDIATOR_MESSAGE)) } fn default_timeout() -> u32 { @@ -159,36 +196,6 @@ fn default_allowed_context_keys() -> BTreeSet { BTreeSet::new() } -impl AuctionConfig { - #[cfg(test)] - pub(crate) fn legacy_provider_map(names: &[&str]) -> BTreeMap { - names - .iter() - .map(|name| { - let id = ProviderId::unchecked_for_legacy_test(name); - ( - id, - ProviderConfig { - protocol: "openrtb-2.6".to_string(), - profile: "standard".to_string(), - endpoint: format!("https://{name}.example/openrtb2/auction"), - timeout_ms: None, - routing: RoutingMode::AllEligible, - notifications: NotificationConfig::default(), - profile_config: serde_json::json!({}), - }, - ) - }) - .collect() - } - - /// Check if this config has a mediator configured. - #[must_use] - pub fn has_mediator(&self) -> bool { - self.mediator.is_some() - } -} - #[cfg(test)] mod tests { use super::*; @@ -283,35 +290,50 @@ mod tests { } #[test] - fn provider_list_shape_is_rejected() { + fn moved_providers_fail_naming_the_demand_table() { + for providers in [ + serde_json::json!(["prebid"]), + serde_json::json!({ "pbs_main": { "endpoint": "https://prebid.example" } }), + ] { + let error = serde_json::from_value::(serde_json::json!({ + "providers": providers + })) + .expect_err("should refuse the moved providers table"); + + assert!( + error.to_string().contains("[demand] modules"), + "should name the new home: {error}" + ); + } + } + + #[test] + fn moved_mediator_fails_naming_the_adserver_table() { let error = serde_json::from_value::(serde_json::json!({ - "providers": ["prebid"] + "mediator": "adserver_mock" })) - .expect_err("should reject the removed provider-list schema"); + .expect_err("should refuse the moved mediator setting"); assert!( - error.to_string().contains("map") || error.to_string().contains("object"), - "should require map-shaped providers: {error}" + error.to_string().contains("[ad-server] module"), + "should name the new home: {error}" ); } #[test] - fn map_schema_round_trips_provider_and_bidder_routes() { + fn bidder_routes_round_trip() { let config: AuctionConfig = serde_json::from_value(serde_json::json!({ - "providers": { - "pbs-main": { - "protocol": "openrtb-2.6", - "profile": "prebid-server", - "endpoint": "https://prebid.example/openrtb2/auction" - } - }, "bidders": { - "example-bidder": { "provider": "pbs-main" } + "example-bidder": { "module": "pbs_main" } } })) - .expect("should parse map-shaped auction config"); + .expect("should parse bidder routes"); - assert_eq!(config.providers.len(), 1); assert_eq!(config.bidders.len(), 1); + let serialized = serde_json::to_value(&config).expect("should serialize"); + assert!( + serialized.get("providers").is_none() && serialized.get("mediator").is_none(), + "should never write the moved settings back" + ); } } diff --git a/crates/trusted-server-core/src/auth.rs b/crates/trusted-server-core/src/auth.rs deleted file mode 100644 index 9bbd40b69a..0000000000 --- a/crates/trusted-server-core/src/auth.rs +++ /dev/null @@ -1,607 +0,0 @@ -use base64::{Engine as _, engine::general_purpose::STANDARD}; -use edgezero_core::body::Body as EdgeBody; -use error_stack::Report; -use http::header; -use http::{Request, Response, StatusCode}; -use sha2::{Digest as _, Sha256}; -use subtle::ConstantTimeEq as _; - -use crate::error::TrustedServerError; -use crate::settings::Settings; - -const BASIC_AUTH_REALM: &str = r#"Basic realm="Trusted Server""#; - -/// Marks the single `Authorization` value Trusted Server validated. -/// -/// The shared template cache may exempt this value from its normal authorization -/// bypass. [`enforce_basic_auth`] clears any existing marker before checking and -/// inserts a digest-bound marker only after successful authentication. -#[derive(Debug, Clone)] -pub(crate) struct EdgeTerminatedAuthorization([u8; 32]); - -impl EdgeTerminatedAuthorization { - fn digest(value: &[u8]) -> [u8; 32] { - Sha256::digest(value).into() - } - - pub(crate) fn matches(&self, headers: &http::HeaderMap) -> bool { - let mut values = headers.get_all(header::AUTHORIZATION).iter(); - let Some(value) = values.next() else { - return false; - }; - values.next().is_none() && self.0 == Self::digest(value.as_bytes()) - } - - /// Builds the marker without performing a credential check. - /// - /// Test-only. Production code obtains this marker exclusively by passing - /// [`enforce_basic_auth`], which is what makes it meaningful. - #[cfg(test)] - #[must_use] - pub(crate) fn for_test(value: &str) -> Self { - Self(Self::digest(value.as_bytes())) - } -} - -/// Enforces HTTP Basic authentication for configured handler paths. -/// -/// Returns `Ok(None)` when the request does not target a protected handler or -/// when the supplied credentials are valid. Returns `Ok(Some(Response))` with -/// the auth challenge when credentials are missing or invalid. -/// -/// Admin endpoints are protected by requiring a handler during settings -/// finalization; see [`Settings::from_toml`]. Credential checks use constant-time -/// comparison for both username and password, and evaluate both regardless of -/// individual match results to avoid timing oracles. Runtime requests within -/// the reserved admin namespace fail closed if no handler matches, providing -/// defense in depth for malformed and parameterized paths. -/// -/// # Request mutation -/// -/// Takes `req` mutably because it owns [`EdgeTerminatedAuthorization`]. Any -/// inherited marker is cleared on entry, and a fresh one is inserted only on the -/// success path, so the marker present after this call always describes this -/// call's own decision. Nothing else about the request is touched — in -/// particular the `Authorization` header is left in place and still reaches the -/// publisher origin. -/// -/// That last point is a stated assumption: a credential this edge terminates is -/// treated as reader-neutral, which holds unless the origin *also* authenticates -/// on the same header. An origin that does so declares `Vary: Authorization`, -/// which the template-cache store refuses as an uncovered `Vary` name. An origin -/// that varies on `Authorization` without declaring it would defeat any HTTP -/// cache, and is out of scope here. -/// -/// # Errors -/// -/// Returns an error when handler configuration is invalid, such as an -/// un-compilable path regex. -pub fn enforce_basic_auth( - settings: &Settings, - req: &mut Request, -) -> Result>, Report> { - // Cleared before any early return so no inherited marker can survive a call - // that did not itself validate a credential. Without this, a request marked - // upstream and then routed to an unprotected path would keep an assertion - // nothing checked. - req.extensions_mut().remove::(); - - let path = req.uri().path(); - let Some(handler) = settings.handler_for_path(path)? else { - if Settings::is_admin_path(path) { - return Err(Report::new(TrustedServerError::Configuration { - message: format!("Admin path `{path}` has no configured handler"), - })); - } - return Ok(None); - }; - - let Some((username, password, authorization_digest)) = extract_credentials(req) else { - return Ok(Some(unauthorized_response())); - }; - - // Hash before comparing to normalise lengths — `ct_eq` on raw byte slices - // short-circuits when lengths differ, which would leak credential length. - // SHA-256 produces fixed-size digests so the comparison is truly constant-time. - // - // Note: constant-time guarantees are best-effort on WASM targets because the - // runtime optimiser/JIT may re-introduce variable-time paths. This is an - // inherent limitation of all constant-time code in managed runtimes. - let username_match = Sha256::digest(handler.username.expose().as_bytes()) - .ct_eq(&Sha256::digest(username.as_bytes())); - let password_match = Sha256::digest(handler.password.expose().as_bytes()) - .ct_eq(&Sha256::digest(password.as_bytes())); - - if bool::from(username_match & password_match) { - // Record that TS itself consumed this credential, so the shared template - // cache can distinguish it from a credential meant for the origin. - req.extensions_mut() - .insert(EdgeTerminatedAuthorization(authorization_digest)); - Ok(None) - } else { - log::warn!("Basic auth failed for path: {}", req.uri().path()); - Ok(Some(unauthorized_response())) - } -} - -/// Username from a basic-auth request, for audit logging. -/// -/// Returns the username alone. The password is a shared static secret and must never reach -/// a log line. -/// -/// This parses a header; it verifies nothing. Call it only on a request -/// [`enforce_basic_auth`] has already accepted, where the username identifies which -/// operator credential was used. -#[must_use] -pub fn authenticated_username(req: &Request) -> Option { - extract_credentials(req).map(|(username, _password, _digest)| username) -} - -fn extract_credentials(req: &Request) -> Option<(String, String, [u8; 32])> { - let mut header_values = req.headers().get_all(header::AUTHORIZATION).iter(); - let header_value = header_values.next()?; - if header_values.next().is_some() { - return None; - } - let authorization_digest = EdgeTerminatedAuthorization::digest(header_value.as_bytes()); - let header_value = header_value.to_str().ok()?; - - let mut parts = header_value.splitn(2, ' '); - let scheme = parts.next()?.trim(); - if !scheme.eq_ignore_ascii_case("basic") { - return None; - } - - let token = parts.next()?.trim(); - if token.is_empty() { - return None; - } - - let decoded = STANDARD.decode(token).ok()?; - let credentials = String::from_utf8(decoded).ok()?; - - let mut credentials_parts = credentials.splitn(2, ':'); - let username = credentials_parts.next()?.to_owned(); - let password = credentials_parts.next()?.to_owned(); - - Some((username, password, authorization_digest)) -} - -fn unauthorized_response() -> Response { - Response::builder() - .status(StatusCode::UNAUTHORIZED) - .header(header::WWW_AUTHENTICATE, BASIC_AUTH_REALM) - .header(header::CONTENT_TYPE, "text/plain; charset=utf-8") - .body(EdgeBody::from(b"Unauthorized".as_ref())) - .expect("should build unauthorized response") -} - -#[cfg(test)] -mod tests { - use super::*; - use base64::engine::general_purpose::STANDARD; - use http::{HeaderValue, Method, header}; - - use crate::test_support::tests::{crate_test_settings_str, create_test_settings}; - - fn build_request(method: Method, uri: &str) -> Request { - Request::builder() - .method(method) - .uri(uri) - .body(EdgeBody::empty()) - .expect("should build request") - } - - fn set_authorization(req: &mut Request, value: &str) { - req.headers_mut().insert( - header::AUTHORIZATION, - HeaderValue::from_str(value).expect("should build authorization header"), - ); - } - - #[test] - fn encoded_admin_separator_path_is_auth_gated() { - // `^/_ts/admin` matches the raw path, so a percent-encoded separator - // still consumes admin credentials. The publisher-fallback boundary - // reserves the same paths so those credentials are never forwarded - // upstream (see `ec::admin::deny_admin_diagnostic_fallback`). - let settings = create_test_settings(); - - for path in ["/_ts/admin%2Fec", "/_ts/admin%2fec"] { - let mut req = build_request(Method::GET, &format!("https://example.com{path}")); - - let response = enforce_basic_auth(&settings, &mut req) - .expect("should evaluate auth") - .unwrap_or_else(|| panic!("should challenge {path}")); - - assert_eq!( - response.status(), - StatusCode::UNAUTHORIZED, - "should require credentials for {path}" - ); - } - } - - #[test] - fn valid_credentials_mark_the_request_as_edge_terminated() { - let settings = create_test_settings(); - let mut req = build_request(Method::GET, "https://example.com/secure"); - let encoded = STANDARD.encode("user:pass"); - set_authorization(&mut req, &format!("Basic {encoded}")); - - assert!( - enforce_basic_auth(&settings, &mut req) - .expect("should evaluate auth") - .is_none(), - "valid credentials should be admitted" - ); - let marker = req - .extensions() - .get::() - .expect("should mark a credential this edge consumed"); - assert!( - marker.matches(req.headers()), - "the marker should match the unchanged validated authorization" - ); - - req.headers_mut().insert( - header::AUTHORIZATION, - HeaderValue::from_static("Bearer publisher-origin-credential"), - ); - let marker = req - .extensions() - .get::() - .expect("should retain the marker after an unrelated mutation"); - assert!( - !marker.matches(req.headers()), - "the marker must not match a replacement authorization value" - ); - } - - #[test] - fn repeated_authorization_values_are_rejected_without_a_marker() { - let settings = create_test_settings(); - let mut req = build_request(Method::GET, "https://example.com/secure"); - let encoded = STANDARD.encode("user:pass"); - set_authorization(&mut req, &format!("Basic {encoded}")); - req.headers_mut().append( - header::AUTHORIZATION, - HeaderValue::from_static("Bearer publisher-origin-credential"), - ); - - let response = enforce_basic_auth(&settings, &mut req) - .expect("should evaluate auth") - .expect("should challenge an ambiguous credential"); - assert_eq!(response.status(), StatusCode::UNAUTHORIZED); - assert!( - req.extensions() - .get::() - .is_none(), - "a repeated authorization field must remain pass-through rather than being marked safe" - ); - } - - #[test] - fn an_inherited_marker_is_cleared_on_an_unprotected_path() { - // The marker asserts "this edge already checked a credential". A request - // routed to a path no handler protects was never checked here, so a marker - // it arrived with must not survive to grant shared-template eligibility. - let settings = create_test_settings(); - let mut req = build_request(Method::GET, "https://example.com/open"); - set_authorization(&mut req, "Basic dXNlcjpwYXNz"); - req.extensions_mut() - .insert(EdgeTerminatedAuthorization::for_test("Basic dXNlcjpwYXNz")); - - assert!( - enforce_basic_auth(&settings, &mut req) - .expect("should evaluate auth") - .is_none(), - "an unprotected path should not challenge" - ); - assert!( - req.extensions() - .get::() - .is_none(), - "a marker no credential check produced must not survive this call" - ); - } - - #[test] - fn an_inherited_marker_is_cleared_when_credentials_are_rejected() { - let settings = create_test_settings(); - let mut req = build_request(Method::GET, "https://example.com/secure"); - let encoded = STANDARD.encode("user:wrong-pass"); - set_authorization(&mut req, &format!("Basic {encoded}")); - req.extensions_mut() - .insert(EdgeTerminatedAuthorization::for_test( - "Basic dXNlcjp3cm9uZy1wYXNz", - )); - - let response = enforce_basic_auth(&settings, &mut req) - .expect("should evaluate auth") - .expect("should challenge"); - assert_eq!(response.status(), StatusCode::UNAUTHORIZED); - assert!( - req.extensions() - .get::() - .is_none(), - "a failed check must strip an inherited marker rather than honour it" - ); - } - - #[test] - fn a_non_protected_path_leaves_authorization_unmarked() { - let settings = create_test_settings(); - let mut req = build_request(Method::GET, "https://example.com/open"); - set_authorization(&mut req, "Basic dXNlcjpwYXNz"); - - assert!( - enforce_basic_auth(&settings, &mut req) - .expect("should evaluate auth") - .is_none(), - "an unprotected path should not challenge" - ); - assert!( - req.extensions() - .get::() - .is_none(), - "a credential no handler consumed is pass-through and must stay disqualifying" - ); - } - - #[test] - fn rejected_credentials_leave_the_request_unmarked() { - let settings = create_test_settings(); - let mut req = build_request(Method::GET, "https://example.com/secure"); - let encoded = STANDARD.encode("user:wrong-pass"); - set_authorization(&mut req, &format!("Basic {encoded}")); - - let response = enforce_basic_auth(&settings, &mut req) - .expect("should evaluate auth") - .expect("should challenge"); - assert_eq!(response.status(), StatusCode::UNAUTHORIZED); - assert!( - req.extensions() - .get::() - .is_none(), - "a failed credential must never be marked as terminated" - ); - } - - #[test] - fn no_challenge_for_non_protected_path() { - let settings = create_test_settings(); - let mut req = build_request(Method::GET, "https://example.com/open"); - - assert!( - enforce_basic_auth(&settings, &mut req) - .expect("should evaluate auth") - .is_none() - ); - } - - #[test] - fn challenge_when_missing_credentials() { - let settings = create_test_settings(); - let mut req = build_request(Method::GET, "https://example.com/secure"); - - let response = enforce_basic_auth(&settings, &mut req) - .expect("should evaluate auth") - .expect("should challenge"); - assert_eq!(response.status(), StatusCode::UNAUTHORIZED); - let realm = response - .headers() - .get(header::WWW_AUTHENTICATE) - .expect("should have WWW-Authenticate header"); - assert_eq!(realm, BASIC_AUTH_REALM); - } - - #[test] - fn allow_when_credentials_match() { - let settings = create_test_settings(); - let mut req = build_request(Method::GET, "https://example.com/secure/data"); - let token = STANDARD.encode("user:pass"); - set_authorization(&mut req, &format!("Basic {token}")); - - assert!( - enforce_basic_auth(&settings, &mut req) - .expect("should evaluate auth") - .is_none() - ); - } - - #[test] - fn challenge_when_both_credentials_wrong() { - let settings = create_test_settings(); - let mut req = build_request(Method::GET, "https://example.com/secure/data"); - let token = STANDARD.encode("wrong:wrong"); - set_authorization(&mut req, &format!("Basic {token}")); - - let response = enforce_basic_auth(&settings, &mut req) - .expect("should evaluate auth") - .expect("should challenge"); - assert_eq!(response.status(), StatusCode::UNAUTHORIZED); - } - - #[test] - fn challenge_when_username_wrong_password_correct() { - // Validates that both fields are always evaluated — no short-circuit username oracle. - let settings = create_test_settings(); - let mut req = build_request(Method::GET, "https://example.com/secure/data"); - let token = STANDARD.encode("wrong-user:pass"); - set_authorization(&mut req, &format!("Basic {token}")); - - let response = enforce_basic_auth(&settings, &mut req) - .expect("should evaluate auth") - .expect("should challenge"); - assert_eq!( - response.status(), - StatusCode::UNAUTHORIZED, - "should reject wrong username even with correct password" - ); - } - - #[test] - fn challenge_when_username_correct_password_wrong() { - let settings = create_test_settings(); - let mut req = build_request(Method::GET, "https://example.com/secure/data"); - let token = STANDARD.encode("user:wrong-pass"); - set_authorization(&mut req, &format!("Basic {token}")); - - let response = enforce_basic_auth(&settings, &mut req) - .expect("should evaluate auth") - .expect("should challenge"); - assert_eq!( - response.status(), - StatusCode::UNAUTHORIZED, - "should reject correct username with wrong password" - ); - } - - #[test] - fn challenge_when_scheme_is_not_basic() { - let settings = create_test_settings(); - let mut req = build_request(Method::GET, "https://example.com/secure"); - set_authorization(&mut req, "Bearer token"); - - let response = enforce_basic_auth(&settings, &mut req) - .expect("should evaluate auth") - .expect("should challenge"); - assert_eq!(response.status(), StatusCode::UNAUTHORIZED); - } - - #[test] - fn returns_error_for_invalid_handler_regex_without_panicking() { - let config = crate_test_settings_str().replace(r#"path = "^/secure""#, r#"path = "(""#); - let err = Settings::from_toml(&config).expect_err("should reject invalid handler regex"); - assert!( - err.to_string() - .contains("Handler path regex `(` failed to compile"), - "should describe the invalid handler regex" - ); - } - - #[test] - fn allow_admin_path_with_valid_credentials() { - let settings = create_test_settings(); - let mut req = build_request(Method::POST, "https://example.com/_ts/admin/keys/rotate"); - let token = STANDARD.encode("admin:admin-pass"); - set_authorization(&mut req, &format!("Basic {token}")); - - assert!( - enforce_basic_auth(&settings, &mut req) - .expect("should evaluate auth") - .is_none(), - "should allow admin path with correct credentials" - ); - } - - #[test] - fn challenge_admin_path_with_wrong_credentials() { - let settings = create_test_settings(); - let mut req = build_request(Method::POST, "https://example.com/_ts/admin/keys/rotate"); - let token = STANDARD.encode("admin:wrong"); - set_authorization(&mut req, &format!("Basic {token}")); - - let response = enforce_basic_auth(&settings, &mut req) - .expect("should evaluate auth") - .expect("should challenge admin path with wrong credentials"); - assert_eq!(response.status(), StatusCode::UNAUTHORIZED); - } - - /// A handler regex is matched against the whole request path, so a broad - /// pattern such as `^/_ts` covers browser-facing endpoints in that - /// namespace — `/_ts/page-bids` and `/_ts/api/v1/identify` — not just the - /// admin routes it was probably meant for. Anonymous browser fetches never - /// carry Basic credentials, so those endpoints answer `401` for every - /// visitor. - /// - /// This pins the behaviour rather than exempting the paths: which paths a - /// handler covers is the operator's decision, and silently carving holes in - /// it would be worse than a documented constraint. Operators must scope - /// handler patterns to the paths they mean (`^/_ts/admin`) — see the - /// configuration guide. The tsjs client's `/__ts/page-bids` fallback keeps - /// affected deployments serving SPA ads until they do, but it disappears - /// with the alias in IABTechLab/trusted-server#970. - #[test] - fn broad_handler_regex_also_covers_browser_facing_endpoints() { - let config = crate_test_settings_str().replace(r#"path = "^/secure""#, r#"path = "^/_ts""#); - let settings = Settings::from_toml(&config).expect("should parse broad handler regex"); - - for path in [ - "https://example.com/_ts/page-bids?path=/article", - "https://example.com/_ts/api/v1/identify", - ] { - let mut req = build_request(Method::GET, path); - - let response = enforce_basic_auth(&settings, &mut req) - .expect("should evaluate auth") - .unwrap_or_else(|| panic!("should challenge {path} under a `^/_ts` handler")); - assert_eq!( - response.status(), - StatusCode::UNAUTHORIZED, - "a `^/_ts` handler should challenge {path}" - ); - } - } - - #[test] - fn challenge_admin_path_with_missing_credentials() { - let settings = create_test_settings(); - let mut req = build_request(Method::POST, "https://example.com/_ts/admin/keys/rotate"); - - let response = enforce_basic_auth(&settings, &mut req) - .expect("should evaluate auth") - .expect("should challenge admin path with missing credentials"); - assert_eq!(response.status(), StatusCode::UNAUTHORIZED); - } - - #[test] - fn concrete_admin_path_without_matching_handler_fails_closed() { - let config = crate_test_settings_str().replace( - r#"path = "^/_ts/admin" - username = "admin" - password = "admin-pass""#, - r#"path = "^/_ts/admin/(keys/rotate|keys/deactivate|ec|eids)$" - username = "admin" - password = "strong-test-password" - - [[handlers]] - path = "^/_ts/admin/ec/[{]id[}]$" - username = "admin" - password = "strong-test-password""#, - ); - let settings: Settings = - toml::from_str(&config).expect("should deserialize settings without finalization"); - let ec_id = format!("{}.abc123", "a".repeat(64)); - let mut req = build_request( - Method::GET, - &format!("https://example.com/_ts/admin/ec/{ec_id}"), - ); - - let error = enforce_basic_auth(&settings, &mut req) - .expect_err("should fail closed without a matching admin handler"); - assert!( - error.to_string().contains("no configured handler"), - "should describe the missing admin handler" - ); - } - - #[test] - fn similar_non_admin_prefix_without_handler_remains_public() { - let config = crate_test_settings_str().replace( - r#"path = "^/_ts/admin""#, - r#"path = "^/_ts/admin/keys/rotate$""#, - ); - let settings: Settings = - toml::from_str(&config).expect("should deserialize settings without finalization"); - let mut req = build_request(Method::GET, "https://example.com/_ts/administrator"); - - assert!( - enforce_basic_auth(&settings, &mut req) - .expect("should evaluate auth") - .is_none(), - "should not classify a similar prefix as the admin namespace" - ); - } -} diff --git a/crates/trusted-server-core/src/build_info.rs b/crates/trusted-server-core/src/build_info.rs new file mode 100644 index 0000000000..38466d2904 --- /dev/null +++ b/crates/trusted-server-core/src/build_info.rs @@ -0,0 +1,89 @@ +//! What this build reports about itself, fixed when it was compiled. +//! +//! `env!` is resolved by the compiler. Some hosts give a WebAssembly guest no +//! custom environment variable when it runs, so a fact read then would read +//! as absent there. +//! +//! | Constant | Build input | Without it | +//! | --- | --- | --- | +//! | [`BUILT_AT`] | `SOURCE_DATE_EPOCH`, in seconds | The time of the build | +//! | [`COMMIT`] | `TRUSTED_SERVER_COMMIT` | `unknown` | +//! | [`BUILD_RUN`] | `TRUSTED_SERVER_BUILD_RUN` | `unknown` | +//! | [`BUILD_TAG`] | `TRUSTED_SERVER_BUILD_TAG` | `unknown` | +//! +//! Each is the builder's own report. Nothing here measures the binary. + +/// The crate version. +pub const VERSION: &str = env!("CARGO_PKG_VERSION"); + +/// When this build was made, RFC 3339 in UTC to the second. +pub const BUILT_AT: &str = env!("TRUSTED_SERVER_BUILT_AT"); + +/// The commit this build was made from, or `unknown`. +pub const COMMIT: &str = env!("TRUSTED_SERVER_COMMIT"); + +/// The build run that made this binary, or `unknown`. +pub const BUILD_RUN: &str = env!("TRUSTED_SERVER_BUILD_RUN"); + +/// The tag this build was made from, as the builder names it, or `unknown`. +pub const BUILD_TAG: &str = env!("TRUSTED_SERVER_BUILD_TAG"); + +/// The build script's date arithmetic, compiled here so a test can call it. +#[cfg(test)] +#[path = "../build_time.rs"] +mod build_time; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_build_time_is_rfc_3339_in_utc() { + let shape = regex::Regex::new(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$") + .expect("should compile the pattern"); + + assert!( + shape.is_match(BUILT_AT), + "should be RFC 3339 in UTC: {BUILT_AT}" + ); + } + + #[test] + fn the_commit_the_run_and_the_tag_are_never_empty() { + for (name, value) in [ + ("commit", COMMIT), + ("build run", BUILD_RUN), + ("build tag", BUILD_TAG), + ] { + assert!( + !value.is_empty() && !value.chars().any(char::is_control), + "the {name} should be the builder's value or `unknown`, got {value:?}" + ); + } + } + + #[test] + fn the_build_script_writes_known_instants_as_rfc_3339() { + for (seconds, expected, instant) in [ + (0, "1970-01-01T00:00:00Z", "the epoch"), + ( + 951_782_400, + "2000-02-29T00:00:00Z", + "the leap day of a 400th year", + ), + (1_000_000_000, "2001-09-09T01:46:40Z", "a time of day"), + (4_102_444_800, "2100-01-01T00:00:00Z", "the start of 2100"), + ( + 4_107_542_400, + "2100-03-01T00:00:00Z", + "the day after 28 February 2100", + ), + ] { + assert_eq!( + build_time::rfc3339_utc(seconds), + expected, + "should write {instant}, {seconds} seconds after the epoch" + ); + } + } +} diff --git a/crates/trusted-server-core/src/cache_purge.rs b/crates/trusted-server-core/src/cache_purge.rs deleted file mode 100644 index f77f43e4dd..0000000000 --- a/crates/trusted-server-core/src/cache_purge.rs +++ /dev/null @@ -1,373 +0,0 @@ -//! The operator-facing purge endpoint for templates and tagged origin responses. -//! -//! `POST /_ts/admin/cache/purge` with `{"scope":"all"}` or -//! `{"scope":"url","url":"https://example.com/page"}`. -//! -//! # Why every method is registered, not just `POST` -//! -//! A named route only claims the methods it lists. A method it does not claim falls through -//! to the publisher, and `enforce_basic_auth` leaves the `Authorization` header in place, so -//! a `GET` to this path would authenticate and then ship the shared admin credential to the -//! publisher origin. The route therefore claims every method and this handler answers the -//! non-`POST` ones with 405 itself. -//! -//! # Why `Content-Type` is enforced exactly -//! -//! Browsers attach basic-auth credentials automatically. A cross-origin form POST with -//! `enctype="text/plain"` is not preflighted, so requiring `POST` alone does not stop CSRF; -//! requiring a `Content-Type` that a form cannot produce does. - -use edgezero_core::body::Body as EdgeBody; -use error_stack::Report; -use http::{Method, Request, Response, StatusCode, header}; -use serde::{Deserialize, Serialize}; - -use crate::error::TrustedServerError; -use crate::http_util::enforce_max_body_size; -use crate::platform::{RuntimeServices, reader_url_surrogate_key}; - -/// Purge bodies name a scope and, for a URL purge, one URL. Nothing here is unbounded. -const PURGE_MAX_BODY_BYTES: usize = 4096; - -/// The request body, before its scope is validated. -/// -/// Deserialized as a flat struct rather than an internally-tagged enum on purpose. -/// `deny_unknown_fields` does not reach the unit variant of such an enum, so -/// `{"scope":"all","url":"…"}` would parse as a full flush — an operator who typed the -/// wrong scope while intending to purge one page would empty the whole cache and be told -/// it succeeded. Validating the pair by hand makes that combination an error. -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct PurgeBody { - scope: String, - url: Option, -} - -/// What an operator asked to purge. -#[derive(Debug, PartialEq, Eq)] -enum PurgeRequest { - /// Every template and tagged origin response this service has cached. - All, - /// One reader-facing URL, as a reader would type it; canonicalized before hashing. - Url(String), -} - -impl PurgeRequest { - /// Parse and validate a purge body. - /// - /// # Errors - /// - /// Returns a message naming the problem when the JSON is malformed, the scope is not - /// one of the two documented values, or the scope and `url` field disagree. - fn parse(bytes: &[u8]) -> Result { - let body: PurgeBody = - serde_json::from_slice(bytes).map_err(|error| format!("invalid JSON: {error}"))?; - - match (body.scope.as_str(), body.url) { - ("all", None) => Ok(Self::All), - ("all", Some(_)) => Err( - "scope \"all\" takes no url; did you mean {\"scope\":\"url\",\"url\":…}?" - .to_owned(), - ), - // Parsed, not merely non-empty. `reader_url_surrogate_key` hashes whatever it - // is given, so a path, a scheme-less host, or an `ftp://` URL would be - // acknowledged as purged under a key nothing was ever stored with — the exact - // silent no-op that key's own documentation calls the failure that matters. - // A CMS webhook sending paths would purge nothing and never find out. - ("url", Some(url)) - if url::Url::parse(&url).is_ok_and(|parsed| { - matches!(parsed.scheme(), "http" | "https") && parsed.host_str().is_some() - }) => - { - Ok(Self::Url(url)) - } - ("url", _) => Err( - "scope \"url\" requires an absolute http(s) url, as a reader addresses the page" - .to_owned(), - ), - (other, _) => Err(format!( - "unknown scope {other:?}; expected \"all\" or \"url\"" - )), - } - } -} - -/// What the purge did, in terms an operator mid-incident can act on. -#[derive(Debug, Serialize)] -struct PurgeResponse { - purged: bool, - scope: &'static str, - /// Present on a URL purge, so an operator can confirm which key was hit. - #[serde(skip_serializing_if = "Option::is_none")] - surrogate_key: Option, - message: &'static str, -} - -/// Handle a purge request. -/// -/// # Errors -/// -/// Returns an error when the body cannot be read, exceeds [`PURGE_MAX_BODY_BYTES`], is not -/// valid JSON for a known scope, or when the platform's purge fails. -pub async fn handle_cache_purge( - services: &RuntimeServices, - req: Request, - authenticated_principal: Option<&str>, -) -> Result, Report> { - if req.method() != Method::POST { - return Ok(method_not_allowed()); - } - - // Checked before the body is read: a request that cannot be a legitimate API call - // should not have its payload parsed at all. - let content_type = req - .headers() - .get(header::CONTENT_TYPE) - .and_then(|value| value.to_str().ok()) - .unwrap_or_default(); - if !is_exactly_json(content_type) { - return Ok(json_response( - StatusCode::UNSUPPORTED_MEDIA_TYPE, - r#"{"purged":false,"error":"Content-Type must be application/json"}"#, - )); - } - - let body = req.into_body(); - if body.is_stream() { - return Err(Report::new(TrustedServerError::BadRequest { - message: "cache-purge request body must be buffered, not streamed".into(), - })); - } - let bytes = body.into_bytes().unwrap_or_default(); - enforce_max_body_size(&bytes, PURGE_MAX_BODY_BYTES, "cache-purge")?; - - let request = - PurgeRequest::parse(&bytes).map_err(|message| TrustedServerError::BadRequest { - message: format!("invalid cache-purge request: {message}"), - })?; - - match request { - PurgeRequest::All => { - // An unbounded flush behind one shared static credential is also an - // origin-stampede lever, and there is no rate-limit primitive on this path, so - // who used it is recorded every time. - log::warn!( - "Cache purge: ALL templates purged by {}", - authenticated_principal.unwrap_or("") - ); - services - .template_cache() - .purge_all() - .await - .map_err(|error| TrustedServerError::Configuration { - message: format!("cache purge failed: {error}"), - })?; - Ok(purge_ok(&PurgeResponse { - purged: true, - scope: "all", - surrogate_key: None, - // One surrogate-key call, so there is no partial state to reason about: - // an error means nothing was purged and the call is safe to retry. - message: "All templates purged. Purge is idempotent and safe to repeat.", - })) - } - PurgeRequest::Url(url) => { - // The reader-facing key, so callers pass the URL a reader would see and never - // have to replay this service's origin rewriting. - let surrogate_key = reader_url_surrogate_key(&url); - log::info!( - "Cache purge: url {url} (key {surrogate_key}) by {}", - authenticated_principal.unwrap_or("") - ); - services - .template_cache() - .purge_url_surrogate_key(&surrogate_key) - .await - .map_err(|error| TrustedServerError::Configuration { - message: format!("cache purge failed: {error}"), - })?; - Ok(purge_ok(&PurgeResponse { - purged: true, - scope: "url", - surrogate_key: Some(surrogate_key), - message: "URL purged. Purge is idempotent and safe to repeat.", - })) - } - } -} - -/// Exactly `application/json`, with optional parameters and whitespace. -/// -/// Deliberately strict: accepting `application/json`-ish media types would readmit the -/// form-post CSRF shape this check exists to close. -fn is_exactly_json(content_type: &str) -> bool { - content_type - .split(';') - .next() - .unwrap_or_default() - .trim() - .eq_ignore_ascii_case("application/json") -} - -fn method_not_allowed() -> Response { - let mut response = json_response( - StatusCode::METHOD_NOT_ALLOWED, - r#"{"purged":false,"error":"cache purge requires POST"}"#, - ); - response - .headers_mut() - .insert(header::ALLOW, header::HeaderValue::from_static("POST")); - response -} - -fn purge_ok(payload: &PurgeResponse) -> Response { - let body = serde_json::to_string(payload) - .unwrap_or_else(|_| r#"{"purged":true,"scope":"unknown"}"#.to_owned()); - json_response(StatusCode::OK, &body) -} - -/// Purge answers are per-operator and per-moment; nothing may store one. -fn json_response(status: StatusCode, body: &str) -> Response { - Response::builder() - .status(status) - .header(header::CONTENT_TYPE, "application/json") - .header(header::CACHE_CONTROL, "private, no-store") - .body(EdgeBody::from(body.to_owned())) - .expect("should build a cache-purge response") -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn content_type_must_be_exactly_json() { - assert!(is_exactly_json("application/json")); - assert!(is_exactly_json("application/json; charset=utf-8")); - assert!(is_exactly_json(" APPLICATION/JSON ")); - } - - #[test] - fn form_postable_content_types_are_refused() { - // These three are the only types a cross-origin form can send, and a form POST - // carries basic-auth credentials without a preflight. - for content_type in [ - "text/plain", - "application/x-www-form-urlencoded", - "multipart/form-data", - "", - ] { - assert!( - !is_exactly_json(content_type), - "{content_type} must not be accepted" - ); - } - } - - #[test] - fn json_lookalike_types_are_refused() { - for content_type in [ - "application/jsonrequest", - "text/json", - "application/ld+json", - ] { - assert!( - !is_exactly_json(content_type), - "{content_type} must not be accepted" - ); - } - } - - #[test] - fn a_non_post_answer_names_the_method_it_accepts() { - let response = method_not_allowed(); - assert_eq!(response.status(), StatusCode::METHOD_NOT_ALLOWED); - assert_eq!( - response - .headers() - .get(header::ALLOW) - .and_then(|value| value.to_str().ok()), - Some("POST") - ); - } - - #[test] - fn every_answer_forbids_storage() { - for response in [method_not_allowed(), json_response(StatusCode::OK, "{}")] { - assert_eq!( - response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("private, no-store"), - "a purge answer must never be stored" - ); - } - } - - #[test] - fn scope_parsing_accepts_both_documented_shapes() { - assert_eq!( - PurgeRequest::parse(br#"{"scope":"all"}"#).expect("should parse"), - PurgeRequest::All - ); - assert_eq!( - PurgeRequest::parse(br#"{"scope":"url","url":"https://example.com/a"}"#) - .expect("should parse"), - PurgeRequest::Url("https://example.com/a".to_owned()) - ); - } - - #[test] - fn a_url_alongside_scope_all_is_refused_rather_than_flushing_everything() { - // The dangerous typo: an operator means to purge one page, mistypes the scope, and - // would otherwise be told a full flush succeeded. - let error = PurgeRequest::parse(br#"{"scope":"all","url":"https://example.com/a"}"#) - .expect_err("should refuse"); - assert!( - error.contains("takes no url"), - "the error must point at the confusion, got: {error}" - ); - } - - #[test] - fn an_unknown_or_malformed_scope_is_refused() { - // A typo must not silently widen into a full flush. - for body in [ - &br#"{"scope":"everything"}"#[..], - br#"{"scope":"url"}"#, - br#"{"scope":"url","url":" "}"#, - // Hashed as given, these would each be acknowledged as purged under a key - // that can never match a stored object. - br#"{"scope":"url","url":"/article"}"#, - br#"{"scope":"url","url":"example.com/article"}"#, - br#"{"scope":"url","url":"ftp://example.com/article"}"#, - br#"{"scope":"all","url":"https://example.com/a"}"#, - br#"{}"#, - br#"{"scope":"ALL"}"#, - br#"{"scope":"all","extra":1}"#, - br#"not json"#, - ] { - assert!( - PurgeRequest::parse(body).is_err(), - "{} must not parse", - String::from_utf8_lossy(body) - ); - } - } - - #[test] - fn a_reader_facing_url_still_parses() { - for body in [ - &br#"{"scope":"url","url":"https://ts.example.com/article"}"#[..], - br#"{"scope":"url","url":"http://ts.example.com/article?a=1"}"#, - br#"{"scope":"url","url":"https://ts.example.com"}"#, - ] { - assert!( - PurgeRequest::parse(body).is_ok(), - "{} is a URL a reader can address and must still purge", - String::from_utf8_lossy(body) - ); - } - } -} diff --git a/crates/trusted-server-core/src/closed_paths.rs b/crates/trusted-server-core/src/closed_paths.rs new file mode 100644 index 0000000000..bac9b398c2 --- /dev/null +++ b/crates/trusted-server-core/src/closed_paths.rs @@ -0,0 +1,256 @@ +//! Paths this server answers itself and never forwards to the publisher's origin. +//! +//! A request to `/_ts/admin`, or to the `/admin/keys` addresses beside it, is +//! an administration request. It can carry an operator's `Authorization` +//! header and a payload meant for this server alone. No route serves either +//! prefix, and forwarding such a request would hand both to the origin, so +//! every adapter asks [`closed_path_response`] before its publisher fallback +//! and the request is answered `404` here. + +use std::borrow::Cow; + +use edgezero_core::body::Body as EdgeBody; +use http::{Request, Response, StatusCode, header}; + +/// The server's administration prefix. +/// +/// It is matched as a bare prefix, with no `/` after it, so that a +/// percent-encoded separator such as `/_ts/admin%2Fec` is closed with the +/// paths it decodes to. A check for a literal slash would miss that spelling. +const ADMIN_NAMESPACE_PREFIX: &str = "/_ts/admin"; + +/// The key administration alias outside `/_ts`. +/// +/// Only the alias itself and its separator descendants are closed. A bare +/// prefix here would also close unrelated publisher paths such as +/// `/admin/keystore`. +const ADMIN_KEYS_PREFIX: &str = "/admin/keys"; + +/// Maximum percent-decoding rounds applied when testing a path. +/// +/// Bounds the work a `%25`-chained path can force while still reaching the +/// fixed point of any separator encoding a proxy chain would plausibly decode. +const MAX_PERCENT_DECODE_ROUNDS: usize = 4; + +/// Returns the local `404 Not Found` for a request to a closed path, and +/// `None` for any other request, which the publisher fallback then answers. +/// +/// The path is tested as sent and after each of a bounded number of +/// percent-decoding rounds, so `/_ts/admin%2Fec` and `/admin%252Fkeys/rotate` +/// are closed with the paths they decode to. +#[must_use] +pub fn closed_path_response(req: &Request) -> Option> { + is_closed(req.uri().path()).then(not_found) +} + +/// Returns whether `path` is closed either as sent or after any bounded number +/// of percent-decoding rounds. +/// +/// A separator encoded twice, as in `/admin%252Fkeys/rotate`, survives one +/// decode as `/admin%2Fkeys/rotate`, so the check is repeated to a fixed point +/// rather than applied once. +fn is_closed(path: &str) -> bool { + if is_closed_as_written(path) { + return true; + } + + let mut current = path.to_owned(); + for _ in 0..MAX_PERCENT_DECODE_ROUNDS { + let Some(decoded) = percent_decoded_path(¤t) else { + return false; + }; + + // A path whose remaining `%` sequences are not decodable escapes is a + // fixed point, so further rounds would repeat the same comparison. + if decoded == current { + return false; + } + + if is_closed_as_written(&decoded) { + return true; + } + + current = decoded; + } + + false +} + +fn is_closed_as_written(path: &str) -> bool { + path.starts_with(ADMIN_NAMESPACE_PREFIX) + || path == ADMIN_KEYS_PREFIX + || path + .strip_prefix(ADMIN_KEYS_PREFIX) + .is_some_and(|remainder| remainder.starts_with('/')) +} + +/// Percent-decodes `path` once, returning `None` when the path contains no +/// escape sequence or decodes to invalid UTF-8. +fn percent_decoded_path(path: &str) -> Option { + if !path.contains('%') { + return None; + } + + urlencoding::decode(path).ok().map(Cow::into_owned) +} + +fn not_found() -> Response { + Response::builder() + .status(StatusCode::NOT_FOUND) + .header(header::CONTENT_TYPE, mime::APPLICATION_JSON.as_ref()) + .header(header::CACHE_CONTROL, "no-store") + .header(header::X_CONTENT_TYPE_OPTIONS, "nosniff") + .body(EdgeBody::from(r#"{"error":"not found"}"#)) + .expect("should build the response for a closed path") +} + +#[cfg(test)] +mod tests { + use http::Method; + + use super::*; + + fn request(method: Method, path: &str) -> Request { + Request::builder() + .method(method) + .uri(format!("https://edge.example.com{path}")) + .body(EdgeBody::empty()) + .expect("should build test request") + } + + fn assert_closed(path: &str) { + for method in [ + Method::GET, + Method::POST, + Method::HEAD, + Method::OPTIONS, + Method::PUT, + Method::PATCH, + Method::DELETE, + ] { + let response = closed_path_response(&request(method.clone(), path)) + .unwrap_or_else(|| panic!("should answer {method} {path} here")); + + assert_eq!( + response.status(), + StatusCode::NOT_FOUND, + "should answer {method} {path} with 404" + ); + assert_eq!( + response.headers().get(header::CACHE_CONTROL), + Some(&http::HeaderValue::from_static("no-store")), + "should prevent caching for {path}" + ); + } + } + + #[test] + fn every_path_beneath_the_administration_prefix_is_closed() { + let ec_id = format!("{}.abc123", "a".repeat(64)); + for path in [ + "/_ts/admin".to_owned(), + "/_ts/admin/".to_owned(), + "/_ts/admin/ec".to_owned(), + "/_ts/admin/ec/".to_owned(), + format!("/_ts/admin/ec/{ec_id}"), + format!("/_ts/admin/ec/{ec_id}/extra"), + "/_ts/admin/eids".to_owned(), + "/_ts/admin/eids.json".to_owned(), + "/_ts/admin/ec;foo".to_owned(), + "/_ts/admin/keys/rotate".to_owned(), + "/_ts/admin/cache/purge".to_owned(), + "/_ts/admin/unknown".to_owned(), + ] { + assert_closed(&path); + } + } + + #[test] + fn an_encoded_separator_does_not_open_a_closed_path() { + // A check for a literal slash would miss `/_ts/admin%2Fec`, and + // reaching the publisher fallback would forward the caller's + // `Authorization` header and body to the origin. + for path in [ + "/_ts/admin%2Fec", + "/_ts/admin%2fec", + "/_ts/admin%2Fkeys/rotate", + "/_ts/admin%252Fec", + "/_ts/admin%5Cec", + "/_ts/adminec", + "/%5Fts/admin/ec", + ] { + assert_closed(path); + } + } + + #[test] + fn the_key_alias_outside_ts_is_closed_with_its_descendants() { + for path in [ + "/admin/keys", + "/admin/keys/", + "/admin/keys/rotate", + "/admin/keys/deactivate", + "/admin/keys/rotate/", + "/admin/keys/rotate/extra", + "/admin/keys%2Frotate", + "/admin/keys%2frotate", + "/admin%2Fkeys/rotate", + "/admin%2fkeys%2Frotate", + ] { + assert_closed(path); + } + } + + #[test] + fn a_separator_encoded_more_than_once_is_still_closed() { + // One decode leaves `/admin%252Fkeys/rotate` as `/admin%2Fkeys/rotate`, + // which no literal check matches. Decoding to a fixed point keeps the + // path closed against a proxy or origin that decodes it more than once. + for path in [ + "/admin%252Fkeys/rotate", + "/admin/keys%252Frotate", + "/admin%25252Fkeys/rotate", + "/_ts%252Fadmin/ec", + ] { + assert_closed(path); + } + } + + #[test] + fn other_publisher_paths_are_left_to_the_fallback() { + for path in [ + "/", + "/articles/example", + "/admin", + "/admin/login", + "/admin/keyboards", + "/admin/keystore", + "/admin/keys%25store", + "/articles/100%25-organic", + "/_ts/api/v1/batch-sync", + "/_ts/data", + ] { + assert!( + closed_path_response(&request(Method::POST, path)).is_none(), + "should leave {path} to the publisher fallback" + ); + } + } + + #[test] + fn the_answer_says_nothing_but_not_found() { + let response = closed_path_response(&request(Method::GET, "/_ts/admin/ec")) + .expect("should answer a closed path"); + + assert_eq!( + response.headers().get(header::CONTENT_TYPE), + Some(&http::HeaderValue::from_static("application/json")) + ); + assert_eq!( + response.headers().get(header::X_CONTENT_TYPE_OPTIONS), + Some(&http::HeaderValue::from_static("nosniff")) + ); + let body = response.into_body().into_bytes().unwrap_or_default(); + assert_eq!(&body[..], br#"{"error":"not found"}"#); + } +} diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index e48f5c2f03..6e939ac7db 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -11,48 +11,16 @@ use std::borrow::Cow; use edgezero_core::app_config::{SecretField, SecretKind, SecretPathSegment}; use error_stack::Report; use serde::{Deserialize, Deserializer, Serialize, Serializer}; -use validator::{Validate, ValidationError, ValidationErrors}; +use validator::{Validate, ValidationError, ValidationErrors, ValidationErrorsKind}; +use crate::ec::module::{HMAC_MODULE_KEY, HOST_SIGNALS_MODULE_KEY}; use crate::ec::registry::PartnerRegistry; use crate::error::TrustedServerError; -use crate::integrations::{ - adserver_mock::AdServerMockConfig, - aps::ApsConfig, - datadome::DataDomeConfig, - didomi::DidomiIntegrationConfig, - google_tag_manager::GoogleTagManagerConfig, - gpt::GptConfig, - gpt_diagnostics::GptDiagnosticsConfig, - js_asset_proxy::{JS_ASSET_PROXY_INTEGRATION_ID, JsAssetProxyConfig}, - lockr::LockrConfig, - nextjs::NextJsIntegrationConfig, - osano::OsanoConfig, - permutive::PermutiveConfig, - prebid, - sourcepoint::SourcepointConfig, - testlight::TestlightConfig, -}; -use crate::settings::{AssetOriginAuth, IntegrationConfig, Settings}; + +use crate::integrations::IntegrationBuilder; +use crate::settings::{AssetOriginAuth, Ec, MODULE_IMPLEMENTATION_KEY, Settings}; const DEPLOY_VALIDATION_FIELD: &str = "trusted_server"; -#[cfg(test)] -const DEPLOY_VALIDATED_INTEGRATION_IDS: &[&str] = &[ - "prebid", - "aps", - "adserver_mock", - "testlight", - "nextjs", - "permutive", - "lockr", - "didomi", - "sourcepoint", - "osano", - "google_tag_manager", - "datadome", - "gpt", - "gpt_diagnostics", - JS_ASSET_PROXY_INTEGRATION_ID, -]; /// Typed app-config root used by the `ts` CLI. /// @@ -116,10 +84,34 @@ impl<'de> Deserialize<'de> for TrustedServerAppConfig { } } +/// The builders [`TrustedServerAppConfig`] validates against besides core's +/// own, set once by the tool that validates. +static DEPLOY_INTEGRATIONS: std::sync::OnceLock> = + std::sync::OnceLock::new(); + +/// Registers the integration builders [`TrustedServerAppConfig`] validates +/// against besides core's own. +/// +/// `EdgeZero` validates an app config through the [`Validate`] trait, which +/// takes no arguments, so a tool that validates a deployment's settings +/// registers that deployment's builders here before it validates anything. +/// The first call wins and a later one changes nothing. +pub fn register_deploy_integrations(builders: Vec) { + let _ = DEPLOY_INTEGRATIONS.set(builders); +} + +/// The builders a tool registered for deploy validation, or none. +fn deploy_integrations() -> &'static [IntegrationBuilder] { + DEPLOY_INTEGRATIONS.get().map_or(&[], Vec::as_slice) +} + impl Validate for TrustedServerAppConfig { fn validate(&self) -> Result<(), ValidationErrors> { let mut errors = self.settings.validate().err().unwrap_or_default(); - if let Err(report) = validate_settings_for_deploy(&self.settings) { + remove_labeled_module_secret_errors(&mut errors, &self.settings.ec); + if let Err(report) = + validate_settings_for_deploy_with(&self.settings, deploy_integrations()) + { errors.add( DEPLOY_VALIDATION_FIELD, report_to_validation_error(&report, "trusted_server_deploy_validation"), @@ -133,6 +125,101 @@ impl Validate for TrustedServerAppConfig { } } +/// Removes the passphrase checks on Edge Cookie module blocks written under +/// a label. +/// +/// Push-time validation reads a configuration whose secret fields hold +/// secret-store key names rather than the secrets themselves, so a value check +/// such as the 32-byte passphrase minimum would be judging a key name. +/// `EdgeZero`'s `validate_excluding_secrets` removes those checks for the +/// leaves [`secret_fields`](edgezero_core::app_config::AppConfigMeta::secret_fields) +/// lists, which covers the `[ec.hmac]` block. A block under a label of the +/// operator's choosing has no fixed path that list can hold, so its check is +/// removed here instead. The check itself is unchanged, and runs wherever +/// settings are loaded with their secrets resolved. +fn remove_labeled_module_secret_errors(errors: &mut ValidationErrors, ec: &Ec) { + let Some(ValidationErrorsKind::Struct(ec_errors)) = errors.errors_mut().get_mut("ec") else { + return; + }; + let labeled = ec + .module_blocks + .hmac_blocks() + .map(|(name, _)| name) + .filter(|name| *name != HMAC_MODULE_KEY) + .chain( + ec.module_blocks + .host_signals_blocks() + .map(|(name, _)| name) + .filter(|name| *name != HOST_SIGNALS_MODULE_KEY), + ); + for name in labeled { + let Some(ValidationErrorsKind::Struct(block_errors)) = ec_errors.errors_mut().get_mut(name) + else { + continue; + }; + block_errors.errors_mut().remove("passphrase"); + if block_errors.errors().is_empty() { + ec_errors.errors_mut().remove(name); + } + } + // An `ec` entry holding nothing would keep the whole result an error, the + // same reason `EdgeZero` prunes emptied containers after its own removals. + let ec_is_empty = ec_errors.errors().is_empty(); + if ec_is_empty { + errors.errors_mut().remove("ec"); + } +} + +impl crate::secret_resolution::ConfiguredSecretFields for TrustedServerAppConfig { + /// The passphrase of every Edge Cookie module block that configures a + /// module built into core under a label. + /// + /// [`secret_fields`](edgezero_core::app_config::AppConfigMeta::secret_fields) + /// lists the passphrases of the `[ec.hmac]` and `[ec.host_signals]` + /// blocks, the one path each of those modules' blocks has when its name + /// is its implementation. The same module under a label of the + /// operator's choosing holds that secret at `ec.kept\ + kept"; + + let replaced = without_bundle_hashes(page); + + assert_eq!( + replaced, + "\ + \ + kept\ + kept" + ); + } } } #[cfg(test)] mod tests { use super::*; - use crate::integrations::{ - AttributeRewriteAction, IntegrationAttributeContext, IntegrationAttributeRewriter, - IntegrationHeadInjector, IntegrationHtmlContext, - }; use crate::streaming_processor::{Compression, PipelineConfig, StreamingPipeline}; use crate::test_support::tests::create_test_settings; use serde_json::json; @@ -739,166 +1167,77 @@ mod tests { fn create_test_config() -> HtmlProcessorConfig { HtmlProcessorConfig { csp_nonce_observed: None, + public_prefix: None, + prepared: Arc::new(Prepared::default()), body_close: BodyCloseInjection::None, origin_host: "origin.example.com".to_owned(), request_host: "test.example.com".to_owned(), request_scheme: "https".to_owned(), integrations: IntegrationRegistry::default(), ad_slots_script: None, + permissions_script: None, ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), } } + /// The client script names its cookies and storage entries after the + /// public prefix, which its tag tells it. A document given no prefix gets + /// no attribute. #[test] - fn integration_attribute_rewriter_can_remove_elements() { - struct RemovingLinkRewriter; - - impl IntegrationAttributeRewriter for RemovingLinkRewriter { - fn integration_id(&self) -> &'static str { - "removing" - } - - fn handles_attribute(&self, attribute: &str) -> bool { - attribute == "href" - } - - fn rewrite( - &self, - _attr_name: &str, - attr_value: &str, - _ctx: &IntegrationAttributeContext<'_>, - ) -> AttributeRewriteAction { - if attr_value.contains("remove-me") { - AttributeRewriteAction::remove_element() - } else { - AttributeRewriteAction::keep() - } - } - } - - let html = r#" - remove - keep - "#; - - let mut config = create_test_config(); - config.integrations = - IntegrationRegistry::from_rewriters(vec![Arc::new(RemovingLinkRewriter)], Vec::new()); - - let processor = create_html_processor(config); - let pipeline_config = PipelineConfig { - input_compression: Compression::None, - output_compression: Compression::None, - chunk_size: 8192, - }; - let mut pipeline = StreamingPipeline::new(pipeline_config, processor); - - let mut output = Vec::new(); - pipeline - .process(Cursor::new(html.as_bytes()), &mut output) - .expect("pipeline should process HTML"); - let processed = String::from_utf8(output).expect("output should be valid UTF-8"); - - assert!(processed.contains("keep-me")); - assert!(!processed.contains("remove-me")); - } - - #[test] - fn integration_head_injector_prepends_after_tsjs_once() { - struct TestHeadInjector; - - impl IntegrationHeadInjector for TestHeadInjector { - fn integration_id(&self) -> &'static str { - "test" - } + fn the_bundle_tag_tells_the_client_script_the_public_prefix() { + fn process(public: Option) -> String { + let settings = create_test_settings(); + let registry = IntegrationRegistry::new(&settings).expect("should build the registry"); + let mut config = create_test_config(); + config.integrations = registry.clone(); + config.public_prefix = public; + let mut processor = test_support::create_page_processor(&settings, ®istry, config); + let output = processor + .process_chunk(b"", true) + .expect("should process HTML"); - fn head_inserts(&self, _ctx: &IntegrationHtmlContext<'_>) -> Vec { - vec!["".to_owned()] - } + String::from_utf8(output).expect("should produce valid UTF-8") } - let html = "Test"; - - let mut config = create_test_config(); - config.integrations = IntegrationRegistry::from_rewriters_with_head_injectors( - Vec::new(), - Vec::new(), - vec![Arc::new(TestHeadInjector)], - ); - - let processor = create_html_processor(config); - let pipeline_config = PipelineConfig { - input_compression: Compression::None, - output_compression: Compression::None, - chunk_size: 8192, - }; - let mut pipeline = StreamingPipeline::new(pipeline_config, processor); - - let mut output = Vec::new(); - pipeline - .process(Cursor::new(html.as_bytes()), &mut output) - .expect("pipeline should process HTML"); - let processed = String::from_utf8(output).expect("output should be valid UTF-8"); - - let tsjs_marker = "id=\"trustedserver-js\""; - let head_marker = "window.__testHeadInjector=true"; - - assert_eq!( - processed.matches(tsjs_marker).count(), - 1, - "should inject unified tsjs tag once" - ); - assert_eq!( - processed.matches(head_marker).count(), - 1, - "should inject head snippet once" - ); - - let tsjs_index = processed - .find(tsjs_marker) - .expect("should include unified tsjs tag"); - let head_index = processed - .find(head_marker) - .expect("should include head snippet"); - let title_index = processed - .find("") - .expect("should keep existing head content"); - - assert!( - head_index < tsjs_index, - "should inject config before tsjs bundle so auto-init can read it" - ); + let told = process(Some( + PublicPrefix::new(["_ex"]).expect("should accept the prefix"), + )); assert!( - tsjs_index < title_index, - "should prepend all injected content before existing head content" + told.contains(r#"data-ts-public-prefix="_ex""#), + "the bundle tag should carry the public prefix: {told}" ); + let untold = process(None); + assert!(!untold.contains(PUBLIC_PREFIX_ATTRIBUTE), "{untold}"); } #[test] - fn integration_head_injector_marks_only_attribution_enabled_gpt_bundle() { - fn process(gpt_config: Option<(bool, bool)>) -> String { - let integrations = if let Some((enabled, gam_attribution_enabled)) = gpt_config { - let mut settings = create_test_settings(); + fn a_registration_marks_only_the_bundle_it_asks_to_mark() { + use crate::integrations::registry_test_support::tag_fixture as tag; + + fn process(mark_bundle: Option<bool>) -> String { + let mut settings = create_test_settings(); + if let Some(mark_bundle) = mark_bundle { settings - .integrations - .insert_config( - "gpt", + .insert_module_config( + "testing", + tag::MODULE, &json!({ - "enabled": enabled, - "gam_attribution_enabled": gam_attribution_enabled + "mark_bundle": mark_bundle }), ) - .expect("should insert GPT config"); - IntegrationRegistry::new(&settings).expect("should build GPT registry") - } else { - IntegrationRegistry::empty_for_tests() - }; + .expect("should insert the stand-in's settings"); + test_support::place_on_every_page( + &mut settings, + crate::middleware::MiddlewarePhase::Fetch, + &[tag::MODULE], + ); + } + let registry = IntegrationRegistry::new(&settings).expect("should build the registry"); let mut config = create_test_config(); - config.integrations = integrations; - let mut processor = create_html_processor(config); + config.integrations = registry.clone(); + let mut processor = test_support::create_page_processor(&settings, ®istry, config); let output = processor .process_chunk(b"<html><head></head><body></body></html>", true) .expect("should process HTML"); @@ -906,77 +1245,57 @@ mod tests { String::from_utf8(output).expect("should produce valid UTF-8") } - let attributed = process(Some((true, true))); - let unattributed = process(Some((true, false))); - let disabled_gpt = process(Some((false, true))); - let without_gpt = process(None); + let marked = process(Some(true)); + let unmarked = process(Some(false)); + let without_the_integration = process(None); - for html in [&attributed, &unattributed, &disabled_gpt, &without_gpt] { + for html in [&marked, &unmarked, &without_the_integration] { assert_eq!( html.matches("id=\"trustedserver-js\"").count(), 1, "should emit exactly one publisher bundle tag: {html}" ); } + let attribute = format!("{}=\"true\"", tag::BUNDLE_ATTRIBUTE); assert!( - attributed.contains("data-ts-gam-attribution=\"true\""), - "should mark only an attribution-enabled GPT publisher bundle" + marked.contains(&attribute), + "should mark the bundle a registration asks to mark" ); assert!( - !unattributed.contains("data-ts-gam-attribution"), - "should leave an attribution-disabled GPT publisher bundle unmarked" + !unmarked.contains(tag::BUNDLE_ATTRIBUTE), + "should leave the bundle unmarked when the registration asks for nothing" ); assert!( - !disabled_gpt.contains("data-ts-gam-attribution"), - "should let the GPT master switch suppress attribution metadata" - ); - assert!( - !without_gpt.contains("data-ts-gam-attribution"), - "should leave a non-GPT publisher bundle unmarked" + !without_the_integration.contains(tag::BUNDLE_ATTRIBUTE), + "should leave the bundle unmarked when no section selects the integration" ); - let head_insert_index = attributed - .find("window.__tsjs_installGptShim") - .expect("should include the GPT head insert"); - let publisher_bundle_index = attributed + let head_insert_index = marked + .find(tag::HEAD_FLAG) + .expect("should include the integration's head insert"); + let publisher_bundle_index = marked .find("id=\"trustedserver-js\"") .expect("should include the publisher bundle"); assert!( head_insert_index < publisher_bundle_index, - "should keep integration head inserts before the publisher bundle" + "should keep a middleware's head markup before the publisher bundle" ); } #[test] - fn active_gpt_diagnostics_loads_standalone_after_unified_bundle_once() { + fn what_a_module_left_on_the_request_is_written_around_the_unified_bundle_once() { + use crate::integrations::registry_test_support::request_fixture; + let html = "<html><head><title>Test"; let mut settings = create_test_settings(); - settings - .integrations - .insert_config("gpt_diagnostics", &json!({ "enabled": true })) - .expect("should insert GPT diagnostics config"); - - let mut request = http::Request::builder() - .method(http::Method::GET) - .uri("https://publisher.example/page?ts_console=1") - .header("sec-fetch-dest", "document") - .body(edgezero_core::body::Body::empty()) - .expect("should build activation request"); - let decision = - crate::integrations::gpt_diagnostics::prepare_request(&settings, &mut request) - .expect("should prepare diagnostics request"); + request_fixture::select_and_place(&mut settings); + let registry = + IntegrationRegistry::new(&settings).expect("should build integration registry"); let mut config = create_test_config(); - config.integrations = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) - .expect("should build integration registry"); - config.gpt_diagnostics = Some(decision); + config.integrations = registry.clone(); + config.request_state = request_fixture::marked(); - let processor = create_html_processor(config); + let processor = test_support::create_page_processor(&settings, ®istry, config); let pipeline_config = PipelineConfig { input_compression: Compression::None, output_compression: Compression::None, @@ -989,14 +1308,14 @@ mod tests { .process(Cursor::new(html.as_bytes()), &mut output) .expect("should process HTML"); let processed = String::from_utf8(output).expect("should produce valid UTF-8"); - let bootstrap_marker = "__tsjs_gpt_diagnostics_active"; + let head_marker = request_fixture::HEAD_FLAG; let bundle_marker = "id=\"trustedserver-js\""; - let diagnostics_marker = "tsjs-gpt_diagnostics.min.js"; + let module_marker = request_fixture::MODULE_FILE; assert_eq!( - processed.matches(bootstrap_marker).count(), + processed.matches(head_marker).count(), 1, - "should inject the diagnostics bootstrap once" + "should write the module's head insert once" ); assert_eq!( processed.matches(bundle_marker).count(), @@ -1004,26 +1323,26 @@ mod tests { "should inject the immediate TSJS bundle once" ); assert_eq!( - processed.matches(diagnostics_marker).count(), + processed.matches(module_marker).count(), 1, - "should inject one standalone diagnostics module" + "should write the module's after-bundle insert once" ); - let bootstrap_index = processed - .find(bootstrap_marker) - .expect("should include diagnostics bootstrap"); + let head_index = processed + .find(head_marker) + .expect("should include the head insert"); let bundle_index = processed .find(bundle_marker) .expect("should include immediate TSJS bundle"); - let diagnostics_index = processed - .find(diagnostics_marker) - .expect("should include standalone diagnostics module"); + let module_index = processed + .find(module_marker) + .expect("should include the after-bundle insert"); assert!( - bootstrap_index < bundle_index, - "should activate before core executes" + head_index < bundle_index, + "should write the head insert before the bundle executes" ); assert!( - bundle_index < diagnostics_index, - "should load diagnostics after core" + bundle_index < module_index, + "should write the after-bundle insert after the bundle" ); } @@ -1066,6 +1385,37 @@ mod tests { assert!(!result.contains("origin.example.com")); } + #[test] + fn the_origin_s_address_is_moved_in_srcset_and_imagesrcset() { + let mut processor = create_html_processor(create_test_config()); + + let output = processor + .process_chunk( + br#""#, + true, + ) + .expect("should process the document"); + let processed = String::from_utf8(output).expect("should stay UTF-8"); + + assert!( + processed.contains( + r#"imagesrcset="https://test.example.com/a.png 1x, https://test.example.com/b.png 2x, //test.example.com/c.png 3x""# + ), + "should move the origin's address in every candidate of an imagesrcset: {processed}" + ); + assert!( + processed.contains( + r#"srcset="https://test.example.com/a.png 1x, https://test.example.com/b.png 2x, //test.example.com/c.png 3x, test.example.com/d.png 4x""# + ), + "should move the origin's address in every candidate of a srcset, a bare host \ + included: {processed}" + ); + assert!( + processed.contains(r#""#), + "should leave a srcset that names another host as it is: {processed}" + ); + } + #[test] fn test_html_processor_config_from_settings() { let settings = create_test_settings(); @@ -1090,62 +1440,6 @@ mod tests { assert_eq!(config.request_scheme, "https"); } - #[test] - fn suppressed_datadome_tag_preserves_and_rewrites_publisher_tag() { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config( - "datadome", - &json!({ - "enabled": true, - "client_side_key": "test-client-key", - }), - ) - .expect("should configure DataDome integration"); - let registry = IntegrationRegistry::new(&settings) - .expect("should create integration registry with DataDome"); - let config = HtmlProcessorConfig::from_settings( - &settings, - ®istry, - "origin.example.com", - "test.example.com", - "https", - ) - .with_datadome_client_tag_suppression(true); - let mut processor = create_html_processor(config); - - let output = processor - .process_chunk( - br#"content"#, - true, - ) - .expect("should process HTML"); - let html = String::from_utf8(output).expect("should produce UTF-8 HTML"); - - assert!( - !html.contains("window.ddjskey"), - "should omit the DataDome client configuration" - ); - assert!( - html.contains("id=\"publisher-datadome\""), - "should preserve the publisher-originated DataDome tag" - ); - assert!( - html.contains("src=\"/integrations/datadome/tags.js\""), - "should rewrite the publisher-originated DataDome tag" - ); - assert!( - !html.contains("https://js.datadome.co/tags.js"), - "should remove the original third-party DataDome URL" - ); - assert_eq!( - html.matches("/integrations/datadome/tags.js").count(), - 1, - "should leave exactly one publisher-originated DataDome tag" - ); - } - #[test] fn test_real_publisher_html() { // Test with publisher HTML from test_publisher.html @@ -1223,61 +1517,6 @@ mod tests { ); } - #[test] - fn test_integration_registry_rewrites_integration_scripts() { - let html = r#" - - "#; - - let mut settings = Settings::default(); - let shim_src = "https://edge.example.com/static/testlight.js".to_owned(); - settings - .integrations - .insert_config( - "testlight", - &json!({ - "enabled": true, - "endpoint": "https://example.com/openrtb2/auction", - "rewrite_scripts": true, - "shim_src": shim_src, - }), - ) - .expect("should insert testlight config"); - - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) - .expect("should create registry"); - let mut config = create_test_config(); - config.integrations = registry; - - let processor = create_html_processor(config); - let pipeline_config = PipelineConfig { - input_compression: Compression::None, - output_compression: Compression::None, - chunk_size: 8192, - }; - let mut pipeline = StreamingPipeline::new(pipeline_config, processor); - - let mut output = Vec::new(); - let result = pipeline.process(Cursor::new(html.as_bytes()), &mut output); - result.unwrap(); - - let processed = String::from_utf8_lossy(&output); - assert!( - processed.contains(&shim_src), - "Integration shim should replace integration script reference" - ); - assert!( - !processed.contains("cdn.testlight.com"), - "Original integration URL should be removed" - ); - } - #[test] fn test_real_publisher_html_with_gzip() { use flate2::Compression as GzCompression; @@ -1541,6 +1780,8 @@ mod tests { fn injects_ad_slots_at_head_open() { let config = HtmlProcessorConfig { csp_nonce_observed: None, + public_prefix: None, + prepared: Arc::new(Prepared::default()), body_close: BodyCloseInjection::None, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), @@ -1550,10 +1791,10 @@ mod tests { r#""# .to_string(), ), + permissions_script: None, ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let mut processor = create_html_processor(config); let output = processor @@ -1619,6 +1860,8 @@ mod tests { let state = std::sync::Arc::new(std::sync::Mutex::new(Some(bids_script.to_string()))); let config = HtmlProcessorConfig { csp_nonce_observed: None, + public_prefix: None, + prepared: Arc::new(Prepared::default()), body_close: BodyCloseInjection::InlineBids, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), @@ -1627,10 +1870,10 @@ mod tests { ad_slots_script: Some( r#""#.to_string(), ), + permissions_script: None, ad_bids_state: state, max_buffered_body_bytes: 16 * 1024 * 1024, - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let mut processor = create_html_processor(config); let output = processor @@ -1658,6 +1901,8 @@ mod tests { let state = std::sync::Arc::new(std::sync::Mutex::new(Some(bids_script.to_string()))); let config = HtmlProcessorConfig { csp_nonce_observed: None, + public_prefix: None, + prepared: Arc::new(Prepared::default()), body_close: BodyCloseInjection::InlineBids, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), @@ -1666,10 +1911,10 @@ mod tests { ad_slots_script: Some( r#""#.to_string(), ), + permissions_script: None, ad_bids_state: state, max_buffered_body_bytes: 16 * 1024 * 1024, - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let mut processor = create_html_processor(config); // Malformed HTML with two elements (common in CMS template pages) @@ -1698,16 +1943,18 @@ mod tests { let request_host = "proxy.test-publisher.example.com"; let config = HtmlProcessorConfig { csp_nonce_observed: None, + public_prefix: None, + prepared: Arc::new(Prepared::default()), body_close: BodyCloseInjection::None, origin_host: "origin.test-publisher.example.com".to_string(), request_host: request_host.to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::default(), ad_slots_script: None, + permissions_script: None, ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let mut processor = create_html_processor(config); let output = processor @@ -1752,6 +1999,8 @@ mod tests { let state = std::sync::Arc::new(std::sync::Mutex::new(None)); let config = HtmlProcessorConfig { csp_nonce_observed: None, + public_prefix: None, + prepared: Arc::new(Prepared::default()), body_close: BodyCloseInjection::InlineBids, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), @@ -1760,10 +2009,10 @@ mod tests { ad_slots_script: Some( r#""#.to_string(), ), + permissions_script: None, ad_bids_state: state, max_buffered_body_bytes: 16 * 1024 * 1024, - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let mut processor = create_html_processor(config); let output = processor @@ -1784,16 +2033,18 @@ mod tests { let state = std::sync::Arc::new(std::sync::Mutex::new(None)); let config = HtmlProcessorConfig { csp_nonce_observed: None, + public_prefix: None, + prepared: Arc::new(Prepared::default()), body_close: BodyCloseInjection::None, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::empty_for_tests(), ad_slots_script: None, + permissions_script: None, ad_bids_state: state, max_buffered_body_bytes: 16 * 1024 * 1024, - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let mut processor = create_html_processor(config); let output = processor @@ -1809,16 +2060,18 @@ mod tests { fn marker_mode_config(marker: &str, observer: Option>) -> HtmlProcessorConfig { HtmlProcessorConfig { csp_nonce_observed: observer, + public_prefix: None, body_close: BodyCloseInjection::Marker(marker.to_string()), origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::empty_for_tests(), ad_slots_script: None, + permissions_script: None, ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + prepared: Arc::new(Prepared::default()), + request_state: IntegrationRequestState::default(), } } @@ -1963,50 +2216,6 @@ mod tests { } } - #[test] - fn nextjs_output_overflow_restores_in_progress_script_at_every_split() { - let mut settings = create_test_settings(); - settings.integrations.insert( - "nextjs".to_owned(), - json!({"enabled": true, "max_combined_payload_bytes": 128}), - ); - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new(crate::auction::compile_auction_plan(&settings).expect("should compile plan")), - ) - .expect("should create registry"); - let first = r#""#; - let script = r#"self.__next_f.push([1,"c"])"#; - let padding = "x".repeat(129); - let expected = format!("{first}{padding}"); - - for split in 1..script.len() { - let mut config = create_test_config(); - config.integrations = registry.clone(); - let mut processor = create_html_processor(config); - let mut output = processor - .process_chunk(first.as_bytes(), false) - .expect("should process unresolved RSC group"); - let second = format!("{padding}", &script[split..]); - output.extend( - processor - .process_chunk(third.as_bytes(), true) - .expect("should finish bypassed script"), - ); - assert_eq!( - String::from_utf8(output).expect("should retain UTF-8"), - expected, - "should restore all original bytes when overflow occurs at script split {split}" - ); - } - } - #[test] fn a_nonce_bearing_meta_policy_is_observed() { let observed = Arc::new(AtomicBool::new(false)); @@ -2070,16 +2279,18 @@ mod tests { const MARKER: &str = ""; let config = HtmlProcessorConfig { csp_nonce_observed: None, + public_prefix: None, + prepared: Arc::new(Prepared::default()), body_close: BodyCloseInjection::Marker(MARKER.to_string()), origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::empty_for_tests(), ad_slots_script: None, + permissions_script: None, ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let source = format!(r#""#); diff --git a/crates/trusted-server-core/src/http_util.rs b/crates/trusted-server-core/src/http_util.rs index 457e503568..2b9bed48bc 100644 --- a/crates/trusted-server-core/src/http_util.rs +++ b/crates/trusted-server-core/src/http_util.rs @@ -152,18 +152,30 @@ impl RequestInfo { /// [`sanitize_forwarded_headers`] at the edge, so `Host` and /// [`ClientInfo`] TLS detection are the only sources that fire. pub fn from_request(req: &Request, client_info: &ClientInfo) -> Self { - let host = extract_request_host(req); - let scheme = detect_request_scheme( - req, - client_info.tls_protocol.as_deref(), - client_info.tls_cipher.as_deref(), - ); - - Self { host, scheme } + Self { + host: request_host(req).to_owned(), + scheme: request_scheme(req, client_info).to_owned(), + } } } -fn extract_request_host(req: &Request) -> String { +/// The host a request was made for, read as [`RequestInfo::from_request`] +/// reads it, borrowed from the request. +pub(crate) fn request_host(req: &Request) -> &str { + extract_request_host(req) +} + +/// The scheme a request was made over, read as +/// [`RequestInfo::from_request`] reads it. +pub(crate) fn request_scheme(req: &Request, client_info: &ClientInfo) -> &'static str { + detect_request_scheme( + req, + client_info.tls_protocol.as_deref(), + client_info.tls_cipher.as_deref(), + ) +} + +fn extract_request_host(req: &Request) -> &str { req.headers() .get("forwarded") .and_then(|h| h.to_str().ok()) @@ -180,7 +192,6 @@ fn extract_request_host(req: &Request) -> String { .and_then(|h| h.to_str().ok()) }) .unwrap_or_default() - .to_owned() } fn parse_forwarded_param<'a>(forwarded: &'a str, param: &str) -> Option<&'a str> { @@ -221,9 +232,15 @@ fn strip_quotes(value: &str) -> &str { } } -fn normalize_scheme(value: &str) -> Option { - let scheme = value.trim().to_ascii_lowercase(); - (scheme == "https" || scheme == "http").then_some(scheme) +fn normalize_scheme(value: &str) -> Option<&'static str> { + let scheme = value.trim(); + if scheme.eq_ignore_ascii_case("https") { + Some("https") + } else if scheme.eq_ignore_ascii_case("http") { + Some("http") + } else { + None + } } /// Detects the request scheme (HTTP or HTTPS) using Fastly SDK methods and headers. @@ -238,17 +255,17 @@ fn detect_request_scheme( req: &Request, tls_protocol: Option<&str>, tls_cipher: Option<&str>, -) -> String { +) -> &'static str { // 1. First try ClientInfo TLS fields populated at the adapter entry point. if let Some(tls_protocol) = tls_protocol { log::debug!("TLS protocol detected: {tls_protocol}"); - return "https".to_owned(); + return "https"; } // Also check TLS cipher - if present, connection is HTTPS. if tls_cipher.is_some() { log::debug!("TLS cipher detected, using HTTPS"); - return "https".to_owned(); + return "https"; } // 2. Try the Forwarded header (RFC 7239) @@ -284,11 +301,11 @@ fn detect_request_scheme( && let Ok(ssl_str) = ssl.to_str() && (ssl_str == "1" || ssl_str.to_lowercase() == "true") { - return "https".to_owned(); + return "https"; } // Default to HTTP - "http".to_owned() + "http" } /// Build a static text response with strong `ETag` and standard caching headers. @@ -483,6 +500,41 @@ pub fn enforce_max_body_size( Ok(()) } +/// The namespace the inspection and attestation endpoints answer under. +pub const ROUTE_NAMESPACE: &str = "/_ts"; + +/// Response headers that describe the deployment and not the page. +const DEPLOYMENT_HEADERS: &[&str] = &["x-cache", "x-cache-hits", "x-served-by"]; + +/// Removes the headers that describe the deployment from a response, unless +/// the request was for a path in `namespace`, which is where a deployment is +/// asked about itself. +/// +/// An adapter calls this before it finalizes a response, so a header that +/// finalizing writes is not removed, an operator's `[response_headers]` +/// among them. The publisher fallback has already removed them from what the +/// origin sent, with [`remove_deployment_headers`], so a response under +/// `namespace` keeps them only when this server answered it. +pub fn strip_deployment_headers(path: &str, namespace: &str, response: &mut Response) { + // A path that only starts with the namespace's text is not in it. `/_tsx` + // is a publisher's page. + let in_namespace = path == namespace + || path + .strip_prefix(namespace) + .is_some_and(|rest| rest.starts_with('/')); + if !in_namespace { + remove_deployment_headers(response); + } +} + +/// Removes the headers that describe the deployment from a response, whatever +/// path it answers. +pub fn remove_deployment_headers(response: &mut Response) { + for name in DEPLOYMENT_HEADERS { + response.headers_mut().remove(*name); + } +} + #[cfg(test)] mod tests { use super::*; @@ -1037,4 +1089,87 @@ mod tests { "should match text/html case-insensitively in fallback" ); } + + const STRIPPED: [&str; 3] = ["x-cache", "x-cache-hits", "x-served-by"]; + + fn response_with(names: &[&'static str]) -> Response<()> { + let mut response = Response::new(()); + for name in names { + response + .headers_mut() + .insert(*name, HeaderValue::from_static("value")); + } + response + } + + #[test] + fn a_publisher_page_loses_every_deployment_header() { + let mut response = response_with(&STRIPPED); + + strip_deployment_headers("/articles/an-article", ROUTE_NAMESPACE, &mut response); + + for name in STRIPPED { + assert!( + response.headers().get(name).is_none(), + "{name} should not reach a reader" + ); + } + } + + #[test] + fn the_servers_own_namespace_keeps_them() { + for path in ["/_ts", "/_ts/permissions", "/_ts/debug/ja4"] { + let mut response = response_with(&STRIPPED); + + strip_deployment_headers(path, ROUTE_NAMESPACE, &mut response); + + for name in STRIPPED { + assert!( + response.headers().get(name).is_some(), + "{path} should keep {name}" + ); + } + } + } + + #[test] + fn a_header_outside_the_list_is_left_alone() { + let kept = ["x-ts-version", "x-geo-info-available", "cache-control"]; + let mut response = response_with(&kept); + + strip_deployment_headers("/articles/an-article", ROUTE_NAMESPACE, &mut response); + + for name in kept { + assert!( + response.headers().get(name).is_some(), + "{name} should be left alone" + ); + } + } + + #[test] + fn a_path_that_only_starts_with_the_namespace_is_not_in_it() { + for path in ["/_tsx", "/_ts-archive", "/_tsfoo/bar"] { + let mut response = response_with(&["x-served-by"]); + + strip_deployment_headers(path, ROUTE_NAMESPACE, &mut response); + + assert!( + response.headers().get("x-served-by").is_none(), + "{path} is a publisher's page" + ); + } + } + + #[test] + fn the_boundary_is_the_namespace_passed_in() { + let mut kept = response_with(&["x-served-by"]); + let mut stripped = response_with(&["x-served-by"]); + + strip_deployment_headers("/_other/permissions", "/_other", &mut kept); + strip_deployment_headers("/_ts/permissions", "/_other", &mut stripped); + + assert!(kept.headers().get("x-served-by").is_some()); + assert!(stripped.headers().get("x-served-by").is_none()); + } } diff --git a/crates/trusted-server-core/src/inspect.rs b/crates/trusted-server-core/src/inspect.rs new file mode 100644 index 0000000000..11ff52fa73 --- /dev/null +++ b/crates/trusted-server-core/src/inspect.rs @@ -0,0 +1,145 @@ +//! What a deployment shows to anyone who asks, with no credential. +//! +//! A publisher can point a reader, a regulator or an auditor at these +//! addresses, and each sees what this deployment decided for their own +//! request without having to take anybody's word for it. Every address is +//! fixed under `/_ts`, so it is the same on every deployment. +//! +//! | Address | What it shows | +//! | --- | --- | +//! | [`permissions::PERMISSIONS_PAGE_PATH`] | The permissions resolved for the request, the signals that produced them and the terms the data is held under | +//! | [`config::CONFIG_PAGE_PATH`] | The settings the deployment is running, with every secret and every value sensitive by default masked | +//! | [`data::DATA_PAGE_PATH`] | What the deployment holds against the Edge Cookie the request carries, with every identifier masked | +//! +//! The first two answer as a page, and as data with `.json` added. The third +//! is about one reader, so it answers a browser opening it as a page and +//! nothing else. +//! +//! Operator data is not here. It is in the platform's own stores, reached +//! with the platform's own authentication, and never through the publisher's +//! domain. + +use serde_json::Value; + +pub mod config; +pub mod data; +pub mod permissions; + +/// What a masked value shows as. +pub const MASK: &str = "XXXX"; + +const DATA_FORM_LEDE: &str = + "Add .json to this address for the same information as data."; + +/// Renders `payload` as a page titled `title`. +#[must_use] +pub fn render_page(title: &str, payload: &Value) -> String { + page(title, DATA_FORM_LEDE, payload) +} + +/// Renders `payload` as [`render_page`] does, under `lede` in place of the +/// line saying where the data form is, for an address that has none. +#[must_use] +pub fn render_page_with_lede(title: &str, lede: &str, payload: &Value) -> String { + page(title, &html_escape(lede), payload) +} + +/// The page, with `lede` written into it as markup. +fn page(title: &str, lede: &str, payload: &Value) -> String { + let pretty = serde_json::to_string_pretty(payload).unwrap_or_else(|_| "{}".to_owned()); + format!( + "\n\ +\n\ +\n\ +\n\ +\n\ +\n\ +{title}\n\ +\n\ +\n\ +\n\ +
\n\ +

{title}

\n\ +

{lede}

\n\ +
{body}
\n\ +
\n\ +\n\ +\n", + title = html_escape(title), + body = html_escape(&pretty), + ) +} + +/// Escapes `value` for HTML text content. +pub(crate) fn html_escape(value: &str) -> String { + value + .replace('&', "&") + .replace('<', "<") + .replace('>', ">") +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn a_page_escapes_whatever_its_payload_and_title_carry() { + let html = render_page( + "A ", + &json!({ "value": "</pre><script>x()</script> & more" }), + ); + + assert!( + !html.contains("<script>x()"), + "should not let a value open a tag" + ); + assert!( + html.contains("</pre><script>x()</script> & more"), + "should show the value as text" + ); + assert!( + html.contains("<title>A <title>"), + "should escape the title as well" + ); + } + + #[test] + fn a_page_tells_the_reader_where_the_data_form_is() { + let html = render_page("Permissions", &json!({})); + + assert!( + html.contains(".json"), + "should say how to get the same information as data" + ); + assert!( + html.contains(""), + "should keep the page out of search indexes" + ); + } + + #[test] + fn a_page_with_a_lede_of_its_own_says_that_instead() { + let html = render_page_with_lede("Your data", "Held for & no other", &json!({})); + + assert!( + html.contains("

Held for <you> & no other

"), + "should show the lede as text" + ); + assert!( + !html.contains(".json"), + "should not point at a data form the address does not have" + ); + } +} diff --git a/crates/trusted-server-core/src/inspect/config.rs b/crates/trusted-server-core/src/inspect/config.rs new file mode 100644 index 0000000000..0e0a5171bf --- /dev/null +++ b/crates/trusted-server-core/src/inspect/config.rs @@ -0,0 +1,714 @@ +//! The configuration a deployment is running, served to anyone at +//! [`CONFIG_PAGE_PATH`] and [`CONFIG_JSON_PATH`]. +//! +//! What is shown is the settings the handler holds, after defaults have been +//! applied, with values masked as [`MASK`]: +//! +//! 1. Every secret, meaning a value the loader filled in from the secret +//! store or a leaf declared as one, whatever the publisher's document says. +//! 2. Every value sensitive by default, being a settings field serialized +//! with [`crate::redacted::sensitive`] and every +//! [`Redacted`](crate::redacted::Redacted) value, unless `[inspect] show` +//! names it. +//! 3. Everything `[inspect] hide` names. + +use std::collections::HashSet; +use std::fmt::{self, Write as _}; + +use edgezero_core::app_config::{AppConfigMeta as _, SecretPathSegment}; +use edgezero_core::body::Body as EdgeBody; +use error_stack::Report; +use http::{HeaderValue, Request, Response, StatusCode, header}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use serde_json::{Map, Value, json}; + +use crate::config::TrustedServerAppConfig; +use crate::error::TrustedServerError; +use crate::integrations::IntegrationBuilder; +use crate::redacted::{SENSITIVE_MARKER, to_marked_value}; +use crate::secret_resolution::unresolved_secrets; +use crate::settings::Settings; + +use super::{MASK, render_page}; + +/// The page form of the configuration. +pub const CONFIG_PAGE_PATH: &str = "/_ts/config"; + +/// The JSON form of the configuration. +pub const CONFIG_JSON_PATH: &str = "/_ts/config.json"; + +/// Both addresses, the page first. +pub const CONFIG_PATHS: [&str; 2] = [CONFIG_PAGE_PATH, CONFIG_JSON_PATH]; + +const NOT_PUBLISHED: &str = "This publisher does not publish its configuration."; + +const PATTERN_FORM: &str = "A pattern is keys joined by `.`, with `[]` for every element of a \ + list or `[N]` for one, such as \ + `proxy.asset_routes[0].origin_url`"; + +/// The `[inspect]` section, which says what the configuration page shows. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct InspectConfig { + /// Whether the configuration is published. `false` answers not found. + #[serde(default = "default_config")] + pub config: bool, + /// Values masked by default that are shown instead. A secret cannot be. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub show: Vec, + /// Values masked as well as the defaults. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub hide: Vec, +} + +fn default_config() -> bool { + true +} + +impl Default for InspectConfig { + fn default() -> Self { + Self { + config: default_config(), + show: Vec::new(), + hide: Vec::new(), + } + } +} + +impl InspectConfig { + /// Whether the section says nothing the defaults do not. + #[must_use] + pub fn is_default(&self) -> bool { + *self == Self::default() + } +} + +/// One step of a concrete path through the view. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub enum PathStep { + /// A key of an object. + Key(String), + /// An index into a list. + Index(usize), +} + +/// A concrete path written the way a pattern is, such as +/// `proxy.asset_routes[0].origin_url`. +#[must_use] +pub fn render_path(steps: &[PathStep]) -> String { + let mut out = String::new(); + for step in steps { + match step { + PathStep::Key(key) => { + if !out.is_empty() { + out.push('.'); + } + out.push_str(key); + } + PathStep::Index(index) => { + let _ = write!(out, "[{index}]"); + } + } + } + out +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum PatternStep { + Key(String), + Each, + Index(usize), +} + +/// Values in the view named by keys joined by `.`, with `[]` for every +/// element of a list and `[N]` for one, such as +/// `proxy.asset_routes[].origin_url` or `proxy.asset_routes[0].prefix`. +/// +/// A pattern names the values at exactly its own depth. A key holding `.`, +/// `[` or `]` cannot be named. +#[derive(Clone, PartialEq, Eq)] +pub struct PathPattern { + text: String, + steps: Vec, +} + +impl PathPattern { + /// Reads a pattern. + /// + /// # Errors + /// + /// A message naming `text` and what is wrong with it. + pub fn parse(text: &str) -> Result { + let malformed = + |why: &str| format!("`{text}` is not a path pattern, because {why}. {PATTERN_FORM}"); + let mut steps = Vec::new(); + let mut rest = text; + loop { + let end = rest.find(['.', '[', ']']).unwrap_or(rest.len()); + let (key, after_key) = rest.split_at(end); + if key.is_empty() { + return Err(malformed("a key is empty")); + } + if key.chars().any(|c| c.is_whitespace() || c.is_control()) { + return Err(malformed("a key holds a space or a control character")); + } + steps.push(PatternStep::Key(key.to_owned())); + rest = after_key; + while let Some(open) = rest.strip_prefix('[') { + let Some((inside, after)) = open.split_once(']') else { + return Err(malformed("a `[` is not closed")); + }; + let step = if inside.is_empty() { + PatternStep::Each + } else if inside.bytes().all(|b| b.is_ascii_digit()) { + PatternStep::Index( + inside + .parse() + .map_err(|_| malformed("an index is too large"))?, + ) + } else { + return Err(malformed("only a number may sit between `[` and `]`")); + }; + steps.push(step); + rest = after; + } + if rest.is_empty() { + break; + } + match rest.strip_prefix('.') { + Some(after) => rest = after, + None if rest.starts_with(']') => return Err(malformed("a `]` has no `[`")), + None => return Err(malformed("a `]` is followed by neither `.` nor `[`")), + } + } + Ok(Self { + text: text.to_owned(), + steps, + }) + } + + /// The pattern a secret or sensitive declaration names, or `None` for a + /// segment kind a pattern cannot express. + #[must_use] + pub fn from_segments(segments: &[SecretPathSegment]) -> Option { + let mut steps = Vec::with_capacity(segments.len()); + let mut text = String::new(); + for segment in segments { + match segment { + SecretPathSegment::Field(name) | SecretPathSegment::OptionalField(name) => { + if !text.is_empty() { + text.push('.'); + } + text.push_str(name); + steps.push(PatternStep::Key(name.to_string())); + } + SecretPathSegment::ArrayEach => { + text.push_str("[]"); + steps.push(PatternStep::Each); + } + _ => return None, + } + } + Some(Self { text, steps }) + } + + /// The pattern as written. + #[must_use] + pub fn as_str(&self) -> &str { + &self.text + } + + /// Whether the concrete path `path` is one the pattern names. + #[must_use] + pub fn matches(&self, path: &[PathStep]) -> bool { + self.steps.len() == path.len() + && self + .steps + .iter() + .zip(path) + .all(|(pattern, step)| match (pattern, step) { + (PatternStep::Key(want), PathStep::Key(got)) => want == got, + (PatternStep::Each, PathStep::Index(_)) => true, + (PatternStep::Index(want), PathStep::Index(got)) => want == got, + _ => false, + }) + } +} + +impl fmt::Debug for PathPattern { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(&self.text) + } +} + +impl fmt::Display for PathPattern { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(&self.text) + } +} + +impl Serialize for PathPattern { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(&self.text) + } +} + +impl<'de> Deserialize<'de> for PathPattern { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let text = String::deserialize(deserializer)?; + Self::parse(&text).map_err(serde::de::Error::custom) + } +} + +/// The configuration as the endpoint shows it. +#[derive(Debug, Clone, PartialEq)] +pub struct ConfigView { + /// Every masked path, concrete and sorted. + pub masked: Vec, + /// The settings, each masked value replaced by [`MASK`] and every + /// object's keys sorted. + pub settings: Value, +} + +/// Builds the view of `settings`. +/// +/// # Errors +/// +/// When the settings cannot be represented as JSON. +pub fn build_view(settings: &Settings) -> Result> { + let marked = marked_settings(settings)?; + Ok(Rules::new(settings).render(&marked)) +} + +fn marked_settings(settings: &Settings) -> Result> { + // The serializer's error is left out, because nothing may carry a value + // out of the settings. + to_marked_value(settings).map_err(|_| { + Report::new(TrustedServerError::Configuration { + message: "the settings could not be represented for the configuration view".to_owned(), + }) + }) +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum List { + Show, + Hide, +} + +impl fmt::Display for List { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Show => "show", + Self::Hide => "hide", + }) + } +} + +/// Everything that decides whether a value is masked. +#[derive(Clone)] +struct Rules<'a> { + secret_paths: &'a [Vec], + secret_patterns: Vec, + secret_values: &'a [String], + show: Vec<&'a PathPattern>, + hide: Vec<&'a PathPattern>, +} + +impl<'a> Rules<'a> { + fn new(settings: &'a Settings) -> Self { + let secrets = settings.resolved_secrets(); + // Core's own declarations are added here as well as recorded by the + // loader, so settings built any other way still mask them. + let secret_patterns = secrets + .patterns() + .iter() + .cloned() + .chain( + TrustedServerAppConfig::secret_fields() + .iter() + .filter_map(|field| PathPattern::from_segments(&field.path)), + ) + .collect(); + Self { + secret_paths: secrets.paths(), + secret_patterns, + secret_values: secrets.values(), + show: settings.inspect.show.iter().collect(), + hide: settings.inspect.hide.iter().collect(), + } + } + + /// These rules less one of the publisher's patterns. + fn without(&self, list: List, index: usize) -> Self { + let mut rules = self.clone(); + let _removed = match list { + List::Show => rules.show.remove(index), + List::Hide => rules.hide.remove(index), + }; + rules + } + + fn is_secret(&self, path: &[PathStep]) -> bool { + self.secret_paths.iter().any(|secret| secret == path) + || self + .secret_patterns + .iter() + .any(|pattern| pattern.matches(path)) + } + + fn render(&self, marked: &Value) -> ConfigView { + let mut masked = Vec::new(); + let mut path = Vec::new(); + let mut settings = self.mask(marked, &mut path, &mut masked); + self.scan(&mut settings, &mut path, &mut masked); + masked.sort(); + masked.dedup(); + ConfigView { + masked: masked.iter().map(|steps| render_path(steps)).collect(), + settings, + } + } + + /// `node` with every sensitive marker resolved, every masked value + /// replaced and every object's keys sorted. + fn mask( + &self, + node: &Value, + path: &mut Vec, + masked: &mut Vec>, + ) -> Value { + let (inner, marked_sensitive) = match unmark(node) { + Some(inner) => (inner, true), + None => (node, false), + }; + if is_empty(inner) { + return inner.clone(); + } + let shown = self.show.iter().any(|p| p.matches(path)); + if self.is_secret(path) + || self.hide.iter().any(|p| p.matches(path)) + || (marked_sensitive && !shown) + { + masked.push(path.clone()); + return Value::String(MASK.to_owned()); + } + match inner { + Value::Object(map) => { + let mut entries: Vec<(&String, &Value)> = map.iter().collect(); + entries.sort_unstable_by(|left, right| left.0.cmp(right.0)); + let mut out = Map::new(); + for (key, value) in entries { + path.push(PathStep::Key(key.clone())); + out.insert(key.clone(), self.mask(value, path, masked)); + path.pop(); + } + Value::Object(out) + } + Value::Array(items) => { + let mut out = Vec::with_capacity(items.len()); + for (index, item) in items.iter().enumerate() { + path.push(PathStep::Index(index)); + out.push(self.mask(item, path, masked)); + path.pop(); + } + Value::Array(out) + } + other => other.clone(), + } + } + + /// Masks every string, and every key, that contains a secret value, + /// wherever it sits. A key holding a secret is renamed [`MASK`], or + /// `XXXX-2` and on where that name is taken, and every path already + /// recorded beneath it is replaced by the renamed one, so no recorded path + /// carries the secret. + fn scan(&self, node: &mut Value, path: &mut Vec, masked: &mut Vec>) { + match node { + Value::String(text) => { + if self.holds_secret(text) { + masked.push(path.clone()); + *node = Value::String(MASK.to_owned()); + } + } + Value::Array(items) => { + for (index, item) in items.iter_mut().enumerate() { + path.push(PathStep::Index(index)); + self.scan(item, path, masked); + path.pop(); + } + } + Value::Object(map) => { + let entries = std::mem::take(map); + let kept: HashSet = entries + .keys() + .filter(|key| !self.holds_secret(key)) + .cloned() + .collect(); + let mut sorted = std::collections::BTreeMap::new(); + for (key, mut value) in entries { + if self.holds_secret(&key) { + let mut renamed = MASK.to_owned(); + let mut copy = 1; + while kept.contains(&renamed) || sorted.contains_key(&renamed) { + copy += 1; + renamed = format!("{MASK}-{copy}"); + } + path.push(PathStep::Key(key)); + masked.retain(|recorded| !recorded.starts_with(path)); + path.pop(); + path.push(PathStep::Key(renamed.clone())); + masked.push(path.clone()); + path.pop(); + sorted.insert(renamed, Value::String(MASK.to_owned())); + continue; + } + path.push(PathStep::Key(key.clone())); + self.scan(&mut value, path, masked); + path.pop(); + sorted.insert(key, value); + } + map.extend(sorted); + } + Value::Null | Value::Bool(_) | Value::Number(_) => {} + } + } + + fn holds_secret(&self, text: &str) -> bool { + self.secret_values + .iter() + .any(|secret| !secret.is_empty() && text.contains(secret.as_str())) + } +} + +/// The value a sensitive marker wraps, or `None` when `node` is not one. +fn unmark(node: &Value) -> Option<&Value> { + match node { + Value::Object(map) if map.len() == 1 => map.get(SENSITIVE_MARKER), + _ => None, + } +} + +/// Whether there is nothing in `value` to hide. +fn is_empty(value: &Value) -> bool { + match value { + Value::Null => true, + Value::String(text) => text.is_empty(), + Value::Array(items) => items.is_empty(), + Value::Object(map) => map.is_empty(), + Value::Bool(_) | Value::Number(_) => false, + } +} + +/// The concrete paths of the values in `marked` that `pattern` names. +fn select(marked: &Value, pattern: &PathPattern) -> Vec> { + fn walk( + node: &Value, + steps: &[PatternStep], + path: &mut Vec, + out: &mut Vec>, + ) { + let node = unmark(node).unwrap_or(node); + let Some((step, rest)) = steps.split_first() else { + out.push(path.clone()); + return; + }; + match (step, node) { + (PatternStep::Key(key), Value::Object(map)) => { + if let Some(child) = map.get(key) { + path.push(PathStep::Key(key.clone())); + walk(child, rest, path, out); + path.pop(); + } + } + (PatternStep::Each, Value::Array(items)) => { + for (index, item) in items.iter().enumerate() { + path.push(PathStep::Index(index)); + walk(item, rest, path, out); + path.pop(); + } + } + (PatternStep::Index(index), Value::Array(items)) => { + if let Some(item) = items.get(*index) { + path.push(PathStep::Index(*index)); + walk(item, rest, path, out); + path.pop(); + } + } + _ => {} + } + } + let mut out = Vec::new(); + walk(marked, &pattern.steps, &mut Vec::new(), &mut out); + out +} + +/// Refuses an `[inspect]` pattern that would not be honored as written, +/// being one that matches nothing, a `show` naming a secret, one another +/// pattern of its list already covers, a `show` a `hide` covers, one that +/// changes nothing the view shows, and any pattern beside `config = false`. +/// Costs nothing when the section has no patterns. +/// +/// # Errors +/// +/// Naming the first pattern at fault. +pub(crate) fn validate_patterns(settings: &Settings) -> Result<(), Report> { + let inspect = &settings.inspect; + if inspect.show.is_empty() && inspect.hide.is_empty() { + return Ok(()); + } + if !inspect.config { + return Err(configuration( + "[inspect] show and hide say what the configuration page shows, and config = false \ + publishes no page, so they change nothing. Remove them, or publish the page" + .to_owned(), + )); + } + let marked = marked_settings(settings)?; + let rules = Rules::new(settings); + let full = rules.render(&marked); + for (list, patterns) in [(List::Show, &inspect.show), (List::Hide, &inspect.hide)] { + for (index, pattern) in patterns.iter().enumerate() { + let selected = select(&marked, pattern); + if selected.is_empty() { + return Err(configuration(format!( + "[inspect] {list} names `{pattern}`, which matches no value in this \ + configuration, so it changes nothing" + ))); + } + if list == List::Show && selected.iter().any(|path| rules.is_secret(path)) { + return Err(configuration(format!( + "[inspect] show names `{pattern}`, which is a secret. A secret is always \ + masked, so it cannot be shown" + ))); + } + let covers = |other: &PathPattern| { + let covered = select(&marked, other); + selected.iter().all(|path| covered.contains(path)) + }; + if let Some(other) = patterns + .iter() + .enumerate() + .find(|(other_index, other)| *other_index != index && covers(other)) + .map(|(_, other)| other) + { + return Err(configuration(format!( + "[inspect] {list} names `{pattern}`, which changes nothing, because \ + `{other}` in the same list already names everything it does" + ))); + } + if list == List::Show + && let Some(hide) = inspect.hide.iter().find(|hide| covers(hide)) + { + return Err(configuration(format!( + "[inspect] show names `{pattern}`, which changes nothing, because hide \ + names it as `{hide}` and hide wins over show" + ))); + } + if rules.without(list, index).render(&marked) == full { + let reason = match list { + List::Show => "nothing it names is masked by default", + List::Hide => "everything it names is masked or empty already", + }; + return Err(configuration(format!( + "[inspect] {list} names `{pattern}`, which changes nothing, because {reason}" + ))); + } + } + } + Ok(()) +} + +/// [`validate_patterns`] for a deployment's settings before they are pushed, +/// when their secret fields still hold key names. The view is handed every +/// secret leaf the load will look for, the ones only the configuration and +/// `extra_integrations` name included, so a pattern the load refuses is +/// refused here. +/// +/// # Errors +/// +/// As [`validate_patterns`]. +pub(crate) fn validate_patterns_for_deploy( + settings: &Settings, + extra_integrations: &[IntegrationBuilder], +) -> Result<(), Report> { + if settings.inspect.show.is_empty() && settings.inspect.hide.is_empty() { + return Ok(()); + } + // The serializer's error is left out, as in `marked_settings`. + let data = serde_json::to_value(settings).map_err(|_| { + configuration("the settings could not be represented for the configuration view".to_owned()) + })?; + let builders = crate::integrations::all_builders(extra_integrations).collect::>(); + let mut settings = settings.clone(); + settings.set_resolved_secrets(unresolved_secrets::( + &data, + crate::module_secrets::secret_fields(&builders), + )); + validate_patterns(&settings) +} + +fn configuration(message: String) -> Report { + Report::new(TrustedServerError::Configuration { message }) +} + +/// Answers [`CONFIG_PAGE_PATH`] with the page and [`CONFIG_JSON_PATH`] with +/// the JSON, or not found when the publisher does not publish its +/// configuration. +#[must_use] +pub fn handle_config(settings: &Settings, req: &Request) -> Response { + let json = req.uri().path() == CONFIG_JSON_PATH; + if !settings.inspect.config { + let payload = json!({ "error": NOT_PUBLISHED }); + return answer(StatusCode::NOT_FOUND, json, &payload); + } + match build_view(settings) { + Ok(view) => answer(StatusCode::OK, json, &view_payload(&view)), + Err(_) => { + log::error!("the configuration view could not be built"); + let payload = json!({ "error": "The configuration could not be shown." }); + answer(StatusCode::INTERNAL_SERVER_ERROR, json, &payload) + } + } +} + +/// What the endpoint serves for `view`, with the version of the core crate +/// that answered. +#[must_use] +pub fn view_payload(view: &ConfigView) -> Value { + json!({ + "version": env!("CARGO_PKG_VERSION"), + "masked": view.masked, + "settings": view.settings, + }) +} + +fn answer(status: StatusCode, json: bool, payload: &Value) -> Response { + let (content_type, body) = if json { + ("application/json", payload.to_string()) + } else { + ( + "text/html; charset=utf-8", + render_page("Configuration", payload), + ) + }; + Response::builder() + .status(status) + .header(header::CONTENT_TYPE, HeaderValue::from_static(content_type)) + .header(header::CACHE_CONTROL, HeaderValue::from_static("no-store")) + .header( + header::ACCESS_CONTROL_ALLOW_ORIGIN, + HeaderValue::from_static("*"), + ) + .body(EdgeBody::from(body.into_bytes())) + .expect("should build configuration response") +} + +#[cfg(test)] +mod tests; diff --git a/crates/trusted-server-core/src/inspect/config/tests.rs b/crates/trusted-server-core/src/inspect/config/tests.rs new file mode 100644 index 0000000000..f58d3f0296 --- /dev/null +++ b/crates/trusted-server-core/src/inspect/config/tests.rs @@ -0,0 +1,926 @@ +use std::collections::BTreeSet; + +use edgezero_core::blob_envelope::BlobEnvelope; + +use super::*; +use crate::config_payload::settings_from_config_blob_with; +use crate::integrations::{IntegrationBuilder, ModuleSecretSetting}; +use crate::platform::{PlatformError, PlatformSecretStore, StoreId, StoreName}; +use crate::secret_resolution::ResolvedSecrets; +use crate::settings::ProxyAssetRoute; +use crate::test_support::tests::crate_test_settings_str; + +/// Every value this store hands back starts with this, so a view can be +/// searched for all of them at once. +const CANARY: &str = "canary-"; + +/// Resolves every key name to a value of its own, long enough for any +/// minimum length a secret has. +struct CanaryStore; + +impl PlatformSecretStore for CanaryStore { + fn get_bytes( + &self, + _store_name: &StoreName, + key: &str, + ) -> Result, Report> { + Ok(format!("{CANARY}{key}-value-0123456789abcdef").into_bytes()) + } + + fn create( + &self, + _store_id: &StoreId, + _name: &str, + _value: &str, + ) -> Result<(), Report> { + Ok(()) + } + + fn delete(&self, _store_id: &StoreId, _name: &str) -> Result<(), Report> { + Ok(()) + } +} + +fn settings() -> Settings { + Settings::from_toml(&crate_test_settings_str()).expect("should parse test settings") +} + +/// The test settings with `extra` appended, as a stored document holds +/// them, so every secret in them is the name of one. +fn document(extra: &str) -> Value { + document_from(&format!("{}\n{extra}\n", crate_test_settings_str())) +} + +/// `toml` as a stored document holds it, with the allowed domains a load +/// needs for the test settings' asset hosts. +fn document_from(toml: &str) -> Value { + let mut settings = Settings::from_toml(toml).expect("should parse the document"); + settings.proxy.allowed_domains = vec!["*.example".to_owned(), "*.example.com".to_owned()]; + serde_json::to_value(&settings).expect("should serialize the document") +} + +fn load_with( + data: Value, + builders: &[IntegrationBuilder], +) -> Result> { + let envelope = BlobEnvelope::new(data, "2026-01-01T00:00:00Z".to_owned()); + let envelope = serde_json::to_string(&envelope).expect("should serialize the envelope"); + settings_from_config_blob_with( + &envelope, + &CanaryStore, + &StoreName::from("ts_secrets"), + builders, + ) +} + +fn load(data: Value) -> Result> { + load_with(data, &[]) +} + +fn view(settings: &Settings) -> ConfigView { + build_view(settings).expect("should build the view") +} + +fn request(uri: &str) -> Request { + Request::builder() + .uri(uri) + .body(EdgeBody::empty()) + .expect("should build a request") +} + +fn body_text(response: Response) -> String { + let bytes = response + .into_body() + .into_bytes() + .expect("a buffered body") + .to_vec(); + String::from_utf8(bytes).expect("a text body") +} + +/// The JSON and the page the endpoint serves for `settings`. +fn served(settings: &Settings) -> (String, String) { + ( + body_text(handle_config(settings, &request(CONFIG_JSON_PATH))), + body_text(handle_config(settings, &request(CONFIG_PAGE_PATH))), + ) +} + +/// The value at a concrete path, such as `proxy.asset_routes[0].origin_url`. +fn at<'v>(value: &'v Value, path: &str) -> Option<&'v Value> { + let pattern = PathPattern::parse(path).expect("a concrete path"); + let mut node = value; + for step in &pattern.steps { + node = match (step, node) { + (PatternStep::Key(key), Value::Object(map)) => map.get(key)?, + (PatternStep::Index(index), Value::Array(items)) => items.get(*index)?, + _ => return None, + }; + } + Some(node) +} + +fn is_masked(view: &ConfigView, path: &str) -> bool { + view.masked.iter().any(|masked| masked == path) +} + +/// `value` with every sensitive marker replaced by what it wraps. +fn unmarked(value: &Value) -> Value { + if let Some(inner) = unmark(value) { + return unmarked(inner); + } + match value { + Value::Object(map) => Value::Object( + map.iter() + .map(|(key, value)| (key.clone(), unmarked(value))) + .collect(), + ), + Value::Array(items) => Value::Array(items.iter().map(unmarked).collect()), + other => other.clone(), + } +} + +#[test] +fn a_pattern_names_keys_every_element_or_one_element() { + let every = PathPattern::parse("proxy.asset_routes[].origin_url").expect("well formed"); + let one = PathPattern::parse("proxy.asset_routes[1].origin_url").expect("well formed"); + let route = |index| { + vec![ + PathStep::Key("proxy".to_owned()), + PathStep::Key("asset_routes".to_owned()), + PathStep::Index(index), + PathStep::Key("origin_url".to_owned()), + ] + }; + + assert!(every.matches(&route(0)) && every.matches(&route(7))); + assert!(one.matches(&route(1))); + assert!(!one.matches(&route(0)), "should name only its own index"); + assert!( + !every.matches(&route(0)[..3]), + "should name only values at its own depth" + ); + assert_eq!(render_path(&route(1)), "proxy.asset_routes[1].origin_url"); +} + +#[test] +fn a_malformed_pattern_is_refused_when_the_document_is_read() { + for pattern in [ + "", + "a..b", + ".a", + "a.", + "a[x]", + "a[", + "a]", + "[0]", + "a b", + "a[0]b", + "a[99999999999999999999999]", + ] { + let toml = format!( + "{}\n[inspect]\nhide = [{pattern:?}]\n", + crate_test_settings_str() + ); + + let error = Settings::from_toml(&toml).expect_err("a malformed pattern is refused"); + + assert!( + format!("{error:?}").contains(&format!("`{pattern}` is not a path pattern")), + "should name the pattern {pattern:?}: {error:?}" + ); + } +} + +#[test] +fn a_key_the_inspect_section_does_not_have_is_refused_naming_it() { + let toml = format!("{}\n[inspect]\nshows = []\n", crate_test_settings_str()); + + let error = Settings::from_toml(&toml).expect_err("a misspelt key is refused"); + + assert!( + format!("{error:?}").contains("unknown field `shows`"), + "should name the key: {error:?}" + ); +} + +/// Documents that between them hold a secret at every path core declares +/// one, each a key name the canary store resolves to a value of its own. +fn documents_holding_every_declared_secret() -> Vec<(&'static str, Value)> { + let mut main = document(""); + main["publisher"]["proxy_secret"] = json!("proxy"); + main["ec"]["hmac"]["passphrase"] = json!("hmac"); + main["ec"]["partners"] = json!([{ + "name": "Example Partner", + "source_domain": "partner.example.com", + "api_token": "partner-api", + "pull_sync_enabled": true, + "pull_sync_url": "https://partner.example.com/sync", + "pull_sync_allowed_domains": ["partner.example.com"], + "ts_pull_token": "partner-pull", + }]); + main["trusted_client_ip"] = json!({ + "ip_header": "x-ts-client-ip", + "auth_header": "x-ts-client-ip-auth", + "shared_secret": "client-ip", + }); + main["proxy"]["asset_routes"] = json!([{ + "prefix": "/assets/", + "origin_url": "https://assets.example.com", + "auth": { + "type": "s3_sigv4", + "region": "us-east-1", + "access_key_id": "route-access", + "secret_access_key": "route-secret", + "session_token": "route-session", + }, + }]); + main["analytics"] = json!({ + "module": "tinybird", + "tinybird": { + "api_host": "api.tinybird.example.com", + "auction_token_secret": "telemetry", + }, + }); + + let mut host_signals = document(""); + let ec = host_signals["ec"].as_object_mut().expect("an ec table"); + ec.remove("hmac"); + ec.insert("module".to_owned(), json!("host_signals")); + ec.insert("host_signals".to_owned(), json!({ "passphrase": "host" })); + + let mut legacy = document(""); + let ec = legacy["ec"].as_object_mut().expect("an ec table"); + ec.remove("hmac"); + ec.remove("module"); + ec.insert("passphrase".to_owned(), json!("legacy")); + + let mut labeled = document(""); + let ec = labeled["ec"].as_object_mut().expect("an ec table"); + ec.remove("hmac"); + ec.insert("module".to_owned(), json!("primary")); + ec.insert( + "primary".to_owned(), + json!({ "implementation": "hmac", "passphrase": "labeled" }), + ); + + vec![ + ("main", main), + ("host signals", host_signals), + ("legacy", legacy), + ("labeled", labeled), + ] +} + +/// A concrete path with every index written as `[]`, as a declaration +/// writes it. +fn generalized(path: &[PathStep]) -> String { + let mut out = String::new(); + for step in path { + match step { + PathStep::Key(key) => { + if !out.is_empty() { + out.push('.'); + } + out.push_str(key); + } + PathStep::Index(_) => out.push_str("[]"), + } + } + out +} + +/// Every secret core declares, loaded the way a deployment loads it, is +/// live in the settings and absent from both forms the endpoint serves, +/// and wherever the view still has its path, that path is masked. +#[test] +fn every_declared_core_secret_is_masked_and_never_served() { + let mut covered = BTreeSet::new(); + for (name, data) in documents_holding_every_declared_secret() { + let settings = load(data).unwrap_or_else(|error| panic!("{name}: {error:?}")); + let record = settings.resolved_secrets(); + let normal = serde_json::to_string(&settings).expect("should serialize"); + assert!( + !record.values().is_empty(), + "{name}: the load should record the secrets it wrote" + ); + for value in record.values() { + assert!( + normal.contains(value.as_str()), + "{name}: each resolved secret should be live in the settings" + ); + } + let view = view(&settings); + for path in record.paths() { + let rendered = render_path(path); + if let Some(value) = at(&view.settings, &rendered) { + assert_eq!(value, MASK, "{name}: {rendered}"); + assert!(is_masked(&view, &rendered), "{name}: {rendered}"); + } + covered.insert(generalized(path)); + } + let (json, page) = served(&settings); + assert!(!json.contains(CANARY), "{name}: the JSON carries a secret"); + assert!(!page.contains(CANARY), "{name}: the page carries a secret"); + } + + let declared: BTreeSet = TrustedServerAppConfig::secret_fields() + .iter() + .filter_map(|field| PathPattern::from_segments(&field.path)) + .map(|pattern| pattern.as_str().to_owned()) + .collect(); + let missing: Vec<&String> = declared.difference(&covered).collect(); + assert!( + missing.is_empty(), + "every declared secret should be exercised, missing {missing:?}" + ); +} + +/// Test settings holding a value of its own in every field marked +/// sensitive, with those paths and values. +fn settings_with_every_sensitive_field() -> (Settings, Vec<(&'static str, &'static str)>) { + let expected = vec![ + ("publisher.origin_url", "https://origin-canary.example.com"), + ( + "publisher.origin_host_header_override", + "host-canary.example.com", + ), + ( + "proxy.asset_routes[0].origin_url", + "https://bucket-canary.example.com", + ), + ("ec.ec_store", "identity-store-canary"), + ("auction.creative_store", "creative-store-canary"), + ]; + let value = |path: &str| { + expected + .iter() + .find(|(at, _)| *at == path) + .map(|(_, value)| (*value).to_owned()) + .expect("an expected path") + }; + let mut settings = settings(); + settings.publisher.origin_url = value("publisher.origin_url"); + settings.publisher.origin_host_header_override = + Some(value("publisher.origin_host_header_override")); + settings.proxy.asset_routes = vec![ProxyAssetRoute::new( + "/assets/", + value("proxy.asset_routes[0].origin_url"), + )]; + settings.ec.ec_store = Some(value("ec.ec_store")); + settings.auction.creative_store = value("auction.creative_store"); + (settings, expected) +} + +#[test] +fn every_value_sensitive_by_default_is_masked() { + let (settings, expected) = settings_with_every_sensitive_field(); + + let view = view(&settings); + let (json, page) = served(&settings); + + for (path, value) in expected { + assert_eq!(at(&view.settings, path), Some(&json!(MASK)), "{path}"); + assert!(is_masked(&view, path), "{path}"); + assert!(!json.contains(value), "the JSON carries {path}"); + assert!(!page.contains(value), "the page carries {path}"); + } + assert!( + is_masked(&view, "publisher.proxy_secret"), + "a Redacted value is masked too: {:?}", + view.masked + ); +} + +#[test] +fn an_ordinary_serialization_is_unchanged() { + let (settings, expected) = settings_with_every_sensitive_field(); + + let normal = serde_json::to_value(&settings).expect("should serialize"); + let marked = to_marked_value(&settings).expect("should serialize"); + + assert_eq!( + normal, + unmarked(&marked), + "should differ from a marked serialization only by the markers" + ); + assert!(!normal.to_string().contains(SENSITIVE_MARKER)); + for (path, value) in expected { + assert_eq!(at(&normal, path), Some(&json!(value)), "{path}"); + } + assert_eq!( + at(&normal, "publisher.proxy_secret"), + Some(&json!("unit-test-proxy-secret")) + ); +} + +#[test] +fn a_stored_document_carries_the_section_only_when_the_publisher_wrote_one() { + let unwritten = serde_json::to_value(settings()).expect("should serialize"); + let written = document("[inspect]\nhide = [\"publisher.domain\"]"); + + assert_eq!( + unwritten.get("inspect"), + None, + "should leave an unwritten section out, so the document is the bytes it was" + ); + assert_eq!( + written.get("inspect"), + Some(&json!({ "config": true, "hide": ["publisher.domain"] })), + "should store a section the publisher wrote" + ); + assert!( + !unwritten.to_string().contains("resolved_secrets"), + "should never store where the loader wrote secrets" + ); +} + +#[test] +fn a_recorded_secret_path_is_masked_without_the_value_scan() { + let mut settings = settings(); + settings.set_resolved_secrets(ResolvedSecrets::recorded( + vec![vec![ + PathStep::Key("publisher".to_owned()), + PathStep::Key("domain".to_owned()), + ]], + Vec::new(), + )); + + let view = view(&settings); + + assert_eq!(at(&view.settings, "publisher.domain"), Some(&json!(MASK))); + assert!(is_masked(&view, "publisher.domain")); +} + +#[test] +fn a_secret_copied_into_an_undeclared_field_is_masked_by_the_value_scan() { + let secret = "canary-copied-secret-value"; + let mut settings = settings(); + settings + .response_headers + .insert("x-debug".to_owned(), format!("prefix {secret} suffix")); + settings + .response_headers + .insert(format!("x-{secret}"), "open".to_owned()); + settings.set_resolved_secrets(ResolvedSecrets::recorded( + Vec::new(), + vec![secret.to_owned()], + )); + + let view = view(&settings); + let (json, page) = served(&settings); + + assert_eq!( + at(&view.settings, "response_headers.x-debug"), + Some(&json!(MASK)) + ); + assert!(is_masked(&view, "response_headers.x-debug")); + assert!( + is_masked(&view, "response_headers.XXXX"), + "a key holding a secret is masked too: {:?}", + view.masked + ); + assert!(!json.contains(secret) && !page.contains(secret)); +} + +#[test] +fn a_path_masked_beneath_a_key_holding_a_secret_lists_the_renamed_key() { + struct Case { + name: &'static str, + real_mask_key: bool, + renamed: [&'static str; 2], + } + let secret = "canary-key-secret-value"; + // Upper case names sort ahead of a real `XXXX` key, so they are + // renamed before it is reached. + let keys = [format!("A-{secret}"), format!("B-{secret}")]; + let path_to = |key: &str| { + vec![ + PathStep::Key("response_headers".to_owned()), + PathStep::Key(key.to_owned()), + ] + }; + for case in [ + Case { + name: "two keys holding the secret", + real_mask_key: false, + renamed: ["response_headers.XXXX", "response_headers.XXXX-2"], + }, + Case { + name: "beside a real key named XXXX", + real_mask_key: true, + renamed: ["response_headers.XXXX-2", "response_headers.XXXX-3"], + }, + ] { + let mut settings = settings(); + for key in &keys { + settings + .response_headers + .insert(key.clone(), "open".to_owned()); + } + if case.real_mask_key { + settings + .response_headers + .insert(MASK.to_owned(), "real".to_owned()); + } + settings.set_resolved_secrets(ResolvedSecrets::recorded( + keys.iter().map(|key| path_to(key)).collect(), + vec![secret.to_owned()], + )); + + let view = view(&settings); + let (json, page) = served(&settings); + + assert!( + !view.masked.iter().any(|path| path.contains(secret)), + "{}: no masked path should carry the secret: {:?}", + case.name, + view.masked + ); + for renamed in case.renamed { + assert!( + is_masked(&view, renamed), + "{}: should list {renamed}: {:?}", + case.name, + view.masked + ); + assert_eq!( + at(&view.settings, renamed), + Some(&json!(MASK)), + "{}", + case.name + ); + } + if case.real_mask_key { + assert_eq!( + at(&view.settings, "response_headers.XXXX"), + Some(&json!("real")), + "{}: should keep the real key's value", + case.name + ); + assert!( + !is_masked(&view, "response_headers.XXXX"), + "{}: should not list the real key as masked: {:?}", + case.name, + view.masked + ); + } + assert!( + !json.contains(secret) && !page.contains(secret), + "{}: should publish no secret", + case.name + ); + } +} + +/// Two asset routes, for a list holding values masked by default and a +/// secret. +const ASSET_ROUTES: &str = r#" + [[proxy.asset_routes]] + prefix = "/assets/" + origin_url = "https://assets.example.com" + + [proxy.asset_routes.auth] + type = "s3_sigv4" + region = "us-east-1" + access_key_id = "route-access" + secret_access_key = "route-secret" + + [[proxy.asset_routes]] + prefix = "/media/" + origin_url = "https://media.example.com" +"#; + +#[test] +fn show_reveals_a_value_masked_by_default() { + let settings = load(document(&format!( + "{ASSET_ROUTES}\n[inspect]\nshow = [\"publisher.origin_url\", \ + \"proxy.asset_routes[].origin_url\"]" + ))) + .expect("should load"); + + let view = view(&settings); + + assert_eq!( + at(&view.settings, "publisher.origin_url"), + Some(&json!("https://origin.test-publisher.com")) + ); + assert_eq!( + at(&view.settings, "proxy.asset_routes[1].origin_url"), + Some(&json!("https://media.example.com")) + ); + assert!(!view.masked.iter().any(|path| path.ends_with(".origin_url"))); + assert!(!is_masked(&view, "publisher.origin_url")); +} + +#[test] +fn hide_masks_a_value_shown_by_default() { + let settings = load(document("[inspect]\nhide = [\"publisher.domain\"]")).expect("should load"); + + let view = view(&settings); + + assert_eq!(at(&view.settings, "publisher.domain"), Some(&json!(MASK))); + assert!(is_masked(&view, "publisher.domain")); +} + +/// A pattern that would not be honored as written is refused when the +/// document loads. Each row's fragment carries the whole reason, so a +/// pattern refused for another reason fails its row. +#[test] +fn a_pattern_not_honored_as_written_is_refused_saying_why() { + for (section, reason) in [ + ( + r#"show = ["publisher.proxy_secret"]"#, + "show names `publisher.proxy_secret`, which is a secret", + ), + ( + r#"show = ["proxy.asset_routes[].auth.secret_access_key"]"#, + "show names `proxy.asset_routes[].auth.secret_access_key`, which is a secret", + ), + ( + r#"hide = ["publisher.no_such_setting"]"#, + "hide names `publisher.no_such_setting`, which matches no value", + ), + ( + r#"show = ["proxy.asset_routes[9].origin_url"]"#, + "show names `proxy.asset_routes[9].origin_url`, which matches no value", + ), + ( + r#"show = ["publisher.domain"]"#, + "show names `publisher.domain`, which changes nothing, because nothing \ + it names is masked by default", + ), + ( + r#"hide = ["publisher.origin_url"]"#, + "hide names `publisher.origin_url`, which changes nothing, because \ + everything it names is masked or empty already", + ), + ( + r#"hide = ["publisher.proxy_secret"]"#, + "hide names `publisher.proxy_secret`, which changes nothing, because \ + everything it names is masked or empty already", + ), + ( + r#"show = ["publisher.origin_url", "publisher.origin_url"]"#, + "show names `publisher.origin_url`, which changes nothing, because \ + `publisher.origin_url` in the same list already names everything it does", + ), + ( + r#"show = ["proxy.asset_routes[].origin_url", "proxy.asset_routes[0].origin_url"]"#, + "show names `proxy.asset_routes[0].origin_url`, which changes nothing, because \ + `proxy.asset_routes[].origin_url` in the same list already names everything \ + it does", + ), + ( + "show = [\"publisher.origin_url\"]\nhide = [\"publisher.origin_url\"]", + "show names `publisher.origin_url`, which changes nothing, because hide \ + names it as `publisher.origin_url` and hide wins over show", + ), + ( + "config = false\nhide = [\"publisher.domain\"]", + "config = false publishes no page, so they change nothing", + ), + ] { + let message = format!( + "{:?}", + load(document(&format!("{ASSET_ROUTES}\n[inspect]\n{section}"))) + .expect_err("the document should be refused") + ); + + assert!(message.contains(reason), "{section}: {message}"); + } +} + +/// The check a deployment's settings pass before they are pushed refuses +/// the same patterns, so a bad one is caught before it is live. +#[test] +fn the_deploy_check_refuses_a_pattern_that_matches_nothing() { + let toml = format!( + "{}\n[inspect]\nhide = [\"publisher.no_such_setting\"]\n", + crate_test_settings_str() + ); + let settings = Settings::from_toml(&toml).expect("should parse the document"); + + let message = format!( + "{:?}", + validate_patterns(&settings).expect_err("the pattern should be refused") + ); + + assert!( + message.contains("hide names `publisher.no_such_setting`, which matches no value"), + "{message}" + ); +} + +/// A secret only the configuration or a deployment's own module names is +/// one the load resolves, so the deploy check refuses showing it as the +/// load does. +#[test] +fn the_deploy_check_refuses_showing_a_secret_only_the_load_resolves() { + let builder = IntegrationBuilder::new( + "probe", + "example-crate", + crate::integrations::registry_test_support::probe_registration, + crate::integrations::registry_test_support::validate_nothing, + ) + .with_module_name("testing.probe") + .with_secret_settings(MODULE_SECRETS); + let labeled = crate::test_support::tests::crate_test_settings_str_with_ec_section( + "[ec] +module = \"primary\" + +[ec.primary] +implementation = \"hmac\" + passphrase = \"labeled-passphrase-key-0123456789\"", + ); + let mut labeled = Settings::from_toml(&format!( + "{labeled} +[inspect] +show = [\"ec.primary.passphrase\"] +" + )) + .expect("should parse the document"); + labeled.proxy.allowed_domains = vec!["*.example".to_owned(), "*.example.com".to_owned()]; + let mut module = settings(); + module.proxy.allowed_domains = vec!["*.example".to_owned(), "*.example.com".to_owned()]; + module + .insert_module_config( + "testing", + "testing.probe", + &json!({ "lock": true, "key_name": "module-key" }), + ) + .expect("should insert the module's table"); + module.inspect.show = vec![PathPattern::parse("testing.probe.key_name").expect("a pattern")]; + + for (settings, pattern) in [ + (labeled, "ec.primary.passphrase"), + (module, "testing.probe.key_name"), + ] { + let reason = format!("show names `{pattern}`, which is a secret"); + let data = serde_json::to_value(&settings).expect("should serialize the document"); + let loaded = format!( + "{:?}", + load_with(data, &[builder]).expect_err("the load should refuse the pattern") + ); + let deployed = format!( + "{:?}", + crate::config::validate_settings_for_deploy_with(&settings, &[builder]) + .expect_err("the deploy check should refuse the pattern") + ); + + assert!(loaded.contains(&reason), "load: {loaded}"); + assert!(deployed.contains(&reason), "deploy: {deployed}"); + } +} + +#[test] +fn the_page_answers_not_found_when_the_publisher_does_not_publish() { + let settings = load(document("[inspect]\nconfig = false")).expect("should load"); + + for (path, content_type) in [ + (CONFIG_PAGE_PATH, "text/html; charset=utf-8"), + (CONFIG_JSON_PATH, "application/json"), + ] { + let response = handle_config(&settings, &request(path)); + + assert_eq!(response.status(), StatusCode::NOT_FOUND, "{path}"); + assert_eq!( + response.headers()[header::CONTENT_TYPE], + content_type, + "{path}" + ); + assert!(body_text(response).contains(NOT_PUBLISHED), "{path}"); + } +} + +#[test] +fn the_page_and_the_json_carry_the_view() { + let settings = load(document("")).expect("should load"); + + for (path, content_type) in [ + (CONFIG_PAGE_PATH, "text/html; charset=utf-8"), + (CONFIG_JSON_PATH, "application/json"), + ] { + let response = handle_config(&settings, &request(path)); + + assert_eq!(response.status(), StatusCode::OK, "{path}"); + assert_eq!(response.headers()[header::CONTENT_TYPE], content_type); + assert_eq!(response.headers()[header::CACHE_CONTROL], "no-store"); + assert_eq!(response.headers()[header::ACCESS_CONTROL_ALLOW_ORIGIN], "*"); + } + let json: Value = serde_json::from_str(&body_text(handle_config( + &settings, + &request(CONFIG_JSON_PATH), + ))) + .expect("a JSON body"); + let keys: Vec<&String> = json.as_object().expect("an object").keys().collect(); + assert_eq!(keys, ["masked", "settings", "version"]); + assert_eq!(json["version"], env!("CARGO_PKG_VERSION")); + assert_eq!( + json["settings"]["publisher"]["domain"], + "test-publisher.com" + ); + assert!( + json["masked"] + .as_array() + .expect("a list") + .contains(&json!("publisher.proxy_secret")), + "{json}" + ); +} + +fn assert_keys_sorted(value: &Value) { + match value { + Value::Object(map) => { + let keys: Vec<&String> = map.keys().collect(); + let mut sorted = keys.clone(); + sorted.sort_unstable(); + assert_eq!(keys, sorted, "keys should be sorted"); + map.values().for_each(assert_keys_sorted); + } + Value::Array(items) => items.iter().for_each(assert_keys_sorted), + _ => {} + } +} + +/// Two loads of one document build hash maps of their own, each iterating +/// in its own order, and the view has to come out the same. +#[test] +fn the_same_settings_give_the_same_view() { + let headers: String = (0..12) + .map(|index| format!("x-header-{index} = \"value-{index}\"\n")) + .collect(); + let data = document_from(&format!( + "{}\n[response_headers]\n{headers}", + crate_test_settings_str() + )); + + let first = load(data.clone()).expect("should load"); + let second = load(data).expect("should load"); + let first_json = view_payload(&view(&first)).to_string(); + let second_json = view_payload(&view(&second)).to_string(); + + assert_eq!(first_json, second_json); + let parsed: Value = serde_json::from_str(&first_json).expect("JSON"); + assert_keys_sorted(&parsed); + assert_eq!( + parsed["settings"]["response_headers"] + .as_object() + .expect("the headers") + .len(), + 12, + "should show every header" + ); + let masked = view(&first).masked; + let mut sorted_masked = masked.clone(); + sorted_masked.sort_unstable(); + assert_eq!(masked, sorted_masked); +} + +fn module_lock_is_on(table: &serde_json::Map) -> bool { + table.get("lock").and_then(Value::as_bool) == Some(true) +} + +const MODULE_SECRETS: &[ModuleSecretSetting] = &[ModuleSecretSetting { + path: &["key_name"], + in_use: module_lock_is_on, +}]; + +/// A secret a module a deployment added declares in its own table is one +/// the loader wrote, so it is masked like any of core's. +#[test] +fn a_secret_a_module_declares_is_masked() { + let builder = IntegrationBuilder::new( + "probe", + "example-crate", + crate::integrations::registry_test_support::probe_registration, + crate::integrations::registry_test_support::validate_nothing, + ) + .with_module_name("testing.probe") + .with_secret_settings(MODULE_SECRETS); + let mut settings = settings(); + settings.proxy.allowed_domains = vec!["*.example".to_owned(), "*.example.com".to_owned()]; + settings + .insert_module_config( + "testing", + "testing.probe", + &json!({ "lock": true, "key_name": "module-key", "label": "open" }), + ) + .expect("should insert the module's table"); + let data = serde_json::to_value(&settings).expect("should serialize the document"); + + let loaded = load_with(data, &[builder]).expect("should load"); + let view = view(&loaded); + let (json, page) = served(&loaded); + + assert!( + is_masked(&view, "testing.probe.key_name"), + "should mask the module's secret: {:?}", + view.masked + ); + assert_eq!( + at(&view.settings, "testing.probe.label"), + Some(&json!("open")), + "should show the module's other settings" + ); + assert!(!json.contains(CANARY) && !page.contains(CANARY)); +} diff --git a/crates/trusted-server-core/src/inspect/data.rs b/crates/trusted-server-core/src/inspect/data.rs new file mode 100644 index 0000000000..1850c1ae7a --- /dev/null +++ b/crates/trusted-server-core/src/inspect/data.rs @@ -0,0 +1,683 @@ +//! What a deployment holds against the reader's own Edge Cookie, at +//! [`DATA_PAGE_PATH`]. +//! +//! 1. Only a browser opening the address as a page is answered, which the +//! browser marks with `Sec-Fetch-Mode: navigate` and +//! `Sec-Fetch-Dest: document`. +//! 2. The page is sent with `Content-Security-Policy: sandbox`, which gives +//! the document an origin of its own, so a page that opened it cannot +//! read it. +//! 3. No identifier is shown. + +use edgezero_core::body::Body as EdgeBody; +use error_stack::Report; +use http::{HeaderValue, Request, Response, StatusCode, header}; +use serde_json::{Value, json}; + +use crate::ec::kv::KvIdentityGraph; +use crate::ec::kv_types::KvEntry; +use crate::ec::module::{AcceptedModules, EdgeCookieModule}; +use crate::error::TrustedServerError; +use crate::response_privacy::RequiredHeaders; + +use super::{MASK, render_page_with_lede}; + +/// The page showing what is held against the reader's own Edge Cookie. +pub const DATA_PAGE_PATH: &str = "/_ts/data"; + +const NOT_OPENED_AS_A_PAGE: &str = "This address shows a reader what is held against their own \ + browser, so it answers only a browser opening it as a page. \ + Open it from the address bar.\n"; + +/// `sandbox` gives the document an origin of its own, so a page that opened +/// it cannot read it, and nothing in it runs or loads but its own style. +const CONTENT_SECURITY_POLICY: &str = "sandbox; default-src 'none'; style-src 'unsafe-inline'; \ + frame-ancestors 'none'; base-uri 'none'; form-action 'none'"; + +/// Answers [`DATA_PAGE_PATH`] for `req`. +/// +/// `kv` is the identity graph the deployment keeps, or `None` when it keeps +/// none, and `module` is the Edge Cookie module it selects. The record is the +/// one stored against the identifier in the request's own Edge Cookie, named +/// after `public`, the request's public prefix. +/// Nothing is created or written for the reader. +/// +/// # Errors +/// +/// When the request's `Cookie` header cannot be read, or the store cannot be +/// read, or what it holds is not a record. +pub fn handle_data( + kv: Option<&KvIdentityGraph>, + module: Option<&dyn EdgeCookieModule>, + public: &crate::prefix::PublicPrefix, + req: &Request, +) -> Result, Report> { + if !is_opened_as_a_page(req) { + return Ok(protected( + StatusCode::FORBIDDEN, + "text/plain; charset=utf-8", + NOT_OPENED_AS_A_PAGE.to_owned(), + )); + } + let Some(kv) = kv else { + return Ok(page(¬hing_held( + "This deployment keeps no record against an Edge Cookie.", + ))); + }; + let Some(ec_id) = crate::ec::request_cookie_ec(req, public)? else { + return Ok(page(¬hing_held( + "The request carried no Edge Cookie, so there is nothing to look up.", + ))); + }; + let Some(kv_key) = AcceptedModules::active(module).canonical_kv_key(&ec_id) else { + return Ok(page(¬hing_held( + "The request's Edge Cookie is not one this deployment issued.", + ))); + }; + let Some((entry, _generation)) = kv.get(&kv_key)? else { + return Ok(page(¬hing_held( + "This deployment holds no record against the request's Edge Cookie. A record \ + made in the last few seconds may not be readable yet.", + ))); + }; + Ok(page(&held_payload(&entry))) +} + +/// Whether `req` is a browser opening the address as a page of its own. +/// +/// A browser sends `Sec-Fetch-*` only to a trustworthy origin, so every +/// browser on a plain `http://` host other than localhost fails this. +fn is_opened_as_a_page(req: &Request) -> bool { + let is = |name: &str, expected: &str| { + req.headers() + .get(name) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.eq_ignore_ascii_case(expected)) + }; + is("sec-fetch-mode", "navigate") && is("sec-fetch-dest", "document") +} + +fn nothing_held(why: &str) -> Value { + json!({ + "held": false, + "why": why, + "version": env!("CARGO_PKG_VERSION"), + }) +} + +/// What the page shows for `entry`, being the record as it is stored with +/// each partner's identifier masked, and the two times in it written out. +fn held_payload(entry: &KvEntry) -> Value { + let mut record = entry.clone(); + for id in record.ids.values_mut() { + MASK.clone_into(&mut id.uid); + } + json!({ + "held": true, + "identifier": MASK, + "created": readable_time(entry.created), + "consent_updated": readable_time(entry.consent.updated), + "withdrawn": !entry.consent.ok, + "record": record, + "version": env!("CARGO_PKG_VERSION"), + }) +} + +/// `unix_seconds` as RFC 3339 in UTC, or `None` beyond what a date can hold. +fn readable_time(unix_seconds: u64) -> Option { + let unix_seconds = i64::try_from(unix_seconds).ok()?; + chrono::DateTime::from_timestamp(unix_seconds, 0) + .map(|time| time.format("%Y-%m-%dT%H:%M:%SZ").to_string()) +} + +fn page(payload: &Value) -> Response { + let lede = format!( + "What this deployment holds against the Edge Cookie your browser sent. Each \ + identifier shows as {MASK}." + ); + protected( + StatusCode::OK, + "text/html; charset=utf-8", + render_page_with_lede("Your data", &lede, payload), + ) +} + +/// A response only the person looking at it can read, being never stored, +/// never framed, in an origin of its own, and offered to no other origin. +/// Its headers are [`RequiredHeaders`], so an operator's own headers of the +/// same names cannot replace them. +fn protected(status: StatusCode, content_type: &'static str, body: String) -> Response { + let mut response = Response::builder() + .status(status) + .header(header::CONTENT_TYPE, HeaderValue::from_static(content_type)) + .header( + header::CACHE_CONTROL, + HeaderValue::from_static("no-store, private"), + ) + .header( + header::CONTENT_SECURITY_POLICY, + HeaderValue::from_static(CONTENT_SECURITY_POLICY), + ) + .header(header::X_FRAME_OPTIONS, HeaderValue::from_static("DENY")) + .header( + header::X_CONTENT_TYPE_OPTIONS, + HeaderValue::from_static("nosniff"), + ) + .header( + header::REFERRER_POLICY, + HeaderValue::from_static("no-referrer"), + ) + .header( + "cross-origin-resource-policy", + HeaderValue::from_static("same-origin"), + ) + .body(EdgeBody::from(body.into_bytes())) + .expect("should build the data response"); + let required = response.headers().clone(); + response.extensions_mut().insert(RequiredHeaders(required)); + response +} + +#[cfg(test)] +mod tests { + use std::collections::{BTreeMap, BTreeSet}; + + use super::*; + use crate::consent::ConsentContext; + use crate::ec::kv_types::{ + KvConsent, KvDevice, KvGeo, KvNetwork, KvPartnerId, KvPubProperties, + }; + use crate::ec::module::HmacModule; + use crate::redacted::Redacted; + + /// An identifier in the form the built-in HMAC module issues. + const EC_ID: &str = + "hmac~0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef.aB3dE9"; + /// The part of [`EC_ID`] that no page may carry. + const EC_HASH: &str = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + const PARTNER_UID: &str = "partner-uid-canary-1234"; + /// 14 March 2025 at midnight UTC. + const CREATED: u64 = 1_741_910_400; + + fn module() -> HmacModule { + HmacModule::new(Redacted::new("test-secret-key-32-bytes-minimum".to_owned())) + } + + fn request(cookie: Option<&str>, mode: Option<&str>, dest: Option<&str>) -> Request { + let mut builder = Request::builder() + .method("GET") + .uri(format!("https://publisher.example{DATA_PAGE_PATH}")); + for (name, value) in [ + ("cookie", cookie), + ("sec-fetch-mode", mode), + ("sec-fetch-dest", dest), + ] { + if let Some(value) = value { + builder = builder.header(name, value); + } + } + builder + .body(EdgeBody::empty()) + .expect("should build the request") + } + + /// A request as a browser sends it when a person opens the address. + fn opened_as_a_page(cookie: Option<&str>) -> Request { + request(cookie, Some("navigate"), Some("document")) + } + + fn body_text(response: Response) -> String { + let bytes = response + .into_body() + .into_bytes() + .expect("a buffered body") + .to_vec(); + String::from_utf8(bytes).expect("a text body") + } + + /// A graph holding one record against [`EC_ID`], with one partner's + /// identifier in it. + fn graph_with_a_record() -> KvIdentityGraph { + let graph = KvIdentityGraph::in_memory("test-ec-store"); + let module = module(); + let key = AcceptedModules::active(Some(&module)) + .canonical_kv_key(EC_ID) + .expect("the test identifier should be one the module issues"); + let mut entry = KvEntry::new( + &ConsentContext::default(), + None, + CREATED, + "publisher.example", + ); + entry.ids.insert( + "partner.example.com".to_owned(), + KvPartnerId { + uid: PARTNER_UID.to_owned(), + }, + ); + graph.create(&key, &entry).expect("should store the record"); + graph + } + + #[test] + fn only_a_browser_opening_the_page_is_answered() { + let graph = graph_with_a_record(); + let module = module(); + let cookie = format!("ts-ec={EC_ID}"); + + for (mode, dest, who) in [ + (None, None, "a tool that sends neither header"), + (Some("cors"), Some("empty"), "a script's fetch"), + (Some("same-origin"), Some("empty"), "a same-origin fetch"), + (Some("navigate"), Some("iframe"), "a frame"), + (Some("navigate"), Some("object"), "an embedded object"), + ( + Some("navigate"), + None, + "a navigation that names no destination", + ), + ( + Some("no-cors"), + Some("document"), + "a request that is not a navigation", + ), + ] { + let response = handle_data( + Some(&graph), + Some(&module), + &crate::prefix::PublicPrefix::default(), + &request(Some(&cookie), mode, dest), + ) + .expect("should answer"); + + assert_eq!( + response.status(), + StatusCode::FORBIDDEN, + "should refuse {who}" + ); + assert_eq!( + body_text(response), + NOT_OPENED_AS_A_PAGE, + "should tell {who} nothing about the record" + ); + } + + let response = handle_data( + Some(&graph), + Some(&module), + &crate::prefix::PublicPrefix::default(), + &opened_as_a_page(Some(&cookie)), + ) + .expect("should answer"); + assert_eq!( + response.status(), + StatusCode::OK, + "should answer a browser opening the page" + ); + } + + #[test] + fn the_record_is_shown_with_every_identifier_masked() { + let graph = graph_with_a_record(); + let module = module(); + let cookie = format!("ts-ec={EC_ID}"); + + let response = handle_data( + Some(&graph), + Some(&module), + &crate::prefix::PublicPrefix::default(), + &opened_as_a_page(Some(&cookie)), + ) + .expect("should answer"); + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response.headers()[header::CONTENT_TYPE], + "text/html; charset=utf-8" + ); + let html = body_text(response); + assert!( + html.contains("\"held\": true"), + "should say a record is held" + ); + assert!( + html.contains("partner.example.com"), + "should name the partner that holds an identifier" + ); + assert!( + html.contains("\"created\": \"2025-03-14T00:00:00Z\""), + "should say when the record was made in a form a person reads" + ); + assert!( + html.contains("publisher.example"), + "should show the site the record was made on" + ); + assert!( + !html.contains(PARTNER_UID), + "should not show the partner's identifier" + ); + assert!( + !html.contains(EC_HASH), + "should not show the Edge Cookie identifier" + ); + assert!( + html.contains("\"uid\": \"XXXX\""), + "should show that the partner's identifier is masked" + ); + assert!( + html.contains("Each identifier shows as XXXX."), + "should tell the reader what the mask means" + ); + } + + #[test] + fn an_identifier_sent_in_another_case_reaches_its_own_record() { + let graph = graph_with_a_record(); + let module = module(); + let cookie = format!("ts-ec=hmac~{}.aB3dE9", EC_HASH.to_ascii_uppercase()); + + let html = body_text( + handle_data( + Some(&graph), + Some(&module), + &crate::prefix::PublicPrefix::default(), + &opened_as_a_page(Some(&cookie)), + ) + .expect("should answer"), + ); + + assert!( + html.contains("\"held\": true"), + "should read the record under the key it is stored under: {html}" + ); + } + + /// Every field the record has is set here and listed below, so a field + /// added to the record fails this test until it is decided whether the + /// field is an identifier. + #[test] + fn every_field_of_the_record_is_shown_or_masked_by_decision() { + fn leaves(value: &Value, path: &str, out: &mut BTreeSet) { + match value { + Value::Object(map) => { + for (key, child) in map { + let key = if path == "ids" { "" } else { key }; + let path = if path.is_empty() { + key.to_owned() + } else { + format!("{path}.{key}") + }; + leaves(child, &path, out); + } + } + _ => { + out.insert(path.to_owned()); + } + } + } + let entry = KvEntry { + v: 1, + created: CREATED, + consent: KvConsent { + tcf: Some("tcf-string".to_owned()), + gpp: Some("gpp-string".to_owned()), + ok: true, + updated: CREATED + 60, + }, + geo: KvGeo { + country: "GB".to_owned(), + region: Some("ENG".to_owned()), + asn: Some(64_496), + dma: Some(807), + }, + pub_properties: Some(KvPubProperties { + origin_domain: "publisher.example".to_owned(), + seen_domains: BTreeSet::from(["publisher.example".to_owned()]), + }), + device: Some(KvDevice { + is_mobile: 0, + ja4_class: Some("t13d1516h2".to_owned()), + platform_class: Some("windows".to_owned()), + h2_fp_hash: Some("0123456789ab".to_owned()), + known_browser: Some(true), + }), + network: Some(KvNetwork { + cluster_size: Some(3), + }), + ids: BTreeMap::from([( + "partner.example.com".to_owned(), + KvPartnerId { + uid: PARTNER_UID.to_owned(), + }, + )]), + }; + + let payload = held_payload(&entry); + + let mut shown = BTreeSet::new(); + leaves(&payload["record"], "", &mut shown); + assert_eq!( + shown.iter().map(String::as_str).collect::>(), + [ + "consent.gpp", + "consent.ok", + "consent.tcf", + "consent.updated", + "created", + "device.h2_fp_hash", + "device.is_mobile", + "device.ja4_class", + "device.known_browser", + "device.platform_class", + "geo.asn", + "geo.country", + "geo.dma", + "geo.region", + "ids..uid", + "network.cluster_size", + "pub_properties.origin_domain", + "pub_properties.seen_domains", + "v", + ], + "a field the record has gained should be shown or masked by decision" + ); + assert_eq!( + payload["record"]["ids"]["partner.example.com"]["uid"], + json!(MASK), + "the one identifier the record holds should be masked" + ); + assert_eq!(payload["identifier"], json!(MASK)); + assert_eq!(payload["consent_updated"], json!("2025-03-14T00:01:00Z")); + assert_eq!(payload["withdrawn"], json!(false)); + assert!( + !payload.to_string().contains(PARTNER_UID), + "should carry no identifier anywhere" + ); + } + + /// The record and the answer the API reference shows. + #[test] + fn a_known_record_gives_the_documented_answer() { + let made = 1_773_480_600; + let entry = KvEntry { + v: 1, + created: made, + consent: KvConsent { + tcf: Some("CP...".to_owned()), + gpp: None, + ok: true, + updated: made, + }, + geo: KvGeo { + country: "GB".to_owned(), + region: Some("ENG".to_owned()), + asn: None, + dma: None, + }, + pub_properties: Some(KvPubProperties { + origin_domain: "example.com".to_owned(), + seen_domains: BTreeSet::from(["example.com".to_owned()]), + }), + device: None, + network: None, + ids: BTreeMap::from([( + "partner.example.com".to_owned(), + KvPartnerId { + uid: PARTNER_UID.to_owned(), + }, + )]), + }; + + assert_eq!( + held_payload(&entry), + json!({ + "consent_updated": "2026-03-14T09:30:00Z", + "created": "2026-03-14T09:30:00Z", + "held": true, + "identifier": "XXXX", + "record": { + "consent": { "ok": true, "tcf": "CP...", "updated": 1_773_480_600 }, + "created": 1_773_480_600, + "geo": { "country": "GB", "region": "ENG" }, + "ids": { "partner.example.com": { "uid": "XXXX" } }, + "pub_properties": { + "origin_domain": "example.com", + "seen_domains": ["example.com"] + }, + "v": 1 + }, + "version": env!("CARGO_PKG_VERSION"), + "withdrawn": false + }) + ); + } + + #[test] + fn a_withdrawal_is_shown_as_what_is_held() { + let payload = held_payload(&KvEntry::tombstone(CREATED)); + + assert_eq!(payload["held"], json!(true)); + assert_eq!( + payload["withdrawn"], + json!(true), + "should say the record is one of a withdrawal" + ); + } + + #[test] + fn a_reader_with_nothing_held_is_told_why() { + let graph = graph_with_a_record(); + let module = module(); + let another = format!("ts-ec=hmac~{}.zZ9yY8", "f".repeat(64)); + + for (kv, cookie, why) in [ + (None, Some("ts-ec=anything"), "keeps no record"), + (Some(&graph), None, "carried no Edge Cookie"), + ( + Some(&graph), + Some("ts-ec=t0op~device-1.issued-monday"), + "is not one this deployment issued", + ), + ( + Some(&graph), + Some(another.as_str()), + "holds no record against the request", + ), + ] { + let response = handle_data( + kv, + Some(&module), + &crate::prefix::PublicPrefix::default(), + &opened_as_a_page(cookie), + ) + .expect("should answer"); + + assert_eq!(response.status(), StatusCode::OK, "{why}"); + let html = body_text(response); + assert!(html.contains("\"held\": false"), "{why}: {html}"); + assert!(html.contains(why), "should say `{why}`: {html}"); + assert!( + !html.contains("partner.example.com"), + "should show nothing of another reader's record" + ); + } + } + + #[test] + fn a_store_that_cannot_be_read_is_an_error_and_not_an_empty_answer() { + let graph = KvIdentityGraph::failing("test-ec-store"); + let module = module(); + let cookie = format!("ts-ec={EC_ID}"); + + let result = handle_data( + Some(&graph), + Some(&module), + &crate::prefix::PublicPrefix::default(), + &opened_as_a_page(Some(&cookie)), + ); + + assert!( + result.is_err(), + "should not tell the reader nothing is held when the store was not read" + ); + } + + #[test] + fn every_answer_is_for_the_reader_alone() { + let graph = graph_with_a_record(); + let module = module(); + let cookie = format!("ts-ec={EC_ID}"); + let answered = handle_data( + Some(&graph), + Some(&module), + &crate::prefix::PublicPrefix::default(), + &opened_as_a_page(Some(&cookie)), + ) + .expect("should answer"); + let refused = handle_data( + Some(&graph), + Some(&module), + &crate::prefix::PublicPrefix::default(), + &request(Some(&cookie), None, None), + ) + .expect("should answer"); + + for (name, response) in [("the page", answered), ("the refusal", refused)] { + let headers = response.headers(); + assert_eq!( + headers[header::CACHE_CONTROL], + "no-store, private", + "{name} should never be stored" + ); + assert!( + headers[header::CONTENT_SECURITY_POLICY] + .to_str() + .expect("a text policy") + .starts_with("sandbox;"), + "{name} should be in an origin of its own" + ); + assert_eq!(headers[header::X_FRAME_OPTIONS], "DENY", "{name}"); + assert_eq!(headers[header::X_CONTENT_TYPE_OPTIONS], "nosniff", "{name}"); + assert_eq!(headers[header::REFERRER_POLICY], "no-referrer", "{name}"); + assert_eq!( + headers["cross-origin-resource-policy"], "same-origin", + "{name}" + ); + assert!( + headers.get(header::ACCESS_CONTROL_ALLOW_ORIGIN).is_none(), + "{name} should be offered to no other origin" + ); + } + } + + #[test] + fn the_page_does_not_offer_a_data_form() { + let html = body_text(page(¬hing_held("nothing"))); + + assert!( + !html.contains(".json"), + "should not point at a form the address does not have" + ); + assert!(html.contains("

Your data

")); + } +} diff --git a/crates/trusted-server-core/src/inspect/permissions.rs b/crates/trusted-server-core/src/inspect/permissions.rs new file mode 100644 index 0000000000..9aaf4925ee --- /dev/null +++ b/crates/trusted-server-core/src/inspect/permissions.rs @@ -0,0 +1,331 @@ +//! The permissions resolved for the request being served, at +//! [`PERMISSIONS_PAGE_PATH`] and [`PERMISSIONS_JSON_PATH`]. +//! +//! The answer is the evaluation the page receives as +//! `window.tsjs.permissions`, offered at an address so client code can fetch +//! it, another front end can render it, and a person can read it. It is +//! resolved for the request that asks, exactly as a page's is, and nothing is +//! written for the reader, because the Edge Cookie lifecycle does not run +//! here. + +use edgezero_core::body::Body as EdgeBody; +use error_stack::Report; +use http::{HeaderValue, Request, Response, StatusCode, header}; +use serde_json::{Value, json}; + +use crate::ec::EcContext; +use crate::error::TrustedServerError; +use crate::permissions::PermissionState; +use crate::platform::RuntimeServices; +use crate::settings::Settings; + +use super::render_page; + +/// The permissions page. +pub const PERMISSIONS_PAGE_PATH: &str = "/_ts/permissions"; + +/// The same resolution as data. +pub const PERMISSIONS_JSON_PATH: &str = "/_ts/permissions.json"; + +/// Both addresses, the page first. +pub const PERMISSIONS_PATHS: [&str; 2] = [PERMISSIONS_PAGE_PATH, PERMISSIONS_JSON_PATH]; + +/// The permission resolution for a request, as the endpoint shows it, being +/// the page shape of [`PermissionState::page_value`] with these fields added. +/// +/// - `storageWithdrawn`: whether the request explicitly withdrew device storage. +/// - `modules.contributed`: the signal modules that produced a signal used here. +/// - `modules.configured`: the signal modules the publisher named, or `null`. +/// - `version`: the version of the core crate that answered. +#[must_use] +pub fn permissions_payload(state: &PermissionState, settings: &Settings) -> Value { + let mut contributed: Vec<&str> = state.signals().iter().map(|signal| signal.module).collect(); + contributed.sort_unstable(); + contributed.dedup(); + let configured = settings + .permission_signal + .modules + .as_ref() + .map(|names| json!(names)); + + let mut payload = state.page_value(); + if let Value::Object(fields) = &mut payload { + fields.insert("version".to_owned(), json!(env!("CARGO_PKG_VERSION"))); + fields.insert( + "storageWithdrawn".to_owned(), + json!(state.storage_withdrawn()), + ); + fields.insert( + "modules".to_owned(), + json!({ + "contributed": contributed, + "configured": configured, + }), + ); + } + payload +} + +/// The permissions page, or its JSON when `path` is +/// [`PERMISSIONS_JSON_PATH`], for a request whose permissions have been +/// resolved. +/// +/// Never stored, because each answer is the asking request's own, and +/// readable from any origin, so a page on another site can render it. A +/// request from another origin carries no cookies, so what that page reads is +/// the resolution of a request with no stored signal. +/// +/// # Panics +/// +/// Never in practice, because the status and every header are fixed, valid +/// values. +#[must_use] +pub fn permissions_response( + state: &PermissionState, + settings: &Settings, + path: &str, +) -> Response { + let payload = permissions_payload(state, settings); + let (content_type, body) = if path == PERMISSIONS_JSON_PATH { + ("application/json", payload.to_string()) + } else { + ( + "text/html; charset=utf-8", + render_page("Permissions", &payload), + ) + }; + Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, HeaderValue::from_static(content_type)) + .header(header::CACHE_CONTROL, HeaderValue::from_static("no-store")) + .header( + header::ACCESS_CONTROL_ALLOW_ORIGIN, + HeaderValue::from_static("*"), + ) + .body(EdgeBody::from(body.into_bytes())) + .expect("should build permissions response") +} + +/// Answers [`PERMISSIONS_PAGE_PATH`] or [`PERMISSIONS_JSON_PATH`] for `req`. +/// +/// The permissions are resolved for this request as a page's would be, from +/// its signals and its location, and then shown. Nothing is created or +/// written for the reader. +/// +/// # Errors +/// +/// When the request's state cannot be read, which is when the selected Edge +/// Cookie module cannot be built or the `Cookie` header is not valid UTF-8, +/// the same as for a page. +pub async fn handle_permissions( + settings: &Settings, + services: &RuntimeServices, + req: &Request, +) -> Result, Report> { + let state = EcContext::read_from_request_resolving_geo(settings, req, services).await?; + Ok(permissions_response( + state.permissions(), + settings, + req.uri().path(), + )) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::permissions::{Permission, PermissionSet, ValidSignal}; + use crate::platform::test_support::noop_services; + use crate::tdl::Tdl; + use crate::test_support::tests::create_test_settings; + + fn state_with_signal() -> PermissionState { + PermissionState::new(PermissionSet::none().with(Permission::StoreOnDevice)) + .with_signals(vec![ValidSignal::new("tcf", "tcf", "CPabc123")].into()) + .with_tdls( + vec![ + Tdl::new("https://terms.example.com/marketing/2.txt") + .expect("should accept the test locator"), + ] + .into(), + ) + } + + fn body_of(response: Response) -> String { + String::from_utf8( + response + .into_body() + .into_bytes() + .unwrap_or_default() + .to_vec(), + ) + .expect("the body should be text") + } + + /// A reader comes to see which signal produced the answer, so the signal + /// and the module that read it must both be there. + #[test] + fn the_payload_names_the_signal_and_the_module_that_read_it() { + let payload = permissions_payload(&state_with_signal(), &create_test_settings()); + + assert_eq!(payload["signals"][0]["module"], "tcf"); + assert_eq!(payload["signals"][0]["scheme"], "tcf"); + assert_eq!(payload["signals"][0]["value"], "CPabc123"); + assert_eq!(payload["modules"]["contributed"], json!(["tcf"])); + assert_eq!( + payload["tdls"], + json!(["https://terms.example.com/marketing/2.txt"]) + ); + assert_eq!(payload["set"], json!(["necessary.operations.storage"])); + assert_eq!(payload["storageWithdrawn"], json!(false)); + assert_eq!(payload["version"], json!(env!("CARGO_PKG_VERSION"))); + } + + /// The page shape is spelled once, on the state, so the endpoint cannot + /// drift from what a page is given. + #[test] + fn the_payload_holds_everything_the_page_is_given() { + let state = state_with_signal(); + let payload = permissions_payload(&state, &create_test_settings()); + + let page = state.page_value(); + for (key, value) in page.as_object().expect("the page shape is an object") { + assert_eq!( + &payload[key], value, + "`{key}` should be what the page is given" + ); + } + } + + /// A module that ran and found nothing and a module that was never + /// configured are different facts, and somebody asking why a permission + /// is not set needs to tell them apart. + #[test] + fn configured_and_contributed_modules_are_reported_separately() { + let mut settings = create_test_settings(); + settings.permission_signal.modules = Some(vec!["tcf".to_owned(), "gpc".to_owned()]); + + let payload = permissions_payload(&state_with_signal(), &settings); + + assert_eq!(payload["modules"]["contributed"], json!(["tcf"])); + assert_eq!(payload["modules"]["configured"], json!(["tcf", "gpc"])); + } + + /// With no modules named the selection is the adapter's, which core + /// cannot see, so it says nothing and does not invent a list. + #[test] + fn configured_is_null_when_the_publisher_named_no_modules() { + let mut settings = create_test_settings(); + settings.permission_signal.modules = None; + + let payload = permissions_payload(&state_with_signal(), &settings); + + assert!( + payload["modules"]["configured"].is_null(), + "should not guess which modules the adapter offers" + ); + } + + /// "No signal was found" is itself the answer somebody came for. + #[test] + fn a_resolution_with_no_signals_still_answers() { + let payload = permissions_payload(&PermissionState::default(), &create_test_settings()); + + assert_eq!(payload["signals"], json!([])); + assert_eq!(payload["set"], json!([])); + assert_eq!(payload["awaiting"], json!([])); + assert_eq!(payload["modules"]["contributed"], json!([])); + } + + #[test] + fn a_withdrawal_of_storage_is_shown_as_one() { + let state = PermissionState::default().with_storage_withdrawn(true); + + let payload = permissions_payload(&state, &create_test_settings()); + + assert_eq!(payload["storageWithdrawn"], json!(true)); + } + + #[test] + fn the_json_address_answers_data_and_the_page_address_a_page() { + let settings = create_test_settings(); + let state = state_with_signal(); + + let data = permissions_response(&state, &settings, PERMISSIONS_JSON_PATH); + assert_eq!(data.status(), StatusCode::OK); + assert_eq!(data.headers()[header::CONTENT_TYPE], "application/json"); + assert_eq!(data.headers()[header::CACHE_CONTROL], "no-store"); + assert_eq!(data.headers()[header::ACCESS_CONTROL_ALLOW_ORIGIN], "*"); + let parsed: Value = + serde_json::from_str(&body_of(data)).expect("the data form should be JSON"); + assert_eq!(parsed, permissions_payload(&state, &settings)); + + let page = permissions_response(&state, &settings, PERMISSIONS_PAGE_PATH); + assert_eq!( + page.headers()[header::CONTENT_TYPE], + "text/html; charset=utf-8" + ); + assert_eq!(page.headers()[header::CACHE_CONTROL], "no-store"); + let html = body_of(page); + assert!(html.contains("

Permissions

"), "should be the page"); + assert!(html.contains("CPabc123"), "should show the signal"); + } + + /// The page carries a signal value straight from the request, so + /// anything that could close the surrounding tag is escaped. + #[test] + fn the_page_escapes_a_hostile_signal_value() { + let state = PermissionState::new(PermissionSet::none()) + .with_signals(vec![ValidSignal::new("tcf", "tcf", " + + + + + +An article on the origin + +A link to the same address + + diff --git a/crates/trusted-server-core/src/integrations/fixtures/js-asset-proxy.recorded.html b/crates/trusted-server-core/src/integrations/fixtures/js-asset-proxy.recorded.html new file mode 100644 index 0000000000..4168885df0 --- /dev/null +++ b/crates/trusted-server-core/src/integrations/fixtures/js-asset-proxy.recorded.html @@ -0,0 +1,17 @@ + + + + +JS asset proxy + + + + + + + +An article on the origin + +A link to the same address + + diff --git a/crates/trusted-server-core/src/integrations/fixtures/js-asset-proxy.settings.toml b/crates/trusted-server-core/src/integrations/fixtures/js-asset-proxy.settings.toml new file mode 100644 index 0000000000..2cedf868f5 --- /dev/null +++ b/crates/trusted-server-core/src/integrations/fixtures/js-asset-proxy.settings.toml @@ -0,0 +1,13 @@ +[proxy] +modules = ["js_asset_proxy"] + +[proxy.js_asset_proxy] + +[[proxy.js_asset_proxy.assets]] +path = "/assets/example-vendor-loader.js" +origin_url = "https://cdn.example.com/vendor-loader.js" +proxy = "enabled" + +[[fetch]] +media_type = "text/html" +middleware = ["js_asset_proxy"] diff --git a/crates/trusted-server-core/src/integrations/js_asset_proxy.rs b/crates/trusted-server-core/src/integrations/js_asset_proxy.rs index 6b301bf935..183288a568 100644 --- a/crates/trusted-server-core/src/integrations/js_asset_proxy.rs +++ b/crates/trusted-server-core/src/integrations/js_asset_proxy.rs @@ -6,6 +6,7 @@ //! proxying, or block matching script tags from publisher HTML. use std::collections::HashSet; +use std::rc::Rc; use std::sync::Arc; use async_trait::async_trait; @@ -22,14 +23,31 @@ use crate::constants::{ }; use crate::error::TrustedServerError; use crate::integrations::{ - AttributeRewriteAction, IntegrationAttributeContext, IntegrationAttributeRewriter, - IntegrationEndpoint, IntegrationProxy, IntegrationRegistration, + AttributeRewriteAction, IntegrationEndpoint, IntegrationProxy, IntegrationRegistration, +}; +use crate::middleware::{ + AttributeRewrite, AttributeRewriteFn, Middleware, MiddlewareAction, MiddlewareContext, + MiddlewarePhase, }; use crate::platform::RuntimeServices; use crate::proxy::{ProxyRequestConfig, proxy_request}; use crate::settings::{IntegrationConfig, Settings}; pub(crate) const JS_ASSET_PROXY_INTEGRATION_ID: &str = "js_asset_proxy"; + +/// The name this module is selected by, in `[proxy]`. +pub const MODULE: &str = "js_asset_proxy"; + +/// The builder the registry runs when a section selects [`MODULE`]. +pub(crate) const BUILDER: crate::integrations::IntegrationBuilder = + crate::integrations::IntegrationBuilder::new( + JS_ASSET_PROXY_INTEGRATION_ID, + crate::integrations::CORE_SOURCE, + register, + validate, + ) + .with_module_name(MODULE) + .selected_in("proxy"); const JS_ASSET_CONTENT_TYPE: &str = "application/javascript; charset=utf-8"; const X_CONTENT_TYPE_OPTIONS_NOSNIFF: &str = "nosniff"; const ERROR_ORIGIN_UNREACHABLE: &str = "js-asset-origin-unreachable"; @@ -37,10 +55,8 @@ const ERROR_ORIGIN_STATUS: &str = "js-asset-origin-status"; /// Configuration for the JavaScript asset proxy integration. #[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct JsAssetProxyConfig { - /// Enables or disables the integration. - #[serde(default)] - pub enabled: bool, /// Optional downstream cache TTL override for every asset. #[serde(default)] pub cache_ttl_seconds: Option, @@ -88,18 +104,16 @@ impl JsAssetProxyConfig { } } -impl IntegrationConfig for JsAssetProxyConfig { - fn is_enabled(&self) -> bool { - self.enabled - } -} +impl IntegrationConfig for JsAssetProxyConfig {} impl Validate for JsAssetProxyConfig { fn validate(&self) -> Result<(), ValidationErrors> { let mut errors = ValidationErrors::new(); errors.merge_self("assets", self.assets.validate()); - if self.enabled && self.assets.is_empty() { + // A selected integration with nothing to serve is a configuration + // mistake, so the block has to list at least one asset. + if self.assets.is_empty() { errors.add("assets", ValidationError::new("empty_assets")); } @@ -268,14 +282,32 @@ impl JsAssetProxyIntegration { fn asset_for_script_src( &self, script_src: &str, - ctx: &IntegrationAttributeContext<'_>, + request_scheme: &str, ) -> Option<&JsAssetProxyAsset> { self.asset_for_origin_url(script_src).or_else(|| { - let normalized_src = normalize_script_src(script_src, ctx.request_scheme)?; + let normalized_src = normalize_script_src(script_src, request_scheme)?; self.asset_for_origin_url(&normalized_src) }) } + /// What happens to a script whose address is `script_src`, on a page + /// served over `request_scheme`. + fn rewrite_script_address( + &self, + script_src: &str, + request_scheme: &str, + ) -> AttributeRewriteAction { + let Some(asset) = self.asset_for_script_src(script_src, request_scheme) else { + return AttributeRewriteAction::keep(); + }; + + match asset.proxy { + JsAssetProxyMode::Enabled => AttributeRewriteAction::replace(asset.path.clone()), + JsAssetProxyMode::Disabled => AttributeRewriteAction::keep(), + JsAssetProxyMode::Blocked => AttributeRewriteAction::remove_element(), + } + } + fn build_proxy_config<'a>( origin_url: &'a str, req: &Request, @@ -446,9 +478,7 @@ impl JsAssetProxyIntegration { fn build( settings: &Settings, ) -> Result>, Report> { - let Some(mut config) = - settings.integration_config::(JS_ASSET_PROXY_INTEGRATION_ID)? - else { + let Some(mut config) = settings.module_config::(MODULE)? else { return Ok(None); }; config.normalize_origin_urls(); @@ -456,11 +486,24 @@ fn build( Ok(Some(JsAssetProxyIntegration::new(config))) } +/// Validates the JavaScript asset proxy configuration for deployment and +/// reports whether a section selects the integration's module. +/// +/// # Errors +/// +/// Returns an error when the configuration cannot be parsed or fails +/// validation. +pub(crate) fn validate(settings: &Settings) -> Result> { + settings + .module_config::(MODULE) + .map(|config| config.is_some()) +} + /// Register the JavaScript asset proxy integration. /// /// # Errors /// -/// Returns an error when the integration is enabled with invalid configuration. +/// Returns an error when the integration runs with invalid configuration. pub fn register( settings: &Settings, ) -> Result, Report> { @@ -471,31 +514,17 @@ pub fn register( Ok(Some( IntegrationRegistration::builder(JS_ASSET_PROXY_INTEGRATION_ID) .with_proxy(integration.clone()) - .with_attribute_rewriter(integration) + .with_middleware(Arc::new(ScriptAddress(integration))) .build(), )) } -#[async_trait(?Send)] -impl IntegrationProxy for JsAssetProxyIntegration { - fn integration_name(&self) -> &'static str { - JS_ASSET_PROXY_INTEGRATION_ID - } - - fn routes(&self) -> Vec { - self.config - .assets - .iter() - .filter(|asset| asset.proxy == JsAssetProxyMode::Enabled) - .map(|asset| IntegrationEndpoint::new(Method::GET, asset.path.clone())) - .collect() - } - - async fn handle( +impl JsAssetProxyIntegration { + async fn route( &self, + req: Request, settings: &Settings, services: &RuntimeServices, - req: Request, ) -> Result, Report> { let request_path = req.uri().path().to_string(); let asset = self.enabled_asset_for_path(&request_path).ok_or_else(|| { @@ -533,33 +562,60 @@ impl IntegrationProxy for JsAssetProxyIntegration { } } -impl IntegrationAttributeRewriter for JsAssetProxyIntegration { - fn integration_id(&self) -> &'static str { +#[async_trait(?Send)] +impl IntegrationProxy for JsAssetProxyIntegration { + fn integration_name(&self) -> &'static str { JS_ASSET_PROXY_INTEGRATION_ID } - fn handles_attribute(&self, attribute: &str) -> bool { - attribute == "src" + fn routes(&self) -> Vec { + self.config + .assets + .iter() + .filter(|asset| asset.proxy == JsAssetProxyMode::Enabled) + .map(|asset| IntegrationEndpoint::new(Method::GET, asset.path.clone())) + .collect() } - fn rewrite( + async fn handle( &self, - attr_name: &str, - attr_value: &str, - ctx: &IntegrationAttributeContext<'_>, - ) -> AttributeRewriteAction { - if attr_name != "src" || !ctx.element_name.eq_ignore_ascii_case("script") { - return AttributeRewriteAction::keep(); - } + call: crate::module_context::ModuleCall<'_>, + req: Request, + ) -> Result, Report> { + call.inject_with(self, req, Self::route)?.await + } +} - let Some(asset) = self.asset_for_script_src(attr_value, ctx) else { - return AttributeRewriteAction::keep(); - }; +/// Points a configured script at its first-party path, or removes one that +/// is blocked, on the pages a `[[fetch]]` entry names [`MODULE`] for. +/// +/// It is asked about a script's address as the middleware named before it +/// left it, so it is named first where it has to see the address the page +/// wrote. +struct ScriptAddress(Arc); + +impl Middleware for ScriptAddress { + fn middleware_id(&self) -> &'static str { + MODULE + } - match asset.proxy { - JsAssetProxyMode::Enabled => AttributeRewriteAction::replace(asset.path.clone()), - JsAssetProxyMode::Disabled => AttributeRewriteAction::keep(), - JsAssetProxyMode::Blocked => AttributeRewriteAction::remove_element(), + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Fetch] + } + + fn create(&self, context: &MiddlewareContext<'_>) -> MiddlewareAction { + let integration = Arc::clone(&self.0); + let request_scheme = context.request_scheme.to_owned(); + let decide: Rc = Rc::new(move |matched| { + integration.rewrite_script_address(matched.value, &request_scheme) + }); + MiddlewareAction { + element_handlers: vec![Box::new(AttributeRewrite::matching( + "script[src]", + "src", + decide, + ))], + ..MiddlewareAction::pass() } } } @@ -571,10 +627,13 @@ mod tests { use std::sync::Arc; use crate::constants::{HEADER_REFERER, HEADER_X_FORWARDED_FOR, HEADER_X_TS_EC}; - use crate::html_processor::{BodyCloseInjection, HtmlProcessorConfig, create_html_processor}; - use crate::integrations::{ - AttributeRewriteAction, IntegrationAttributeRewriter, IntegrationRegistry, + use crate::html_processor::test_support::{place_on_every_page, processed_page}; + use crate::html_processor::{ + BodyCloseInjection, HtmlProcessorConfig, create_html_processor_with_middleware, }; + use crate::integrations::registry_test_support::tag_fixture as tag; + use crate::integrations::{AttributeRewriteAction, IntegrationRegistry}; + use crate::middleware::MiddlewareChain; use crate::platform::test_support::{StubHttpClient, build_services_with_http_client}; use crate::streaming_processor::{Compression, PipelineConfig, StreamingPipeline}; use crate::test_support::tests::create_test_settings; @@ -600,47 +659,39 @@ mod tests { fn config_with_assets(assets: Vec) -> JsAssetProxyConfig { JsAssetProxyConfig { - enabled: true, cache_ttl_seconds: None, assets, } } - fn rewrite_context() -> IntegrationAttributeContext<'static> { - IntegrationAttributeContext { - attribute_name: "src", - element_name: "script", - request_host: "publisher.example.com", - request_scheme: "https", - origin_host: "origin.example.com", - } - } - + /// The page a reader of `html` receives when `integration`'s middleware + /// alone runs on it. fn process_html_with_integration( html: &str, integration: Arc, ) -> String { - let rewriter: Arc = integration; - process_html_with_registry( - html, - IntegrationRegistry::from_rewriters(vec![rewriter], Vec::new()), - ) - } - - fn process_html_with_registry(html: &str, integrations: IntegrationRegistry) -> String { - let processor = create_html_processor(HtmlProcessorConfig { + let chain = MiddlewareChain::new( + MiddlewarePhase::Fetch, + crate::middleware::HTML_MEDIA_TYPE, + vec![Arc::new(ScriptAddress(integration))], + ); + let config = HtmlProcessorConfig { csp_nonce_observed: None, + public_prefix: None, + prepared: std::sync::Arc::new(crate::middleware::Prepared::default()), body_close: BodyCloseInjection::None, origin_host: "origin.example.com".to_string(), request_host: "publisher.example.com".to_string(), request_scheme: "https".to_string(), - integrations, + integrations: IntegrationRegistry::default(), ad_slots_script: None, ad_bids_state: Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, - }); + request_state: crate::integrations::IntegrationRequestState::default(), + permissions_script: None, + }; + let (processor, _) = create_html_processor_with_middleware(config, &chain, None) + .expect("should plan the asset proxy's middleware"); let pipeline_config = PipelineConfig { input_compression: Compression::None, output_compression: Compression::None, @@ -655,35 +706,50 @@ mod tests { String::from_utf8(output).expect("should produce UTF-8 HTML") } - #[test] - fn disabled_config_does_not_register_routes() { + /// The page a reader of `html` receives from a deployment running + /// `settings`. + fn process_html_with_settings(html: &str, settings: &Settings) -> String { + let registry = IntegrationRegistry::new(settings).expect("should build registry"); + processed_page(settings, ®istry, html, 8192) + } + + /// Settings that run the tag stand-in and proxy its script in `mode`, + /// with the two middleware named in the order given. + fn settings_proxying_the_tag(mode: &str, middleware: &[&str]) -> Settings { let mut settings = create_test_settings(); + settings.select_module("testing", tag::MODULE); settings - .integrations - .insert_config( - JS_ASSET_PROXY_INTEGRATION_ID, + .insert_module_config( + "proxy", + "js_asset_proxy", &json!({ - "enabled": false, "assets": [{ - "path": "/assets/vendor.js", - "origin_url": "https://cdn.example.com/vendor.js" + "path": "/assets/sdk.js", + "origin_url": tag::SCRIPT_URL, + "proxy": mode }] }), ) - .expect("should insert integration config"); + .expect("should insert JS asset proxy config"); + place_on_every_page(&mut settings, MiddlewarePhase::Fetch, middleware); + settings + } + + #[test] + fn an_integration_that_is_not_named_registers_no_routes() { + let settings = create_test_settings(); let registry = IntegrationRegistry::new(&settings).expect("should build registry"); assert!( !registry.has_route(&Method::GET, "/assets/vendor.js"), - "disabled integration should not register asset route" + "an integration that is not named should register no asset route" ); } #[test] fn enabled_config_requires_at_least_one_asset() { let config = JsAssetProxyConfig { - enabled: true, cache_ttl_seconds: None, assets: Vec::new(), }; @@ -723,17 +789,16 @@ mod tests { assert_eq!(routes[0].method, Method::GET); assert_eq!(routes[0].path, "/assets/enabled.js"); - let ctx = rewrite_context(); assert!(matches!( - integration.rewrite("src", "https://cdn.example.com/enabled.js", &ctx), + integration.rewrite_script_address("https://cdn.example.com/enabled.js", "https"), AttributeRewriteAction::Replace(ref value) if value == "/assets/enabled.js" )); assert!(matches!( - integration.rewrite("src", "https://cdn.example.com/disabled.js", &ctx), + integration.rewrite_script_address("https://cdn.example.com/disabled.js", "https"), AttributeRewriteAction::Keep )); assert!(matches!( - integration.rewrite("src", "https://cdn.example.com/blocked.js", &ctx), + integration.rewrite_script_address("https://cdn.example.com/blocked.js", "https"), AttributeRewriteAction::RemoveElement )); } @@ -745,46 +810,13 @@ mod tests { "https://cdn.example.com/vendor.js", JsAssetProxyMode::Enabled, )])); - let ctx = rewrite_context(); assert!(matches!( - integration.rewrite("src", "https://cdn.example.com/vendor.js?v=1", &ctx), + integration.rewrite_script_address("https://cdn.example.com/vendor.js?v=1", "https"), AttributeRewriteAction::Keep )); assert!(matches!( - integration.rewrite("src", "https://cdn.example.com/other.js", &ctx), - AttributeRewriteAction::Keep - )); - } - - #[test] - fn non_script_src_matches_are_not_rewritten_or_blocked() { - let integration = JsAssetProxyIntegration::new(config_with_assets(vec![ - asset( - "/assets/enabled.js", - "https://cdn.example.com/enabled.js", - JsAssetProxyMode::Enabled, - ), - asset( - "/assets/blocked.js", - "https://cdn.example.com/blocked.js", - JsAssetProxyMode::Blocked, - ), - ])); - let ctx = IntegrationAttributeContext { - attribute_name: "src", - element_name: "img", - request_host: "publisher.example.com", - request_scheme: "https", - origin_host: "origin.example.com", - }; - - assert!(matches!( - integration.rewrite("src", "https://cdn.example.com/enabled.js", &ctx), - AttributeRewriteAction::Keep - )); - assert!(matches!( - integration.rewrite("src", "https://cdn.example.com/blocked.js", &ctx), + integration.rewrite_script_address("https://cdn.example.com/other.js", "https"), AttributeRewriteAction::Keep )); } @@ -818,6 +850,25 @@ mod tests { assert!(processed.contains(r#""#)); } + #[test] + fn a_script_tag_written_in_upper_case_is_a_script() { + let integration = JsAssetProxyIntegration::new(config_with_assets(vec![asset( + "/assets/enabled.js", + "https://cdn.example.com/enabled.js", + JsAssetProxyMode::Enabled, + )])); + + let processed = process_html_with_integration( + r#""#, + integration, + ); + + assert!( + processed.contains(r#"SRC="/assets/enabled.js""#), + "should rewrite a script whatever case its tag is written in: {processed}" + ); + } + #[test] fn script_src_matching_normalizes_common_browser_url_forms() { let integration = JsAssetProxyIntegration::new(config_with_assets(vec![asset( @@ -825,7 +876,6 @@ mod tests { "https://cdn.example.com/vendor.js", JsAssetProxyMode::Enabled, )])); - let ctx = rewrite_context(); for script_src in [ "//cdn.example.com/vendor.js", @@ -834,7 +884,7 @@ mod tests { ] { assert!( matches!( - integration.rewrite("src", script_src, &ctx), + integration.rewrite_script_address(script_src, "https"), AttributeRewriteAction::Replace(ref value) if value == "/assets/vendor.js" ), "script src {script_src} should normalize to the configured origin URL" @@ -843,106 +893,84 @@ mod tests { } #[test] - fn js_asset_proxy_rewriter_takes_precedence_over_native_rewriters() { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config("gpt", &json!({ "enabled": true })) - .expect("should insert GPT config"); - settings - .integrations - .insert_config( - JS_ASSET_PROXY_INTEGRATION_ID, - &json!({ - "enabled": true, - "assets": [{ - "path": "/assets/gpt.js", - "origin_url": "https://securepubads.g.doubleclick.net/tag/js/gpt.js", - "proxy": "enabled" - }] - }), - ) - .expect("should insert JS asset proxy config"); - let registry = IntegrationRegistry::new(&settings).expect("should build registry"); - let html = r#""#; + fn the_asset_proxy_named_first_rewrites_a_script_ahead_of_a_module() { + let settings = settings_proxying_the_tag("enabled", &[MODULE, tag::MODULE]); + let html = format!( + r#""#, + tag::SCRIPT_URL + ); - let processed = process_html_with_registry(html, registry); + let processed = process_html_with_settings(&html, &settings); assert!( - processed.contains(r#""#), - "JS asset proxy should rewrite before GPT native rewriter: {processed}" + processed.contains(r#""#), + "JS asset proxy should rewrite before the module named after it: {processed}" ); assert!( - !processed.contains("/integrations/gpt/script"), - "GPT native rewrite should not override JS asset proxy" + !processed.contains(tag::FIRST_PARTY_SCRIPT), + "the module's own rewrite should not override JS asset proxy" ); } #[test] - fn js_asset_proxy_blocking_takes_precedence_over_native_rewriters() { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config("gpt", &json!({ "enabled": true })) - .expect("should insert GPT config"); - settings - .integrations - .insert_config( - JS_ASSET_PROXY_INTEGRATION_ID, - &json!({ - "enabled": true, - "assets": [{ - "path": "/assets/gpt.js", - "origin_url": "https://securepubads.g.doubleclick.net/tag/js/gpt.js", - "proxy": "blocked" - }] - }), - ) - .expect("should insert JS asset proxy config"); - let registry = IntegrationRegistry::new(&settings).expect("should build registry"); - let html = r#""#; + fn a_module_named_ahead_of_the_asset_proxy_rewrites_the_script_first() { + let settings = settings_proxying_the_tag("enabled", &[tag::MODULE, MODULE]); + let html = format!( + r#""#, + tag::SCRIPT_URL + ); + + let processed = process_html_with_settings(&html, &settings); + + assert!( + processed.contains(&format!( + r#""#, + tag::FIRST_PARTY_SCRIPT + )), + "the module named first should rewrite the script: {processed}" + ); + assert!( + !processed.contains("/assets/sdk.js"), + "JS asset proxy should not know an address the module has already moved" + ); + } + + #[test] + fn the_asset_proxy_named_first_blocks_a_script_ahead_of_a_module() { + let settings = settings_proxying_the_tag("blocked", &[MODULE, tag::MODULE]); + let html = format!( + r#""#, + tag::SCRIPT_URL + ); - let processed = process_html_with_registry(html, registry); + let processed = process_html_with_settings(&html, &settings); assert!( - !processed.contains("googletag.cmd"), - "blocked JS asset should remove the script element before GPT can rewrite it" + !processed.contains("vendor.cmd"), + "blocked JS asset should remove the script element before the module can rewrite it" ); assert!( - !processed.contains("/integrations/gpt/script"), - "GPT native rewrite should not keep a blocked script" + !processed.contains(tag::FIRST_PARTY_SCRIPT), + "the module's own rewrite should not keep a blocked script" ); } #[test] fn disabled_js_asset_proxy_candidate_allows_native_rewriters() { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config("gpt", &json!({ "enabled": true })) - .expect("should insert GPT config"); - settings - .integrations - .insert_config( - JS_ASSET_PROXY_INTEGRATION_ID, - &json!({ - "enabled": true, - "assets": [{ - "path": "/assets/gpt.js", - "origin_url": "https://securepubads.g.doubleclick.net/tag/js/gpt.js", - "proxy": "disabled" - }] - }), - ) - .expect("should insert JS asset proxy config"); - let registry = IntegrationRegistry::new(&settings).expect("should build registry"); - let html = r#""#; + let settings = settings_proxying_the_tag("disabled", &[MODULE, tag::MODULE]); + let html = format!( + r#""#, + tag::SCRIPT_URL + ); - let processed = process_html_with_registry(html, registry); + let processed = process_html_with_settings(&html, &settings); assert!( - processed.contains(r#""#), - "disabled JS asset proxy entries should not suppress native integration rewrites" + processed.contains(&format!( + r#""#, + tag::FIRST_PARTY_SCRIPT + )), + "disabled JS asset proxy entries should not suppress a module's own rewrite" ); } @@ -1032,16 +1060,6 @@ mod tests { #[test] fn rejects_unknown_proxy_mode() { let toml = r#" - [[handlers]] - path = "^/secure" - username = "user" - password = "pass" - - [[handlers]] - path = "^/_ts/admin" - username = "admin" - password = "admin-pass" - [publisher] domain = "test-publisher.com" cookie_domain = ".test-publisher.com" @@ -1051,14 +1069,16 @@ mod tests { [ec] passphrase = "test-secret-key-32-bytes-minimum" + [geo] + assume_single_jurisdiction = true + [request_signing] - config_store_id = "test-config-store-id" - secret_store_id = "test-secret-store-id" + enabled = false - [integrations.js_asset_proxy] - enabled = true + [proxy] + modules = ["js_asset_proxy"] - [[integrations.js_asset_proxy.assets]] + [[proxy.js_asset_proxy.assets]] path = "/assets/vendor.js" origin_url = "https://cdn.example.com/vendor.js" proxy = "passthrough" @@ -1067,7 +1087,7 @@ mod tests { assert!( settings - .integration_config::(JS_ASSET_PROXY_INTEGRATION_ID) + .module_config::(MODULE) .is_err(), "unknown proxy mode should fail deserialization" ); @@ -1077,11 +1097,10 @@ mod tests { fn exact_configured_routes_are_registered() { let mut settings = create_test_settings(); settings - .integrations - .insert_config( - JS_ASSET_PROXY_INTEGRATION_ID, + .insert_module_config( + "proxy", + "js_asset_proxy", &json!({ - "enabled": true, "assets": [ { "path": "/assets/vendor.js", @@ -1278,11 +1297,10 @@ mod tests { fn configured_origin_urls_are_canonicalized_for_matching_and_duplicates() { let mut settings = create_test_settings(); settings - .integrations - .insert_config( - JS_ASSET_PROXY_INTEGRATION_ID, + .insert_module_config( + "proxy", + "js_asset_proxy", &json!({ - "enabled": true, "assets": [{ "path": "/assets/vendor.js", "origin_url": "HTTPS://CDN.EXAMPLE.COM:443/vendor.js" @@ -1290,10 +1308,10 @@ mod tests { }), ) .expect("should insert integration config"); - let registry = IntegrationRegistry::new(&settings).expect("should build registry"); - let processed = process_html_with_registry( + place_on_every_page(&mut settings, MiddlewarePhase::Fetch, &[MODULE]); + let processed = process_html_with_settings( r#""#, - registry, + &settings, ); assert!(processed.contains(r#""#)); @@ -1385,7 +1403,7 @@ mod tests { ); let response = integration - .handle(&settings, &services, request) + .route(request, &settings, &services) .await .expect("should proxy conditional asset response"); @@ -1435,13 +1453,13 @@ mod tests { Arc::clone(&buffered_stub) as Arc ); let success = integration - .handle( - &settings, - &buffered_services, + .route( build_http_request( Method::GET, "https://publisher.example.com/assets/vendor.js", ), + &settings, + &buffered_services, ) .await .expect("should proxy buffered asset response"); @@ -1455,13 +1473,13 @@ mod tests { Arc::clone(&streaming_stub) as Arc ); let success = integration - .handle( - &settings, - &streaming_services, + .route( build_http_request( Method::GET, "https://publisher.example.com/assets/vendor.js", ), + &settings, + &streaming_services, ) .await .expect("should proxy streaming asset response"); @@ -1483,13 +1501,13 @@ mod tests { Arc::clone(&unavailable_stub) as Arc, ); let unavailable = integration - .handle( - &settings, - &unavailable_services, + .route( build_http_request( Method::GET, "https://publisher.example.com/assets/vendor.js", ), + &settings, + &unavailable_services, ) .await .expect("should map unavailable origin response"); @@ -1508,13 +1526,13 @@ mod tests { Arc::clone(&status_stub) as Arc ); let status = integration - .handle( - &settings, - &status_services, + .route( build_http_request( Method::GET, "https://publisher.example.com/assets/vendor.js", ), + &settings, + &status_services, ) .await .expect("should map non-success origin response"); @@ -1664,4 +1682,21 @@ mod tests { "should identify the unknown asset field" ); } + + /// The page a reader receives with the script proxy running, kept as a + /// file so that changing how the page change is made can be shown to + /// leave the page as it was. + #[test] + fn the_page_a_reader_receives_is_the_recorded_one() { + crate::html_processor::test_support::assert_page_is_recorded( + include_str!("fixtures/js-asset-proxy.settings.toml"), + &[], + include_str!("fixtures/js-asset-proxy.input.html"), + include_str!("fixtures/js-asset-proxy.recorded.html"), + concat!( + env!("CARGO_MANIFEST_DIR"), + "/src/integrations/fixtures/js-asset-proxy.recorded.html" + ), + ); + } } diff --git a/crates/trusted-server-core/src/integrations/mod.rs b/crates/trusted-server-core/src/integrations/mod.rs index 820e2c24b8..cf07cee5d8 100644 --- a/crates/trusted-server-core/src/integrations/mod.rs +++ b/crates/trusted-server-core/src/integrations/mod.rs @@ -1,43 +1,32 @@ //! Integration module registry and sample implementations. +use std::sync::Arc; use std::time::Duration; use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt}; use futures::StreamExt as _; +use http::{Request, Response}; use url::Url; +use crate::auction::AuctionPlan; +use crate::auction::demand::{AdServerImplementation, DemandImplementation}; use crate::error::TrustedServerError; use crate::platform::{DEFAULT_FIRST_BYTE_TIMEOUT, PlatformBackendSpec, RuntimeServices}; use crate::settings::Settings; -pub mod adserver_mock; -pub mod aps; -pub mod datadome; -pub mod didomi; -pub mod google_tag_manager; -pub mod gpt; -pub mod gpt_diagnostics; pub mod js_asset_proxy; -pub mod lockr; -pub mod nextjs; -pub mod osano; -pub mod permutive; -pub mod prebid; mod registry; -pub mod sourcepoint; -pub mod testlight; +#[cfg(test)] +pub(crate) use registry::test_support as registry_test_support; pub use registry::{ - AttributeRewriteAction, AttributeRewriteOutcome, HeaderMutation, HeaderMutationMode, - IntegrationAttributeContext, IntegrationAttributeRewriter, IntegrationDocumentState, - IntegrationEndpoint, IntegrationHeadInjector, IntegrationHtmlContext, - IntegrationHtmlStreamContext, IntegrationHtmlStreamProcessorFactory, IntegrationMetadata, - IntegrationProxy, IntegrationRegistration, IntegrationRegistrationBuilder, IntegrationRegistry, - IntegrationRequestFilter, IntegrationScriptContext, IntegrationScriptRewriter, - ProxyDispatchInput, RequestFilterDecision, RequestFilterEffects, RequestFilterInput, - RequestFilterRegistryInput, RequestFilterRegistryOutcome, ScriptRewriteAction, - ScriptTextAccumulator, + AttributeRewriteAction, CarriedJsModule, HeaderMutation, HeaderMutationMode, + IntegrationDocumentState, IntegrationEndpoint, IntegrationMetadata, IntegrationProxy, + IntegrationRegistration, IntegrationRegistrationBuilder, IntegrationRegistry, + IntegrationRequestFilter, IntegrationRequestState, ProxyDispatchInput, RequestFilterDecision, + RequestFilterEffects, RequestFilterInput, RequestFilterRegistryInput, + RequestFilterRegistryOutcome, ScriptRewriteAction, }; /// Registers or retrieves a platform backend for the given URL. @@ -46,11 +35,13 @@ pub use registry::{ /// 15-second first-byte timeout, and delegates to /// [`crate::platform::PlatformBackend::ensure`]. /// +/// Public for the same reason as [`ensure_integration_backend_with_timeout`]. +/// /// # Errors /// /// Returns an error when `url` cannot be parsed, is missing a host, or the /// backend registration fails. -pub(crate) fn ensure_integration_backend( +pub fn ensure_integration_backend( services: &RuntimeServices, url: &str, integration: &'static str, @@ -77,11 +68,15 @@ pub(crate) fn ensure_integration_backend( /// given `first_byte_timeout`, and delegates to /// [`crate::platform::PlatformBackend::ensure`]. /// +/// Public because an integration crate outside this one cannot reach an +/// upstream without registering a backend first, and a crate rolling its own +/// registration would produce a different backend name for the same URL. +/// /// # Errors /// /// Returns an error when `url` cannot be parsed, is missing a host, or the /// backend registration fails. -pub(crate) fn ensure_integration_backend_with_timeout( +pub fn ensure_integration_backend_with_timeout( services: &RuntimeServices, url: &str, integration: &'static str, @@ -106,11 +101,15 @@ pub(crate) fn ensure_integration_backend_with_timeout( /// Parses `url`, builds a [`PlatformBackendSpec`], and delegates to /// [`crate::platform::PlatformBackend::predict_name`]. /// +/// Public for the same reason as [`ensure_integration_backend_with_timeout`], +/// and the two have to agree, so a crate outside this one uses this function +/// and does not derive a name of its own. +/// /// # Errors /// /// Returns an error when the URL cannot be parsed, is missing a host, or the /// platform backend cannot predict a name for the spec. -pub(crate) fn predict_integration_backend_name( +pub fn predict_integration_backend_name( services: &RuntimeServices, url: &str, integration: &'static str, @@ -164,22 +163,34 @@ fn integration_backend_spec( } /// Maximum body size accepted by integration proxy endpoints (256 KiB). -pub(crate) const INTEGRATION_MAX_BODY_BYTES: usize = 256 * 1024; +/// +/// Public so every integration crate bounds a request body at one size. +pub const INTEGRATION_MAX_BODY_BYTES: usize = 256 * 1024; -/// Maximum response body size from RTB providers (prebid, aps, mediator). -pub(crate) const UPSTREAM_RTB_MAX_RESPONSE_BYTES: usize = 2 * 1024 * 1024; +/// Maximum response body size from RTB providers (prebid, aps, ad server). +/// +/// Public so an integration crate outside this one bounds an upstream +/// response at the size the built-in ones do. +pub const UPSTREAM_RTB_MAX_RESPONSE_BYTES: usize = 2 * 1024 * 1024; /// Maximum response body size from SDK/proxy integrations. -pub(crate) const UPSTREAM_SDK_MAX_RESPONSE_BYTES: usize = 16 * 1024 * 1024; +/// +/// Public so every integration crate bounds an upstream script or proxied +/// response at one size. +pub const UPSTREAM_SDK_MAX_RESPONSE_BYTES: usize = 16 * 1024 * 1024; /// Drains an [`EdgeBody`] into a byte vector, rejecting bodies larger than /// `max_bytes` with [`TrustedServerError::RequestTooLarge`]. /// +/// Public because an integration crate outside this one reads request bodies +/// too, and an unbounded read of a client's body is not a fault each crate +/// should solve again. +/// /// # Errors /// /// Returns an error when: /// - The body exceeds `max_bytes`. /// - A streaming body chunk cannot be read (mapped to an `Integration` error). -pub(crate) async fn collect_body_bounded( +pub async fn collect_body_bounded( body: EdgeBody, max_bytes: usize, integration: &'static str, @@ -226,7 +237,7 @@ pub(crate) async fn collect_body_bounded( /// /// Use this for upstream (provider/integration) response bodies to bound /// memory usage when a third-party server misbehaves. Unlike -/// [`collect_body_bounded`], oversized bodies are classified as +/// `collect_body_bounded`, oversized bodies are classified as /// [`TrustedServerError::Integration`] (502 `BAD_GATEWAY`) rather than /// [`TrustedServerError::RequestTooLarge`] (413). /// @@ -234,12 +245,16 @@ pub(crate) async fn collect_body_bounded( /// because the size check runs after each chunk is materialized. Fastly /// H2/H3 chunks are ≤ 16 KiB in practice, making the overshoot negligible. /// +/// Public because an integration crate outside this one reads upstream +/// responses too, and an unbounded read of a misbehaving upstream is not a +/// fault each crate should solve again. +/// /// # Errors /// /// Returns an error when: /// - The body exceeds `max_bytes` (mapped to [`TrustedServerError::Integration`]). /// - A streaming body chunk cannot be read (same error type). -pub(crate) async fn collect_response_bounded( +pub async fn collect_response_bounded( body: EdgeBody, max_bytes: usize, integration: &'static str, @@ -281,69 +296,737 @@ pub(crate) async fn collect_response_bounded( } } -type IntegrationBuilderFn = +/// Builds an integration's registration from settings, or `None` when the +/// settings give it nothing to register. +/// +/// The registry calls this only for an integration a section selects, so an integration runs exactly when an operator names it. +pub type IntegrationBuilderFn = fn(&Settings) -> Result, Report>; -pub(crate) struct IntegrationBuilder { +/// Validates an integration's configuration for deployment and reports +/// whether a section selects it. +/// +/// Runs for every builder, named or not, so one builder's rules cannot be +/// skipped by the order the builders happen to be in. At deploy time secret +/// fields hold secret-store key names rather than values, so a validator must +/// not depend on a resolved secret. +pub type IntegrationValidateFn = fn(&Settings) -> Result>; + +/// Prepares a request before routing, for every routed request except the +/// health check. +/// +/// Runs whether or not a section selects the integration, so one +/// can strip its own reserved query or cookie in a deployment that does not +/// run it. +pub type IntegrationPrepareRequestFn = + fn(&Settings, &mut Request) -> Result<(), Report>; + +/// Finishes the response the page path returns for one request, from what +/// the module's request hooks left for it in the request's +/// [`IntegrationRequestState`]. +/// +/// Runs whether or not a section selects the integration, as the preparer +/// does, and has nothing to do for a request its module left nothing on. +pub type IntegrationFinalizeResponseFn = fn(&IntegrationRequestState, &mut Response); + +/// Builds a module's registration from the settings and the compiled auction +/// plan, or `None` when the two give it nothing to register. +/// +/// Runs for every builder that has one, whether or not a section selects the +/// module, because a module registered this way follows what the plan +/// selects and decides for itself whether it runs. +pub type IntegrationPlanRegistrationFn = + fn( + &Settings, + &AuctionPlan, + ) -> Result, Report>; + +/// Checks a module's configuration against the compiled auction plan, for a +/// rule that depends on what the plan selects. +/// +/// Runs for every builder that has one, when a deployment is validated and +/// as the settings load. +pub type IntegrationPlanValidateFn = + fn(&Settings, &AuctionPlan) -> Result<(), Report>; + +/// A setting in a module's own table that holds the name of a secret. +/// +/// The name is looked up in the default secret store as the settings load, +/// and the setting holds the secret from then on. +#[derive(Clone, Copy, Debug)] +pub struct ModuleSecretSetting { + /// Where the setting sits inside the module's table, one name for each + /// level. + pub path: &'static [&'static str], + /// Whether the module's table, as written, puts the setting to use. A + /// setting not in use is cleared as the settings load and is not looked + /// up, and one in use has to name a key before a deployment is accepted. + pub in_use: fn(&serde_json::Map) -> bool, +} + +/// Source label for the built-in integrations. +pub const CORE_SOURCE: &str = "trusted-server-core"; + +/// A named factory for one integration, the unit an adapter or a vendor crate +/// hands to [`IntegrationRegistry::with_plan_and_registrations`]. +/// +/// # Examples +/// +/// ``` +/// use error_stack::Report; +/// use trusted_server_core::error::TrustedServerError; +/// use std::sync::Arc; +/// +/// use trusted_server_core::auction::compile_auction_plan; +/// use trusted_server_core::integrations::{ +/// IntegrationBuilder, IntegrationRegistration, IntegrationRegistry, +/// }; +/// use trusted_server_core::settings::Settings; +/// +/// fn build( +/// _settings: &Settings, +/// ) -> Result, Report> { +/// Ok(Some(IntegrationRegistration::builder("example").build())) +/// } +/// +/// fn validate(_settings: &Settings) -> Result> { +/// Ok(true) +/// } +/// +/// # fn demo(settings: &Settings) -> Result<(), Report> { +/// let builder = IntegrationBuilder::new("example", "example-crate", build, validate) +/// .with_module_name("testing.example"); +/// // The registry builds a module a section selects, so a deployment that +/// // wants this one writes `[testing] modules = ["example"]`. +/// let mut settings = settings.clone(); +/// settings.select_module("testing", "testing.example"); +/// let plan = Arc::new(compile_auction_plan(&settings)?); +/// let registry = IntegrationRegistry::with_plan_and_registrations(&settings, plan, &[builder])?; +/// assert!(registry.integration_runs("example")); +/// # Ok(()) +/// # } +/// ``` +#[derive(Clone, Copy, Debug)] +pub struct IntegrationBuilder { id: &'static str, + source: &'static str, build: IntegrationBuilderFn, + validate: IntegrationValidateFn, + prepare_request: Option, + finalize_response: Option, + plan_registration: Option, + plan_validator: Option, + reads_auction_token: bool, + secret_settings: &'static [ModuleSecretSetting], + supplies_integration: bool, + demand: Option<&'static DemandImplementation>, + adserver: Option<&'static AdServerImplementation>, + module: Option<&'static str>, + section: Option<&'static str>, + loading_cache: Option, + shared: Option<&'static str>, +} + +/// Gives the loading cache a module supplies, see +/// [`IntegrationBuilder::with_loading_cache`]. +pub type LoadingCacheFactory = fn() -> Arc; + +/// The build function of a builder that supplies no page integration. +fn no_registration( + _settings: &Settings, +) -> Result, Report> { + Ok(None) +} + +/// The validate function of a builder that supplies no page integration. +fn nothing_to_validate(_settings: &Settings) -> Result> { + Ok(false) +} + +impl IntegrationBuilder { + /// Creates a builder for the integration `id`, attributed to `source` + /// (a crate or package name used in duplicate-id errors). + #[must_use] + pub const fn new( + id: &'static str, + source: &'static str, + build: IntegrationBuilderFn, + validate: IntegrationValidateFn, + ) -> Self { + Self { + id, + source, + build, + validate, + prepare_request: None, + finalize_response: None, + plan_registration: None, + plan_validator: None, + reads_auction_token: false, + secret_settings: &[], + supplies_integration: true, + demand: None, + adserver: None, + module: None, + section: None, + loading_cache: None, + shared: None, + } + } + + /// Creates a builder that supplies only implementations, such as a demand + /// or ad server implementation, and no page integration. + /// + /// Its `id` still claims a place among builder ids, so two crates cannot + /// register under one id, but it is not an integration a deployment can + /// name. + #[must_use] + pub const fn implementations(id: &'static str, source: &'static str) -> Self { + Self { + id, + source, + build: no_registration, + validate: nothing_to_validate, + prepare_request: None, + finalize_response: None, + plan_registration: None, + plan_validator: None, + reads_auction_token: false, + secret_settings: &[], + supplies_integration: false, + demand: None, + adserver: None, + module: None, + section: None, + loading_cache: None, + shared: None, + } + } + + /// Registers a demand implementation, which `[demand] modules` or an + /// `implementation` line can name. + #[must_use] + pub const fn with_demand(mut self, demand: &'static DemandImplementation) -> Self { + self.demand = Some(demand); + self + } + + /// Registers an ad server implementation, which `[ad-server] module` or + /// an `implementation` line can name. + #[must_use] + pub const fn with_adserver(mut self, adserver: &'static AdServerImplementation) -> Self { + self.adserver = Some(adserver); + self + } + + /// Whether this builder supplies a page integration, as opposed to only + /// implementations. + #[must_use] + pub const fn supplies_integration(&self) -> bool { + self.supplies_integration + } + + /// The demand implementation this builder registers, when it registers + /// one. + #[must_use] + pub const fn demand(&self) -> Option<&'static DemandImplementation> { + self.demand + } + + /// The ad server implementation this builder registers, when it registers + /// one. + #[must_use] + pub const fn adserver(&self) -> Option<&'static AdServerImplementation> { + self.adserver + } + + /// Attaches a request preparation function that runs before routing on + /// every request, whether or not the integration runs. + #[must_use] + pub const fn with_request_preparer(mut self, prepare: IntegrationPrepareRequestFn) -> Self { + self.prepare_request = Some(prepare); + self + } + + /// Attaches a function that finishes the response the page path returns, + /// from what the module's request hooks left for the request. + #[must_use] + pub const fn with_response_finalizer( + mut self, + finalize: IntegrationFinalizeResponseFn, + ) -> Self { + self.finalize_response = Some(finalize); + self + } + + /// Registers the module from the compiled auction plan as well as the + /// settings, for a module whose page support follows what the plan + /// selects. + /// + /// The function runs whether or not a section selects the module, and + /// what it registers has its hooks run ahead of the modules the sections + /// select. + #[must_use] + pub const fn with_plan_registration(mut self, register: IntegrationPlanRegistrationFn) -> Self { + self.plan_registration = Some(register); + self + } + + /// Attaches a check of the module's configuration against the compiled + /// auction plan. + #[must_use] + pub const fn with_plan_validator(mut self, validate: IntegrationPlanValidateFn) -> Self { + self.plan_validator = Some(validate); + self + } + + /// Declares that the module's browser script reads the token an auction + /// publishes with its winning bids. + /// + /// A token is made for each auction when a section selects such a + /// module, and none is made in a deployment where nothing reads it. + #[must_use] + pub const fn with_auction_token(mut self) -> Self { + self.reads_auction_token = true; + self + } + + /// Whether the module's browser script reads the auction token. + #[must_use] + pub const fn reads_auction_token(&self) -> bool { + self.reads_auction_token + } + + /// Declares the settings in the module's own table that hold the name of + /// a secret, so each is looked up as the settings load and checked when + /// a deployment is validated. + #[must_use] + pub const fn with_secret_settings(mut self, secrets: &'static [ModuleSecretSetting]) -> Self { + self.secret_settings = secrets; + self + } + + /// The settings in the module's own table that hold the name of a + /// secret. + #[must_use] + pub const fn secret_settings(&self) -> &'static [ModuleSecretSetting] { + self.secret_settings + } + + /// Runs this builder when a section selects the module `name`, which is a + /// crate's path under `crates` with `.` between the parts, such as + /// `cmp.example`. An integration still in core holds the name its crate + /// will have as a constant. + #[must_use] + pub const fn with_module_name(mut self, name: &'static str) -> Self { + self.module = Some(name); + self + } + + /// The name a section selects this builder's module by, when it has one. + #[must_use] + pub const fn module_name(&self) -> Option<&'static str> { + self.module + } + + /// Offers the loading cache this crate supplies, see + /// [`crate::platform::PlatformLoadingCache`]. + /// + /// The offer is not a selection. A cache is selected by naming the module + /// in its own `[cache.]` table, and an adapter resolves every table's + /// selection with [`loading_caches_from`] when it builds its state. A cache + /// whose table selects none asks each source itself. + #[must_use] + pub const fn with_loading_cache(mut self, factory: LoadingCacheFactory) -> Self { + self.loading_cache = Some(factory); + self + } + + /// Runs this builder when `[shared.]` is present, for a vendor + /// whose modules all read that one table and cannot be built without it. + /// A builder the table runs is selected by no section, so the registry + /// refuses one that also has a module name. + #[must_use] + pub const fn with_shared_settings(mut self, vendor: &'static str) -> Self { + self.shared = Some(vendor); + self + } + + /// The vendor whose `[shared.]` table runs this builder, when one + /// does. + #[must_use] + pub const fn shared_settings(&self) -> Option<&'static str> { + self.shared + } + + /// The loading cache this builder offers, when it offers one. + #[must_use] + pub const fn loading_cache(&self) -> Option { + self.loading_cache + } + + /// Names the section that selects this module, for one selected outside + /// the section of its type, such as a module of core's own in a section + /// of core's. + #[must_use] + pub const fn selected_in(mut self, section: &'static str) -> Self { + self.section = Some(section); + self + } + + /// The section that selects this builder's module: the one + /// [`selected_in`](Self::selected_in) named, or else its type's. + #[must_use] + pub fn section(&self) -> Option<&'static str> { + self.section.or_else(|| { + self.module + .and_then(|name| name.split_once('.').map(|(folder, _)| folder)) + }) + } + + /// The integration id this builder produces. + #[must_use] + pub const fn id(&self) -> &'static str { + self.id + } + + /// The source label used in diagnostics. + #[must_use] + pub const fn source(&self) -> &'static str { + self.source + } + + /// Builds the registration, or `None` when the settings give the + /// integration nothing to register. + /// + /// # Errors + /// + /// Returns an error when the integration runs with invalid configuration. + pub(crate) fn build( + &self, + settings: &Settings, + ) -> Result, Report> { + (self.build)(settings) + } + + /// Validates the integration's configuration for deployment and reports + /// whether a section selects the integration's module. + /// + /// # Errors + /// + /// Returns an error when the configuration cannot be parsed or fails + /// validation. + pub(crate) fn validate(&self, settings: &Settings) -> Result> { + (self.validate)(settings) + } + + /// The request preparation function, when one is attached. + pub(crate) fn prepare_request(&self) -> Option { + self.prepare_request + } + + /// The response finalizer, when one is attached. + pub(crate) fn finalize_response(&self) -> Option { + self.finalize_response + } + + /// The registration from the auction plan, when one is attached. + pub(crate) fn plan_registration(&self) -> Option { + self.plan_registration + } + + /// The check against the auction plan, when one is attached. + pub(crate) fn plan_validator(&self) -> Option { + self.plan_validator + } } +/// The built-in integrations, in the order `trusted-server-modules` lists them. +const BUILT_IN_BUILDERS: &[IntegrationBuilder] = &[ + // This stays the first module a section selects, so its middleware is + // the first of theirs in the order the registry lists them. + js_asset_proxy::BUILDER, + // A stand-in for an integration that streams, which core's own tests + // select where they need one. + #[cfg(test)] + registry_test_support::payload_fixture::BUILDER, + // A stand-in for an integration that tags a page, for the same tests. + #[cfg(test)] + registry_test_support::tag_fixture::BUILDER, + // A stand-in for an integration that acts on one request, for the same + // tests. + #[cfg(test)] + registry_test_support::request_fixture::BUILDER, + // A stand-in for an integration whose browser module loads deferred, for + // the same tests. + #[cfg(test)] + registry_test_support::deferred_fixture::BUILDER, + // A stand-in for a module that changes a page through middleware, for + // the same tests. + #[cfg(test)] + registry_test_support::middleware_fixture::BUILDER, + // A stand-in for the plainest demand implementation there can be, which + // core's own tests name where they need a source. + #[cfg(test)] + IntegrationBuilder::implementations( + crate::auction::test_support::plain_fixture::MODULE, + CORE_SOURCE, + ) + .with_demand(&crate::auction::test_support::plain_fixture::DEMAND), + // A stand-in demand implementation, which core's own tests name where + // they need one that departs from the plain one. + #[cfg(test)] + IntegrationBuilder::implementations( + crate::auction::test_support::demand_fixture::MODULE, + CORE_SOURCE, + ) + .with_demand(&crate::auction::test_support::demand_fixture::DEMAND), + // A stand-in ad server, which core's own tests select where they need + // one. + #[cfg(test)] + IntegrationBuilder::implementations( + crate::auction::test_support::adserver_fixture::MODULE, + CORE_SOURCE, + ) + .with_adserver(&crate::auction::test_support::adserver_fixture::ADSERVER), +]; + +/// The built-in integration builders, in the order `trusted-server-modules` +/// lists them. pub(crate) fn builders() -> &'static [IntegrationBuilder] { - &[ - // This must remain first: attribute rewriters chain replacements and short-circuit removals. - IntegrationBuilder { - id: js_asset_proxy::JS_ASSET_PROXY_INTEGRATION_ID, - build: js_asset_proxy::register, - }, - IntegrationBuilder { - id: "testlight", - build: testlight::register, - }, - IntegrationBuilder { - id: "nextjs", - build: nextjs::register, - }, - IntegrationBuilder { - id: "permutive", - build: permutive::register, - }, - IntegrationBuilder { - id: "lockr", - build: lockr::register, - }, - IntegrationBuilder { - id: "didomi", - build: didomi::register, - }, - IntegrationBuilder { - id: "sourcepoint", - build: sourcepoint::register, - }, - IntegrationBuilder { - id: "osano", - build: osano::register, - }, - IntegrationBuilder { - id: "google_tag_manager", - build: google_tag_manager::register, - }, - IntegrationBuilder { - id: "datadome", - build: datadome::register, - }, - IntegrationBuilder { - id: "gpt", - build: gpt::register, - }, - IntegrationBuilder { - id: "gpt_diagnostics", - build: gpt_diagnostics::register, - }, - ] + BUILT_IN_BUILDERS +} + +/// Every builder the registry will consider: the built-in set followed by +/// `extra`, in that order, so the built-ins keep their order. +pub(crate) fn all_builders( + extra: &[IntegrationBuilder], +) -> impl Iterator + '_ { + builders().iter().copied().chain(extra.iter().copied()) +} + +/// The loading cache each `[cache.]` table selects, from the caches the +/// builders a deployment links offer. A table that names no module keeps no +/// copy, and a module's cache is made once however many tables select it. +/// +/// # Errors +/// +/// When a table names a module no linked builder offers a cache from, naming +/// the table and the modules that do. +pub fn loading_caches_from( + settings: &Settings, + builders: &[IntegrationBuilder], +) -> Result, Report> { + let offered: Vec<(&'static str, LoadingCacheFactory)> = builders + .iter() + .filter_map(|builder| { + builder + .loading_cache() + .map(|factory| (builder.id(), factory)) + }) + .collect(); + let mut made: Vec<(&'static str, Arc)> = Vec::new(); + let mut caches = crate::platform::LoadingCaches::default(); + for (table, config) in settings.cache.loading_caches() { + // A table naming no module takes the one loading cache module the + // build links, so answers are held without a line in the document. + // With none linked, or several, it holds nothing. + let written = match config.module.as_deref() { + Some(written) => written, + None if offered.len() == 1 => offered[0].0, + None => continue, + }; + let Some((name, factory)) = offered.iter().find(|(name, _)| { + *name == written || crate::module_name::short_form(LOADING_CACHE_TYPE, name) == written + }) else { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "[cache.{table}] module names `{written}`, which no linked crate offers \ + a loading cache from. The loading cache modules this build has are: {}", + offered_cache_names(&offered) + ), + })); + }; + let cache = match made.iter().find(|(made_name, _)| made_name == name) { + Some((_, cache)) => Arc::clone(cache), + None => { + let cache = factory(); + made.push((*name, Arc::clone(&cache))); + cache + } + }; + caches.insert(table, cache); + } + Ok(Arc::new(caches)) +} + +/// The type folder loading cache modules live under, which their short +/// names leave out. +const LOADING_CACHE_TYPE: &str = "loading-cache"; + +/// The short names of the modules offering a cache, for a refusal. +fn offered_cache_names(offered: &[(&'static str, LoadingCacheFactory)]) -> String { + if offered.is_empty() { + return "none".to_owned(); + } + offered + .iter() + .map(|(name, _)| crate::module_name::short_form(LOADING_CACHE_TYPE, name)) + .collect::>() + .join(", ") } #[cfg(test)] -pub(crate) fn registered_builder_ids() -> impl Iterator { - builders().iter().map(|builder| builder.id) +mod loading_caches_from_tests { + use bytes::Bytes; + + use super::*; + use crate::platform::{CacheSpec, LoadError, Loader, PlatformLoadingCache, StoreKey}; + use crate::settings::ROBOTS_TXT_CACHE; + + fn no_build( + _settings: &Settings, + ) -> Result, Report> { + Ok(None) + } + + fn no_validate(_settings: &Settings) -> Result> { + Ok(true) + } + + /// A cache that says it keeps values, so a test can tell it from the + /// default, which keeps none. + struct Keeping; + + #[async_trait::async_trait(?Send)] + impl PlatformLoadingCache for Keeping { + async fn get_or_load( + &self, + _services: &crate::platform::RuntimeServices, + _spec: &CacheSpec, + _key: &StoreKey, + load: Loader, + ) -> Result { + load().await.map(|loaded| loaded.bytes) + } + + async fn remove( + &self, + _services: &crate::platform::RuntimeServices, + _spec: &CacheSpec, + _key: &StoreKey, + ) { + } + } + + fn keeping() -> Arc { + Arc::new(Keeping) + } + + fn nothing() -> Arc { + Arc::new(crate::platform::NoLoadingCache) + } + + fn builder(source: &'static str) -> IntegrationBuilder { + IntegrationBuilder::new("an_id", source, no_build, no_validate) + } + + /// A module `id` offering a cache that keeps values. + fn offering(id: &'static str) -> IntegrationBuilder { + IntegrationBuilder::implementations(id, "a-crate").with_loading_cache(keeping) + } + + /// A module `id` offering a cache that keeps nothing. + fn offering_nothing(id: &'static str) -> IntegrationBuilder { + IntegrationBuilder::implementations(id, "a-crate").with_loading_cache(nothing) + } + + fn settings_selecting(module: Option<&str>) -> Settings { + let mut settings = crate::test_support::tests::create_test_settings(); + settings.cache.robots_txt.module = module.map(str::to_owned); + settings + } + + #[test] + fn a_feature_selecting_nothing_takes_the_one_module_offered() { + let caches = loading_caches_from( + &settings_selecting(None), + &[builder("first"), offering("loading-cache.one")], + ) + .expect("no selection is not an error"); + + assert!(caches.for_feature(ROBOTS_TXT_CACHE).keeps_values()); + assert!(!caches.is_empty()); + } + + #[test] + fn a_feature_selecting_nothing_keeps_nothing_when_two_are_offered() { + let caches = loading_caches_from( + &settings_selecting(None), + &[offering("loading-cache.one"), offering("loading-cache.two")], + ) + .expect("no selection is not an error"); + + assert!(!caches.for_feature(ROBOTS_TXT_CACHE).keeps_values()); + assert!(caches.is_empty()); + } + + #[test] + fn a_feature_selecting_nothing_keeps_nothing_when_none_is_offered() { + let caches = loading_caches_from(&settings_selecting(None), &[builder("first")]) + .expect("no selection is not an error"); + + assert!(!caches.for_feature(ROBOTS_TXT_CACHE).keeps_values()); + assert!(caches.is_empty()); + } + + #[test] + fn a_feature_selects_a_module_by_its_short_name_or_in_full() { + for written in ["one", "loading-cache.one"] { + let caches = loading_caches_from( + &settings_selecting(Some(written)), + &[ + offering_nothing("loading-cache.other"), + offering("loading-cache.one"), + ], + ) + .expect("a selected module should be found"); + + assert!( + caches.for_feature(ROBOTS_TXT_CACHE).keeps_values(), + "{written} should select the cache that keeps values, not the first offered" + ); + assert!(!caches.is_empty()); + } + } + + #[test] + fn a_feature_naming_a_module_nothing_offers_is_refused_naming_what_is_offered() { + let error = loading_caches_from( + &settings_selecting(Some("none")), + &[offering("loading-cache.one"), offering("loading-cache.two")], + ) + .expect_err("a module nothing offers should be refused"); + + let message = format!("{error:?}"); + assert!( + message.contains( + "[cache.robots-txt] module names `none`, which no linked crate offers a \ + loading cache from. The loading cache modules this build has are: one, two" + ), + "{message}" + ); + } + + #[test] + fn with_nothing_offered_the_refusal_says_so() { + let error = loading_caches_from(&settings_selecting(Some("one")), &[builder("first")]) + .expect_err("a selection with nothing offered should be refused"); + + let message = format!("{error:?}"); + assert!( + message.contains("The loading cache modules this build has are: none"), + "{message}" + ); + } } diff --git a/crates/trusted-server-core/src/integrations/osano.rs b/crates/trusted-server-core/src/integrations/osano.rs deleted file mode 100644 index 9fc3aa66c1..0000000000 --- a/crates/trusted-server-core/src/integrations/osano.rs +++ /dev/null @@ -1,116 +0,0 @@ -//! Osano integration for client-side consent mirroring. -//! -//! The Rust side of this integration intentionally only provides explicit -//! enablement for the `tsjs-osano` browser module. Osano consent extraction runs -//! in JavaScript because the relevant CMP APIs (`__uspapi`, `__gpp`, and -//! `__tcfapi`) are browser-only. - -use error_stack::Report; -use serde::Deserialize; -use validator::Validate; - -use crate::error::TrustedServerError; -use crate::settings::{IntegrationConfig, Settings}; - -use super::IntegrationRegistration; - -const OSANO_INTEGRATION_ID: &str = "osano"; - -/// Configuration for the Osano consent mirror integration. -#[derive(Debug, Clone, Deserialize, Validate)] -#[serde(deny_unknown_fields)] -pub struct OsanoConfig { - /// Whether the Osano browser consent mirror is enabled. - #[serde(default)] - pub enabled: bool, -} - -impl IntegrationConfig for OsanoConfig { - fn is_enabled(&self) -> bool { - self.enabled - } -} - -/// Register the Osano JS integration when enabled. -/// -/// # Errors -/// -/// Returns an error when the Osano integration configuration cannot be parsed or -/// fails validation. -pub fn register( - settings: &Settings, -) -> Result, Report> { - let Some(_config) = settings.integration_config::(OSANO_INTEGRATION_ID)? else { - return Ok(None); - }; - - Ok(Some( - IntegrationRegistration::builder(OSANO_INTEGRATION_ID).build(), - )) -} - -#[cfg(test)] -mod tests { - use serde_json::json; - - use super::{OsanoConfig, register}; - use crate::test_support::tests::create_test_settings; - - #[test] - fn register_returns_none_when_disabled() { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config("osano", &json!({ "enabled": false })) - .expect("should insert osano config"); - - let registration = register(&settings).expect("should parse disabled osano config"); - - assert!( - registration.is_none(), - "disabled Osano integration should not register" - ); - } - - #[test] - fn register_returns_js_module_registration_when_enabled() { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config("osano", &json!({ "enabled": true })) - .expect("should insert osano config"); - - let registration = register(&settings) - .expect("should parse enabled osano config") - .expect("enabled Osano integration should register"); - - assert_eq!(registration.integration_id, "osano"); - assert!( - registration.proxies.is_empty(), - "Osano v1 should not register Rust proxy routes" - ); - assert!( - registration.head_injectors.is_empty(), - "Osano v1 should not inject HTML from Rust" - ); - } - - #[test] - fn config_rejects_unknown_fields() { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config("osano", &json!({ "enabled": true, "typo": true })) - .expect("should insert osano config"); - - let err = settings - .integration_config::("osano") - .expect_err("should reject unknown Osano config fields"); - let error_text = format!("{err:?}"); - - assert!( - error_text.contains("typo") || error_text.contains("unknown field"), - "error should mention the unknown field: {err:?}" - ); - } -} diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index d3c09cd7d7..d48651bf8c 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -1,25 +1,33 @@ use std::any::{Any, TypeId}; use std::collections::BTreeMap; -use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; +use std::sync::{Arc, Mutex}; use async_trait::async_trait; use edgezero_core::body::Body as EdgeBody; use error_stack::Report; use http::{Method, Request, Response}; use matchit::Router; +use sha2::{Digest as _, Sha256}; +use crate::attestation::PlatformIdentity; use crate::auction::AuctionPlan; use crate::constants::HEADER_X_TS_EC; use crate::ec::EcContext; +use crate::ec::device::DeviceModule; use crate::ec::kv::KvIdentityGraph; +use crate::ec::module::{EcModuleSelection, EdgeCookieModule}; use crate::error::TrustedServerError; use crate::geo::GeoInfo; use crate::http_util::is_navigation_request; -use crate::platform::RuntimeServices; +use crate::middleware::{Middleware, MiddlewareChain, MiddlewarePhase, PhaseEntries}; +use crate::module_context::{ModuleCall, ModuleContext, ModuleRequest, ResolvedRequest}; +use crate::platform::{DisabledGeo, PlatformGeo, RuntimeServices}; +use crate::prefix::{PrefixModule, Prefixes, SelectedPrefixModule}; +use crate::robots_txt::{ALLOW_ALL, REFUSE_ALL, RobotsTxtContributor}; use crate::settings::Settings; -use crate::streaming_processor::StreamProcessor; -/// Action returned by attribute rewriters to describe how the runtime should mutate the element. +/// What a middleware's element handler decides about one attribute of an +/// element. #[derive(Debug, Clone, PartialEq, Eq)] pub enum AttributeRewriteAction { /// Leave the attribute and element untouched. @@ -47,15 +55,8 @@ impl AttributeRewriteAction { } } -/// Outcome returned by the registry after running every matching attribute rewriter. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum AttributeRewriteOutcome { - Unchanged, - Replaced(String), - RemoveElement, -} - -/// Action returned by inline script rewriters to describe how to mutate the node. +/// What a middleware's text handler decides about one chunk of the text +/// inside an element. #[derive(Debug, Clone, PartialEq, Eq)] pub enum ScriptRewriteAction { Keep, @@ -80,33 +81,13 @@ impl ScriptRewriteAction { } } -/// Context provided to integration HTML attribute rewriters. -#[derive(Debug)] -pub struct IntegrationAttributeContext<'a> { - pub attribute_name: &'a str, - pub element_name: &'a str, - pub request_host: &'a str, - pub request_scheme: &'a str, - pub origin_host: &'a str, -} - -/// Context passed to script/text rewriters for inline HTML handling. -#[derive(Debug)] -pub struct IntegrationScriptContext<'a> { - pub selector: &'a str, - pub request_host: &'a str, - pub request_scheme: &'a str, - pub origin_host: &'a str, - pub is_last_in_text_node: bool, - pub max_buffered_script_bytes: usize, - pub document_state: &'a IntegrationDocumentState, -} - type IntegrationDocumentStateMap = BTreeMap<(&'static str, TypeId), Arc>; -/// Per-document state shared between HTML/script rewriters and post-processors. +/// State shared by the middleware working on one document in one phase. /// -/// This exists to support multi-phase HTML processing without requiring a second HTML parse. +/// What one handler leaves here a later handler or a stream processor reads, +/// so the document is parsed once. In the serve phase it starts with what +/// the request's hooks left, see [`IntegrationRequestState`]. #[derive(Clone, Default)] pub struct IntegrationDocumentState { inner: Arc>, @@ -191,35 +172,101 @@ impl IntegrationDocumentState { } } -/// Per-document buffer for script text fragments split across chunks. +/// Values a module's request hooks leave for its own serve middleware, for +/// one request. /// -/// `lol_html` can deliver one text node as several chunks, so a rewriter that -/// needs the whole script must accumulate until `is_last_in_text_node`. +/// A request preparer or a request filter leaves a value under its +/// integration id with [`IntegrationRequestState::insert`]. Where the +/// request's reader is served an HTML document, every value is copied into +/// the [`IntegrationDocumentState`] the `[[serve]]` entry's middleware share, +/// so the module's serve middleware reads it there, and the same values are +/// handed to the module's response finalizer. /// -/// This lives in [`IntegrationDocumentState`] rather than on the rewriter. -/// Rewriters are registered once as `Arc` and -/// live as long as the [`IntegrationRegistry`], so a buffer owned by a -/// rewriter is shared by every document that registry serves. A document whose -/// stream ends before the final fragment — client disconnect, origin error, -/// truncated body — leaves its partial script in that buffer, and the next -/// document prepends the residue to its own accumulation. That corrupts the -/// response and can disclose the previous document's content. -/// -/// Keyed per integration id, so each integration gets its own buffer, and -/// dropped with the document state at end of document. -#[derive(Debug, Default)] -pub struct ScriptTextAccumulator { - buffer: Mutex, +/// A request that carries any value keeps to the origin path. Its document is +/// never read from a shared template and never stored as one, and an HTML +/// response with a body is sent `private, no-store`. A fetch middleware is +/// handed none of these values. +#[derive(Clone, Default)] +pub struct IntegrationRequestState { + values: IntegrationDocumentStateMap, +} + +impl std::fmt::Debug for IntegrationRequestState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("IntegrationRequestState") + .field("keys", &self.values.keys().collect::>()) + .finish() + } } -impl ScriptTextAccumulator { - /// Locks the buffer. +impl IntegrationRequestState { + /// Leaves `value` on `request` for the serve middleware of + /// `integration_id`, in place of a value of the same type left earlier. + pub fn insert(request: &mut Request, integration_id: &'static str, value: T) + where + T: Any + Send + Sync + 'static, + { + let mut state = request + .extensions_mut() + .remove::() + .unwrap_or_default(); + state.set(integration_id, value); + request.extensions_mut().insert(state); + } + + /// Holds `value` for the serve middleware of `integration_id`, in place + /// of a value of the same type held earlier. + pub fn set(&mut self, integration_id: &'static str, value: T) + where + T: Any + Send + Sync + 'static, + { + self.values + .insert((integration_id, TypeId::of::()), Arc::new(value)); + } + + /// The values left on `request`, which is none for a request no module + /// left one on. + #[must_use] + pub fn of(request: &Request) -> Self { + request + .extensions() + .get::() + .cloned() + .unwrap_or_default() + } + + /// The value of type `T` left for `integration_id`. + #[must_use] + pub fn get(&self, integration_id: &'static str) -> Option> + where + T: Any + Send + Sync + 'static, + { + let value = self.values.get(&(integration_id, TypeId::of::()))?; + Arc::clone(value).downcast::().ok() + } + + /// Whether no module left a value. + #[must_use] + pub fn is_empty(&self) -> bool { + self.values.is_empty() + } + + /// Copies every value into the state of a document that is starting. + /// + /// # Panics /// - /// Recovers from poisoning rather than panicking: a poisoned buffer holds - /// at worst a partial script, and the caller's `is_last_in_text_node` - /// handling already tolerates unexpected contents. - pub fn buffer(&self) -> MutexGuard<'_, String> { - self.buffer.lock().unwrap_or_else(PoisonError::into_inner) + /// Panics if the document state's lock is poisoned. + pub(crate) fn seed(&self, document: &IntegrationDocumentState) { + if self.values.is_empty() { + return; + } + let mut guard = document + .inner + .lock() + .expect("should lock integration document state"); + for (key, value) in &self.values { + guard.insert(*key, Arc::clone(value)); + } } } @@ -315,11 +362,21 @@ pub trait IntegrationProxy: Send + Sync { /// or define routes manually for backwards compatibility. fn routes(&self) -> Vec; + /// The permissions this route declares, which decide whether a gated + /// value, such as the Edge Cookie identifier, is passed to it. None by + /// default. + fn required_permissions(&self) -> crate::permissions::PermissionSet { + crate::permissions::PermissionSet::none() + } + /// Handle the proxied request. + /// + /// Hand a function to [`ModuleCall::inject_with`] with `req` as its + /// argument. A route whose use needs a permission declares it and checks + /// it is granted, because nothing gates the route itself. async fn handle( &self, - settings: &Settings, - services: &RuntimeServices, + call: ModuleCall<'_>, req: Request, ) -> Result, Report>; @@ -365,6 +422,11 @@ pub struct RequestFilterInput<'a> { pub services: &'a RuntimeServices, pub request: &'a mut Request, pub geo_info: Option<&'a GeoInfo>, + /// The permission state resolved for this request at the start of the + /// request cycle, so a filter reads the same permissions the rest of the + /// request uses rather than resolving its own. `None` only on paths that + /// build no EC context, such as batch sync. + pub permissions: Option<&'a crate::permissions::PermissionState>, /// Whether the request matches a registered integration proxy route. pub is_integration_route: bool, } @@ -445,6 +507,10 @@ pub struct RequestFilterRegistryInput<'a> { pub services: &'a RuntimeServices, pub req: &'a mut Request, pub geo_info: Option<&'a GeoInfo>, + /// The permission state resolved for this request at the start of the + /// request cycle, passed on to every filter. `None` only on paths that + /// build no EC context, such as batch sync. + pub permissions: Option<&'a crate::permissions::PermissionState>, } /// Outcome returned by [`IntegrationRegistry::filter_request`]. @@ -548,74 +614,16 @@ fn apply_header_mutation_to_response(response: &mut Response, mutation } } -/// Trait for integration-provided HTML attribute rewrite hooks. -pub trait IntegrationAttributeRewriter: Send + Sync { - /// Identifier for logging/diagnostics. - fn integration_id(&self) -> &'static str; - /// Return true when this rewriter wants to inspect a given attribute. - fn handles_attribute(&self, attribute: &str) -> bool; - /// Attempt to rewrite the attribute value. Return `AttributeRewriteAction::Replace` - /// to update the attribute, `Keep` to leave it untouched, or `RemoveElement` to drop the node. - fn rewrite( - &self, - attr_name: &str, - attr_value: &str, - ctx: &IntegrationAttributeContext<'_>, - ) -> AttributeRewriteAction; -} - -/// Trait for integration-provided inline script/text rewrite hooks. -pub trait IntegrationScriptRewriter: Send + Sync { - /// Identifier for logging/diagnostics. - fn integration_id(&self) -> &'static str; - /// CSS selector (e.g. `script#__NEXT_DATA__`) that should trigger this rewriter. - fn selector(&self) -> &'static str; - /// Attempt to rewrite the inline text content for the selector. - fn rewrite(&self, content: &str, ctx: &IntegrationScriptContext<'_>) -> ScriptRewriteAction; -} - -/// Context for HTML post-processors. -#[derive(Debug)] -pub struct IntegrationHtmlContext<'a> { - pub request_host: &'a str, - pub request_scheme: &'a str, - pub origin_host: &'a str, - pub document_state: &'a IntegrationDocumentState, -} - -/// Owned request data supplied when an integration creates an HTML stream processor. -#[derive(Clone)] -pub struct IntegrationHtmlStreamContext { - /// Publisher-facing host used for rewritten URLs. - pub request_host: String, - /// Publisher-facing scheme used for rewritten URLs. - pub request_scheme: String, - /// Origin host whose URLs may be rewritten. - pub origin_host: String, - /// Request-local state shared with the document's integration rewriters. - pub document_state: IntegrationDocumentState, -} - -/// Creates one mutable HTML output processor for each document. -pub trait IntegrationHtmlStreamProcessorFactory: Send + Sync { - /// Identifier for logging and diagnostics. - fn integration_id(&self) -> &'static str; - - /// Create a request-local streaming processor. - fn create(&self, context: IntegrationHtmlStreamContext) -> Box; -} - -/// Trait for integration-provided HTML head injections. -pub trait IntegrationHeadInjector: Send + Sync { - /// Identifier for logging/diagnostics. - fn integration_id(&self) -> &'static str; - /// Return HTML snippets to insert at the start of ``. - fn head_inserts(&self, ctx: &IntegrationHtmlContext<'_>) -> Vec; - - /// Return attributes to add to the publisher TSJS bundle tag. - fn tsjs_script_tag_attributes(&self) -> Vec<(&'static str, &'static str)> { - Vec::new() - } +/// A browser module a registration carries, for a module built outside +/// `trusted-server-js`. The crate embeds its built IIFE with `include_str!` +/// and states its SHA-256 as a literal next to it; the registry verifies the +/// two agree when it is built, and the served `?v=` hash is derived from it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CarriedJsModule { + /// The built IIFE. + pub source: &'static str, + /// SHA-256 of `source`, hex encoded, lower case. + pub sha256: &'static str, } /// Registration payload returned by integration builders. @@ -623,12 +631,56 @@ pub struct IntegrationRegistration { pub integration_id: &'static str, pub js_deferred: bool, pub js_disabled: bool, + /// Browser module carried by the registration, when the module is not + /// compiled into `trusted-server-js`. + pub js_module: Option, + /// Serve the module only on its own `/static/tsjs=tsjs-.min.js` path, + /// never in the unified bundle and never as a deferred tag; the + /// integration injects the tag itself when it decides to. + pub js_standalone: bool, pub proxies: Vec>, - pub attribute_rewriters: Vec>, - pub script_rewriters: Vec>, - pub html_stream_processors: Vec>, - pub head_injectors: Vec>, pub request_filters: Vec>, + /// The page changes this registration supplies, see [`crate::middleware`]. + /// + /// Declaring one does not make it run, because a middleware changes a + /// page only where an entry of the settings names it. + pub middleware: Vec>, + /// Attributes this registration puts on the script bundle's tag, each a + /// name and a value, for a browser module that reads a setting from the + /// tag it was loaded by. + pub bundle_tag_attributes: Vec<(&'static str, &'static str)>, + /// Geo module this registration supplies, with the name `[geo] module` + /// selects it by. + /// + /// Declaring one does not make it active, because the module is only asked + /// to resolve location when `[geo] module` names it. + pub geo_module: Option<(&'static str, Arc)>, + /// Edge Cookie module this registration supplies, with the name + /// `[ec] module` selects it by. + /// + /// Declaring one does not make it active, because the module is only asked + /// to create an identifier when `[ec] module` names it. This is the same + /// route geo takes, so identity is not a second extension mechanism + /// sitting beside the integration system. + pub ec_module: Option<(&'static str, Arc)>, + /// Device module this registration supplies, with the name + /// `[device] module` selects it by. + /// + /// Declaring one does not make it active, because the module is only asked + /// to classify a request when `[device] module` names it. + pub device_module: Option<(&'static str, Arc)>, + /// Prefix module this registration supplies, with the name + /// `[prefix] module` selects it by. + /// + /// Declaring one does not make it active, because the module only gives + /// the deployment's prefixes when `[prefix] module` names it. + pub prefix_module: Option<(&'static str, Arc)>, + /// The rules this registration adds to `/robots.txt`, with the name + /// `[robots-txt] modules` selects them by. + /// + /// Declaring them does not put them in the file, because the file is only + /// built from the contributors `[robots-txt] modules` names. + pub robots_txt_contributor: Option<(&'static str, Arc)>, } impl IntegrationRegistration { @@ -649,12 +701,17 @@ impl IntegrationRegistrationBuilder { integration_id, js_deferred: false, js_disabled: false, + js_module: None, + js_standalone: false, proxies: Vec::new(), - attribute_rewriters: Vec::new(), - script_rewriters: Vec::new(), - html_stream_processors: Vec::new(), - head_injectors: Vec::new(), request_filters: Vec::new(), + middleware: Vec::new(), + bundle_tag_attributes: Vec::new(), + geo_module: None, + ec_module: None, + device_module: None, + prefix_module: None, + robots_txt_contributor: None, }, } } @@ -666,38 +723,100 @@ impl IntegrationRegistrationBuilder { } #[must_use] - pub fn with_attribute_rewriter( - mut self, - rewriter: Arc, - ) -> Self { - self.registration.attribute_rewriters.push(rewriter); + pub fn with_request_filter(mut self, filter: Arc) -> Self { + self.registration.request_filters.push(filter); self } + /// Declare a page change this registration supplies, see + /// [`crate::middleware`]. Called once for each one. + /// + /// The middleware changes a page only where an entry of the settings + /// names it, and one no entry names is logged as a warning when the + /// registry is built. #[must_use] - pub fn with_script_rewriter(mut self, rewriter: Arc) -> Self { - self.registration.script_rewriters.push(rewriter); + pub fn with_middleware(mut self, middleware: Arc) -> Self { + self.registration.middleware.push(middleware); self } + /// Put an attribute on the script bundle's tag, on every page the bundle + /// is written into. + /// + /// The name is lower case letters, digits and hyphens, and the value + /// holds none of `"`, `&`, `<` and `>`, or startup is refused. Where two + /// registrations give one name different values the first is kept, and + /// the other is logged as a warning. #[must_use] - pub fn with_html_stream_processor( - mut self, - processor: Arc, - ) -> Self { - self.registration.html_stream_processors.push(processor); + pub fn with_bundle_tag_attribute(mut self, name: &'static str, value: &'static str) -> Self { + self.registration.bundle_tag_attributes.push((name, value)); self } + /// Declare the geo module this registration supplies, under its `name`. + /// + /// The name is the path under `crates` of the crate the module lives in, + /// as [`crate::module_name!`] gives it, so a module from + /// `crates/geo/example` is selected by `[geo] module = "example"`. One + /// registration can therefore supply a module of each type, each under + /// the name of its own crate. The module only resolves location when + /// `[geo] module` names it, and a module the selector does not choose is + /// logged as a warning when the registry is built. #[must_use] - pub fn with_head_injector(mut self, injector: Arc) -> Self { - self.registration.head_injectors.push(injector); + pub fn with_geo_module(mut self, name: &'static str, module: Arc) -> Self { + self.registration.geo_module = Some((name, module)); self } + /// Declare the Edge Cookie module this registration supplies, under its + /// `name`, which is also what the module's + /// [`id`](EdgeCookieModule::id) returns. + /// + /// The module only creates identifiers when `[ec] module` names it, and + /// a module the selector does not choose is logged as a warning when the + /// registry is built, the same as geo. #[must_use] - pub fn with_request_filter(mut self, filter: Arc) -> Self { - self.registration.request_filters.push(filter); + pub fn with_ec_module(mut self, name: &'static str, module: Arc) -> Self { + self.registration.ec_module = Some((name, module)); + self + } + + /// Declare the device module this registration supplies, under its + /// `name`. + /// + /// The module only classifies requests when `[device] module` names it, + /// and a module the selector does not choose is logged as a warning when + /// the registry is built, the same as geo. + #[must_use] + pub fn with_device_module(mut self, name: &'static str, module: Arc) -> Self { + self.registration.device_module = Some((name, module)); + self + } + + /// Declare the prefix module this registration supplies, with the name + /// `[prefix] module` selects it by. + /// + /// It only gives the deployment's prefixes when `[prefix] module` names + /// it, see [`crate::prefix`], and a module the selector does not choose is + /// logged as a warning when the registry is built, the same as geo. + #[must_use] + pub fn with_prefix_module(mut self, name: &'static str, module: Arc) -> Self { + self.registration.prefix_module = Some((name, module)); + self + } + + /// Declare the rules this registration adds to `/robots.txt`, under their + /// `name`. + /// + /// They are only asked for when `[robots-txt] modules` names them. Core + /// holds what they return and writes the file, see [`crate::robots_txt`]. + #[must_use] + pub fn with_robots_txt_contributor( + mut self, + name: &'static str, + contributor: Arc, + ) -> Self { + self.registration.robots_txt_contributor = Some((name, contributor)); self } @@ -706,6 +825,8 @@ impl IntegrationRegistrationBuilder { #[must_use] pub fn with_deferred_js(mut self) -> Self { self.registration.js_deferred = true; + self.registration.js_disabled = false; + self.registration.js_standalone = false; self } @@ -714,6 +835,28 @@ impl IntegrationRegistrationBuilder { pub fn without_js(mut self) -> Self { self.registration.js_disabled = true; self.registration.js_deferred = false; + self.registration.js_standalone = false; + self + } + + /// Carry a browser module built outside `trusted-server-js`. + #[must_use] + pub fn with_js_module(mut self, module: CarriedJsModule) -> Self { + self.registration.js_module = Some(module); + self + } + + /// Serve the module standalone only; see + /// [`IntegrationRegistration::js_standalone`]. + /// + /// The three delivery flags are exclusive and the last builder call wins, + /// so this clears the disabled and deferred flags as those methods clear + /// this one. + #[must_use] + pub fn with_standalone_js(mut self) -> Self { + self.registration.js_standalone = true; + self.registration.js_disabled = false; + self.registration.js_deferred = false; self } @@ -725,6 +868,10 @@ impl IntegrationRegistrationBuilder { type RouteValue = (Arc, &'static str); +/// Marks a request the preparers have run for. +#[derive(Clone, Copy)] +struct RequestPrepared; + struct IntegrationRegistryInner { // Method-specific routers for O(log n) lookups get_router: Router, @@ -737,14 +884,63 @@ struct IntegrationRegistryInner { // Metadata for introspection routes: Vec<(IntegrationEndpoint, &'static str)>, - enabled_integration_ids: Vec<&'static str>, + // Every builder considered at construction, named or not, in order. + builder_ids: Vec<(&'static str, &'static str)>, + running_integration_ids: Vec<&'static str>, deferred_js_ids: Vec<&'static str>, disabled_js_ids: Vec<&'static str>, - html_rewriters: Vec>, - script_rewriters: Vec>, - html_stream_processors: Vec>, - head_injectors: Vec>, + // Modules that run, served only on their own path, never in the bundle. + standalone_js_ids: Vec<&'static str>, + // Modules carried by their registrations, verified against their + // declared hash at construction. + carried_js: Vec<(&'static str, CarriedJsModule)>, request_filters: Vec>, + // The middleware the running registrations supply, each against the + // integration that supplied it, in registration order. + middleware: Vec<(&'static str, Arc)>, + // The attributes the running registrations put on the script bundle's + // tag, each against the integration that asked for it, in registration + // order. + bundle_tag_attributes: Vec<(&'static str, (&'static str, &'static str))>, + /// JS module IDs to include in the bundle that come from a source other than + /// a registered integration, for example a module tied to the selected Edge + /// Cookie module. Populated in [`IntegrationRegistry::new`] from settings. + extra_js_module_ids: Vec<&'static str>, + // Preparers from every builder, named or not, in registration order. + request_preparers: Vec, + // Finalizers from every builder, named or not, in registration order. + response_finalizers: Vec, + // Geo modules the running registrations supply, each with the name + // `[geo] module` selects it by, in registration order. Declaring one does + // not activate it. + geo_modules: Vec<(&'static str, Arc)>, + // Edge Cookie modules the running registrations supply, each with its + // name, in registration order. `[ec] module` picks at most one of them. + ec_modules: Vec<(&'static str, Arc)>, + // Device modules the running registrations supply, each with its name, in + // registration order. `[device] module` picks at most one of them. + device_modules: Vec<(&'static str, Arc)>, + // The robots.txt contributors the running registrations supply, each with + // its name, in registration order. `[robots-txt] modules` picks any of + // them, in its own order. + robots_txt_contributors: Vec<(&'static str, Arc)>, + // Every builder's module name, with whether a section selects it, so the + // refusal of a name a selector wrote can say what that name is. + builder_modules: Vec<(&'static str, bool)>, + // Vendors whose shared table is absent, for the refusal note. + absent_shared: Vec<&'static str>, + // The module `[geo] module` resolved to, or `None` when the selector + // is `platform` and the adapter's own host lookup stands. Unset and `none` + // both resolve the disabled module. + geo_module: Option>, + ec_module: Option>, + device_module: Option>, + // The prefix modules the running registrations supply, each with its + // name, in registration order. `[prefix] module` picks at most one. + prefix_modules: Vec<(&'static str, Arc)>, + // The prefix module `[prefix] module` resolved to, core's own when + // nothing is selected. + prefix_module: SelectedPrefixModule, } impl Default for IntegrationRegistryInner { @@ -758,16 +954,442 @@ impl Default for IntegrationRegistryInner { head_router: Router::new(), options_router: Router::new(), routes: Vec::new(), - enabled_integration_ids: Vec::new(), + builder_ids: Vec::new(), + running_integration_ids: Vec::new(), deferred_js_ids: Vec::new(), disabled_js_ids: Vec::new(), - html_rewriters: Vec::new(), - script_rewriters: Vec::new(), - html_stream_processors: Vec::new(), - head_injectors: Vec::new(), + standalone_js_ids: Vec::new(), + carried_js: Vec::new(), request_filters: Vec::new(), + middleware: Vec::new(), + bundle_tag_attributes: Vec::new(), + extra_js_module_ids: Vec::new(), + request_preparers: Vec::new(), + response_finalizers: Vec::new(), + geo_modules: Vec::new(), + ec_modules: Vec::new(), + device_modules: Vec::new(), + robots_txt_contributors: Vec::new(), + builder_modules: Vec::new(), + absent_shared: Vec::new(), + geo_module: None, + ec_module: None, + device_module: None, + prefix_modules: Vec::new(), + prefix_module: SelectedPrefixModule::default(), + } + } +} + +/// Reserved value of `[geo] module` that resolves no location at all. +const GEO_MODULE_NONE: &str = "none"; + +/// `[geo] module` value opting in to the adapter's own host geo lookup. +const GEO_MODULE_PLATFORM: &str = "platform"; + +/// `[device] module` value naming the User-Agent-only module core supplies. +/// +/// The same choice as leaving the selector unset, spelled explicitly. +const DEVICE_MODULE_BUILTIN: &str = "builtin"; + +/// `[device] module` value opting in to the host module the adapter builds. +/// +/// Resolved by `build_device_module` in [`device`](crate::ec::device) rather +/// than by a module, so the registry supplies nothing for it. +const DEVICE_MODULE_FASTLY: &str = "fastly"; + +/// The type folder each of these kinds of module is selected in, which a name +/// written in its section may leave off. +const EDGECOOKIE_TYPE: &str = crate::ec::module::MODULE_TYPE; +const DEVICE_TYPE: &str = "device"; +const GEO_TYPE: &str = "geo"; +const PREFIX_TYPE: &str = crate::prefix::MODULE_TYPE; +const ROBOTS_TXT_TYPE: &str = crate::robots_txt::MODULE_TYPE; + +impl IntegrationRegistryInner { + /// The registered middleware an entry means by `name`. + fn middleware_named(&self, name: &str) -> Option<&Arc> { + self.middleware + .iter() + .map(|(_, middleware)| middleware) + .find(|middleware| middleware.middleware_id() == name) + } + + /// A sentence to follow the refusal of a middleware name an entry wrote, + /// when the name is that of a module no section selects, or begins with + /// one. Empty otherwise. + fn unselected_module_note(&self, name: &str) -> String { + self.builder_modules + .iter() + .find(|(module, selected)| { + !*selected + && name + .strip_prefix(*module) + .is_some_and(|rest| rest.is_empty() || rest.starts_with('.')) + }) + .map(|(module, _)| { + format!( + ". `{module}` is a module no section selects, so nothing it supplies is \ + running" + ) + }) + .unwrap_or_default() + } + + /// A sentence to follow the refusal of a name a selector wrote, when the + /// name is a builder's module: one no section selects, which is why + /// nothing it supplies is running, or one that is selected and supplies no + /// module of this kind. Empty when no builder's module answers to the + /// name. + fn builder_note(&self, type_folder: &str, kind: &str, written: &str) -> String { + let names: Vec<&str> = self.builder_modules.iter().map(|(name, _)| *name).collect(); + let Some(name) = crate::module_name::resolve(type_folder, written, &names) else { + return String::new(); + }; + let selected = self + .builder_modules + .iter() + .any(|(module, selected)| *module == name && *selected); + if selected { + format!(". `{written}` is selected and supplies no {kind} module") + } else { + format!( + ". `{written}` is a module no section selects, so nothing it supplies is running" + ) + } + } + + /// A sentence naming the shared tables whose absence kept their vendors' + /// modules from running, to follow a refusal of a module nothing supplies, + /// or nothing when every builder a shared table runs is running. + fn absent_shared_note(&self) -> String { + if self.absent_shared.is_empty() { + return String::new(); + } + let tables = self + .absent_shared + .iter() + .map(|vendor| format!("[shared.{vendor}]")) + .collect::>() + .join(" or "); + let those = if self.absent_shared.len() == 1 { + "that table runs" + } else { + "those tables run" + }; + format!(". There is no {tables}, so the modules {those} are not running") + } +} + +/// The module a name written in a section means, among those the running +/// registrations supply, written in full or with the section's type folder +/// left off. +fn find_named<'a, T: ?Sized>( + type_folder: &str, + written: &str, + offered: &'a [(&'static str, Arc)], +) -> Option<&'a Arc> { + let names: Vec<&str> = offered.iter().map(|(name, _)| *name).collect(); + let name = crate::module_name::resolve(type_folder, written, &names)?; + offered + .iter() + .find(|(offered_name, _)| *offered_name == name) + .map(|(_, module)| module) +} + +/// Warns about each module on offer that the section does not select, so an +/// operator can see a capability the deployment never uses. +fn warn_unselected( + section: &str, + type_folder: &str, + selected: Option<&Arc>, + offered: &[(&'static str, Arc)], +) { + for (name, module) in offered { + if !selected.is_some_and(|chosen| Arc::ptr_eq(chosen, module)) { + log::warn!( + "the module `{}` is offered, and `[{section}] module` does not select it", + crate::module_name::short_form(type_folder, name) + ); + } + } +} + +/// The registered module a section names. +/// +/// # Errors +/// +/// A configuration error naming the section and the modules on offer when no +/// registered module answers to the name, because otherwise a mistyped name +/// would fall back to core's own module with nothing said. +fn named_module( + type_folder: &str, + section: &str, + written: &str, + offered: &[(&'static str, Arc)], + inner: &IntegrationRegistryInner, +) -> Result, Report> { + if let Some(module) = find_named(type_folder, written, offered) { + return Ok(Arc::clone(module)); + } + let names: Vec<&str> = offered + .iter() + .map(|(name, _)| crate::module_name::short_form(type_folder, name)) + .collect(); + let runs = if names.is_empty() { + format!("It runs no {section} module") + } else { + format!("The {section} modules it runs are [{}]", names.join(", ")) + }; + Err(Report::new(TrustedServerError::Configuration { + message: format!( + "`[{section}] module` names `{written}`, which no module this deployment runs \ + supplies. {runs}{}{}", + inner.builder_note(type_folder, section, written), + inner.absent_shared_note() + ), + })) +} + +/// Records that `integration` supplies the `kind` module `name`. +/// +/// # Errors +/// +/// When another registration already supplies a module of that kind under the +/// name, naming both, because a selector could then mean either. +fn claim_module_name( + claimed: &mut Vec<(&'static str, &'static str, &'static str)>, + kind: &'static str, + name: &'static str, + integration: &'static str, +) -> Result<(), Report> { + if let Some((_, _, first)) = claimed + .iter() + .find(|(claimed_kind, claimed_name, _)| *claimed_kind == kind && *claimed_name == name) + { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "the {kind} module `{name}` is declared twice, by integration `{first}` and by \ + integration `{integration}`" + ), + })); + } + claimed.push((kind, name, integration)); + Ok(()) +} + +/// Resolves the Edge Cookie module `[ec] module` names, when a registered +/// module supplies it. +/// +/// Core has Edge Cookie modules of its own, so a name no registered module +/// answers to is not an error here. This returns `Some` only for a registered +/// module, and core resolves the rest. The name looked for is the +/// implementation the selection's `[ec.]` block names, or the selection +/// itself, which is the name core checks the module's id against. +fn resolve_ec_module( + settings: &Settings, + inner: &IntegrationRegistryInner, +) -> Option> { + // `None` spells statelessness and never names a module, so only a named + // selection can match one. + let implementation = match settings.ec.module.as_ref() { + Some(EcModuleSelection::Named(key)) => Some(settings.ec.module_blocks.implementation(key)), + Some(EcModuleSelection::None) | None => None, + }; + let resolved = implementation + .and_then(|name| find_named(EDGECOOKIE_TYPE, name, &inner.ec_modules)) + .map(Arc::clone); + warn_unselected("ec", EDGECOOKIE_TYPE, resolved.as_ref(), &inner.ec_modules); + resolved +} + +/// Resolves the device module `[device] module` names, when a registered +/// module supplies it. +/// +/// As with identity, core has a device module of its own, so a selector naming +/// it is not an error here. +/// +/// # Errors +/// +/// Returns [`TrustedServerError::Configuration`] when the selector names no +/// registered module, or names one that declares a permission nothing can +/// enforce. +fn resolve_device_module( + settings: &Settings, + inner: &IntegrationRegistryInner, +) -> Result>, Report> { + let selector = settings.device.module.as_deref(); + let resolved = match selector { + // Unset and `builtin` both name the module core supplies itself, and + // `fastly` names the host module the adapter builds, so the registry + // supplies nothing for any of the three. + None | Some(DEVICE_MODULE_BUILTIN) | Some(DEVICE_MODULE_FASTLY) => None, + Some(written) => Some(named_module( + DEVICE_TYPE, + "device", + written, + &inner.device_modules, + inner, + )?), + }; + + // A module-supplied device module may declare the permissions its data + // use requires, but nothing enforces that declaration yet. Device + // classification runs before the permission set for the request is + // assembled, so there is no per-request gate to check it against. Selecting + // a module is an operator decision and not a per-request permission + // decision, so honoring the declaration by silently ignoring it would let a + // vendor state a requirement that never binds. Refuse the selection instead, + // loudly and at startup, until a real per-request device gate exists. + if let Some(module) = resolved.as_ref() + && module.required_permissions() != crate::permissions::PermissionSet::none() + { + let declared = module + .required_permissions() + .iter() + .map(crate::permissions::Permission::as_str) + .collect::>() + .join(", "); + let message = format!( + "`[device] module` selects `{}`, a device module requiring `{declared}`, and \ + Trusted Server has no per-request gate that can enforce that yet, so the \ + selection is refused rather than silently ignored", + selector.unwrap_or_default(), + ); + return Err(Report::new(TrustedServerError::Configuration { message })); + } + + warn_unselected( + "device", + DEVICE_TYPE, + resolved.as_ref(), + &inner.device_modules, + ); + + Ok(resolved) +} + +/// Resolves `[prefix] module` against the registered prefix modules, or to +/// core's own when nothing or `ts` is selected, and asks the module for the prefixes +/// of the publisher the settings describe, so a module that names something +/// its call never carries is refused here rather than on every request. +/// +/// # Errors +/// +/// A configuration error when the selector names no registered module, or +/// when the module names a value other than the request and the settings. +fn resolve_prefix_module( + settings: &Settings, + inner: &IntegrationRegistryInner, +) -> Result> { + let selected = match settings.prefix.module.as_deref() { + None | Some(crate::prefix::TRUSTED_SERVER_MODULE) => SelectedPrefixModule::trusted_server(), + Some(written) => { + let module = + named_module(PREFIX_TYPE, "prefix", written, &inner.prefix_modules, inner)?; + let name = inner + .prefix_modules + .iter() + .find(|(_, offered)| Arc::ptr_eq(offered, &module)) + .map_or(PREFIX_TYPE, |(name, _)| *name); + SelectedPrefixModule::new(name, module) + } + }; + let chosen = inner + .prefix_modules + .iter() + .find(|(name, _)| *name == selected.name()) + .map(|(_, module)| module); + warn_unselected("prefix", PREFIX_TYPE, chosen, &inner.prefix_modules); + let trial = ModuleRequest::new(&Method::GET, &settings.publisher.domain, "https", "/"); + selected.prefixes(settings, trial).map_err(|withheld| { + Report::new(TrustedServerError::Configuration { + message: format!( + "`[prefix] module` selects `{}`, which cannot give its prefixes: {withheld}. A \ + prefix module is handed the request and the settings, and nothing else", + selected.name() + ), + }) + })?; + Ok(selected) +} + +/// Refuses a `[robots-txt] modules` that names something nothing this +/// deployment runs supplies. +/// +/// Only knowable here, where every registration has been handed over, so it +/// is checked when the registry is built rather than when the settings load. +/// The file would otherwise fail on the first request for it rather than at +/// deploy. +fn check_robots_txt_selection( + settings: &Settings, + inner: &IntegrationRegistryInner, +) -> Result<(), Report> { + let Some(robots_txt) = settings.robots_txt.as_ref() else { + return Ok(()); + }; + for written in &robots_txt.modules { + if written == REFUSE_ALL + || written == ALLOW_ALL + || find_named(ROBOTS_TXT_TYPE, written, &inner.robots_txt_contributors).is_some() + { + continue; } + let offered: Vec<&str> = [REFUSE_ALL, ALLOW_ALL] + .into_iter() + .chain( + inner + .robots_txt_contributors + .iter() + .map(|(name, _)| crate::module_name::short_form(ROBOTS_TXT_TYPE, name)), + ) + .collect(); + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "`[robots-txt] modules` names `{written}`, which nothing this deployment runs \ + supplies. What it can name is [{}]{}", + offered.join(", "), + inner.builder_note(ROBOTS_TXT_TYPE, "robots.txt", written) + ), + })); } + Ok(()) +} + +/// Resolves `[geo] module` against the registered geo modules. +/// +/// Returns the disabled module when the selector is unset or `none`, `None` +/// when it is `platform` so the adapter's own host lookup stands, and the +/// registered module otherwise. A module on offer that the selector does not +/// choose is logged as a warning when the registry is built. +/// +/// # Errors +/// +/// Returns [`TrustedServerError::Configuration`] when the selector names no +/// registered module. +fn resolve_geo_module( + settings: &Settings, + inner: &IntegrationRegistryInner, +) -> Result>, Report> { + let selector = settings.geo.module.as_deref(); + let resolved = match selector { + // Unset resolves nothing and makes no host geo call, so a default + // deployment is not tied to any host geo service. `none` spells the + // same choice explicitly. + None | Some(GEO_MODULE_NONE) => Some(Arc::new(DisabledGeo) as Arc), + // `platform` opts in to the adapter's own host lookup, so the registry + // supplies nothing and the adapter's module stands. + Some(GEO_MODULE_PLATFORM) => None, + Some(written) => Some(named_module( + GEO_TYPE, + "geo", + written, + &inner.geo_modules, + inner, + )?), + }; + warn_unselected("geo", GEO_TYPE, resolved.as_ref(), &inner.geo_modules); + Ok(resolved) } /// Summary of registered integration capabilities. @@ -775,9 +1397,8 @@ impl Default for IntegrationRegistryInner { pub struct IntegrationMetadata { pub id: &'static str, pub routes: Vec, - pub attribute_rewriters: usize, - pub script_selectors: Vec<&'static str>, - pub head_injectors: usize, + /// The names of the middleware the integration supplies. + pub middleware: Vec<&'static str>, pub request_filters: usize, } @@ -786,9 +1407,7 @@ impl IntegrationMetadata { Self { id, routes: Vec::new(), - attribute_rewriters: 0, - script_selectors: Vec::new(), - head_injectors: 0, + middleware: Vec::new(), request_filters: 0, } } @@ -809,19 +1428,203 @@ pub struct ProxyDispatchInput<'a> { pub req: Request, } +/// Refuses a registered middleware no entry could name, and an entry this +/// deployment cannot run. +/// +/// A middleware no entry names is left alone and logged. +/// +/// # Errors +/// +/// Naming the module or the entry at fault. +fn check_middleware( + settings: &Settings, + inner: &IntegrationRegistryInner, +) -> Result<(), Report> { + let refuse = |message: String| Report::new(TrustedServerError::Configuration { message }); + let phases_of = |middleware: &Arc| -> Vec { + middleware + .phases() + .iter() + .map(|phase| format!("[[{phase}]]")) + .collect() + }; + for (position, (integration, middleware)) in inner.middleware.iter().enumerate() { + let id = middleware.middleware_id(); + if !crate::module_name::is_valid(id) { + return Err(refuse(format!( + "module `{integration}` supplies a middleware named `{id}`, which is not a \ + name an entry can write. A name is parts joined by `.`, each in lower case \ + letters, digits, `_` or `-`" + ))); + } + if middleware.phases().is_empty() { + return Err(refuse(format!( + "module `{integration}` supplies the middleware `{id}`, which says it runs \ + in no phase, so no entry could name it" + ))); + } + if let Some((earlier, _)) = inner.middleware[..position] + .iter() + .find(|(_, earlier)| earlier.middleware_id() == id) + { + return Err(refuse(format!( + "modules `{earlier}` and `{integration}` both supply a middleware named \ + `{id}`, so an entry could not say which one it means" + ))); + } + } + for phase in MiddlewarePhase::ALL { + for (index, entry) in settings.phase_entries(phase).entries().iter().enumerate() { + let at = format!("[[{phase}]] entry {}", index + 1); + for name in &entry.middleware { + let Some(middleware) = inner.middleware_named(name) else { + let supplied: Vec<&str> = inner + .middleware + .iter() + .map(|(_, middleware)| middleware.middleware_id()) + .collect(); + return Err(refuse(format!( + "{at} names `{name}`, which no module that runs supplies. The \ + middleware the running modules supply is [{}]{}", + supplied.join(", "), + inner.unselected_module_note(name) + ))); + }; + if !middleware.phases().contains(&phase) { + return Err(refuse(format!( + "{at} names `{name}`, which does not run in that phase. It runs in {}", + phases_of(middleware).join(" and ") + ))); + } + if !middleware.handles_media_type(&entry.media_type) { + return Err(refuse(format!( + "{at} covers `{}` and names `{name}`, which does not work on that media \ + type", + entry.media_type + ))); + } + } + } + } + for (integration, middleware) in unnamed_middleware(settings, inner) { + log::warn!( + "Module `{integration}` supplies the middleware `{}` and no {} entry names \ + it, so it changes no page", + middleware.middleware_id(), + phases_of(middleware).join(" or ") + ); + } + Ok(()) +} + +/// The middleware no entry names in a phase it runs in, each with the +/// integration that supplies it, in registration order. Core's own +/// middleware is offered to every deployment, so one that no entry names +/// is not a module running for nothing, and is left out. +fn unnamed_middleware<'a>( + settings: &Settings, + inner: &'a IntegrationRegistryInner, +) -> Vec<(&'static str, &'a Arc)> { + inner + .middleware + .iter() + .filter(|(integration, _)| *integration != crate::service_version::MODULE) + .filter(|(_, middleware)| { + let id = middleware.middleware_id(); + !middleware + .phases() + .iter() + .any(|phase| settings.phase_entries(*phase).names().contains(&id)) + }) + .map(|(integration, middleware)| (*integration, middleware)) + .collect() +} + /// In-memory registry of integrations discovered from settings. #[derive(Clone, Default)] pub struct IntegrationRegistry { inner: Arc, plan: Option>, + /// What the adapter reports about the platform the deployment runs on, + /// which a middleware is told about every document. + platform: Arc, +} + +/// Refuses a name a section selects that no module in this deployment +/// supplies, listing what the section could select instead. +/// +/// # Errors +/// +/// Naming the first section and name at fault. +fn check_section_selections( + settings: &Settings, + extra: &[crate::integrations::IntegrationBuilder], + inner: &IntegrationRegistryInner, +) -> Result<(), Report> { + let offered: Vec<&'static str> = crate::integrations::all_builders(extra) + .filter_map(|builder| builder.module_name()) + .collect(); + for (section, modules) in settings.module_sections() { + for written in modules.selected() { + if crate::module_name::resolve(section, written, &offered).is_some() { + continue; + } + let of_type: Vec<&str> = offered + .iter() + .map(|name| crate::module_name::short_form(section, name)) + .filter(|short| !offered.contains(short)) + .collect(); + let supplied = if of_type.is_empty() { + let mut types: Vec<&str> = offered + .iter() + .filter_map(|name| name.split_once('.').map(|(folder, _)| folder)) + .collect(); + types.sort_unstable(); + types.dedup(); + format!( + "It supplies no {section} module. The types it supplies modules of are [{}]", + types.join(", ") + ) + } else { + format!( + "The {section} modules it supplies are [{}]", + of_type.join(", ") + ) + }; + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "[{section}] selects `{written}`, which no module this deployment \ + supplies. {supplied}{}", + inner.absent_shared_note() + ), + })); + } + } + Ok(()) } impl IntegrationRegistry { - /// Build a registry and auction plan from the provided settings for tests. - /// - /// Runtime adapters should compile one plan and pass it to [`Self::with_plan`]. - /// - /// # Errors + /// Tells the registry what the adapter reports about the platform, which + /// a middleware is told about every document. + #[must_use] + pub fn with_platform(mut self, platform: PlatformIdentity) -> Self { + self.platform = Arc::new(platform); + self + } + + /// What the adapter reports about the platform the deployment runs on. + #[must_use] + pub fn platform(&self) -> &PlatformIdentity { + &self.platform + } + + /// Build a registry and auction plan from the provided settings for tests. + /// + /// Runtime adapters should compile one plan and pass it to + /// [`Self::with_plan`], or to [`Self::with_plan_and_registrations`] when + /// they also supply integrations of their own. + /// + /// # Errors /// /// Returns an error if the auction plan or integration registry is invalid. #[cfg(test)] @@ -830,41 +1633,196 @@ impl IntegrationRegistry { Self::with_plan(settings, plan) } - /// Build a registry from the provided settings. + /// Build a registry from the built-in integrations followed by `extra`, + /// compiling the auction plan from `settings`, for tests. + /// + /// Runtime adapters compile one plan and pass it to + /// [`Self::with_plan_and_registrations`], so the plan is shared rather than + /// compiled twice. + /// + /// # Errors + /// + /// Returns an error if the auction plan or the integration registry is + /// invalid. + #[cfg(any(test, feature = "test-utils"))] + pub fn with_registrations( + settings: &Settings, + extra: &[crate::integrations::IntegrationBuilder], + ) -> Result> { + let plan = Arc::new(crate::auction::compile_auction_plan_with(settings, extra)?); + Self::with_plan_and_registrations(settings, plan, extra) + } + + /// Build a registry from the built-in integrations, against one shared + /// compiled auction plan. + /// + /// # Errors + /// + /// Returns an error if route registration fails due to duplicate routes or + /// invalid paths, or when a registration carries a browser module whose + /// declared SHA-256 does not match its source. + pub fn with_plan( + settings: &Settings, + plan: Arc, + ) -> Result> { + Self::with_plan_and_registrations(settings, plan, &[]) + } + + /// Build a registry from the built-in integrations followed by `extra`, + /// the builders an adapter or a vendor crate supplies, against one shared + /// compiled auction plan. + /// + /// The plan-backed auction providers, Prebid then APS, register before the + /// builders, so opening the builder table changes no existing hook order. + /// Their ids are reserved for core whether or not they run, so an + /// outside builder claiming either is refused the way two builders claiming + /// one id are. /// /// # Errors /// - /// Returns an error if route registration fails due to duplicate routes or invalid paths. + /// Returns an error when two builders claim the same integration id, when + /// route registration fails due to duplicate routes or invalid paths, when + /// a builder fails, or when a registration carries a browser module whose + /// declared SHA-256 does not match its source. /// /// # Panics /// /// Panics if a route path ends with `/*` but `strip_suffix` unexpectedly fails (invariant violation). - pub fn with_plan( + pub fn with_plan_and_registrations( settings: &Settings, plan: Arc, + extra: &[crate::integrations::IntegrationBuilder], ) -> Result> { let mut inner = IntegrationRegistryInner::default(); - let mut registrations = Vec::new(); - if let Some(registration) = crate::integrations::prebid::register_for_plan(settings, &plan)? - { - registrations.push(registration); - } - if let Some(registration) = crate::integrations::aps::register_for_plan(settings, &plan)? { - registrations.push(registration); + // What a builder registers from the auction plan goes first, so its + // hooks run ahead of every module a section selects. + let mut plan_registrations: Vec = Vec::new(); + let mut registrations: Vec = Vec::new(); + + // A vendor's shared table is read by that vendor's builder alone, so + // one no builder reads is a table the deployment cannot use. + let unread: Vec<&str> = settings + .shared + .vendors() + .filter(|vendor| { + !crate::integrations::all_builders(extra) + .any(|builder| builder.shared_settings() == Some(*vendor)) + }) + .collect(); + if !unread.is_empty() { + let read: Vec<&str> = crate::integrations::all_builders(extra) + .filter_map(|builder| builder.shared_settings()) + .collect(); + let reads = if read.is_empty() { + "It reads none".to_owned() + } else { + format!("The shared tables it reads are [{}]", read.join(", ")) + }; + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "[shared.{}] is read by no module this deployment supplies. {reads}", + unread.join("] and [shared."), + ), + })); } - for builder in crate::integrations::builders() { - if let Some(registration) = (builder.build)(settings)? { + + for builder in crate::integrations::all_builders(extra) { + if let Some((_, first_source)) = + inner.builder_ids.iter().find(|(id, _)| *id == builder.id()) + { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "integration id `{}` is registered twice, by `{first_source}` and by `{}`", + builder.id(), + builder.source() + ), + })); + } + inner.builder_ids.push((builder.id(), builder.source())); + // A builder its shared table runs answers to no section, so a + // module name would let a section select what it does not run. + if let (Some(vendor), Some(name)) = (builder.shared_settings(), builder.module_name()) { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "integration `{}` from `{}` is run by [shared.{vendor}] and also names \ + the module `{name}`. A builder its shared table runs is selected by \ + no section, so it takes no module name", + builder.id(), + builder.source() + ), + })); + } + // Preparers are collected before the selection check, so an + // integration can sanitize its own reserved query or cookie in a + // deployment that does not run it. + if let Some(prepare) = builder.prepare_request() { + inner.request_preparers.push(prepare); + } + if let Some(finalize) = builder.finalize_response() { + inner.response_finalizers.push(finalize); + } + // A registration from the plan is made whatever the sections + // select, because the function decides from the plan and the + // settings whether the module runs. + if let Some(register) = builder.plan_registration() + && let Some(registration) = register(settings, &plan)? + { + debug_assert_eq!( + registration.integration_id, + builder.id(), + "integration builder ID should match registration ID" + ); + plan_registrations.push(registration); + } + + // Only a builder whose module a section selects is built, so an + // integration runs exactly when an operator selects it, whatever + // its builder would otherwise make of the settings. + let selected = builder + .module_name() + .is_some_and(|name| settings.selects_module(name)); + if let Some(name) = builder.module_name() { + inner.builder_modules.push((name, selected)); + } + // A builder a vendor's shared table runs is built when the table + // is present, whatever any section selects. + let runs = match builder.shared_settings() { + Some(vendor) => { + let present = settings.shared.contains(vendor); + if !present && !inner.absent_shared.contains(&vendor) { + inner.absent_shared.push(vendor); + } + present + } + None => selected, + }; + if !runs { + continue; + } + + if let Some(registration) = builder.build(settings)? { debug_assert_eq!( - registration.integration_id, builder.id, + registration.integration_id, + builder.id(), "integration builder ID should match registration ID" ); registrations.push(registration); } } - for registration in registrations { + // Which names a section can select is only knowable here, where the + // adapter's and a vendor crate's builders have been handed over, so + // the selections are checked against them rather than in the + // settings. Deploy validation deliberately does not make this check, + // because a vendor crate the CLI never links may supply the name. + check_section_selections(settings, extra, &inner)?; + + // The geo, Edge Cookie and device module names taken so far, with the + // integration that supplies each. + let mut claimed = Vec::new(); + for registration in plan_registrations.into_iter().chain(registrations) { inner - .enabled_integration_ids + .running_integration_ids .push(registration.integration_id); for proxy in registration.proxies { @@ -915,25 +1873,154 @@ impl IntegrationRegistry { inner.routes.push((route, registration.integration_id)); } } - inner - .html_rewriters - .extend(registration.attribute_rewriters); - inner.script_rewriters.extend(registration.script_rewriters); - inner - .html_stream_processors - .extend(registration.html_stream_processors); - inner.head_injectors.extend(registration.head_injectors); inner.request_filters.extend(registration.request_filters); + for middleware in registration.middleware { + inner + .middleware + .push((registration.integration_id, middleware)); + } + for (name, value) in registration.bundle_tag_attributes { + // The tag is written as markup with no escaping, so a name + // or a value that could end the attribute is refused. + let name_is_plain = !name.is_empty() + && name.bytes().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-' + }); + let value_is_plain = !value + .bytes() + .any(|byte| matches!(byte, b'"' | b'&' | b'<' | b'>')); + if !name_is_plain || !value_is_plain { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "integration `{}` puts the attribute `{name}` on the script \ + bundle's tag with the value `{value}`. A name is lower case \ + letters, digits and hyphens, and a value holds none of `\"`, `&`, \ + `<` and `>`", + registration.integration_id + ), + })); + } + inner + .bundle_tag_attributes + .push((registration.integration_id, (name, value))); + } + if let Some((name, module)) = registration.geo_module { + claim_module_name(&mut claimed, "geo", name, registration.integration_id)?; + inner.geo_modules.push((name, module)); + } + if let Some((name, module)) = registration.ec_module { + claim_module_name( + &mut claimed, + "Edge Cookie", + name, + registration.integration_id, + )?; + // Core checks the selected module by its own id, so a module + // declared under another name would be selected here and + // refused there. + if module.id() != name { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "integration `{}` declares an Edge Cookie module under `{name}`, \ + and the module's own id is `{}`. The two must be the same name", + registration.integration_id, + module.id() + ), + })); + } + inner.ec_modules.push((name, module)); + } + if let Some((name, module)) = registration.device_module { + claim_module_name(&mut claimed, "device", name, registration.integration_id)?; + inner.device_modules.push((name, module)); + } + if let Some((name, module)) = registration.prefix_module { + claim_module_name(&mut claimed, "prefix", name, registration.integration_id)?; + inner.prefix_modules.push((name, module)); + } + if let Some((name, contributor)) = registration.robots_txt_contributor { + claim_module_name( + &mut claimed, + "robots.txt", + name, + registration.integration_id, + )?; + inner.robots_txt_contributors.push((name, contributor)); + } if registration.js_disabled { inner.disabled_js_ids.push(registration.integration_id); } else if registration.js_deferred { inner.deferred_js_ids.push(registration.integration_id); } + if registration.js_standalone { + inner.standalone_js_ids.push(registration.integration_id); + } + if let Some(module) = registration.js_module { + // The served `?v=` hash and its memo trust this value, so a + // stale literal is a startup error rather than a stale + // script. The cost is one SHA-256 of each carried module + // per registry build, and only the Axum dev server builds + // the registry once, because its `main` builds the router + // before serving. Fastly starts a fresh Wasm instance per + // request, and `edgezero_adapter_cloudflare::run_app` and + // `edgezero_adapter_spin::run_app` both call `build_app` + // inside the per-request entry point, so those three pay + // it on every request. + let actual = hex::encode(Sha256::digest(module.source.as_bytes())); + if actual != module.sha256 { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "Integration `{}` carries a browser module whose declared SHA-256 does not match its source (declared {}, actual {actual})", + registration.integration_id, module.sha256 + ), + })); + } + inner.carried_js.push((registration.integration_id, module)); + } + } + + // A client-cycle Edge Cookie module ships a page script that posts its + // result to the resolve endpoint. The script rides the tsjs bundle, so + // include its module when that module is selected. The same module + // list drives both the served bundle and the injected `")], + after_bundle_inserts: vec![format!( + "", + crate::tsjs::tsjs_single_module_script_src(&module) + )], + ..MiddlewareAction::pass() + } + } } - fn head_inserts(&self, _ctx: &IntegrationHtmlContext<'_>) -> Vec { - Vec::new() + /// Has `settings` select the stand-in and run its middleware, and no + /// other, on every reader's copy of a page. + pub(crate) fn select_and_place(settings: &mut Settings) { + settings.select_module("testing", MODULE); + crate::html_processor::test_support::place_on_every_page( + settings, + MiddlewarePhase::Serve, + &[MODULE], + ); } - fn tsjs_script_tag_attributes(&self) -> Vec<(&'static str, &'static str)> { - vec![ - ("data-ts-gam-attribution", "false"), - ("data-third-attribute", "third"), - ] + fn register( + settings: &Settings, + ) -> Result, Report> { + let Some(_config) = settings.module_config::(MODULE)? else { + return Ok(None); + }; + Ok(Some( + IntegrationRegistration::builder(ID) + .with_js_module(CarriedJsModule { + source: JS, + sha256: JS_SHA256, + }) + .with_standalone_js() + .with_middleware(Arc::new(Head)) + .with_request_filter(Arc::new(Filter)) + .build(), + )) + } + + fn validate(settings: &Settings) -> Result> { + settings + .module_config::(MODULE) + .map(|config| config.is_some()) } } - #[test] - fn tsjs_script_tag_attributes_preserve_registration_order_and_default_empty() { - let registry = IntegrationRegistry::from_rewriters_with_head_injectors( - Vec::new(), - Vec::new(), - vec![ - Arc::new(DefaultMetadataHeadInjector), - Arc::new(StaticMetadataHeadInjector), - Arc::new(ConflictingMetadataHeadInjector), - ], - ); + /// A stand-in for an integration whose browser module loads deferred, for + /// core's own tests of how modules are divided between the bundle and + /// deferred tags, and of how a module's settings reach a page template. + /// + /// Selected, it carries a deferred browser module and supplies one fetch + /// middleware under its own name, which writes its two settings into + /// ``. + pub(crate) mod deferred_fixture { + use std::sync::Arc; + + use error_stack::Report; + + use crate::error::TrustedServerError; + use crate::integrations::registry::{CarriedJsModule, IntegrationRegistration}; + use crate::integrations::{CORE_SOURCE, IntegrationBuilder}; + use crate::middleware::{Middleware, MiddlewareAction, MiddlewareContext, MiddlewarePhase}; + use crate::settings::Settings; + + /// The integration id the stand-in registers under. + pub(crate) const ID: &str = "deferred_fixture"; + /// The name a test's settings select the stand-in by, in `[testing]`. + pub(crate) const MODULE: &str = "testing.deferred-fixture"; + /// The file of the stand-in's deferred module. + pub(crate) const MODULE_FILE: &str = "tsjs-deferred_fixture.min.js"; + + const JS: &str = "(function(){window.__ts_deferred_fixture_loaded=1;})();"; + // SHA-256 of JS, hex. The registry refuses a literal that is not. + const JS_SHA256: &str = "3947bc5e93ffb5d057f0d500d20ee25d53612b6c8b59cd3f9512c1b64b45f9f6"; + + /// The builder core's test build lists beside its own. + pub(crate) const BUILDER: IntegrationBuilder = + IntegrationBuilder::new(ID, CORE_SOURCE, register, validate).with_module_name(MODULE); + + #[derive(Debug, serde::Deserialize, validator::Validate)] + #[serde(deny_unknown_fields)] + struct FixtureSettings { + #[serde(default)] + label: String, + #[serde(default)] + timeout_ms: u32, + } - assert_eq!( - registry.tsjs_script_tag_attributes(), - vec![ - ("data-ts-gam-attribution", "true"), - ("data-test-order", "second"), - ("data-third-attribute", "third"), - ], - "should keep the first value for duplicate names and preserve attribute order" - ); - } + impl crate::settings::IntegrationConfig for FixtureSettings {} - // Mock integration proxy for testing - struct MockProxy; + struct Head { + label: String, + timeout_ms: u32, + } - #[async_trait(?Send)] - impl IntegrationProxy for MockProxy { - fn integration_name(&self) -> &'static str { - "test" + impl Middleware for Head { + fn middleware_id(&self) -> &'static str { + MODULE + } + + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Fetch] + } + + fn create(&self, _context: &MiddlewareContext<'_>) -> MiddlewareAction { + let config = serde_json::json!({ + "label": self.label, + "timeoutMs": self.timeout_ms, + }); + MiddlewareAction { + head_inserts: vec![format!( + "" + )], + ..MiddlewareAction::pass() + } + } } - fn routes(&self) -> Vec { - vec![] + /// Has `settings` run the stand-in's middleware, and no other, on + /// every page. + pub(crate) fn place(settings: &mut Settings) { + crate::html_processor::test_support::place_on_every_page( + settings, + MiddlewarePhase::Fetch, + &[MODULE], + ); } - async fn handle( - &self, - _settings: &Settings, - _services: &RuntimeServices, - _req: Request, - ) -> Result, Report> { - Ok(Response::new(EdgeBody::empty())) + fn register( + settings: &Settings, + ) -> Result, Report> { + let Some(config) = settings.module_config::(MODULE)? else { + return Ok(None); + }; + Ok(Some( + IntegrationRegistration::builder(ID) + .with_js_module(CarriedJsModule { + source: JS, + sha256: JS_SHA256, + }) + .with_deferred_js() + .with_middleware(Arc::new(Head { + label: config.label, + timeout_ms: config.timeout_ms, + })) + .build(), + )) + } + + fn validate(settings: &Settings) -> Result> { + settings + .module_config::(MODULE) + .map(|config| config.is_some()) } } - struct EnrichingRequestFilter; - #[derive(Clone, Copy)] - struct RequestAnnotation; + /// A stand-in for a module that changes a page through middleware, for + /// core's own tests of the entries and of the page path. + /// + /// Selected, it supplies three fetch middleware. The one under the + /// module's own name writes a marker carrying the module's `label` + /// setting at the start of ``. The one named `.links` points every + /// link that carries `data-fixture` at a fixed path. The one named + /// `.broken` asks for a selector that does not parse. + /// + /// It supplies two serve middleware as well. The one named `.reader` + /// writes a marker ahead of the script bundle, saying which origin it was + /// told of and whether the request stand-in left its mark, and a script + /// straight after the bundle. The one named `.broken-reader` asks for a + /// selector that does not parse. + pub(crate) mod middleware_fixture { + use std::rc::Rc; + use std::sync::Arc; + + use error_stack::Report; + + use crate::error::TrustedServerError; + use crate::integrations::registry::{AttributeRewriteAction, IntegrationRegistration}; + use crate::integrations::{CORE_SOURCE, IntegrationBuilder}; + use crate::middleware::{ + AttributeRewrite, Middleware, MiddlewareAction, MiddlewareContext, MiddlewarePhase, + Preparation, Refusal, + }; + use crate::module_context::ModuleCall; + use crate::settings::Settings; + + /// The integration id the stand-in registers under. + pub(crate) const ID: &str = "middleware_fixture"; + /// The name a test's settings select the stand-in by, in `[testing]`. + pub(crate) const MODULE: &str = "testing.middleware-fixture"; + /// The name of the middleware that marks the head. + pub(crate) const HEAD: &str = MODULE; + /// The name of the middleware that moves links. + pub(crate) const LINKS: &str = "testing.middleware-fixture.links"; + /// The name of the middleware whose selector does not parse. + pub(crate) const BROKEN: &str = "testing.middleware-fixture.broken"; + /// The name of the serve middleware that marks a reader's copy. + pub(crate) const READER: &str = "testing.middleware-fixture.reader"; + /// The name of the serve middleware whose selector does not parse. + pub(crate) const BROKEN_READER: &str = "testing.middleware-fixture.broken-reader"; + /// What the reader middleware writes straight after the bundle. + pub(crate) const READER_SCRIPT: &str = ""; + /// Where the links middleware points a link. + pub(crate) const LINK_TARGET: &str = "/fixture/link"; + /// The selector the broken middleware asks for. + pub(crate) const BROKEN_SELECTOR: &str = "a["; + /// The name of the fetch middleware that works on `text/plain`. + pub(crate) const TEXT: &str = "testing.middleware-fixture.text"; + /// The media type the text middleware works on. + pub(crate) const TEXT_MEDIA_TYPE: &str = "text/plain"; + /// The line the text middleware adds at the end of a text body. + pub(crate) const TEXT_LINE: &str = "# added by the middleware fixture\n"; + /// The name of the fetch middleware that prepares. + pub(crate) const PREPARING: &str = "testing.middleware-fixture.preparing"; + /// The name of the fetch middleware that refuses every request. + pub(crate) const REFUSING: &str = "testing.middleware-fixture.refusing"; + /// How long the refusing middleware asks the caller to wait. + pub(crate) const REFUSAL_RETRY_AFTER: std::time::Duration = + std::time::Duration::from_secs(30); + /// What the refusing middleware says. + pub(crate) const REFUSAL_MESSAGE: &str = "the middleware fixture refuses"; + /// What the preparing middleware writes when nothing was prepared. + pub(crate) const NOTHING_PREPARED: &str = + ""; + + /// The marker the preparing middleware writes, naming the module it + /// was called as and the host it was told of. + pub(crate) fn prepared_marker(module: &str, request_host: &str) -> String { + format!( + "" + ) + } - #[async_trait(?Send)] - impl IntegrationRequestFilter for EnrichingRequestFilter { - fn integration_id(&self) -> &'static str { - "enriching" + struct Preparing; + + impl Middleware for Preparing { + fn middleware_id(&self) -> &'static str { + PREPARING + } + + fn phases(&self) -> &[MiddlewarePhase] { + &MiddlewarePhase::ALL + } + + fn prepare<'a>( + &'a self, + call: ModuleCall<'a>, + context: &'a MiddlewareContext<'a>, + ) -> Option> { + Some(Box::pin(async move { + let marker = prepared_marker(call.module(), context.request_host); + Ok(Box::new(marker) as Box) + })) + } + + fn create(&self, context: &MiddlewareContext<'_>) -> MiddlewareAction { + let marker: String = context + .take_prepared(PREPARING) + .unwrap_or_else(|| NOTHING_PREPARED.to_owned()); + MiddlewareAction { + head_inserts: vec![marker], + ..MiddlewareAction::pass() + } + } } - async fn filter_request( - &self, - input: RequestFilterInput<'_>, - ) -> Result> { - input.request.extensions_mut().insert(RequestAnnotation); - Ok(RequestFilterDecision::Continue(RequestFilterEffects { - request_headers: vec![HeaderMutation::set("x-datadome-isbot", "1")], - response_headers: vec![HeaderMutation::set("x-dd-b", "allowed")], - })) + struct Refusing; + + impl Middleware for Refusing { + fn middleware_id(&self) -> &'static str { + REFUSING + } + + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Fetch] + } + + fn prepare<'a>( + &'a self, + _call: ModuleCall<'a>, + _context: &'a MiddlewareContext<'a>, + ) -> Option> { + Some(Box::pin(async { + Err(Refusal::service_unavailable( + REFUSAL_RETRY_AFTER, + REFUSAL_MESSAGE, + )) + })) + } + + fn create(&self, _context: &MiddlewareContext<'_>) -> MiddlewareAction { + MiddlewareAction::pass() + } + } + /// The name of the serve middleware that writes a header on a text + /// response. + pub(crate) const TEXT_HEADER: &str = "testing.middleware-fixture.text-header"; + /// The header the text header middleware writes, name and value. + pub(crate) const TEXT_HEADER_NAME: &str = "x-fixture-text"; + pub(crate) const TEXT_HEADER_VALUE: &str = "served"; + + /// The builder core's test build lists beside its own. + pub(crate) const BUILDER: IntegrationBuilder = + IntegrationBuilder::new(ID, CORE_SOURCE, register, validate).with_module_name(MODULE); + + #[derive(Debug, serde::Deserialize, validator::Validate)] + #[serde(deny_unknown_fields)] + struct FixtureSettings { + #[serde(default)] + label: String, } - } - struct EchoProxy; + impl crate::settings::IntegrationConfig for FixtureSettings {} - #[async_trait(?Send)] - impl IntegrationProxy for EchoProxy { - fn integration_name(&self) -> &'static str { - "echo" + /// The marker the head middleware writes for `label`. + pub(crate) fn head_marker(label: &str) -> String { + format!("") } - fn routes(&self) -> Vec { - vec![] + struct Head { + label: String, } - async fn handle( - &self, - _settings: &Settings, - _services: &RuntimeServices, - req: http::Request, - ) -> Result, Report> { - let response = http::Response::builder() - .status(http::StatusCode::OK) - .header("x-echo-path", req.uri().path()) - .body(EdgeBody::empty()) - .expect("should build echo response"); - Ok(response) + impl Middleware for Head { + fn middleware_id(&self) -> &'static str { + HEAD + } + + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Fetch] + } + + fn create(&self, _context: &MiddlewareContext<'_>) -> MiddlewareAction { + MiddlewareAction { + head_inserts: vec![head_marker(&self.label)], + ..MiddlewareAction::pass() + } + } } - } - #[test] - fn document_state_keeps_multiple_types_for_one_integration() { - let state = IntegrationDocumentState::default(); - let number = state.get_or_insert_with("test", || 7_u32); - let label = state.get_or_insert_with("test", || "first".to_string()); - let repeated_number = state.get_or_insert_with("test", || 99_u32); + /// The marker the reader middleware writes when told of `origin_host`, + /// for a request the request stand-in marked or did not. + pub(crate) fn reader_marker(origin_host: &str, marked: bool) -> String { + format!( + "" + ) + } - assert!( - Arc::ptr_eq(&number, &repeated_number), - "repeated insertion should preserve the original typed state" - ); - assert_eq!( - *state.get::("test").expect("should retrieve number"), - 7, - "should retain numeric state" - ); - assert_eq!( - state - .get::("test") - .expect("should retrieve label") - .as_str(), - "first", - "should retain string state under the same integration ID" - ); - assert_eq!( - label.as_str(), - "first", - "should return inserted string state" - ); - } + struct Reader; - struct CountingStreamFactory(&'static str); + impl Middleware for Reader { + fn middleware_id(&self) -> &'static str { + READER + } - impl IntegrationHtmlStreamProcessorFactory for CountingStreamFactory { - fn integration_id(&self) -> &'static str { - self.0 - } - - fn create(&self, _context: IntegrationHtmlStreamContext) -> Box { - struct CountingStreamProcessor(usize); - - impl StreamProcessor for CountingStreamProcessor { - fn process_chunk( - &mut self, - chunk: &[u8], - _is_last: bool, - ) -> std::io::Result> { - self.0 += 1; - let mut output = self.0.to_string().into_bytes(); - output.extend_from_slice(chunk); - Ok(output) + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Serve] + } + + fn create(&self, context: &MiddlewareContext<'_>) -> MiddlewareAction { + use super::request_fixture; + + let marked = context + .document_state + .get::(request_fixture::ID) + .is_some(); + MiddlewareAction { + head_inserts: vec![reader_marker(context.origin_host, marked)], + after_bundle_inserts: vec![READER_SCRIPT.to_owned()], + ..MiddlewareAction::pass() } } + } + + struct BrokenReader; + + impl Middleware for BrokenReader { + fn middleware_id(&self) -> &'static str { + BROKEN_READER + } - Box::new(CountingStreamProcessor(0)) + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Serve] + } + + fn create(&self, _context: &MiddlewareContext<'_>) -> MiddlewareAction { + MiddlewareAction { + element_handlers: vec![Box::new(AttributeRewrite::matching( + BROKEN_SELECTOR, + "href", + Rc::new(|_matched| AttributeRewriteAction::keep()), + ))], + ..MiddlewareAction::pass() + } + } } - } - #[test] - fn html_stream_factories_preserve_order_and_create_isolated_sessions() { - let registration = IntegrationRegistration::builder("test") - .with_html_stream_processor(Arc::new(CountingStreamFactory("first"))) - .with_html_stream_processor(Arc::new(CountingStreamFactory("second"))) - .build(); - let identifiers: Vec<_> = registration - .html_stream_processors - .iter() - .map(|factory| factory.integration_id()) - .collect(); - assert_eq!( - identifiers, - ["first", "second"], - "should preserve factory registration order", - ); + struct Links; - let context = IntegrationHtmlStreamContext { - request_host: "proxy.example.com".to_owned(), - request_scheme: "https".to_owned(), - origin_host: "origin.example.com".to_owned(), - document_state: IntegrationDocumentState::default(), - }; - let factory = ®istration.html_stream_processors[0]; - let mut first = factory.create(context.clone()); - let mut second = factory.create(context); + impl Middleware for Links { + fn middleware_id(&self) -> &'static str { + LINKS + } - assert_eq!( - first - .process_chunk(b"a", false) - .expect("should process first session"), - b"1a", - "should initialize the first session counter", - ); - assert_eq!( - second - .process_chunk(b"b", true) - .expect("should process second session"), - b"1b", - "should initialize an independent second session counter", - ); - } + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Fetch] + } - #[test] - fn handle_proxy_passes_http_request_without_fastly_round_trip() { - let settings = create_test_settings(); - let registry = IntegrationRegistry::from_routes(vec![( - http::Method::GET, - "/integrations/test/echo", - (Arc::new(EchoProxy) as Arc, "echo"), - )]); - let req = http::Request::builder() - .method(http::Method::GET) - .uri("https://test.example.com/integrations/test/echo?x=1") - .body(EdgeBody::empty()) - .expect("should build request"); + fn create(&self, _context: &MiddlewareContext<'_>) -> MiddlewareAction { + MiddlewareAction { + element_handlers: vec![Box::new(AttributeRewrite::matching( + "a[data-fixture]", + "href", + Rc::new(|_matched| AttributeRewriteAction::replace(LINK_TARGET)), + ))], + ..MiddlewareAction::pass() + } + } + } - let mut ec_context = - EcContext::new_for_test(None, crate::consent::ConsentContext::default()); - let response = futures::executor::block_on(registry.handle_proxy(ProxyDispatchInput { - method: &http::Method::GET, - path: "/integrations/test/echo", - settings: &settings, - kv: None, - ec_context: &mut ec_context, - services: &noop_services(), - req, - })) - .expect("should match route") - .expect("proxy should succeed"); + /// Passes a text body through and adds a line at its end. + struct AppendLine; + + impl crate::streaming_processor::StreamProcessor for AppendLine { + fn process_chunk( + &mut self, + chunk: &[u8], + is_last: bool, + ) -> Result, std::io::Error> { + let mut output = chunk.to_vec(); + if is_last { + output.extend_from_slice(TEXT_LINE.as_bytes()); + } + Ok(output) + } + } - assert_eq!( + struct Text; + + impl Middleware for Text { + fn middleware_id(&self) -> &'static str { + TEXT + } + + fn handles_media_type(&self, media_type: &str) -> bool { + media_type == TEXT_MEDIA_TYPE + } + + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Fetch] + } + + fn create(&self, _context: &MiddlewareContext<'_>) -> MiddlewareAction { + MiddlewareAction { + stream: Some(Box::new(AppendLine)), + ..MiddlewareAction::pass() + } + } + } + + struct TextHeader; + + impl Middleware for TextHeader { + fn middleware_id(&self) -> &'static str { + TEXT_HEADER + } + + fn handles_media_type(&self, media_type: &str) -> bool { + media_type == TEXT_MEDIA_TYPE + } + + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Serve] + } + + fn create(&self, _context: &MiddlewareContext<'_>) -> MiddlewareAction { + MiddlewareAction { + response_headers: vec![( + http::HeaderName::from_static(TEXT_HEADER_NAME), + http::HeaderValue::from_static(TEXT_HEADER_VALUE), + )], + ..MiddlewareAction::pass() + } + } + } + + struct Broken; + + impl Middleware for Broken { + fn middleware_id(&self) -> &'static str { + BROKEN + } + + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Fetch] + } + + fn create(&self, _context: &MiddlewareContext<'_>) -> MiddlewareAction { + MiddlewareAction { + element_handlers: vec![Box::new(AttributeRewrite::matching( + BROKEN_SELECTOR, + "href", + Rc::new(|_matched| AttributeRewriteAction::keep()), + ))], + ..MiddlewareAction::pass() + } + } + } + + fn register( + settings: &Settings, + ) -> Result, Report> { + let Some(config) = settings.module_config::(MODULE)? else { + return Ok(None); + }; + Ok(Some( + IntegrationRegistration::builder(ID) + .without_js() + .with_middleware(Arc::new(Head { + label: config.label, + })) + .with_middleware(Arc::new(Links)) + .with_middleware(Arc::new(Broken)) + .with_middleware(Arc::new(Reader)) + .with_middleware(Arc::new(BrokenReader)) + .with_middleware(Arc::new(Text)) + .with_middleware(Arc::new(Preparing)) + .with_middleware(Arc::new(Refusing)) + .with_middleware(Arc::new(TextHeader)) + .build(), + )) + } + + fn validate(settings: &Settings) -> Result> { + settings + .module_config::(MODULE) + .map(|config| config.is_some()) + } + } + + /// A stand-in for an integration that tags a page, for core's own tests + /// of the HTML processor and the JavaScript asset proxy. + /// + /// Selected, it supplies one fetch middleware under its own name, which + /// writes one script in `` ahead of the bundle and rewrites the + /// address of its own script to a first-party path. With `mark_bundle` + /// set it also asks for an attribute on the publisher bundle tag. + pub(crate) mod tag_fixture { + use std::rc::Rc; + use std::sync::Arc; + + use error_stack::Report; + + use crate::error::TrustedServerError; + use crate::integrations::registry::{AttributeRewriteAction, IntegrationRegistration}; + use crate::integrations::{CORE_SOURCE, IntegrationBuilder}; + use crate::middleware::{ + AttributeRewrite, AttributeRewriteFn, Middleware, MiddlewareAction, MiddlewareContext, + MiddlewarePhase, + }; + use crate::settings::Settings; + + /// The integration id the stand-in registers under. + pub(crate) const ID: &str = "tag_fixture"; + /// The name a test's settings select the stand-in by, in `[testing]`. + pub(crate) const MODULE: &str = "testing.tag-fixture"; + /// The script the stand-in's vendor would serve. + pub(crate) const SCRIPT_URL: &str = "https://cdn.tag-fixture.example/sdk.js"; + /// The first-party path the stand-in rewrites that script to. + pub(crate) const FIRST_PARTY_SCRIPT: &str = "/integrations/tag_fixture/script"; + /// What the stand-in's head insert sets, so a test can find it. + pub(crate) const HEAD_FLAG: &str = "window.__ts_tag_fixture=true;"; + /// The attribute the stand-in asks for on the publisher bundle tag. + pub(crate) const BUNDLE_ATTRIBUTE: &str = "data-ts-tag-fixture"; + + /// The builder core's test build lists beside its own. + pub(crate) const BUILDER: IntegrationBuilder = + IntegrationBuilder::new(ID, CORE_SOURCE, register, validate).with_module_name(MODULE); + + #[derive(Debug, serde::Deserialize, validator::Validate)] + #[serde(deny_unknown_fields)] + struct FixtureSettings { + #[serde(default)] + mark_bundle: bool, + } + + impl crate::settings::IntegrationConfig for FixtureSettings {} + + struct Tag; + + impl Middleware for Tag { + fn middleware_id(&self) -> &'static str { + MODULE + } + + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Fetch] + } + + fn create(&self, _context: &MiddlewareContext<'_>) -> MiddlewareAction { + let decide: Rc = Rc::new(|matched| { + if matched.value == SCRIPT_URL { + AttributeRewriteAction::Replace(FIRST_PARTY_SCRIPT.to_owned()) + } else { + AttributeRewriteAction::Keep + } + }); + MiddlewareAction { + head_inserts: vec![format!("")], + element_handlers: vec![Box::new(AttributeRewrite::new("src", decide))], + ..MiddlewareAction::pass() + } + } + } + + fn register( + settings: &Settings, + ) -> Result, Report> { + let Some(config) = settings.module_config::(MODULE)? else { + return Ok(None); + }; + let mut registration = IntegrationRegistration::builder(ID) + .without_js() + .with_middleware(Arc::new(Tag)); + if config.mark_bundle { + registration = registration.with_bundle_tag_attribute(BUNDLE_ATTRIBUTE, "true"); + } + Ok(Some(registration.build())) + } + + fn validate(settings: &Settings) -> Result> { + settings + .module_config::(MODULE) + .map(|config| config.is_some()) + } + } + + /// A stand-in for an integration that rewrites script payloads in two + /// passes, for core's own tests of the page pipeline. + /// + /// Selected, it supplies one fetch middleware under its own name. Its + /// text handler swaps the payload of each `fixture_payload("...")` call + /// for a placeholder that carries a namespace made per document, and + /// keeps the payload. Its stream processor swaps each placeholder back + /// for its payload with the origin host rewritten. A payload pushed with + /// `fixture_payload_open` leaves its group unresolved, and the processor + /// holds its output from that placeholder on until a + /// `fixture_payload_close` arrives. + pub(crate) mod payload_fixture { + use std::io; + use std::sync::{Arc, Mutex, PoisonError}; + + use error_stack::Report; + + use crate::error::TrustedServerError; + use crate::integrations::registry::{IntegrationRegistration, ScriptRewriteAction}; + use crate::integrations::{CORE_SOURCE, IntegrationBuilder}; + use crate::middleware::{ + Middleware, MiddlewareAction, MiddlewareContext, MiddlewarePhase, TextHandler, + }; + use crate::settings::Settings; + use crate::streaming_processor::StreamProcessor; + + /// The integration id the stand-in registers under. + pub(crate) const ID: &str = "payload_fixture"; + /// The name a test's settings select the stand-in by, in `[testing]`. + pub(crate) const MODULE: &str = "testing.payload-fixture"; + /// What every placeholder starts with, so a test can check that none + /// reaches a reader. + pub(crate) const PLACEHOLDER_PREFIX: &str = "__ts_fixture_"; + const PLACEHOLDER_END: &str = "__"; + + /// The builder core's test build lists beside its own. + pub(crate) const BUILDER: IntegrationBuilder = + IntegrationBuilder::new(ID, CORE_SOURCE, register, validate).with_module_name(MODULE); + + struct Payload { + placeholder: String, + original: String, + /// Whether the group this payload opened is still waiting for its + /// close. + unresolved: bool, + } + + /// What one document's text handler has captured so far. + struct Captured { + namespace: String, + payloads: Vec, + } + + impl Default for Captured { + fn default() -> Self { + Self { + namespace: uuid::Uuid::new_v4().simple().to_string(), + payloads: Vec::new(), + } + } + } + + /// The scripts of one document, each held until its last chunk. + struct Scripts { + captured: Arc>, + held: String, + } + + impl Scripts { + /// Swaps the payload of a whole script for a placeholder, or + /// leaves a script that pushes no payload as it is. + fn rewrite_whole(&self, script: &str) -> Option { + let (call, opens, closes) = [ + ("fixture_payload_open(\"", true, false), + ("fixture_payload_close(\"", false, true), + ("fixture_payload(\"", false, false), + ] + .into_iter() + .find(|(call, _, _)| script.contains(*call))?; + let start = script.find(call)? + call.len(); + let end = script.rfind("\")")?; + if end < start { + return None; + } + + let mut captured = self.captured.lock().unwrap_or_else(PoisonError::into_inner); + let placeholder = format!( + "{PLACEHOLDER_PREFIX}{}_{}{PLACEHOLDER_END}", + captured.namespace, + captured.payloads.len() + ); + if closes { + for payload in &mut captured.payloads { + payload.unresolved = false; + } + } + captured.payloads.push(Payload { + placeholder: placeholder.clone(), + original: script[start..end].to_owned(), + unresolved: opens, + }); + + let mut rewritten = script.to_owned(); + rewritten.replace_range(start..end, &placeholder); + Some(rewritten) + } + } + + impl TextHandler for Scripts { + fn selector(&self) -> &str { + "script" + } + + fn decide(&mut self, text: &str, is_last: bool) -> ScriptRewriteAction { + let claimed = !self.held.is_empty() || text.contains("fixture_payload"); + if !claimed { + return ScriptRewriteAction::Keep; + } + self.held.push_str(text); + if !is_last { + return ScriptRewriteAction::RemoveNode; + } + let script = std::mem::take(&mut self.held); + let rewritten = self.rewrite_whole(&script).unwrap_or(script); + ScriptRewriteAction::replace(rewritten) + } + } + + struct Payloads; + + impl Middleware for Payloads { + fn middleware_id(&self) -> &'static str { + MODULE + } + + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Fetch] + } + + /// The text handler and the stream processor of one document + /// share what the handler captures. + fn create(&self, context: &MiddlewareContext<'_>) -> MiddlewareAction { + let captured = Arc::new(Mutex::new(Captured::default())); + MiddlewareAction { + text_handlers: vec![Box::new(Scripts { + captured: Arc::clone(&captured), + held: String::new(), + })], + stream: Some(Box::new(Processor { + captured, + origin_host: context.origin_host.to_owned(), + request_host: context.request_host.to_owned(), + held: Vec::new(), + })), + ..MiddlewareAction::pass() + } + } + } + + struct Processor { + captured: Arc>, + origin_host: String, + request_host: String, + /// Output not yet released, because it ends inside a placeholder + /// or starts at one whose group is unresolved. + held: Vec, + } + + fn find(haystack: &[u8], needle: &[u8], from: usize) -> Option { + haystack + .get(from..)? + .windows(needle.len()) + .position(|window| window == needle) + .map(|at| at + from) + } + + impl StreamProcessor for Processor { + fn process_chunk(&mut self, chunk: &[u8], is_last: bool) -> Result, io::Error> { + self.held.extend_from_slice(chunk); + let captured = self.captured.lock().unwrap_or_else(PoisonError::into_inner); + let prefix = PLACEHOLDER_PREFIX.as_bytes(); + let mut out = Vec::with_capacity(self.held.len()); + let mut at = 0; + + while let Some(start) = find(&self.held, prefix, at) { + let Some(end) = + find(&self.held, PLACEHOLDER_END.as_bytes(), start + prefix.len()) + else { + // The placeholder runs past this chunk. + out.extend_from_slice(&self.held[at..start]); + at = start; + break; + }; + let end = end + PLACEHOLDER_END.len(); + let found = std::str::from_utf8(&self.held[start..end]) + .ok() + .and_then(|text| { + captured + .payloads + .iter() + .find(|payload| payload.placeholder == text) + }); + match found { + Some(payload) if payload.unresolved && !is_last => { + out.extend_from_slice(&self.held[at..start]); + at = start; + break; + } + Some(payload) => { + out.extend_from_slice(&self.held[at..start]); + out.extend_from_slice( + payload + .original + .replace(&self.origin_host, &self.request_host) + .as_bytes(), + ); + at = end; + } + None => { + out.extend_from_slice(&self.held[at..end]); + at = end; + } + } + } + + let stopped_at_placeholder = find(&self.held, prefix, at) == Some(at); + if is_last { + out.extend_from_slice(&self.held[at..]); + self.held.clear(); + } else if stopped_at_placeholder { + self.held.drain(..at); + } else { + // Keep back a tail that could be the start of a + // placeholder split across chunks. + let rest = &self.held[at..]; + let keep = (1..prefix.len().min(rest.len() + 1)) + .rev() + .find(|len| rest.ends_with(&prefix[..*len])) + .unwrap_or(0); + out.extend_from_slice(&rest[..rest.len() - keep]); + let tail = rest[rest.len() - keep..].to_vec(); + self.held = tail; + } + Ok(out) + } + } + + /// Has `settings` select the stand-in and run its middleware, and no + /// other, on every page. + pub(crate) fn select_and_place(settings: &mut Settings) { + settings.select_module("testing", MODULE); + crate::html_processor::test_support::place_on_every_page( + settings, + MiddlewarePhase::Fetch, + &[MODULE], + ); + } + + fn register( + settings: &Settings, + ) -> Result, Report> { + if settings.module_config::(MODULE)?.is_none() { + return Ok(None); + } + Ok(Some( + IntegrationRegistration::builder(ID) + .with_middleware(Arc::new(Payloads)) + .build(), + )) + } + + /// The stand-in takes no settings, and refuses one it does not know + /// as any module does. + #[derive(Debug, serde::Deserialize, validator::Validate)] + #[serde(deny_unknown_fields)] + struct FixtureSettings {} + + impl crate::settings::IntegrationConfig for FixtureSettings {} + + fn validate(settings: &Settings) -> Result> { + settings + .module_config::(MODULE) + .map(|config| config.is_some()) + } + } +} + +#[cfg(test)] +mod tests { + use super::test_support::middleware_fixture as fixture; + use super::test_support::{ + PROBE_JS, PROBE_JS_SHA256, carried_probe_registration, probe_registration, validate_nothing, + }; + use super::*; + use crate::constants::COOKIE_EC; + use crate::middleware::{HTML_MEDIA_TYPE, MiddlewareAction, MiddlewareContext, PhaseEntry}; + use crate::permissions::{Permission, PermissionSet, PermissionState}; + use crate::platform::test_support::noop_services; + use http::{HeaderValue, StatusCode, header}; + + // Mock integration proxy for testing + struct MockProxy; + + #[async_trait(?Send)] + impl IntegrationProxy for MockProxy { + fn integration_name(&self) -> &'static str { + "test" + } + + fn routes(&self) -> Vec { + vec![] + } + + async fn handle( + &self, + _call: crate::module_context::ModuleCall<'_>, + _req: Request, + ) -> Result, Report> { + Ok(Response::new(EdgeBody::empty())) + } + } + + struct EnrichingRequestFilter; + #[derive(Clone, Copy)] + struct RequestAnnotation; + + #[async_trait(?Send)] + impl IntegrationRequestFilter for EnrichingRequestFilter { + fn integration_id(&self) -> &'static str { + "enriching" + } + + async fn filter_request( + &self, + input: RequestFilterInput<'_>, + ) -> Result> { + input.request.extensions_mut().insert(RequestAnnotation); + Ok(RequestFilterDecision::Continue(RequestFilterEffects { + request_headers: vec![HeaderMutation::set("x-probe-isbot", "1")], + response_headers: vec![HeaderMutation::set("x-dd-b", "allowed")], + })) + } + } + + /// Records the permission state each filter invocation received, so a test + /// can assert what the registry handed to the filter. + #[derive(Default)] + struct RecordingPermissionsFilter { + seen: std::sync::Mutex>>, + } + + impl RecordingPermissionsFilter { + /// The permission state observed by the last invocation, or `None` when + /// the filter has not run. + fn seen(&self) -> Option> { + self.seen + .lock() + .expect("should lock the recorded permission state") + .clone() + } + } + + #[async_trait(?Send)] + impl IntegrationRequestFilter for RecordingPermissionsFilter { + fn integration_id(&self) -> &'static str { + "recording-permissions" + } + + async fn filter_request( + &self, + input: RequestFilterInput<'_>, + ) -> Result> { + *self + .seen + .lock() + .expect("should lock the recorded permission state") = + Some(input.permissions.cloned()); + Ok(RequestFilterDecision::Continue( + RequestFilterEffects::default(), + )) + } + } + + struct EchoProxy; + + #[async_trait(?Send)] + impl IntegrationProxy for EchoProxy { + fn integration_name(&self) -> &'static str { + "echo" + } + + fn routes(&self) -> Vec { + vec![] + } + + async fn handle( + &self, + _call: crate::module_context::ModuleCall<'_>, + req: http::Request, + ) -> Result, Report> { + let response = http::Response::builder() + .status(http::StatusCode::OK) + .header("x-echo-path", req.uri().path()) + .body(EdgeBody::empty()) + .expect("should build echo response"); + Ok(response) + } + } + + #[test] + fn document_state_keeps_multiple_types_for_one_integration() { + let state = IntegrationDocumentState::default(); + let number = state.get_or_insert_with("test", || 7_u32); + let label = state.get_or_insert_with("test", || "first".to_string()); + let repeated_number = state.get_or_insert_with("test", || 99_u32); + + assert!( + Arc::ptr_eq(&number, &repeated_number), + "repeated insertion should preserve the original typed state" + ); + assert_eq!( + *state.get::("test").expect("should retrieve number"), + 7, + "should retain numeric state" + ); + assert_eq!( + state + .get::("test") + .expect("should retrieve label") + .as_str(), + "first", + "should retain string state under the same integration ID" + ); + assert_eq!( + label.as_str(), + "first", + "should return inserted string state" + ); + } + + #[test] + fn handle_proxy_passes_http_request_without_fastly_round_trip() { + let settings = create_test_settings(); + let registry = IntegrationRegistry::from_routes(vec![( + http::Method::GET, + "/integrations/test/echo", + (Arc::new(EchoProxy) as Arc, "echo"), + )]); + let req = http::Request::builder() + .method(http::Method::GET) + .uri("https://test.example.com/integrations/test/echo?x=1") + .body(EdgeBody::empty()) + .expect("should build request"); + + let mut ec_context = + EcContext::new_for_test(None, crate::consent::ConsentContext::default()); + let response = futures::executor::block_on(registry.handle_proxy(ProxyDispatchInput { + method: &http::Method::GET, + path: "/integrations/test/echo", + settings: &settings, + kv: None, + ec_context: &mut ec_context, + services: &noop_services(), + req, + })) + .expect("should match route") + .expect("proxy should succeed"); + + assert_eq!( response.status(), http::StatusCode::OK, "should preserve HTTP status" ); assert_eq!( - response.headers()["x-echo-path"], - "/integrations/test/echo", - "should expose the HTTP request path to the proxy" + response.headers()["x-echo-path"], + "/integrations/test/echo", + "should expose the HTTP request path to the proxy" + ); + } + + #[test] + fn filter_request_applies_request_headers_and_returns_response_headers() { + let registry = + IntegrationRegistry::from_request_filters(vec![Arc::new(EnrichingRequestFilter)]); + let settings = crate::test_support::tests::create_test_settings(); + let services = crate::platform::test_support::noop_services(); + let mut req = Request::builder() + .method(Method::GET) + .uri("https://example.com/page") + .body(EdgeBody::empty()) + .expect("should build request"); + + let outcome = + futures::executor::block_on(registry.filter_request(RequestFilterRegistryInput { + settings: &settings, + services: &services, + req: &mut req, + geo_info: None, + permissions: None, + })) + .expect("should run request filter"); + + assert_eq!( + req.headers() + .get("x-probe-isbot") + .and_then(|value| value.to_str().ok()), + Some("1"), + "should apply a filter's request enrichment before routing" + ); + assert!( + req.extensions().get::().is_some(), + "should preserve private request annotations for downstream routing" + ); + match outcome { + RequestFilterRegistryOutcome::Continue(effects) => { + assert_eq!( + effects.response_headers, + vec![HeaderMutation::set("x-dd-b", "allowed")], + "should return downstream response header effects for finalization" + ); + } + RequestFilterRegistryOutcome::Respond { .. } => panic!("should continue routing"), + } + } + + #[test] + fn filter_request_passes_the_resolved_permission_state_to_each_filter() { + let filter = Arc::new(RecordingPermissionsFilter::default()); + let registry = IntegrationRegistry::from_request_filters(vec![ + filter.clone() as Arc + ]); + let settings = crate::test_support::tests::create_test_settings(); + let services = crate::platform::test_support::noop_services(); + let permissions = + PermissionState::new(PermissionSet::none().with(Permission::StoreOnDevice)); + let mut req = Request::builder() + .method(Method::GET) + .uri("https://example.com/page") + .body(EdgeBody::empty()) + .expect("should build request"); + + futures::executor::block_on(registry.filter_request(RequestFilterRegistryInput { + settings: &settings, + services: &services, + req: &mut req, + geo_info: None, + permissions: Some(&permissions), + })) + .expect("should run request filter"); + + assert_eq!( + filter.seen(), + Some(Some(permissions)), + "the filter should observe exactly the permission state resolved for the request" + ); + + // A path that builds no EC context passes no permissions, and the + // filter must see that absence rather than an empty state. + futures::executor::block_on(registry.filter_request(RequestFilterRegistryInput { + settings: &settings, + services: &services, + req: &mut req, + geo_info: None, + permissions: None, + })) + .expect("should run request filter"); + + assert_eq!( + filter.seen(), + Some(None), + "an absent permission state should reach the filter as `None`" + ); + } + + #[test] + fn test_exact_route_matching() { + let routes = vec![( + Method::GET, + "/integrations/test/exact", + (Arc::new(MockProxy) as Arc, "test"), + )]; + + let registry = IntegrationRegistry::from_routes(routes); + + // Should match exact route + assert!(registry.has_route(&Method::GET, "/integrations/test/exact")); + + // Should not match different paths + assert!(!registry.has_route(&Method::GET, "/integrations/test/other")); + assert!(!registry.has_route(&Method::GET, "/integrations/test/exact/nested")); + + // Should not match different methods + assert!(!registry.has_route(&Method::POST, "/integrations/test/exact")); + } + + #[test] + fn test_wildcard_route_matching() { + let routes = vec![( + Method::GET, + "/integrations/lockr/api/*", + (Arc::new(MockProxy) as Arc, "lockr"), + )]; + + let registry = IntegrationRegistry::from_routes(routes); + + // Should match paths under the wildcard prefix + assert!(registry.has_route(&Method::GET, "/integrations/lockr/api/settings")); + assert!(registry.has_route( + &Method::GET, + "/integrations/lockr/api/publisher/app/v1/identityLockr/settings" + )); + assert!(registry.has_route(&Method::GET, "/integrations/lockr/api/page-view")); + assert!(registry.has_route(&Method::GET, "/integrations/lockr/api/a/b/c/d/e")); + + // Should not match paths that don't start with the prefix + assert!(!registry.has_route(&Method::GET, "/integrations/lockr/sdk")); + assert!(!registry.has_route(&Method::GET, "/integrations/lockr/other")); + assert!(!registry.has_route(&Method::GET, "/integrations/other/api/settings")); + + // Should not match different methods + assert!(!registry.has_route(&Method::POST, "/integrations/lockr/api/settings")); + } + + #[test] + fn test_wildcard_and_exact_routes_coexist() { + let routes = vec![ + ( + Method::GET, + "/integrations/test/api/*", + (Arc::new(MockProxy) as Arc, "test"), + ), + ( + Method::GET, + "/integrations/test/exact", + (Arc::new(MockProxy) as Arc, "test"), + ), + ]; + + let registry = IntegrationRegistry::from_routes(routes); + + // Exact route should match + assert!(registry.has_route(&Method::GET, "/integrations/test/exact")); + + // Wildcard routes should match + assert!(registry.has_route(&Method::GET, "/integrations/test/api/anything")); + assert!(registry.has_route(&Method::GET, "/integrations/test/api/nested/path")); + + // Non-matching should fail + assert!(!registry.has_route(&Method::GET, "/integrations/test/other")); + } + + #[test] + fn test_multiple_wildcard_routes() { + let routes = vec![ + ( + Method::GET, + "/integrations/lockr/api/*", + (Arc::new(MockProxy) as Arc, "lockr"), + ), + ( + Method::POST, + "/integrations/lockr/api/*", + (Arc::new(MockProxy) as Arc, "lockr"), + ), + ( + Method::GET, + "/integrations/testlight/api/*", + ( + Arc::new(MockProxy) as Arc, + "testlight", + ), + ), + ]; + + let registry = IntegrationRegistry::from_routes(routes); + + // Lockr GET routes should match + assert!(registry.has_route(&Method::GET, "/integrations/lockr/api/settings")); + + // Lockr POST routes should match + assert!(registry.has_route(&Method::POST, "/integrations/lockr/api/settings")); + + // Testlight routes should match + assert!(registry.has_route(&Method::GET, "/integrations/testlight/api/auction")); + assert!(registry.has_route(&Method::GET, "/integrations/testlight/api/any-path")); + + // Cross-integration paths should not match + assert!(!registry.has_route(&Method::GET, "/integrations/lockr/other-endpoint")); + assert!(!registry.has_route(&Method::GET, "/integrations/other/api/test")); + } + + #[test] + fn test_wildcard_preserves_casing() { + let routes = vec![( + Method::GET, + "/integrations/lockr/api/*", + (Arc::new(MockProxy) as Arc, "lockr"), + )]; + + let registry = IntegrationRegistry::from_routes(routes); + + // Should match with camelCase preserved + assert!(registry.has_route( + &Method::GET, + "/integrations/lockr/api/publisher/app/v1/identityLockr/settings" + )); + assert!(registry.has_route( + &Method::GET, + "/integrations/lockr/api/publisher/app/v1/identitylockr/settings" + )); + } + + #[test] + fn test_wildcard_edge_cases() { + let routes = vec![( + Method::GET, + "/api/*", + (Arc::new(MockProxy) as Arc, "test"), + )]; + + let registry = IntegrationRegistry::from_routes(routes); + + // Should match paths under /api/ + assert!(registry.has_route(&Method::GET, "/api/v1")); + assert!(registry.has_route(&Method::GET, "/api/v1/users")); + + // Should not match /api without trailing content + // The current implementation requires a / after the prefix + assert!(!registry.has_route(&Method::GET, "/api")); + + // Should not match partial prefix matches + assert!(!registry.has_route(&Method::GET, "/apiv1")); + } + + #[test] + fn test_helper_methods_create_namespaced_routes() { + let proxy = Arc::new(MockProxy); + + // Test all HTTP method helpers + let get_endpoint = proxy.get("/users"); + assert_eq!(get_endpoint.method, Method::GET); + assert_eq!(get_endpoint.path, "/integrations/test/users"); + + let post_endpoint = proxy.post("/users"); + assert_eq!(post_endpoint.method, Method::POST); + assert_eq!(post_endpoint.path, "/integrations/test/users"); + + let put_endpoint = proxy.put("/users"); + assert_eq!(put_endpoint.method, Method::PUT); + assert_eq!(put_endpoint.path, "/integrations/test/users"); + + let delete_endpoint = proxy.delete("/users"); + assert_eq!(delete_endpoint.method, Method::DELETE); + assert_eq!(delete_endpoint.path, "/integrations/test/users"); + + let patch_endpoint = proxy.patch("/users"); + assert_eq!(patch_endpoint.method, Method::PATCH); + assert_eq!(patch_endpoint.path, "/integrations/test/users"); + } + + #[test] + fn test_put_delete_patch_routes() { + let routes = vec![ + ( + Method::PUT, + "/integrations/test/users", + (Arc::new(MockProxy) as Arc, "test"), + ), + ( + Method::DELETE, + "/integrations/test/users", + (Arc::new(MockProxy) as Arc, "test"), + ), + ( + Method::PATCH, + "/integrations/test/users", + (Arc::new(MockProxy) as Arc, "test"), + ), + ]; + + let registry = IntegrationRegistry::from_routes(routes); + + // Should match PUT, DELETE, and PATCH routes + assert!(registry.has_route(&Method::PUT, "/integrations/test/users")); + assert!(registry.has_route(&Method::DELETE, "/integrations/test/users")); + assert!(registry.has_route(&Method::PATCH, "/integrations/test/users")); + + // Should not match other methods on same path + assert!(!registry.has_route(&Method::GET, "/integrations/test/users")); + assert!(!registry.has_route(&Method::POST, "/integrations/test/users")); + } + + // Tests for EC ID header on proxy responses + use crate::test_support::tests::create_test_settings; + + /// Mock proxy that returns a simple 200 OK response + struct EcTestProxy; + + #[async_trait(?Send)] + impl IntegrationProxy for EcTestProxy { + fn integration_name(&self) -> &'static str { + "ec_test" + } + + fn routes(&self) -> Vec { + vec![ + IntegrationEndpoint { + method: Method::GET, + path: "/integrations/test/ec".to_owned(), + }, + IntegrationEndpoint { + method: Method::POST, + path: "/integrations/test/ec".to_owned(), + }, + ] + } + + async fn handle( + &self, + _call: crate::module_context::ModuleCall<'_>, + req: Request, + ) -> Result, Report> { + let mut response = Response::builder() + .status(StatusCode::OK) + .body(EdgeBody::from("test response")) + .expect("should build test response"); + if let Some(ec) = req.headers().get(HEADER_X_TS_EC.clone()) { + response + .headers_mut() + .insert(http::HeaderName::from_static("x-echo-ts-ec"), ec.clone()); + } + Ok(response) + } + } + + #[test] + fn handle_proxy_removes_ec_id_header_on_request() { + let settings = create_test_settings(); + let routes = vec![( + Method::GET, + "/integrations/test/ec", + ( + Arc::new(EcTestProxy) as Arc, + "ec_test", + ), + )]; + let registry = IntegrationRegistry::from_routes(routes); + + let mut req = Request::builder() + .method(Method::GET) + .uri("https://test-publisher.com/integrations/test/ec") + .body(EdgeBody::empty()) + .expect("should build request"); + req.headers_mut().insert( + HEADER_X_TS_EC.clone(), + HeaderValue::from_static("some-ec-value"), + ); + let mut ec_context = + EcContext::new_for_test(None, crate::consent::ConsentContext::default()); + let services = noop_services(); + + // Call handle_proxy (uses futures executor in test environment) + let result = futures::executor::block_on(registry.handle_proxy(ProxyDispatchInput { + method: &Method::GET, + path: "/integrations/test/ec", + settings: &settings, + kv: None, + ec_context: &mut ec_context, + services: &services, + req, + })); + + // Should have matched and returned a response + assert!(result.is_some(), "should find route and handle request"); + let response = result.unwrap(); + assert!(response.is_ok(), "handler should succeed"); + + let response = response.unwrap(); + + assert!( + response.headers().get("x-echo-ts-ec").is_none(), + "should not have x-ts-ec header on integration request" + ); + } + + /// A route that answers with what its module call handed it. + struct StateRoute { + declared: PermissionSet, + } + + impl StateRoute { + fn report( + &self, + _req: Request, + request: crate::module_context::ModuleRequest<'_>, + permissions: &PermissionState, + edge_cookie: Option>, + ) -> Result, Report> { + let body = format!( + "{} storage={} id={}", + request.host(), + permissions.is_set(Permission::StoreOnDevice), + edge_cookie.map_or("none", |id| id.as_str()), + ); + Ok(Response::builder() + .status(StatusCode::OK) + .body(EdgeBody::from(body)) + .expect("should build the response")) + } + } + + #[async_trait(?Send)] + impl IntegrationProxy for StateRoute { + fn integration_name(&self) -> &'static str { + "state" + } + + fn routes(&self) -> Vec { + vec![IntegrationEndpoint::get("/integrations/test/state")] + } + + fn required_permissions(&self) -> PermissionSet { + self.declared + } + + async fn handle( + &self, + call: ModuleCall<'_>, + req: Request, + ) -> Result, Report> { + call.inject_with(self, req, Self::report)? + } + } + + /// A route is handed the permissions resolved for its request, and the + /// request's Edge Cookie identifier when it declares the permission the + /// identifier's use needs. + #[test] + fn a_route_is_handed_its_requests_permissions_and_identity() { + let settings = create_test_settings(); + let storage = PermissionSet::none().with(Permission::StoreOnDevice); + let answer = |declared: PermissionSet| { + let registry = IntegrationRegistry::from_routes(vec![( + Method::GET, + "/integrations/test/state", + ( + Arc::new(StateRoute { declared }) as Arc, + "state", + ), + )]); + let req = Request::builder() + .method(Method::GET) + .uri("https://test-publisher.com/integrations/test/state") + .header(header::HOST, "test-publisher.com") + .body(EdgeBody::empty()) + .expect("should build request"); + let mut ec_context = EcContext::new_for_test( + Some("an-id".to_owned()), + crate::consent::ConsentContext::default(), + ) + .with_module_for_test(Arc::new(crate::ec::module::HmacModule::new( + crate::redacted::Redacted::new("test-secret-key-32-bytes-minimum".to_owned()), + ))); + let services = noop_services(); + let response = futures::executor::block_on(registry.handle_proxy(ProxyDispatchInput { + method: &Method::GET, + path: "/integrations/test/state", + settings: &settings, + kv: None, + ec_context: &mut ec_context, + services: &services, + req, + })) + .expect("should find the route") + .expect("the route should answer"); + String::from_utf8( + response + .into_body() + .into_bytes() + .unwrap_or_default() + .to_vec(), + ) + .expect("the answer is text") + }; + + assert_eq!( + answer(storage), + "test-publisher.com storage=true id=an-id", + "a route declaring storage is handed the identifier" + ); + assert_eq!( + answer(PermissionSet::none()), + "test-publisher.com storage=true id=none", + "and one declaring nothing still sees the permissions but not the identifier" + ); + } + + #[test] + fn handle_proxy_rejects_invalid_ec_request_header() { + let settings = create_test_settings(); + let routes = vec![( + Method::GET, + "/integrations/test/ec", + ( + Arc::new(EcTestProxy) as Arc, + "ec_test", + ), + )]; + let registry = IntegrationRegistry::from_routes(routes); + + let mut req = Request::builder() + .method(Method::GET) + .uri("https://test-publisher.com/integrations/test/ec") + .body(EdgeBody::empty()) + .expect("should build request"); + req.headers_mut().insert( + HEADER_X_TS_EC.clone(), + HeaderValue::from_static("evil;injected"), + ); + let mut ec_context = + EcContext::new_for_test(None, crate::consent::ConsentContext::default()); + + let services = crate::platform::test_support::noop_services(); + + let result = futures::executor::block_on(registry.handle_proxy(ProxyDispatchInput { + method: &Method::GET, + path: "/integrations/test/ec", + settings: &settings, + kv: None, + ec_context: &mut ec_context, + services: &services, + req, + })) + .expect("should handle proxy request"); + + let response = result.expect("handler should succeed"); + + assert!( + response.headers().get("x-echo-ts-ec").is_none(), + "should not reflect the tampered request header to the integration" + ); + } + + #[test] + fn handle_proxy_removes_request_ec_header_even_when_consent_denied() { + let settings = create_test_settings(); + let routes = vec![( + Method::GET, + "/integrations/test/ec", + (Arc::new(EcTestProxy) as Arc, "test"), + )]; + + let registry = IntegrationRegistry::from_routes(routes); + + let mut req = Request::builder() + .method(Method::GET) + .uri("https://test.example.com/integrations/test/ec") + .body(EdgeBody::empty()) + .expect("should build request"); + req.headers_mut().insert( + header::COOKIE, + HeaderValue::from_str(&format!( + "ts-{COOKIE_EC}={}", + crate::test_support::tests::VALID_SYNTHETIC_ID + )) + .expect("should build Cookie header"), + ); + let mut ec_context = + EcContext::new_for_test(None, crate::consent::ConsentContext::default()); + + let services = crate::platform::test_support::noop_services(); + + let result = futures::executor::block_on(registry.handle_proxy(ProxyDispatchInput { + method: &Method::GET, + path: "/integrations/test/ec", + settings: &settings, + kv: None, + ec_context: &mut ec_context, + services: &services, + req, + })) + .expect("should handle proxy request"); + + let response = result.expect("proxy handle should succeed"); + + assert!( + response.headers().get("x-echo-ts-ec").is_none(), + "should not set x-ts-ec on integration request" + ); + } + + #[test] + fn handle_proxy_works_with_post_method() { + let settings = create_test_settings(); + let routes = vec![( + Method::POST, + "/integrations/test/ec", + ( + Arc::new(EcTestProxy) as Arc, + "ec_test", + ), + )]; + let registry = IntegrationRegistry::from_routes(routes); + + let mut req = Request::builder() + .method(Method::POST) + .uri("https://test-publisher.com/integrations/test/ec") + .body(EdgeBody::from("test body")) + .expect("should build POST request"); + req.headers_mut().insert( + HEADER_X_TS_EC.clone(), + HeaderValue::from_static("some-ec-value"), + ); + let mut ec_context = + EcContext::new_for_test(None, crate::consent::ConsentContext::default()); + + let services = crate::platform::test_support::noop_services(); + + let result = futures::executor::block_on(registry.handle_proxy(ProxyDispatchInput { + method: &Method::POST, + path: "/integrations/test/ec", + settings: &settings, + kv: None, + ec_context: &mut ec_context, + services: &services, + req, + })); + + assert!(result.is_some(), "Should find POST route"); + let response = result.unwrap(); + assert!(response.is_ok(), "Handler should succeed"); + + let response = response.unwrap(); + assert!( + response.headers().get("x-echo-ts-ec").is_none(), + "POST integration request should not include x-ts-ec" + ); + } + + #[test] + fn js_module_ids_defer_a_deferred_module_and_include_core_js_only_modules() { + let mut settings = crate::test_support::tests::create_test_settings(); + enable_deferred_module(&mut settings); + + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); + + let all = registry.js_module_ids(); + let immediate = registry.js_module_ids_immediate(); + let deferred = registry.js_module_ids_deferred(); + let module = test_support::deferred_fixture::ID; + + assert!( + all.contains(&module), + "should include the deferred module in the module IDs" + ); + assert!( + immediate.contains(&"creative"), + "should include creative in immediate IDs" + ); + assert!( + !immediate.contains(&"sourcepoint"), + "should not include Sourcepoint unless it is named" + ); + assert!( + !immediate.contains(&module), + "should not include the deferred module in immediate IDs" + ); + assert!( + deferred.contains(&module), + "should serve the module as a deferred module" + ); + } + + #[test] + fn js_module_ids_skip_named_integrations_without_generated_js_module() { + let settings = settings_naming("testing.probe"); + let extra = [crate::integrations::IntegrationBuilder::new( + "probe", + "seam-probe", + probe_registration, + validate_nothing, + ) + .with_module_name("testing.probe")]; + + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should create registry"); + let all = registry.js_module_ids(); + + assert!( + !all.contains(&"probe"), + "should not include named integrations without generated JS modules" + ); + + let metadata = registry.registered_integrations(); + assert!( + metadata.iter().any(|integration| integration.id == "probe"), + "should still register named Rust-only integrations" + ); + } + + #[test] + fn js_module_ids_include_client_fixed_when_module_selected() { + let mut settings = crate::test_support::tests::create_test_settings(); + settings.ec.module = Some(crate::ec::module::EcModuleSelection::from( + crate::ec::module::CLIENT_FIXED_MODULE_KEY, + )); + let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + + assert!( + registry + .js_module_ids_immediate() + .contains(&"ec_client_fixed"), + "selecting the `client_fixed` module should inject its demo page script" + ); + } + + #[test] + fn js_module_ids_exclude_client_fixed_without_module() { + let registry = + IntegrationRegistry::new(&crate::test_support::tests::create_test_settings()) + .expect("should create registry"); + + assert!( + !registry + .js_module_ids_immediate() + .contains(&"ec_client_fixed"), + "the demo page script should not ship unless the `client_fixed` module is selected" + ); + } + + #[test] + fn js_module_ids_deferred_empty_when_prebid_is_not_named() { + let mut settings = crate::test_support::tests::create_test_settings(); + // The shared fixture names prebid, and this asks what the registry + // serves when it does not. + settings.auction.modules.clear(); + + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); + + let deferred = registry.js_module_ids_deferred(); + assert!( + deferred.is_empty(), + "should have no deferred IDs when prebid is not named" + ); + } + + #[test] + fn js_module_ids_split_is_exhaustive() { + let mut settings = crate::test_support::tests::create_test_settings(); + enable_deferred_module(&mut settings); + + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); + + let all = registry.js_module_ids(); + assert!( + !registry.js_module_ids_deferred().is_empty(), + "should have a deferred module to divide from the bundle" + ); + let mut recombined = registry.js_module_ids_immediate(); + recombined.extend(registry.js_module_ids_deferred()); + recombined.sort_unstable(); + + let mut all_sorted = all; + all_sorted.sort_unstable(); + + assert_eq!( + recombined, all_sorted, + "should reconstruct full module list from immediate + deferred" + ); + } + + /// Claims the id of the JavaScript asset proxy, which is core's own. + fn duplicate_core_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder( + crate::integrations::js_asset_proxy::JS_ASSET_PROXY_INTEGRATION_ID, + ) + .build(), + )) + } + + /// The shared fixture with the module `name` selected in its section, so a + /// test builder is built the way any integration an operator names is. + fn settings_naming(name: &str) -> Settings { + let mut settings = crate::test_support::tests::create_test_settings(); + let section = name.split_once('.').map_or("testing", |(folder, _)| folder); + settings.select_module(section, name); + settings + } + + #[test] + fn with_registrations_adds_an_external_builder_after_the_built_ins() { + let settings = settings_naming("probe"); + let extra = [crate::integrations::IntegrationBuilder::new( + "probe", + "seam-probe", + probe_registration, + validate_nothing, + ) + .with_module_name("testing.probe")]; + + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should build registry with an external builder"); + + assert!( + registry.integration_runs("probe"), + "should register the external integration" + ); + let ids = registry.registered_builder_ids(); + assert_eq!( + ids.last().copied(), + Some("probe"), + "should order the external builder after every built-in" + ); + } + + const SHARED_VENDOR: &str = "example"; + + fn shared_probe_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("shared_probe").build(), + )) + } + + /// The `shared_probe` builder, run by `[shared.example]`. + fn shared_probe_builder() -> crate::integrations::IntegrationBuilder { + crate::integrations::IntegrationBuilder::new( + "shared_probe", + "seam-probe", + shared_probe_registration, + validate_nothing, + ) + .with_shared_settings(SHARED_VENDOR) + } + + fn with_shared_table(mut settings: Settings) -> Settings { + settings.shared = + serde_json::from_value(serde_json::json!({ SHARED_VENDOR: { "key": "value" } })) + .expect("should read the shared tables"); + settings + } + + #[test] + fn a_builder_its_shared_table_runs_is_built_only_when_the_table_is_present() { + let extra = [shared_probe_builder()]; + + let without = IntegrationRegistry::with_registrations( + &crate::test_support::tests::create_test_settings(), + &extra, + ) + .expect("nothing is built, which is not an error"); + assert!(!without.integration_runs("shared_probe")); + + let with = IntegrationRegistry::with_registrations( + &with_shared_table(crate::test_support::tests::create_test_settings()), + &extra, + ) + .expect("should build the builder its table runs"); + assert!(with.integration_runs("shared_probe")); + } + + #[test] + fn a_shared_table_no_builder_reads_is_refused() { + let refused = IntegrationRegistry::with_registrations( + &with_shared_table(crate::test_support::tests::create_test_settings()), + &[], + ) + .err() + .expect("nothing reads the table") + .to_string(); + + assert!( + refused.contains("[shared.example] is read by no module"), + "{refused}" + ); + } + + #[test] + fn a_builder_its_shared_table_runs_answers_to_no_selection() { + let refused = IntegrationRegistry::with_registrations( + &with_shared_table(settings_naming("shared_probe")), + &[shared_probe_builder()], + ) + .err() + .expect("the table runs the builder, so no section can select it") + .to_string(); + + assert!(refused.contains("selects `shared_probe`"), "{refused}"); + } + + #[test] + fn a_selection_nothing_supplies_names_the_shared_table_that_is_absent() { + let mut settings = crate::test_support::tests::create_test_settings(); + settings.geo.module = Some("example".to_owned()); + + let refused = IntegrationRegistry::with_registrations(&settings, &[shared_probe_builder()]) + .err() + .expect("no geo module is supplied") + .to_string(); + + assert!( + refused.contains("There is no [shared.example]"), + "the message has to name the table to add: {refused}" + ); + } + + #[test] + fn a_list_selection_nothing_supplies_names_the_shared_table_that_is_absent() { + let mut settings = crate::test_support::tests::create_test_settings(); + settings.select_module("identity", "example.id"); + + let refused = IntegrationRegistry::with_registrations(&settings, &[shared_probe_builder()]) + .err() + .expect("no identity module is supplied") + .to_string(); + + assert!( + refused.contains("[identity] selects `example.id`") + && refused.contains("There is no [shared.example]"), + "the message has to name the table to add: {refused}" + ); + } + + #[test] + fn a_builder_its_shared_table_runs_is_refused_a_module_name() { + let mut settings = with_shared_table(crate::test_support::tests::create_test_settings()); + settings.geo.module = Some("example".to_owned()); + + let refused = IntegrationRegistry::with_registrations( + &settings, + &[shared_probe_builder().with_module_name("geo.example")], + ) + .err() + .expect("a section could select what the table runs") + .to_string(); + + assert!( + refused.contains("is run by [shared.example] and also names the module `geo.example`"), + "{refused}" + ); + } + + #[test] + fn with_registrations_rejects_a_duplicate_integration_id_naming_both_sources() { + let settings = crate::test_support::tests::create_test_settings(); + let extra = [crate::integrations::IntegrationBuilder::new( + crate::integrations::js_asset_proxy::JS_ASSET_PROXY_INTEGRATION_ID, + "seam-probe", + duplicate_core_registration, + validate_nothing, + ) + .with_module_name("testing.duplicate")]; + + let error = IntegrationRegistry::with_registrations(&settings, &extra) + .err() + .expect("should reject a duplicate integration id"); + + let message = error.to_string(); + assert!( + message.contains(crate::integrations::js_asset_proxy::JS_ASSET_PROXY_INTEGRATION_ID) + && message.contains("trusted-server-core") + && message.contains("seam-probe"), + "error should name the id and both sources: {message}" + ); + } + + /// A builder that registers from the auction plan holds its id like any + /// other, so a second builder claiming it is refused, naming both sources, + /// whether or not the plan-backed integration runs. + #[test] + fn with_registrations_rejects_a_second_builder_claiming_a_plan_backed_id() { + fn register_nothing( + _settings: &Settings, + ) -> Result, Report> { + Ok(None) + } + fn nothing_from_the_plan( + _settings: &Settings, + _plan: &crate::auction::plan::AuctionPlan, + ) -> Result, Report> { + Ok(None) + } + + let settings = crate::test_support::tests::create_test_settings(); + let id = "plan_backed"; + let extra = [ + crate::integrations::IntegrationBuilder::new( + id, + "plan-backed-crate", + register_nothing, + validate_nothing, + ) + .with_plan_registration(nothing_from_the_plan), + crate::integrations::IntegrationBuilder::new( + id, + "seam-probe", + register_nothing, + validate_nothing, + ), + ]; + + let Err(error) = IntegrationRegistry::with_registrations(&settings, &extra) else { + panic!("should refuse a second builder claiming `{id}`"); + }; + + let message = error.to_string(); + assert!( + message.contains(id) + && message.contains("plan-backed-crate") + && message.contains("seam-probe"), + "error should name `{id}` and both sources: {message}" + ); + } + + /// Selects core's stand-in, which registers a deferred browser module. + fn enable_deferred_module(settings: &mut Settings) { + settings.select_module("testing", test_support::deferred_fixture::MODULE); + } + + /// Selects core's stand-in, which registers a standalone browser module. + fn enable_standalone_module(settings: &mut Settings) { + settings.select_module("testing", test_support::request_fixture::MODULE); + } + + fn carried_probe_builder() -> crate::integrations::IntegrationBuilder { + crate::integrations::IntegrationBuilder::new( + "probe", + "seam-probe", + carried_probe_registration, + validate_nothing, + ) + .with_module_name("testing.probe") + } + + const ZERO_SHA256: &str = "0000000000000000000000000000000000000000000000000000000000000000"; + + fn lying_probe_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("probe") + .with_js_module(CarriedJsModule { + source: PROBE_JS, + sha256: ZERO_SHA256, + }) + .build(), + )) + } + + fn disabled_carried_probe_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("probe") + .with_js_module(CarriedJsModule { + source: PROBE_JS, + sha256: PROBE_JS_SHA256, + }) + .without_js() + .build(), + )) + } + + #[test] + fn probe_js_hash_literal_matches_its_source() { + assert_eq!( + hex::encode(Sha256::digest(PROBE_JS.as_bytes())), + PROBE_JS_SHA256, + "should keep the PROBE_JS_SHA256 literal equal to the hash of PROBE_JS" + ); + } + + #[test] + fn a_carried_js_module_is_served_in_the_immediate_parts() { + let settings = settings_naming("probe"); + let extra = [carried_probe_builder()]; + + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should build registry with a carried module"); + + let parts = registry.js_parts_immediate(); + assert_eq!( + parts.first().map(|part| part.id), + Some("core"), + "should put core first in the immediate parts" + ); + let probe = parts + .iter() + .find(|part| part.id == "probe") + .expect("should include the carried probe module in the immediate parts"); + assert_eq!( + probe.source, PROBE_JS, + "should serve the carried source verbatim" + ); + assert_eq!( + probe.sha256, PROBE_JS_SHA256, + "should carry the declared hash on the part" + ); + assert!( + registry.js_module_ids_immediate().contains(&"probe"), + "should list the carried module among the immediate module ids" + ); + } + + #[test] + fn a_carried_js_module_with_a_wrong_hash_is_rejected_at_registry_build() { + let settings = settings_naming("probe"); + let extra = [crate::integrations::IntegrationBuilder::new( + "probe", + "seam-probe", + lying_probe_registration, + validate_nothing, + ) + .with_module_name("testing.probe")]; + + let error = IntegrationRegistry::with_registrations(&settings, &extra) + .err() + .expect("should reject a carried module whose declared hash is wrong"); + + let message = error.to_string(); + assert!( + message.contains("probe") && message.contains("does not match"), + "error should name the integration and say the hash does not match: {message}" + ); + assert!( + message.contains(ZERO_SHA256) && message.contains(PROBE_JS_SHA256), + "error should quote the declared and the actual hash: {message}" + ); + } + + #[test] + fn js_part_is_none_for_an_integration_registered_without_js() { + // The carried lookup would answer `Some` on its own, so this proves the + // without-JS check runs first. No built-in integration can stand in, because + // none registers without a browser script. + let settings = settings_naming("probe"); + let extra = [crate::integrations::IntegrationBuilder::new( + "probe", + "seam-probe", + disabled_carried_probe_registration, + validate_nothing, + ) + .with_module_name("testing.probe")]; + + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should build registry"); + + assert!( + registry.integration_runs("probe"), + "should still register the integration" + ); + assert!( + registry.js_part("probe").is_none(), + "should not serve a module for an integration registered without JS" + ); + assert!( + !registry.js_module_ids().contains(&"probe"), + "should keep a without-JS integration out of the bundle module ids" + ); + } + + #[test] + fn the_last_js_delivery_flag_set_on_the_builder_wins() { + let disabled_last = IntegrationRegistration::builder("probe") + .with_standalone_js() + .without_js() + .build(); + assert!( + disabled_last.js_disabled && !disabled_last.js_standalone && !disabled_last.js_deferred, + "without_js after with_standalone_js should leave only disabled set" + ); + + let deferred_last = IntegrationRegistration::builder("probe") + .with_standalone_js() + .with_deferred_js() + .build(); + assert!( + deferred_last.js_deferred && !deferred_last.js_standalone && !deferred_last.js_disabled, + "with_deferred_js after with_standalone_js should leave only deferred set" + ); + + let deferred_after_disabled = IntegrationRegistration::builder("probe") + .without_js() + .with_deferred_js() + .build(); + assert!( + deferred_after_disabled.js_deferred + && !deferred_after_disabled.js_disabled + && !deferred_after_disabled.js_standalone, + "with_deferred_js after without_js should leave only deferred set" + ); + + let standalone_last = IntegrationRegistration::builder("probe") + .without_js() + .with_deferred_js() + .with_standalone_js() + .build(); + assert!( + standalone_last.js_standalone + && !standalone_last.js_disabled + && !standalone_last.js_deferred, + "with_standalone_js last should leave only standalone set" + ); + } + + #[test] + fn a_standalone_js_module_is_served_alone_and_not_in_the_bundle() { + let mut settings = crate::test_support::tests::create_test_settings(); + enable_standalone_module(&mut settings); + + let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + + assert!( + !registry + .js_module_ids() + .contains(&test_support::request_fixture::ID), + "should keep a standalone module out of the bundle module ids" + ); + assert!( + registry + .js_part(test_support::request_fixture::ID) + .is_some(), + "should serve the standalone module as a part" + ); + assert_eq!( + registry.js_standalone_ids(), + vec![test_support::request_fixture::ID], + "should list the standalone module that runs" + ); + assert!( + registry.js_part("lockr").is_none(), + "should not serve a module for an integration that does not run" + ); + } + + #[test] + fn js_parts_all_covers_bundle_deferred_and_standalone_modules() { + let mut settings = settings_naming("probe"); + enable_deferred_module(&mut settings); + enable_standalone_module(&mut settings); + let extra = [carried_probe_builder()]; + + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should build registry"); + + let ids = registry + .js_parts_all() + .into_iter() + .map(|part| part.id) + .collect::>(); + for expected in [ + "core", + "creative", + "probe", + test_support::deferred_fixture::ID, + test_support::request_fixture::ID, + ] { + assert_eq!( + ids.iter().filter(|id| **id == expected).count(), + 1, + "should list `{expected}` exactly once in {ids:?}" + ); + } + } + + /// A builder function for an integration that never registers, so its + /// preparer is the only thing the registry can take from it. + fn never_registering_builder( + _settings: &Settings, + ) -> Result, Report> { + Ok(None) + } + + /// Preparers cannot capture, because a builder holds plain function + /// pointers, so they record the order they ran in here. + static PREPARER_ORDER: Mutex> = Mutex::new(Vec::new()); + + fn record_first_preparer( + _settings: &Settings, + _request: &mut Request, + ) -> Result<(), Report> { + PREPARER_ORDER + .lock() + .expect("should lock the preparer order") + .push("first"); + Ok(()) + } + + fn record_second_preparer( + _settings: &Settings, + _request: &mut Request, + ) -> Result<(), Report> { + PREPARER_ORDER + .lock() + .expect("should lock the preparer order") + .push("second"); + Ok(()) + } + + /// Records whether the preparer registered after the failing one ran. + static PREPARER_AFTER_FAILURE: Mutex> = Mutex::new(Vec::new()); + + const FAILING_PREPARER_MESSAGE: &str = "probe preparer refused the request"; + + fn failing_preparer( + _settings: &Settings, + _request: &mut Request, + ) -> Result<(), Report> { + Err(Report::new(TrustedServerError::Configuration { + message: FAILING_PREPARER_MESSAGE.to_owned(), + })) + } + + fn record_after_failure_preparer( + _settings: &Settings, + _request: &mut Request, + ) -> Result<(), Report> { + PREPARER_AFTER_FAILURE + .lock() + .expect("should lock the after-failure record") + .push("after"); + Ok(()) + } + + fn plain_request() -> Request { + Request::builder() + .method(Method::GET) + .uri("https://publisher.example.com/article") + .body(EdgeBody::empty()) + .expect("should build request") + } + + #[test] + fn prepare_request_runs_every_preparer_in_registration_order_named_or_not() { + let settings = crate::test_support::tests::create_test_settings(); + let extra = [ + crate::integrations::IntegrationBuilder::new( + "probe-first", + "seam-probe", + never_registering_builder, + validate_nothing, + ) + .with_module_name("testing.probe-first") + .with_request_preparer(record_first_preparer), + crate::integrations::IntegrationBuilder::new( + "probe-second", + "seam-probe", + never_registering_builder, + validate_nothing, + ) + .with_module_name("testing.probe-second") + .with_request_preparer(record_second_preparer), + ]; + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should build registry with request preparers"); + assert!( + !registry.integration_runs("probe-first"), + "should leave the first probe integration unnamed, so it does not run" + ); + assert!( + !registry.integration_runs("probe-second"), + "should leave the second probe integration unnamed, so it does not run" + ); + PREPARER_ORDER + .lock() + .expect("should lock the preparer order") + .clear(); + let mut request = plain_request(); + + registry + .prepare_request(&settings, &mut request) + .expect("should run every request preparer"); + + assert_eq!( + *PREPARER_ORDER + .lock() + .expect("should lock the preparer order"), + vec!["first", "second"], + "should run both preparers in registration order even though neither integration runs" + ); + } + + #[test] + fn prepare_request_surfaces_the_first_preparer_error() { + let settings = crate::test_support::tests::create_test_settings(); + let extra = [ + crate::integrations::IntegrationBuilder::new( + "probe-failing", + "seam-probe", + never_registering_builder, + validate_nothing, + ) + .with_module_name("testing.probe-failing") + .with_request_preparer(failing_preparer), + crate::integrations::IntegrationBuilder::new( + "probe-after-failure", + "seam-probe", + never_registering_builder, + validate_nothing, + ) + .with_module_name("testing.probe-after-failure") + .with_request_preparer(record_after_failure_preparer), + ]; + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should build registry with request preparers"); + PREPARER_AFTER_FAILURE + .lock() + .expect("should lock the after-failure record") + .clear(); + let mut request = plain_request(); + + let error = registry + .prepare_request(&settings, &mut request) + .expect_err("should surface the failing preparer's error"); + + assert!( + error.to_string().contains(FAILING_PREPARER_MESSAGE), + "should keep the preparer's message intact: {error}" + ); + assert!( + PREPARER_AFTER_FAILURE + .lock() + .expect("should lock the after-failure record") + .is_empty(), + "should not run a preparer registered after the failing one" + ); + } + + // Each probe supplies a middleware that changes nothing, so a test can + // read the order registrations were made in from the names the registry + // lists. + const SECTION_MIDDLEWARE: &str = "testing.probe-section"; + const PLAN_MIDDLEWARE: &str = "testing.probe-plan"; + + fn section_probe_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("probe-section") + .with_middleware(Arc::new(Named { + id: SECTION_MIDDLEWARE, + phases: FETCH_ONLY, + })) + .build(), + )) + } + + /// Registers when the plan's auction is enabled, whatever the sections + /// select. + fn plan_probe_registration( + _settings: &Settings, + plan: &AuctionPlan, + ) -> Result, Report> { + Ok(plan.enabled().then(|| { + IntegrationRegistration::builder("probe-plan") + .with_middleware(Arc::new(Named { + id: PLAN_MIDDLEWARE, + phases: FETCH_ONLY, + })) + .build() + })) + } + + #[test] + fn a_registration_from_the_plan_runs_unselected_and_ahead_of_a_section_s_module() { + let mut settings = settings_naming("testing.probe-section"); + // The section's module is listed first, and what the plan registers + // still goes ahead of it. + let extra = [ + crate::integrations::IntegrationBuilder::new( + "probe-section", + "seam-probe", + section_probe_registration, + validate_nothing, + ) + .with_module_name("testing.probe-section"), + crate::integrations::IntegrationBuilder::new( + "probe-plan", + "seam-probe", + never_registering_builder, + validate_nothing, + ) + .with_module_name("testing.probe-plan") + .with_plan_registration(plan_probe_registration), + ]; + let probe_middleware = |registry: &IntegrationRegistry| { + registry + .middleware_ids() + .into_iter() + .filter(|name| *name == SECTION_MIDDLEWARE || *name == PLAN_MIDDLEWARE) + .collect::>() + }; + + settings.auction.enabled = false; + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should build registry"); + assert!( + !registry.integration_runs("probe-plan"), + "should register nothing where the function finds nothing in the plan" + ); + assert_eq!(probe_middleware(®istry), vec![SECTION_MIDDLEWARE]); + + settings.auction.enabled = true; + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should build registry"); + assert!( + registry.integration_runs("probe-plan"), + "should register from the plan a module no section selects" + ); + assert_eq!( + probe_middleware(®istry), + vec![PLAN_MIDDLEWARE, SECTION_MIDDLEWARE], + "should register what the plan supplies ahead of a section's module" + ); + } + + /// Appends to a header on each run, so a test can count the runs on the + /// request itself. + fn counting_preparer( + _settings: &Settings, + request: &mut Request, + ) -> Result<(), Report> { + request + .headers_mut() + .append("x-probe-prepared", HeaderValue::from_static("1")); + Ok(()) + } + + #[test] + fn prepare_request_runs_the_preparers_once_for_a_request() { + let settings = crate::test_support::tests::create_test_settings(); + let extra = [crate::integrations::IntegrationBuilder::new( + "probe-counting", + "seam-probe", + never_registering_builder, + validate_nothing, + ) + .with_module_name("testing.probe-counting") + .with_request_preparer(counting_preparer)]; + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should build registry with a request preparer"); + let mut request = plain_request(); + + registry + .prepare_request(&settings, &mut request) + .expect("should prepare the request"); + registry + .prepare_request(&settings, &mut request) + .expect("should accept a request that is already prepared"); + + assert_eq!( + request.headers().get_all("x-probe-prepared").iter().count(), + 1, + "should run a preparer once however often the request is prepared" + ); + + let mut next = plain_request(); + registry + .prepare_request(&settings, &mut next) + .expect("should prepare the next request"); + assert_eq!( + next.headers().get_all("x-probe-prepared").iter().count(), + 1, + "should prepare each request" + ); + } + + /// What the second probe's request hook leaves for its finalizer. + #[derive(Debug, PartialEq)] + struct ProbeNote(&'static str); + + fn first_finalizer(_state: &IntegrationRequestState, response: &mut Response) { + response + .headers_mut() + .append("x-probe-finalized", HeaderValue::from_static("first")); + } + + fn second_finalizer(state: &IntegrationRequestState, response: &mut Response) { + let value = match state.get::("probe-second") { + Some(_) => "second-with-its-note", + None => "second", + }; + response + .headers_mut() + .append("x-probe-finalized", HeaderValue::from_static(value)); + } + + #[test] + fn finalize_response_runs_every_finalizer_in_registration_order_with_the_request_state() { + let settings = crate::test_support::tests::create_test_settings(); + let extra = [ + crate::integrations::IntegrationBuilder::new( + "probe-first", + "seam-probe", + never_registering_builder, + validate_nothing, + ) + .with_module_name("testing.probe-first") + .with_response_finalizer(first_finalizer), + crate::integrations::IntegrationBuilder::new( + "probe-second", + "seam-probe", + never_registering_builder, + validate_nothing, + ) + .with_module_name("testing.probe-second") + .with_response_finalizer(second_finalizer), + ]; + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should build registry with response finalizers"); + let finalized = |response: &Response| { + response + .headers() + .get_all("x-probe-finalized") + .iter() + .map(|value| value.to_str().expect("should be text").to_owned()) + .collect::>() + }; + + let mut request = plain_request(); + IntegrationRequestState::insert(&mut request, "probe-second", ProbeNote("left")); + let mut response = Response::builder() + .body(EdgeBody::empty()) + .expect("should build response"); + registry.finalize_response(&IntegrationRequestState::of(&request), &mut response); + + assert_eq!( + finalized(&response), + vec!["first", "second-with-its-note"], + "should run both finalizers in registration order, neither integration \ + running, and hand each what was left on the request" + ); + + let mut unmarked = Response::builder() + .body(EdgeBody::empty()) + .expect("should build response"); + registry.finalize_response(&IntegrationRequestState::default(), &mut unmarked); + assert_eq!( + finalized(&unmarked), + vec!["first", "second"], + "should hand a finalizer no state for a request nothing was left on" + ); + } + + #[test] + fn what_a_module_leaves_on_a_request_reaches_a_document_when_it_is_seeded() { + let mut request = plain_request(); + assert!( + IntegrationRequestState::of(&request).is_empty(), + "should hold nothing for a request no module left a value on" + ); + + IntegrationRequestState::insert(&mut request, "probe", ProbeNote("first")); + IntegrationRequestState::insert(&mut request, "probe", ProbeNote("second")); + IntegrationRequestState::insert(&mut request, "other", ProbeNote("other")); + let state = IntegrationRequestState::of(&request); + + assert!(!state.is_empty(), "should hold what was left"); + assert_eq!( + state.get::("probe").as_deref(), + Some(&ProbeNote("second")), + "should keep the last value of one type left for an integration" + ); + assert!( + state.get::("absent").is_none(), + "should hold nothing for an integration that left nothing" + ); + + let document = IntegrationDocumentState::default(); + assert!( + document.get::("probe").is_none(), + "should start a document with none of the request's values" + ); + state.seed(&document); + assert_eq!( + document.get::("probe").as_deref(), + Some(&ProbeNote("second")), + "should copy the value into the document's state" + ); + assert_eq!( + document.get::("other").as_deref(), + Some(&ProbeNote("other")), + "should copy every integration's value" ); } #[test] - fn filter_request_applies_request_headers_and_returns_response_headers() { - let registry = - IntegrationRegistry::from_request_filters(vec![Arc::new(EnrichingRequestFilter)]); + fn the_request_stand_in_marks_a_navigation_and_strips_its_query() { + use super::test_support::request_fixture; + let settings = crate::test_support::tests::create_test_settings(); - let services = crate::platform::test_support::noop_services(); - let mut req = Request::builder() + let registry = IntegrationRegistry::new(&settings).expect("should build registry"); + let mut navigation = Request::builder() .method(Method::GET) - .uri("https://example.com/page") + .uri("https://publisher.example.com/article?fixture_request=1&keep=yes") + .header("sec-fetch-dest", "document") + .body(EdgeBody::empty()) + .expect("should build request"); + let mut subresource = Request::builder() + .method(Method::GET) + .uri("https://publisher.example.com/data.json?fixture_request=1") + .header("sec-fetch-dest", "empty") .body(EdgeBody::empty()) .expect("should build request"); - let outcome = - futures::executor::block_on(registry.filter_request(RequestFilterRegistryInput { - settings: &settings, - services: &services, - req: &mut req, - geo_info: None, - })) - .expect("should run request filter"); + registry + .prepare_request(&settings, &mut navigation) + .expect("should prepare the navigation"); + registry + .prepare_request(&settings, &mut subresource) + .expect("should prepare the subresource request"); assert_eq!( - req.headers() - .get("x-datadome-isbot") - .and_then(|value| value.to_str().ok()), - Some("1"), - "should apply DataDome-style request enrichment before routing" + navigation.uri().query(), + Some("keep=yes"), + "should strip the stand-in's query and keep the rest" ); assert!( - req.extensions().get::().is_some(), - "should preserve private request annotations for downstream routing" + IntegrationRequestState::of(&navigation) + .get::(request_fixture::ID) + .is_some(), + "should leave the mark on a document navigation, selected or not" ); - match outcome { - RequestFilterRegistryOutcome::Continue(effects) => { - assert_eq!( - effects.response_headers, - vec![HeaderMutation::set("x-dd-b", "allowed")], - "should return downstream response header effects for finalization" - ); + assert_eq!( + subresource.uri().query(), + None, + "should strip the stand-in's query from any request" + ); + assert!( + IntegrationRequestState::of(&subresource).is_empty(), + "should leave no mark on a request for something other than a document" + ); + } + + /// Example country code returned by the test geo module. `ZZ` is the + /// user-assigned code, so it names no real place. + const GEO_PROBE_COUNTRY: &str = "ZZ"; + + /// A geo module that resolves one fixed location, so a test can tell the + /// module's module apart from the "no location" one. + #[derive(Debug)] + struct FixedCountryGeo; + + #[async_trait::async_trait(?Send)] + impl crate::platform::PlatformGeo for FixedCountryGeo { + async fn lookup( + &self, + _client_ip: Option, + _services: &crate::platform::RuntimeServices, + ) -> Result, Report> { + Ok(Some(GeoInfo { + city: "Example City".to_owned(), + country: GEO_PROBE_COUNTRY.to_owned(), + continent: "Example".to_owned(), + latitude: 0.0, + longitude: 0.0, + metro_code: 0, + region: None, + asn: None, + })) + } + } + + /// The name the `geo-probe` registration's geo module is selected by. + const GEO_PROBE_MODULE: &str = "testing.geo-probe"; + + /// Builds a `geo-probe` registration that declares a geo module. + fn geo_probe_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("geo-probe") + .with_geo_module(GEO_PROBE_MODULE, Arc::new(FixedCountryGeo)) + .build(), + )) + } + + /// A device module that declares it needs a permission, which nothing can + /// enforce per request yet. + #[derive(Debug)] + struct PermissionDemandingDevice; + + /// The name the `device-probe` registration's device module is selected + /// by. + const DEVICE_PROBE_MODULE: &str = "testing.device-probe"; + + #[async_trait::async_trait(?Send)] + impl crate::ec::device::DeviceModule for PermissionDemandingDevice { + fn id(&self) -> &'static str { + DEVICE_PROBE_MODULE + } + + async fn detect( + &self, + _call: crate::module_context::ModuleCall<'_>, + ) -> crate::ec::device::DeviceSignals { + crate::ec::device::DeviceSignals { + is_mobile: 2, + ja4_class: None, + platform_class: None, + h2_fp_hash: None, + known_browser: None, + looks_like_browser: false, } - RequestFilterRegistryOutcome::Respond { .. } => panic!("should continue routing"), } + + fn required_permissions(&self) -> crate::permissions::PermissionSet { + crate::permissions::PermissionSet::none() + .with(crate::permissions::Permission::StoreOnDevice) + } + } + + fn device_probe_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("device-probe") + .with_device_module(DEVICE_PROBE_MODULE, Arc::new(PermissionDemandingDevice)) + .build(), + )) + } + + fn device_probe_builders() -> [crate::integrations::IntegrationBuilder; 1] { + [crate::integrations::IntegrationBuilder::new( + "device-probe", + "seam-probe", + device_probe_registration, + validate_nothing, + ) + .with_module_name(DEVICE_PROBE_MODULE)] } #[test] - fn test_exact_route_matching() { - let routes = vec![( - Method::GET, - "/integrations/test/exact", - (Arc::new(MockProxy) as Arc, "test"), - )]; + fn a_modules_device_module_declaring_permissions_is_refused_at_startup() { + let mut settings = settings_naming("device-probe"); + settings.device.module = Some(DEVICE_PROBE_MODULE.to_owned()); - let registry = IntegrationRegistry::from_routes(routes); + let error = IntegrationRegistry::with_registrations(&settings, &device_probe_builders()) + .err() + .expect("should refuse a device module whose permissions nothing can enforce"); - // Should match exact route - assert!(registry.has_route(&Method::GET, "/integrations/test/exact")); + let rendered = format!("{error:?}"); + assert!( + rendered.contains("device-probe"), + "the failure should name the module, got: {rendered}" + ); + assert!( + rendered.contains(&format!( + "requiring `{}`", + crate::permissions::Permission::StoreOnDevice.as_str() + )), + "the failure should say why it was refused, got: {rendered}" + ); + } - // Should not match different paths - assert!(!registry.has_route(&Method::GET, "/integrations/test/other")); - assert!(!registry.has_route(&Method::GET, "/integrations/test/exact/nested")); + fn geo_probe_builders() -> [crate::integrations::IntegrationBuilder; 1] { + [crate::integrations::IntegrationBuilder::new( + "geo-probe", + "seam-probe", + geo_probe_registration, + validate_nothing, + ) + .with_module_name(GEO_PROBE_MODULE)] + } - // Should not match different methods - assert!(!registry.has_route(&Method::POST, "/integrations/test/exact")); + /// The names the two prefix test modules run under. + const PREFIX_PROBE_MODULE: &str = "testing.prefix-probe"; + const PREFIX_READER_MODULE: &str = "testing.prefix-reader"; + + /// A prefix module giving `_ex` and `ex.`, so a test can tell it + /// apart from core's own. + #[derive(Debug)] + struct ExamplePrefix; + + impl ExamplePrefix { + fn server_key(&self, settings: &Settings) -> crate::platform::ServerKeyPrefix { + crate::platform::ServerKeyPrefix::new("ex", &settings.publisher.domain) + } } - #[test] - fn test_wildcard_route_matching() { - let routes = vec![( - Method::GET, - "/integrations/lockr/api/*", - (Arc::new(MockProxy) as Arc, "lockr"), - )]; + impl PrefixModule for ExamplePrefix { + fn public_prefix( + &self, + _call: ModuleCall<'_>, + ) -> Result { + Ok(crate::prefix::PublicPrefix::new(["_ex"]).expect("should accept the prefix")) + } - let registry = IntegrationRegistry::from_routes(routes); + fn server_key_prefix( + &self, + call: ModuleCall<'_>, + ) -> Result { + call.inject(self, Self::server_key) + } + } - // Should match paths under the wildcard prefix - assert!(registry.has_route(&Method::GET, "/integrations/lockr/api/settings")); - assert!(registry.has_route( - &Method::GET, - "/integrations/lockr/api/publisher/app/v1/identityLockr/settings" - )); - assert!(registry.has_route(&Method::GET, "/integrations/lockr/api/page-view")); - assert!(registry.has_route(&Method::GET, "/integrations/lockr/api/a/b/c/d/e")); + /// A prefix module naming the reader's evidence, which its call never + /// carries. + #[derive(Debug)] + struct EvidenceReadingPrefix; - // Should not match paths that don't start with the prefix - assert!(!registry.has_route(&Method::GET, "/integrations/lockr/sdk")); - assert!(!registry.has_route(&Method::GET, "/integrations/lockr/other")); - assert!(!registry.has_route(&Method::GET, "/integrations/other/api/settings")); + impl EvidenceReadingPrefix { + fn public( + &self, + evidence: &dyn crate::evidence::RequestInfo, + ) -> crate::prefix::PublicPrefix { + crate::prefix::PublicPrefix::new([evidence.user_agent()]) + .expect("should accept the prefix") + } + } - // Should not match different methods - assert!(!registry.has_route(&Method::POST, "/integrations/lockr/api/settings")); + impl PrefixModule for EvidenceReadingPrefix { + fn public_prefix( + &self, + call: ModuleCall<'_>, + ) -> Result { + call.inject(self, Self::public) + } + + fn server_key_prefix( + &self, + _call: ModuleCall<'_>, + ) -> Result { + Ok(crate::platform::ServerKeyPrefix::new("ev", "example.com")) + } } - #[test] - fn test_wildcard_and_exact_routes_coexist() { - let routes = vec![ - ( - Method::GET, - "/integrations/test/api/*", - (Arc::new(MockProxy) as Arc, "test"), - ), - ( - Method::GET, - "/integrations/test/exact", - (Arc::new(MockProxy) as Arc, "test"), - ), - ]; + fn prefix_probe_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("prefix-probe") + .with_prefix_module(PREFIX_PROBE_MODULE, Arc::new(ExamplePrefix)) + .build(), + )) + } - let registry = IntegrationRegistry::from_routes(routes); + fn prefix_reader_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("prefix-reader") + .with_prefix_module(PREFIX_READER_MODULE, Arc::new(EvidenceReadingPrefix)) + .build(), + )) + } - // Exact route should match - assert!(registry.has_route(&Method::GET, "/integrations/test/exact")); + fn prefix_probe_builders() -> [crate::integrations::IntegrationBuilder; 2] { + [ + crate::integrations::IntegrationBuilder::new( + "prefix-probe", + "seam-probe", + prefix_probe_registration, + validate_nothing, + ) + .with_module_name(PREFIX_PROBE_MODULE), + crate::integrations::IntegrationBuilder::new( + "prefix-reader", + "seam-probe", + prefix_reader_registration, + validate_nothing, + ) + .with_module_name(PREFIX_READER_MODULE), + ] + } - // Wildcard routes should match - assert!(registry.has_route(&Method::GET, "/integrations/test/api/anything")); - assert!(registry.has_route(&Method::GET, "/integrations/test/api/nested/path")); + /// Settings running both probe modules on `www.example.com`, with + /// `[prefix] module` naming `selected` when given. + fn settings_selecting_prefix_module(selected: Option<&str>) -> Settings { + let mut settings = crate::test_support::tests::create_test_settings(); + settings.publisher.domain = "www.example.com".to_owned(); + for id in ["prefix-probe", "prefix-reader"] { + settings + .insert_module_config("testing", &format!("testing.{id}"), &serde_json::json!({})) + .expect("should select the test module"); + } + settings.prefix.module = selected.map(str::to_owned); + settings + } - // Non-matching should fail - assert!(!registry.has_route(&Method::GET, "/integrations/test/other")); + #[test] + fn with_no_prefix_module_selected_the_registry_gives_core_ts() { + let settings = settings_selecting_prefix_module(None); + let registry = IntegrationRegistry::with_registrations(&settings, &prefix_probe_builders()) + .expect("should build the registry"); + + let prefixes = + registry.prefixes(&settings, crate::module_context::test_support::request("/")); + + assert_eq!(registry.prefix_module().name(), "ts"); + assert_eq!(prefixes.public().as_str(), "ts"); + assert_eq!(prefixes.public().cookie_name("ec"), "ts-ec"); + assert_eq!(prefixes.server_key().as_str(), "ts.example.com"); } #[test] - fn test_multiple_wildcard_routes() { - let routes = vec![ - ( - Method::GET, - "/integrations/lockr/api/*", - (Arc::new(MockProxy) as Arc, "lockr"), - ), - ( - Method::POST, - "/integrations/lockr/api/*", - (Arc::new(MockProxy) as Arc, "lockr"), - ), - ( - Method::GET, - "/integrations/testlight/api/*", - ( - Arc::new(MockProxy) as Arc, - "testlight", - ), - ), - ]; + fn a_prefix_selector_naming_ts_selects_cores_own() { + let settings = settings_selecting_prefix_module(Some("ts")); + let registry = IntegrationRegistry::with_registrations(&settings, &prefix_probe_builders()) + .expect("should accept the name the registry reports core's module by"); - let registry = IntegrationRegistry::from_routes(routes); + let prefixes = + registry.prefixes(&settings, crate::module_context::test_support::request("/")); - // Lockr GET routes should match - assert!(registry.has_route(&Method::GET, "/integrations/lockr/api/settings")); + assert_eq!(registry.prefix_module().name(), "ts"); + assert_eq!(prefixes.public().cookie_name("ec"), "ts-ec"); + assert_eq!(prefixes.server_key().as_str(), "ts.example.com"); + } - // Lockr POST routes should match - assert!(registry.has_route(&Method::POST, "/integrations/lockr/api/settings")); + #[test] + fn the_selected_prefix_module_gives_the_prefixes() { + let settings = settings_selecting_prefix_module(Some(PREFIX_PROBE_MODULE)); + let registry = IntegrationRegistry::with_registrations(&settings, &prefix_probe_builders()) + .expect("should build the registry"); - // Testlight routes should match - assert!(registry.has_route(&Method::GET, "/integrations/testlight/api/auction")); - assert!(registry.has_route(&Method::GET, "/integrations/testlight/api/any-path")); + let prefixes = + registry.prefixes(&settings, crate::module_context::test_support::request("/")); - // Cross-integration paths should not match - assert!(!registry.has_route(&Method::GET, "/integrations/lockr/other-endpoint")); - assert!(!registry.has_route(&Method::GET, "/integrations/other/api/test")); + assert_eq!(registry.prefix_module().name(), PREFIX_PROBE_MODULE); + assert_eq!(prefixes.public().cookie_name("ec"), "_ex-ec"); + assert_eq!(prefixes.server_key().as_str(), "ex.example.com"); } #[test] - fn test_wildcard_preserves_casing() { - let routes = vec![( - Method::GET, - "/integrations/lockr/api/*", - (Arc::new(MockProxy) as Arc, "lockr"), - )]; + fn a_prefix_module_nothing_supplies_is_refused() { + let settings = settings_selecting_prefix_module(Some("missing")); - let registry = IntegrationRegistry::from_routes(routes); + let error = IntegrationRegistry::with_registrations(&settings, &prefix_probe_builders()) + .err() + .expect("a prefix module nothing supplies should be refused"); - // Should match with camelCase preserved - assert!(registry.has_route( - &Method::GET, - "/integrations/lockr/api/publisher/app/v1/identityLockr/settings" - )); - assert!(registry.has_route( - &Method::GET, - "/integrations/lockr/api/publisher/app/v1/identitylockr/settings" - )); + assert!( + error.to_string().contains("[prefix] module") && error.to_string().contains("missing"), + "{error}" + ); } #[test] - fn test_wildcard_edge_cases() { - let routes = vec![( - Method::GET, - "/api/*", - (Arc::new(MockProxy) as Arc, "test"), - )]; + fn a_prefix_module_naming_what_its_call_never_carries_is_refused() { + let settings = settings_selecting_prefix_module(Some(PREFIX_READER_MODULE)); - let registry = IntegrationRegistry::from_routes(routes); + let error = IntegrationRegistry::with_registrations(&settings, &prefix_probe_builders()) + .err() + .expect("a prefix module that cannot answer should be refused at startup"); + + assert!( + error.to_string().contains(PREFIX_READER_MODULE) + && error.to_string().contains("request evidence"), + "{error}" + ); + } + + /// Settings whose `[geo] module` names `module`, which may be a value + /// core resolves itself, such as `none` or `platform`. + fn settings_with_geo_selector(module: &str) -> Settings { + let mut settings = crate::test_support::tests::create_test_settings(); + settings.geo.module = Some(module.to_owned()); + settings + } + + /// The same, with the module selected in its section as well, which is + /// what a deployment running a module's geo module writes. + fn settings_selecting_geo_module(name: &str) -> Settings { + let mut settings = settings_with_geo_selector(name); + let section = name.split_once('.').map_or("testing", |(folder, _)| folder); + settings.select_module(section, name); + settings + } + + #[tokio::test] + async fn geo_selector_resolves_the_geo_module_the_selected_module_declares() { + let settings = settings_selecting_geo_module(GEO_PROBE_MODULE); + + let registry = IntegrationRegistry::with_registrations(&settings, &geo_probe_builders()) + .expect("should build registry with a module geo module"); + + let module = registry + .geo_module() + .expect("should resolve the module's geo module"); + let resolved = module + .lookup(None, &noop_services()) + .await + .expect("should look up without failing") + .expect("should resolve a location"); + assert_eq!( + resolved.country, GEO_PROBE_COUNTRY, + "should resolve through the module's own module" + ); + } + + #[tokio::test] + async fn geo_module_none_resolves_no_location() { + let settings = settings_with_geo_selector("none"); + + let registry = IntegrationRegistry::with_registrations(&settings, &geo_probe_builders()) + .expect("should build registry with the disabled geo module"); + + let module = registry + .geo_module() + .expect("should resolve the disabled geo module"); + assert!( + module + .lookup(None, &noop_services()) + .await + .expect("should look up without failing") + .is_none(), + "should resolve no location when the selector is `none`" + ); + } + + #[tokio::test] + async fn an_unset_geo_selector_resolves_the_disabled_module_and_platform_opts_in() { + // Unset is the permission model's privacy default: it resolves the + // disabled module, so no client IP ever reaches a host geo service. + // It is deliberately not the same as leaving the adapter's own lookup + // in place, which is what `platform` spells. + let settings = crate::test_support::tests::create_test_settings(); + assert!( + settings.geo.module.is_none(), + "the shared test settings should leave the geo selector unset" + ); + + let registry = IntegrationRegistry::with_registrations(&settings, &geo_probe_builders()) + .expect("should build registry with an unset geo selector"); + + let module = registry + .geo_module() + .expect("an unset selector should resolve the disabled module, not nothing"); + assert!( + module + .lookup(None, &noop_services()) + .await + .expect("should look up without failing") + .is_none(), + "the disabled module should resolve no location and make no host call" + ); + + // `platform` is the explicit opt-in to the adapter's own lookup, and it + // is the one case that resolves nothing here so the adapter's module + // stands. + let platform = settings_with_geo_selector(GEO_MODULE_PLATFORM); + let registry = IntegrationRegistry::with_registrations(&platform, &geo_probe_builders()) + .expect("should build registry with the platform geo selector"); + assert!( + registry.geo_module().is_none(), + "`platform` should leave the adapter's own host lookup in place" + ); + } - // Should match paths under /api/ - assert!(registry.has_route(&Method::GET, "/api/v1")); - assert!(registry.has_route(&Method::GET, "/api/v1/users")); + #[test] + fn geo_selector_rejects_a_module_that_declares_no_geo_module() { + let settings = settings_selecting_geo_module("testing.probe"); + let extra = [crate::integrations::IntegrationBuilder::new( + "probe", + "seam-probe", + probe_registration, + validate_nothing, + ) + .with_module_name("testing.probe")]; - // Should not match /api without trailing content - // The current implementation requires a / after the prefix - assert!(!registry.has_route(&Method::GET, "/api")); + let error = IntegrationRegistry::with_registrations(&settings, &extra) + .err() + .expect("should reject a module that declares no geo module"); - // Should not match partial prefix matches - assert!(!registry.has_route(&Method::GET, "/apiv1")); + let message = error.to_string(); + assert!( + message.contains("`[geo] module` names `testing.probe`") + && message.contains("It runs no geo module") + && message.contains("is selected and supplies no geo module"), + "error should name the module and say it supplies no geo module: {message}" + ); } #[test] - fn test_helper_methods_create_namespaced_routes() { - let proxy = Arc::new(MockProxy); + fn geo_selector_rejects_a_module_no_section_selects() { + // The module is registered, but no section selects it, so its geo + // module is not there to select. + let settings = settings_with_geo_selector("testing.probe-unnamed"); + let extra = [crate::integrations::IntegrationBuilder::new( + "probe-unnamed", + "seam-probe", + never_registering_builder, + validate_nothing, + ) + .with_module_name("testing.probe-unnamed")]; - // Test all HTTP method helpers - let get_endpoint = proxy.get("/users"); - assert_eq!(get_endpoint.method, Method::GET); - assert_eq!(get_endpoint.path, "/integrations/test/users"); + let error = IntegrationRegistry::with_registrations(&settings, &extra) + .err() + .expect("should reject a module no section selects"); - let post_endpoint = proxy.post("/users"); - assert_eq!(post_endpoint.method, Method::POST); - assert_eq!(post_endpoint.path, "/integrations/test/users"); + let message = error.to_string(); + assert!( + message.contains("`[geo] module` names `testing.probe-unnamed`") + && message.contains("is a module no section selects"), + "error should name the module and say no section selects it: {message}" + ); + } - let put_endpoint = proxy.put("/users"); - assert_eq!(put_endpoint.method, Method::PUT); - assert_eq!(put_endpoint.path, "/integrations/test/users"); + /// An id no builder supplies is refused here, where the adapter's and a + /// vendor crate's builders are known, rather than in the settings. + #[test] + fn an_id_no_builder_supplies_is_refused_at_registry_build() { + let settings = settings_naming("a_vendors_own_integration"); + let extra = [crate::integrations::IntegrationBuilder::new( + "probe", + "seam-probe", + probe_registration, + validate_nothing, + ) + .with_module_name("testing.probe")]; - let delete_endpoint = proxy.delete("/users"); - assert_eq!(delete_endpoint.method, Method::DELETE); - assert_eq!(delete_endpoint.path, "/integrations/test/users"); + let error = IntegrationRegistry::with_registrations(&settings, &extra) + .err() + .expect("should refuse an id no builder in this deployment supplies"); - let patch_endpoint = proxy.patch("/users"); - assert_eq!(patch_endpoint.method, Method::PATCH); - assert_eq!(patch_endpoint.path, "/integrations/test/users"); + let message = error.to_string(); + assert!( + message.contains("a_vendors_own_integration"), + "should name the id nothing supplies: {message}" + ); + assert!( + message.contains("The testing modules it supplies are") && message.contains("probe"), + "should list the modules of that type this deployment does supply: {message}" + ); } + /// A demand or ad server implementation is not a page integration, so a + /// section naming one is refused like any name no builder supplies. #[test] - fn test_put_delete_patch_routes() { - let routes = vec![ - ( - Method::PUT, - "/integrations/test/users", - (Arc::new(MockProxy) as Arc, "test"), - ), - ( - Method::DELETE, - "/integrations/test/users", - (Arc::new(MockProxy) as Arc, "test"), - ), - ( - Method::PATCH, - "/integrations/test/users", - (Arc::new(MockProxy) as Arc, "test"), - ), - ]; + fn an_auction_implementation_named_in_a_section_is_refused() { + for (section, name) in [ + ("auction", "fixture"), + ("auction", "ad-server.fixture"), + ("auction", "plain-fixture"), + ] { + let mut settings = crate::test_support::tests::create_test_settings(); + settings.select_module(section, name); + + let error = IntegrationRegistry::new(&settings) + .err() + .expect("should refuse an implementation named as a page module"); + + let message = error.to_string(); + assert!( + message.contains(&format!("[{section}] selects `{name}`")), + "should name the section and the implementation: {message}" + ); + } + } - let registry = IntegrationRegistry::from_routes(routes); + /// The same id is accepted once a builder supplies it, so a vendor crate + /// an adapter composes in is named the same way a built-in is. + #[test] + fn an_id_a_supplied_builder_claims_is_accepted() { + let settings = settings_naming("probe"); + let extra = [crate::integrations::IntegrationBuilder::new( + "probe", + "seam-probe", + probe_registration, + validate_nothing, + ) + .with_module_name("testing.probe")]; - // Should match PUT, DELETE, and PATCH routes - assert!(registry.has_route(&Method::PUT, "/integrations/test/users")); - assert!(registry.has_route(&Method::DELETE, "/integrations/test/users")); - assert!(registry.has_route(&Method::PATCH, "/integrations/test/users")); + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should accept an id a supplied builder claims"); - // Should not match other methods on same path - assert!(!registry.has_route(&Method::GET, "/integrations/test/users")); - assert!(!registry.has_route(&Method::POST, "/integrations/test/users")); + assert!( + registry.integration_runs("probe"), + "the named module should run" + ); } - // Tests for EC ID header on proxy responses - use crate::test_support::tests::create_test_settings; + #[test] + fn geo_selector_rejects_a_name_no_module_supplies() { + // Nothing supplies the name, as a geo module or as a module at all. + let settings = settings_with_geo_selector("absent-module"); - /// Mock proxy that returns a simple 200 OK response - struct EcTestProxy; + let error = IntegrationRegistry::with_registrations(&settings, &geo_probe_builders()) + .err() + .expect("should reject a selector naming nothing this deployment runs"); - #[async_trait(?Send)] - impl IntegrationProxy for EcTestProxy { - fn integration_name(&self) -> &'static str { - "ec_test" - } + let message = error.to_string(); + assert!( + message.contains("`[geo] module` names `absent-module`") + && message.contains("which no module this deployment runs supplies") + && !message.contains("no section selects"), + "error should name what was written and say nothing supplies it: {message}" + ); + } - fn routes(&self) -> Vec { - vec![ - IntegrationEndpoint { - method: Method::GET, - path: "/integrations/test/ec".to_owned(), - }, - IntegrationEndpoint { - method: Method::POST, - path: "/integrations/test/ec".to_owned(), - }, - ] + /// A device module that needs no permission, for a registration that + /// supplies more than one type of module. + #[derive(Debug)] + struct ExampleDevice; + + #[async_trait::async_trait(?Send)] + impl crate::ec::device::DeviceModule for ExampleDevice { + fn id(&self) -> &'static str { + "device.example" } - async fn handle( + async fn detect( &self, - _settings: &Settings, - _services: &RuntimeServices, - req: Request, - ) -> Result, Report> { - let mut response = Response::builder() - .status(StatusCode::OK) - .body(EdgeBody::from("test response")) - .expect("should build test response"); - if let Some(ec) = req.headers().get(HEADER_X_TS_EC.clone()) { - response - .headers_mut() - .insert(http::HeaderName::from_static("x-echo-ts-ec"), ec.clone()); + _call: crate::module_context::ModuleCall<'_>, + ) -> crate::ec::device::DeviceSignals { + crate::ec::device::DeviceSignals { + is_mobile: 1, + ja4_class: None, + platform_class: Some("example".to_owned()), + h2_fp_hash: None, + known_browser: None, + looks_like_browser: true, } - Ok(response) } } - #[test] - fn handle_proxy_removes_ec_id_header_on_request() { - let settings = create_test_settings(); - let routes = vec![( - Method::GET, - "/integrations/test/ec", - ( - Arc::new(EcTestProxy) as Arc, - "ec_test", - ), - )]; - let registry = IntegrationRegistry::from_routes(routes); + /// An Edge Cookie module that answers to `edgecookie.example`. + #[derive(Debug)] + struct ExampleEc; - let mut req = Request::builder() - .method(Method::GET) - .uri("https://test-publisher.com/integrations/test/ec") - .body(EdgeBody::empty()) - .expect("should build request"); - req.headers_mut().insert( - HEADER_X_TS_EC.clone(), - HeaderValue::from_static("some-ec-value"), - ); - let mut ec_context = - EcContext::new_for_test(None, crate::consent::ConsentContext::default()); - let services = noop_services(); + #[async_trait::async_trait(?Send)] + impl EdgeCookieModule for ExampleEc { + fn id(&self) -> &'static str { + "edgecookie.example" + } - // Call handle_proxy (uses futures executor in test environment) - let result = futures::executor::block_on(registry.handle_proxy(ProxyDispatchInput { - method: &Method::GET, - path: "/integrations/test/ec", - settings: &settings, - kv: None, - ec_context: &mut ec_context, - services: &services, - req, - })); + fn code(&self) -> crate::ec::module::ModuleCode { + crate::module_code!("t0rg") + } - // Should have matched and returned a response - assert!(result.is_some(), "should find route and handle request"); - let response = result.unwrap(); - assert!(response.is_ok(), "handler should succeed"); + async fn generate( + &self, + _call: crate::module_context::ModuleCall<'_>, + ) -> Result> { + Ok(crate::ec::module::GeneratedEdgeCookie::default()) + } + } - let response = response.unwrap(); + /// Builds one registration that supplies a geo, a device and an Edge + /// Cookie module, each under the name of its own type. + fn example_vendor_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("example-vendor") + .with_geo_module("geo.example", Arc::new(FixedCountryGeo)) + .with_device_module("device.example", Arc::new(ExampleDevice)) + .with_ec_module("edgecookie.example", Arc::new(ExampleEc)) + .build(), + )) + } - assert!( - response.headers().get("x-echo-ts-ec").is_none(), - "should not have x-ts-ec header on integration request" - ); + fn example_vendor_builders() -> [crate::integrations::IntegrationBuilder; 1] { + [crate::integrations::IntegrationBuilder::new( + "example-vendor", + "seam-probe", + example_vendor_registration, + validate_nothing, + ) + .with_module_name("testing.example-vendor")] } + /// One registration can supply a module of each type, and each selector + /// finds its own by the short name, the type folder left off. #[test] - fn handle_proxy_rejects_invalid_ec_request_header() { - let settings = create_test_settings(); - let routes = vec![( - Method::GET, - "/integrations/test/ec", - ( - Arc::new(EcTestProxy) as Arc, - "ec_test", - ), - )]; - let registry = IntegrationRegistry::from_routes(routes); + fn each_selector_resolves_its_own_type_of_module_from_one_registration() { + let mut settings = settings_naming("example-vendor"); + settings.geo.module = Some("example".to_owned()); + settings.device.module = Some("example".to_owned()); + settings.ec.module = Some(EcModuleSelection::from("example")); - let mut req = Request::builder() - .method(Method::GET) - .uri("https://test-publisher.com/integrations/test/ec") - .body(EdgeBody::empty()) - .expect("should build request"); - req.headers_mut().insert( - HEADER_X_TS_EC.clone(), - HeaderValue::from_static("evil;injected"), - ); - let mut ec_context = - EcContext::new_for_test(None, crate::consent::ConsentContext::default()); + let registry = + IntegrationRegistry::with_registrations(&settings, &example_vendor_builders()) + .expect("should build registry with one module of each type"); - let services = crate::platform::test_support::noop_services(); + assert!( + registry.geo_module().is_some(), + "`[geo] module = \"example\"` should resolve `geo.example`" + ); + assert_eq!( + registry + .device_module() + .expect("`[device] module = \"example\"` should resolve `device.example`") + .id(), + "device.example", + "should resolve the registration's device module" + ); + assert_eq!( + registry + .ec_module() + .expect("`[ec] module = \"example\"` should resolve `edgecookie.example`") + .id(), + "edgecookie.example", + "should resolve the registration's Edge Cookie module" + ); + } - let result = futures::executor::block_on(registry.handle_proxy(ProxyDispatchInput { - method: &Method::GET, - path: "/integrations/test/ec", - settings: &settings, - kv: None, - ec_context: &mut ec_context, - services: &services, - req, - })) - .expect("should handle proxy request"); + /// A name written in full selects the same module, and a short name is + /// read within the selector's own type only. + #[test] + fn a_selector_reads_a_full_name_and_keeps_a_short_name_within_its_type() { + let mut settings = settings_naming("example-vendor"); + settings.geo.module = Some("geo.example".to_owned()); - let response = result.expect("handler should succeed"); + let registry = + IntegrationRegistry::with_registrations(&settings, &example_vendor_builders()) + .expect("should resolve a geo module written in full"); + assert!( + registry.geo_module().is_some(), + "`[geo] module = \"geo.example\"` should resolve the module" + ); + // `device.example` is a device module, so `[geo]` does not find it + // under the short name of another type. + let mut settings = settings_naming("example-vendor"); + settings.geo.module = Some("device.example".to_owned()); + let error = IntegrationRegistry::with_registrations(&settings, &example_vendor_builders()) + .err() + .expect("should refuse a device module named as a geo module"); + let message = error.to_string(); assert!( - response.headers().get("x-echo-ts-ec").is_none(), - "should not reflect the tampered request header to the integration" + message.contains("`[geo] module` names `device.example`") + && message.contains("The geo modules it runs are [example]"), + "error should list the geo modules by the name the section writes: {message}" ); } + /// `[ec] module` may name a label whose block names the implementation, + /// and the registry resolves the implementation, as core does. #[test] - fn handle_proxy_removes_request_ec_header_even_when_consent_denied() { - let settings = create_test_settings(); - let routes = vec![( - Method::GET, - "/integrations/test/ec", - (Arc::new(EcTestProxy) as Arc, "test"), - )]; + fn ec_selector_resolves_the_implementation_a_labelled_block_names() { + let mut settings = settings_naming("example-vendor"); + settings.ec.module = Some(EcModuleSelection::from("primary")); + settings.ec.module_blocks.insert( + "primary".to_owned(), + crate::settings::EcModuleBlock { + implementation: Some("example".to_owned()), + settings: crate::settings::EcModuleSettings::Injected(serde_json::Map::new()), + }, + ); - let registry = IntegrationRegistry::from_routes(routes); + let registry = + IntegrationRegistry::with_registrations(&settings, &example_vendor_builders()) + .expect("should build registry with a labelled Edge Cookie selection"); - let mut req = Request::builder() - .method(Method::GET) - .uri("https://test.example.com/integrations/test/ec") - .body(EdgeBody::empty()) - .expect("should build request"); - req.headers_mut().insert( - header::COOKIE, - HeaderValue::from_str(&format!( - "{}={}", - COOKIE_TS_EC, - crate::test_support::tests::VALID_SYNTHETIC_ID - )) - .expect("should build Cookie header"), + assert_eq!( + registry + .ec_module() + .expect("should resolve the implementation the block names") + .id(), + "edgecookie.example", + "should resolve the registration's Edge Cookie module" ); - let mut ec_context = - EcContext::new_for_test(None, crate::consent::ConsentContext::default()); - - let services = crate::platform::test_support::noop_services(); + } - let result = futures::executor::block_on(registry.handle_proxy(ProxyDispatchInput { - method: &Method::GET, - path: "/integrations/test/ec", - settings: &settings, - kv: None, - ec_context: &mut ec_context, - services: &services, - req, - })) - .expect("should handle proxy request"); + /// Core's own Edge Cookie modules are not the registry's to supply, so a + /// selection naming one resolves nothing here and is not an error. + #[test] + fn ec_selector_naming_a_module_of_cores_resolves_nothing_here() { + let settings = settings_naming("example-vendor"); + assert!( + settings + .ec + .module + .as_ref() + .is_some_and(|selection| selection.key() == crate::ec::module::HMAC_MODULE_KEY), + "the shared test settings should select core's HMAC module" + ); - let response = result.expect("proxy handle should succeed"); + let registry = + IntegrationRegistry::with_registrations(&settings, &example_vendor_builders()) + .expect("should build registry with core's own Edge Cookie module selected"); assert!( - response.headers().get("x-echo-ts-ec").is_none(), - "should not set x-ts-ec on integration request" + registry.ec_module().is_none(), + "core resolves its own module, so the registry should supply none" ); } + fn misnamed_ec_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("misnamed") + .with_ec_module("edgecookie.another", Arc::new(ExampleEc)) + .build(), + )) + } + + /// Core checks the selected Edge Cookie module by its own id, so one + /// declared under another name is refused where the mistake is made. #[test] - fn handle_proxy_works_with_post_method() { - let settings = create_test_settings(); - let routes = vec![( - Method::POST, - "/integrations/test/ec", - ( - Arc::new(EcTestProxy) as Arc, - "ec_test", - ), - )]; - let registry = IntegrationRegistry::from_routes(routes); + fn an_edge_cookie_module_declared_under_another_name_is_refused() { + let settings = settings_naming("misnamed"); + let extra = [crate::integrations::IntegrationBuilder::new( + "misnamed", + "seam-probe", + misnamed_ec_registration, + validate_nothing, + ) + .with_module_name("testing.misnamed")]; - let mut req = Request::builder() - .method(Method::POST) - .uri("https://test-publisher.com/integrations/test/ec") - .body(EdgeBody::from("test body")) - .expect("should build POST request"); - req.headers_mut().insert( - HEADER_X_TS_EC.clone(), - HeaderValue::from_static("some-ec-value"), + let error = IntegrationRegistry::with_registrations(&settings, &extra) + .err() + .expect("should refuse an Edge Cookie module declared under another name"); + + let message = error.to_string(); + assert!( + message.contains("`edgecookie.another`") && message.contains("`edgecookie.example`"), + "error should give both names: {message}" ); - let mut ec_context = - EcContext::new_for_test(None, crate::consent::ConsentContext::default()); + } - let services = crate::platform::test_support::noop_services(); + fn second_geo_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("geo-second") + .with_geo_module(GEO_PROBE_MODULE, Arc::new(FixedCountryGeo)) + .build(), + )) + } - let result = futures::executor::block_on(registry.handle_proxy(ProxyDispatchInput { - method: &Method::POST, - path: "/integrations/test/ec", - settings: &settings, - kv: None, - ec_context: &mut ec_context, - services: &services, - req, - })); + /// Two registrations supplying a module of one type under one name would + /// leave a selector meaning either, so the pair is refused. + #[test] + fn two_registrations_declaring_one_geo_module_name_are_refused() { + let mut settings = settings_naming("geo-probe"); + settings.select_module("testing", "testing.geo-second"); + let extra = [ + geo_probe_builders()[0], + crate::integrations::IntegrationBuilder::new( + "geo-second", + "seam-probe", + second_geo_registration, + validate_nothing, + ) + .with_module_name("testing.geo-second"), + ]; - assert!(result.is_some(), "Should find POST route"); - let response = result.unwrap(); - assert!(response.is_ok(), "Handler should succeed"); + let error = IntegrationRegistry::with_registrations(&settings, &extra) + .err() + .expect("should refuse one geo module name declared twice"); - let response = response.unwrap(); + let message = error.to_string(); assert!( - response.headers().get("x-echo-ts-ec").is_none(), - "POST integration request should not include x-ts-ec" + message.contains("the geo module `testing.geo-probe` is declared twice") + && message.contains("`geo-probe`") + && message.contains("`geo-second`"), + "error should name the module and both integrations: {message}" ); } - #[test] - fn js_module_ids_defer_prebid_and_include_core_js_only_modules() { - let settings = crate::test_support::tests::create_test_settings(); - let mut settings_with_prebid = settings; - settings_with_prebid - .integrations - .insert_config( - "prebid", - &serde_json::json!({ - "enabled": true, - "external_bundle_url": "https://assets.example/prebid/trusted-prebid.js", - "timeout_ms": 1000, - "debug": false - }), - ) - .expect("should insert prebid config"); + fn html_entry(path: Option<&str>, names: &[&str]) -> PhaseEntry { + PhaseEntry { + media_type: HTML_MEDIA_TYPE.to_owned(), + path: path.map(str::to_owned), + middleware: names.iter().map(|name| (*name).to_owned()).collect(), + } + } - let registry = IntegrationRegistry::with_plan( - &settings_with_prebid, - Arc::new( - crate::auction::compile_auction_plan(&settings_with_prebid) - .expect("should compile auction plan"), - ), - ) - .expect("should create registry"); + /// Settings that select core's middleware stand-in and hold `entries`. + fn fixture_settings_with_entries(entries: Vec) -> Settings { + let mut settings = settings_naming(fixture::MODULE); + settings.fetch = PhaseEntries::new(entries); + settings + } - let all = registry.js_module_ids(); - let immediate = registry.js_module_ids_immediate(); - let deferred = registry.js_module_ids_deferred(); + #[test] + fn a_selected_module_s_middleware_run_where_an_entry_names_them() { + let settings = fixture_settings_with_entries(vec![ + html_entry(Some("/news/"), &[fixture::LINKS, fixture::HEAD]), + html_entry(None, &[fixture::HEAD]), + ]); - assert!( - all.contains(&"prebid"), - "should include the prebid shim in embedded TSJS module IDs" + let registry = IntegrationRegistry::new(&settings) + .expect("should build a registry whose entries name registered middleware"); + + assert_eq!( + registry.middleware_ids(), + [ + fixture::HEAD, + fixture::LINKS, + fixture::BROKEN, + fixture::READER, + fixture::BROKEN_READER, + fixture::TEXT, + fixture::PREPARING, + fixture::REFUSING, + fixture::TEXT_HEADER, + crate::service_version::MODULE, + ], + "should list the middleware the selected module supplies, and core's own after them" ); - assert!( - immediate.contains(&"creative"), - "should include creative in immediate IDs" + assert_eq!( + registry.middleware_in(MiddlewarePhase::Fetch), + [ + fixture::HEAD, + fixture::LINKS, + fixture::BROKEN, + fixture::TEXT, + fixture::PREPARING, + fixture::REFUSING + ], + "should list the middleware that run in the fetch phase" ); - assert!( - !immediate.contains(&"sourcepoint"), - "should not include Sourcepoint unless explicitly enabled" + assert_eq!( + registry.middleware_in(MiddlewarePhase::Serve), + [ + fixture::READER, + fixture::BROKEN_READER, + fixture::PREPARING, + fixture::TEXT_HEADER, + crate::service_version::MODULE + ], + "should list the middleware that run in the serve phase, core's own last" ); - assert!( - !immediate.contains(&"prebid"), - "should not include prebid in immediate IDs" + let chain_for = |media_type: &str, path: &str| { + registry + .middleware_chain(&settings.fetch, MiddlewarePhase::Fetch, media_type, path) + .ids() + }; + assert_eq!( + chain_for(HTML_MEDIA_TYPE, "/news/today"), + [fixture::LINKS, fixture::HEAD], + "should run the first covering entry's middleware in the order it names them" ); + assert_eq!(chain_for(HTML_MEDIA_TYPE, "/"), [fixture::HEAD]); assert!( - deferred.contains(&"prebid"), - "should serve the prebid shim as a deferred module" + chain_for("text/css", "/news/site.css").is_empty(), + "should run nothing on a media type no entry covers" ); } + /// An entry for `text/plain` naming the middleware that works on it. + fn text_entry(names: &[&str]) -> PhaseEntry { + PhaseEntry { + media_type: fixture::TEXT_MEDIA_TYPE.to_owned(), + path: None, + middleware: names.iter().map(|name| (*name).to_owned()).collect(), + } + } + #[test] - fn js_module_ids_skip_enabled_integrations_without_generated_js_module() { - let mut settings = crate::test_support::tests::create_test_settings(); - settings - .integrations - .insert_config("nextjs", &serde_json::json!({ "enabled": true })) - .expect("should insert nextjs config"); + fn an_entry_runs_a_middleware_on_the_media_type_it_handles() { + let settings = fixture_settings_with_entries(vec![ + html_entry(None, &[fixture::HEAD]), + text_entry(&[fixture::TEXT]), + ]); + + let registry = IntegrationRegistry::new(&settings).expect("should build a registry"); + + let chain_for = |media_type: &str| { + registry + .middleware_chain( + &settings.fetch, + MiddlewarePhase::Fetch, + media_type, + "/a.txt", + ) + .ids() + }; + assert_eq!( + chain_for(fixture::TEXT_MEDIA_TYPE), + [fixture::TEXT], + "should run the text middleware on a text response" + ); + assert_eq!( + chain_for(HTML_MEDIA_TYPE), + [fixture::HEAD], + "should run the page middleware on a page and not the text one" + ); + } - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) - .expect("should create registry"); - let all = registry.js_module_ids(); + #[test] + fn an_entry_naming_a_middleware_under_a_media_type_it_does_not_handle_is_refused() { + for (media_type, name) in [ + (fixture::TEXT_MEDIA_TYPE, fixture::HEAD), + (HTML_MEDIA_TYPE, fixture::TEXT), + ] { + let settings = fixture_settings_with_entries(vec![PhaseEntry { + media_type: media_type.to_owned(), + path: None, + middleware: vec![name.to_owned()], + }]); + + let error = IntegrationRegistry::new(&settings) + .err() + .expect("should refuse a middleware under a media type it does not handle"); + + let message = error.to_string(); + assert!( + message.contains(&format!( + "[[fetch]] entry 1 covers `{media_type}` and names `{name}`, which does not \ + work on that media type" + )), + "should name the entry, the media type and the middleware: {message}" + ); + } + } - assert!( - !all.contains(&"nextjs"), - "should not include enabled integrations without generated JS modules" - ); + #[test] + fn a_module_no_section_selects_supplies_no_middleware() { + let settings = crate::test_support::tests::create_test_settings(); + + let registry = IntegrationRegistry::new(&settings).expect("should build a registry"); - let metadata = registry.registered_integrations(); assert!( - metadata - .iter() - .any(|integration| integration.id == "nextjs"), - "should still register enabled Rust-only integrations" + !registry.middleware_ids().contains(&fixture::HEAD), + "should register nothing of a module that does not run" ); } #[test] - fn js_module_ids_include_explicitly_enabled_cmp_mirrors() { - let mut settings = crate::test_support::tests::create_test_settings(); - settings - .integrations - .insert_config("sourcepoint", &serde_json::json!({ "enabled": true })) - .expect("should insert sourcepoint config"); - settings - .integrations - .insert_config("osano", &serde_json::json!({ "enabled": true })) - .expect("should insert osano config"); + fn an_entry_naming_a_middleware_nothing_supplies_is_refused_with_what_is_supplied() { + let settings = fixture_settings_with_entries(vec![ + html_entry(Some("/news/"), &[fixture::HEAD]), + html_entry(None, &[fixture::HEAD, "testing.nothing"]), + ]); - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) - .expect("should create registry"); - let immediate = registry.js_module_ids_immediate(); + let error = IntegrationRegistry::new(&settings) + .err() + .expect("should refuse an entry naming a middleware nothing supplies"); + let message = error.to_string(); assert!( - immediate.contains(&"sourcepoint"), - "should include Sourcepoint when explicitly enabled" - ); - assert!( - immediate.contains(&"osano"), - "should include Osano when explicitly enabled" + message.contains( + "[[fetch]] entry 2 names `testing.nothing`, which no module that runs supplies" + ) && message.contains(&format!( + "[{}, {}, {}, {}, {}, {}, {}, {}, {}, {}]", + fixture::HEAD, + fixture::LINKS, + fixture::BROKEN, + fixture::READER, + fixture::BROKEN_READER, + fixture::TEXT, + fixture::PREPARING, + fixture::REFUSING, + fixture::TEXT_HEADER, + crate::service_version::MODULE + )), + "should name the entry and list what could be named: {message}" ); - - let metadata = registry.registered_integrations(); assert!( - metadata.iter().any(|integration| integration.id == "osano"), - "should include JS-only Osano registration in metadata" + !message.contains("no section selects"), + "should not blame a selection when the name is no module's: {message}" ); } #[test] - fn js_module_ids_deferred_empty_when_prebid_disabled() { + fn an_entry_naming_the_middleware_of_a_module_no_section_selects_says_so() { let mut settings = crate::test_support::tests::create_test_settings(); - settings - .integrations - .insert_config( - "prebid", - &serde_json::json!({ - "enabled": false, - "external_bundle_url": "https://assets.example/prebid/trusted-prebid.js", - }), - ) - .expect("should update prebid config"); + settings.fetch = PhaseEntries::new(vec![html_entry(None, &[fixture::LINKS])]); - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) - .expect("should create registry"); + let error = IntegrationRegistry::new(&settings) + .err() + .expect("should refuse an entry naming the middleware of a module that does not run"); - let deferred = registry.js_module_ids_deferred(); + let message = error.to_string(); assert!( - deferred.is_empty(), - "should have no deferred IDs when prebid is disabled" + message.contains(&format!("[[fetch]] entry 1 names `{}`", fixture::LINKS)) + && message.contains(&format!( + "`{}` is a module no section selects, so nothing it supplies is running", + fixture::MODULE + )), + "should say the module is not selected: {message}" ); } #[test] - fn js_module_ids_defer_prebid_shim_when_external_bundle_is_configured() { - let mut settings = crate::test_support::tests::create_test_settings(); - settings - .integrations - .insert_config( - "prebid", - &serde_json::json!({ - "enabled": true, - "external_bundle_url": "https://assets.example/prebid/trusted-prebid.js" - }), - ) - .expect("should update prebid config"); + fn a_middleware_no_entry_names_is_listed_for_the_startup_warning() { + let unnamed = |entries: Vec| -> Vec<&'static str> { + let settings = fixture_settings_with_entries(entries); + let registry = IntegrationRegistry::new(&settings).expect("should build a registry"); + unnamed_middleware(&settings, ®istry.inner) + .into_iter() + .map(|(integration, middleware)| { + assert_eq!(integration, fixture::ID, "should name the supplier"); + middleware.middleware_id() + }) + .collect() + }; - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) - .expect("should create registry"); + assert_eq!( + unnamed(Vec::new()), + [ + fixture::HEAD, + fixture::LINKS, + fixture::BROKEN, + fixture::READER, + fixture::BROKEN_READER, + fixture::TEXT, + fixture::PREPARING, + fixture::REFUSING, + fixture::TEXT_HEADER, + ], + "should list every middleware when there are no entries" + ); + assert_eq!( + unnamed(vec![ + html_entry(Some("/news/"), &[fixture::LINKS]), + html_entry(None, &[fixture::HEAD]), + ]), + [ + fixture::BROKEN, + fixture::READER, + fixture::BROKEN_READER, + fixture::TEXT, + fixture::PREPARING, + fixture::REFUSING, + fixture::TEXT_HEADER + ], + "should leave out a middleware any entry names" + ); + } + #[test] + fn an_entry_naming_a_middleware_in_a_phase_it_does_not_run_in_is_refused() { + let mut settings = settings_naming(fixture::MODULE); + settings.serve = PhaseEntries::new(vec![html_entry(None, &[fixture::HEAD])]); + let error = IntegrationRegistry::new(&settings) + .err() + .expect("should refuse a fetch middleware named in a serve entry"); + let message = error.to_string(); assert!( - registry.js_module_ids().contains(&"prebid"), - "external bundle mode should include the prebid shim in embedded TSJS modules" + message.contains(&format!("[[serve]] entry 1 names `{}`", fixture::HEAD)) + && message.contains("does not run in that phase. It runs in [[fetch]]"), + "should say which phase the middleware runs in: {message}" ); + + let mut settings = settings_naming(fixture::MODULE); + settings.fetch = PhaseEntries::new(vec![html_entry(None, &[fixture::READER])]); + let error = IntegrationRegistry::new(&settings) + .err() + .expect("should refuse a serve middleware named in a fetch entry"); + let message = error.to_string(); assert!( - !registry.js_module_ids_immediate().contains(&"prebid"), - "the prebid shim should not load in the immediate TSJS bundle" + message.contains(&format!("[[fetch]] entry 1 names `{}`", fixture::READER)) + && message.contains("does not run in that phase. It runs in [[serve]]"), + "should say which phase the middleware runs in: {message}" ); - assert!( - registry.js_module_ids_deferred().contains(&"prebid"), - "the prebid shim should load as a deferred TSJS module" + } + + #[test] + fn each_phase_has_entries_of_its_own() { + let mut settings = settings_naming(fixture::MODULE); + settings.fetch = PhaseEntries::new(vec![html_entry(None, &[fixture::HEAD])]); + settings.serve = PhaseEntries::new(vec![html_entry(Some("/news/"), &[fixture::READER])]); + let registry = IntegrationRegistry::new(&settings) + .expect("should build a registry with an entry in each phase"); + + let chain_for = |phase: MiddlewarePhase, path: &str| { + registry + .middleware_chain(settings.phase_entries(phase), phase, HTML_MEDIA_TYPE, path) + .ids() + }; + assert_eq!( + chain_for(MiddlewarePhase::Fetch, "/news/today"), + [fixture::HEAD] + ); + assert_eq!( + chain_for(MiddlewarePhase::Serve, "/news/today"), + [fixture::READER] ); assert!( - registry.has_route(&Method::GET, "/integrations/prebid/bundle.js"), - "external bundle mode should register the first-party bundle route" + chain_for(MiddlewarePhase::Serve, "/sport/today").is_empty(), + "should run no serve middleware on a path only the fetch entries cover" + ); + let unnamed: Vec<&str> = unnamed_middleware(&settings, ®istry.inner) + .into_iter() + .map(|(_, middleware)| middleware.middleware_id()) + .collect(); + assert_eq!( + unnamed, + [ + fixture::LINKS, + fixture::BROKEN, + fixture::BROKEN_READER, + fixture::TEXT, + fixture::PREPARING, + fixture::REFUSING, + fixture::TEXT_HEADER + ], + "should count a middleware as named only by an entry of a phase it runs in" ); } - #[test] - fn js_module_ids_split_is_exhaustive() { - let settings = crate::test_support::tests::create_test_settings(); - let mut settings_with_prebid = settings; - settings_with_prebid - .integrations - .insert_config( - "prebid", - &serde_json::json!({ - "enabled": true, - "external_bundle_url": "https://assets.example/prebid/trusted-prebid.js", - "timeout_ms": 1000, - "debug": false - }), - ) - .expect("should insert prebid config"); + /// A middleware under a name of the test's choosing, running in the + /// phases given. + struct Named { + id: &'static str, + phases: &'static [MiddlewarePhase], + } - let registry = IntegrationRegistry::with_plan( - &settings_with_prebid, - Arc::new( - crate::auction::compile_auction_plan(&settings_with_prebid) - .expect("should compile auction plan"), - ), + impl Middleware for Named { + fn middleware_id(&self) -> &'static str { + self.id + } + + fn phases(&self) -> &[MiddlewarePhase] { + self.phases + } + + fn create(&self, _context: &MiddlewareContext<'_>) -> MiddlewareAction { + MiddlewareAction::pass() + } + } + + const FETCH_ONLY: &[MiddlewarePhase] = &[MiddlewarePhase::Fetch]; + + fn first_supplier( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("first_supplier") + .without_js() + .with_middleware(Arc::new(Named { + id: "testing.shared-name", + phases: FETCH_ONLY, + })) + .build(), + )) + } + + fn second_supplier( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("second_supplier") + .without_js() + .with_middleware(Arc::new(Named { + id: "testing.shared-name", + phases: FETCH_ONLY, + })) + .build(), + )) + } + + fn misnamed_supplier( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("misnamed_supplier") + .without_js() + .with_middleware(Arc::new(Named { + id: "Not A Name", + phases: FETCH_ONLY, + })) + .build(), + )) + } + + fn phaseless_supplier( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("phaseless_supplier") + .without_js() + .with_middleware(Arc::new(Named { + id: "testing.phaseless", + phases: &[], + })) + .build(), + )) + } + + /// A builder for `register`, selected in `settings` under a module name + /// made from its id. + fn selected_supplier( + settings: &mut Settings, + id: &'static str, + module: &'static str, + register: crate::integrations::IntegrationBuilderFn, + ) -> crate::integrations::IntegrationBuilder { + settings.select_module("testing", module); + crate::integrations::IntegrationBuilder::new( + id, + "a-vendor-crate", + register, + validate_nothing, ) - .expect("should create registry"); + .with_module_name(module) + } - let all = registry.js_module_ids(); - let mut recombined = registry.js_module_ids_immediate(); - recombined.extend(registry.js_module_ids_deferred()); - recombined.sort_unstable(); + #[test] + fn two_modules_supplying_one_middleware_name_are_refused() { + let mut settings = crate::test_support::tests::create_test_settings(); + let extra = [ + selected_supplier( + &mut settings, + "first_supplier", + "testing.first-supplier", + first_supplier, + ), + selected_supplier( + &mut settings, + "second_supplier", + "testing.second-supplier", + second_supplier, + ), + ]; - let mut all_sorted = all; - all_sorted.sort_unstable(); + let error = IntegrationRegistry::with_registrations(&settings, &extra) + .err() + .expect("should refuse one middleware name supplied twice"); + + let message = error.to_string(); + assert!( + message.contains("`first_supplier` and `second_supplier` both supply a middleware") + && message.contains("`testing.shared-name`"), + "should name both suppliers and the name: {message}" + ); + } + + fn marking_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("marking") + .without_js() + .with_bundle_tag_attribute("data-example-mode", "first") + .with_bundle_tag_attribute("data-example-flag", "true") + .build(), + )) + } + + fn disagreeing_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("disagreeing") + .without_js() + .with_bundle_tag_attribute("data-example-mode", "second") + .with_bundle_tag_attribute("data-example-other", "kept") + .build(), + )) + } + + fn unsafe_name_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("unsafe_name") + .without_js() + .with_bundle_tag_attribute("onload=alert(1) data-x", "true") + .build(), + )) + } + + fn unsafe_value_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("unsafe_value") + .without_js() + .with_bundle_tag_attribute("data-example-mode", "\"> + "#; + + let mut plain = create_html_processor(config()); + let without = String::from_utf8( + crate::streaming_processor::StreamProcessor::process_chunk( + &mut plain, + html.as_bytes(), + true, + ) + .expect("should process the document"), + ) + .expect("should leave the document UTF-8"); + + assert_eq!(page(&chain_of(Vec::new()), html), without); +} + +#[test] +fn head_markup_goes_ahead_of_the_bundle_in_the_order_the_middleware_are_named() { + let chain = chain_of(vec![ + head("example.first", ""), + head("example.second", ""), + ]); + + let page = page(&chain, PAGE); + + let first = index_of(&page, "name=\"first\""); + let second = index_of(&page, "name=\"second\""); + let bundle = index_of(&page, BUNDLE_TAG); + let title = index_of(&page, ""); + assert!( + first < second && second < bundle && bundle < title, + "should write head markup in the order named, ahead of the bundle and of the \ + page's own head: {page}" + ); + assert_eq!( + page.matches("name=\"first\"").count(), + 1, + "should write head markup once" + ); +} + +#[test] +fn markup_for_after_the_bundle_goes_between_the_bundle_and_the_page_s_own_head() { + let chain = chain_of(vec![middleware("example.after", |_| MiddlewareAction { + after_bundle_inserts: vec!["<script id=\"after\"></script>".to_owned()], + ..MiddlewareAction::pass() + })]); + + let page = page(&chain, PAGE); + + let bundle = index_of(&page, BUNDLE_TAG); + let after = index_of(&page, "id=\"after\""); + let title = index_of(&page, "<title>"); + assert!( + bundle < after && after < title, + "should write the markup straight after the bundle: {page}" + ); +} + +#[test] +fn a_page_with_no_head_gets_no_head_markup() { + let chain = chain_of(vec![head("example.first", "<meta name=\"first\">")]); + + let page = page(&chain, "<p>a fragment</p>"); + + assert_eq!( + page, "<p>a fragment</p>", + "should leave a document with no head as it arrived" + ); +} + +#[test] +fn an_element_handler_is_asked_about_an_address_as_it_will_be_served() { + let chain = chain_of(vec![element("example.seen", "a.x", "href", |matched| { + AttributeRewriteAction::replace(format!( + "seen:{}:{}:{}", + matched.element_name, matched.attribute_name, matched.value + )) + })]); + + let page = page( + &chain, + r#"<html><body><a class="x" href="https://origin.example.com/page">x</a><a href="https://origin.example.com/other">y</a></body></html>"#, + ); + + assert!( + page.contains(r#"<a class="x" href="seen:a:href:https://test.example.com/page">"#), + "should hand the handler the address after core moved it to the reader's host: {page}" + ); + assert!( + page.contains(r#"<a href="https://test.example.com/other">"#), + "should leave an element the selector does not match to core: {page}" + ); +} + +#[test] +fn one_decision_can_be_asked_about_each_of_several_attributes() { + let chain = chain_of(vec![middleware("example.each", |_| { + let decide: Rc<AttributeRewriteFn> = Rc::new(|matched| { + if matched.value == "/vendor.js" { + AttributeRewriteAction::replace(format!( + "/first-party.js#{}", + matched.attribute_name + )) + } else { + AttributeRewriteAction::keep() + } + }); + MiddlewareAction { + element_handlers: AttributeRewrite::each(&["src", "href"], &decide), + ..MiddlewareAction::pass() + } + })]); + + let page = page( + &chain, + r#"<html><body><script src="/vendor.js"></script><link href="/vendor.js"><img src="/other.png"><a data-src="/vendor.js">x</a></body></html>"#, + ); + + assert!( + page.contains(r#"<script src="/first-party.js#src"></script>"#) + && page.contains(r#"<link href="/first-party.js#href">"#), + "should ask the one decision about each attribute named: {page}" + ); + assert!( + page.contains(r#"<img src="/other.png">"#) && page.contains(r#"<a data-src="/vendor.js">"#), + "should leave another value and another attribute alone: {page}" + ); +} + +#[test] +fn a_module_s_tests_can_call_a_middleware_with_a_test_page_s_context() { + use crate::html_processor::test_support::{ORIGIN_HOST, REQUEST_HOST}; + + let document_state = crate::integrations::IntegrationDocumentState::default(); + let context = test_support::context(MiddlewarePhase::Serve, &document_state); + + assert_eq!(context.phase, MiddlewarePhase::Serve); + assert_eq!( + ( + context.request_scheme, + context.request_host, + context.origin_host + ), + ("https", REQUEST_HOST, ORIGIN_HOST), + "should describe the page the recording helpers serve" + ); +} + +#[test] +fn an_element_handler_can_remove_the_element() { + let chain = chain_of(vec![element("example.remove", "a.gone", "href", |_| { + AttributeRewriteAction::remove_element() + })]); + + let page = page( + &chain, + r#"<html><body><a class="gone" href="/x">removed</a><a class="gone">no address</a><a href="/y">kept</a></body></html>"#, + ); + + assert!( + !page.contains("removed"), + "should remove the element and what it holds: {page}" + ); + assert!( + page.contains(r#"<a class="gone">no address</a>"#), + "should leave an element that carries no such attribute as it is: {page}" + ); + assert!(page.contains(r#"<a href="/y">kept</a>"#)); +} + +#[test] +fn handlers_on_one_element_are_asked_in_order_each_seeing_the_last_value() { + let chain = chain_of(vec![ + element("example.one", "a", "href", |_| { + AttributeRewriteAction::replace("/one") + }), + element("example.two", "a[href]", "href", |matched| { + AttributeRewriteAction::replace(format!("{}/two", matched.value)) + }), + ]); + + let page = page( + &chain, + r#"<html><body><a href="/start">x</a></body></html>"#, + ); + + assert!( + page.contains(r#"<a href="/one/two">"#), + "should hand the second handler the value the first left: {page}" + ); +} + +#[test] +fn an_element_an_earlier_handler_removed_is_not_asked_about_again() { + let asked = Arc::new(AtomicUsize::new(0)); + let counted = Arc::clone(&asked); + let chain = chain_of(vec![ + element("example.remove", "a", "href", |_| { + AttributeRewriteAction::remove_element() + }), + element("example.count", "a", "href", move |_| { + counted.fetch_add(1, Ordering::SeqCst); + AttributeRewriteAction::keep() + }), + ]); + + let page = page(&chain, r#"<html><body><a href="/x">x</a></body></html>"#); + + assert!(!page.contains("<a "), "should remove the element: {page}"); + assert_eq!( + asked.load(Ordering::SeqCst), + 0, + "should not ask a later handler about an element that is already gone" + ); +} + +#[test] +fn a_text_handler_is_handed_the_text_as_the_one_before_it_left_it() { + let chain = chain_of(vec![ + text("example.one", "script.x", || { + |text: &str, _is_last: bool| { + if text.contains("alpha") { + ScriptRewriteAction::replace(text.replace("alpha", "beta")) + } else { + ScriptRewriteAction::keep() + } + } + }), + text("example.two", "script", || { + |text: &str, _is_last: bool| { + if text.contains("beta") { + ScriptRewriteAction::replace(text.replace("beta", "gamma")) + } else { + ScriptRewriteAction::keep() + } + } + }), + ]); + + let page = page( + &chain, + r#"<html><body><script class="x">var a = "alpha";</script><script>var b = "alpha";</script></body></html>"#, + ); + + assert!( + page.contains(r#"<script class="x">var a = "gamma";</script>"#), + "should hand the second handler what the first wrote: {page}" + ); + assert!( + page.contains(r#"<script>var b = "alpha";</script>"#), + "should leave a script only the second matches, which holds nothing it changes: {page}" + ); +} + +#[test] +fn text_a_handler_removed_is_empty_for_the_next_which_may_write_in_its_place() { + let chain = chain_of(vec![ + text("example.remove", "script", || { + |_text: &str, _is_last: bool| ScriptRewriteAction::remove_node() + }), + text("example.write", "script", || { + |text: &str, is_last: bool| { + assert_eq!(text, "", "should hand over nothing where text was removed"); + if is_last { + ScriptRewriteAction::replace("written()") + } else { + ScriptRewriteAction::keep() + } + } + }), + ]); + + let page = page( + &chain, + "<html><body><script>removed()</script></body></html>", + ); + + assert!( + page.contains("<script>written()</script>"), + "should write the later handler's text in place of what was removed: {page}" + ); +} + +#[test] +fn a_text_handler_s_replacement_is_written_as_it_is() { + let chain = chain_of(vec![text("example.script", "script", || { + |text: &str, _is_last: bool| { + if text.is_empty() { + ScriptRewriteAction::keep() + } else { + ScriptRewriteAction::replace("if (a && b < c || d > e) { go(\"?x=1&y=2\"); }") + } + } + })]); + + let page = page(&chain, "<html><body><script>old()</script></body></html>"); + + assert!( + page.contains("<script>if (a && b < c || d > e) { go(\"?x=1&y=2\"); }</script>"), + "should write a script's text with no character escaped: {page}" + ); +} + +#[test] +fn a_handler_that_holds_a_script_writes_it_back_whole() { + // The script reaches the handler in pieces, because the document is read + // seven bytes at a time, and the word to change lies across pieces. + let chain = chain_of(vec![text("example.hold", "script", || { + let mut held = String::new(); + move |text: &str, is_last: bool| { + held.push_str(text); + if !is_last { + return ScriptRewriteAction::remove_node(); + } + ScriptRewriteAction::replace(std::mem::take(&mut held).replace("ORIGINAL", "CHANGED")) + } + })]); + + let page = page_with( + config(), + &chain, + "<html><body><script>one(); two(\"ORIGINAL\"); three();</script><p>after</p></body></html>", + 7, + ); + + assert!( + page.contains("<script>one(); two(\"CHANGED\"); three();</script><p>after</p>"), + "should write the whole script once, in order, with the word changed: {page}" + ); +} + +/// Appends a comment to the document, and swaps one word for another. +struct Rewording { + from: &'static str, + to: &'static str, + comment: &'static str, +} + +impl crate::streaming_processor::StreamProcessor for Rewording { + fn process_chunk(&mut self, chunk: &[u8], is_last: bool) -> io::Result<Vec<u8>> { + let mut text = String::from_utf8_lossy(chunk).replace(self.from, self.to); + if is_last { + text.push_str(self.comment); + } + Ok(text.into_bytes()) + } +} + +#[test] +fn a_stream_processor_is_handed_the_document_the_handlers_left() { + let chain = chain_of(vec![ + middleware("example.first", |_| MiddlewareAction { + element_handlers: vec![Box::new(AttributeRewrite::matching( + "a", + "href", + Rc::new(|_| AttributeRewriteAction::replace("/from-handler")), + ))], + stream: Some(Box::new(Rewording { + from: "/from-handler", + to: "/from-first", + comment: "<!--first-->", + })), + ..MiddlewareAction::pass() + }), + middleware("example.second", |_| MiddlewareAction { + stream: Some(Box::new(Rewording { + from: "/from-first", + to: "/from-second", + comment: "<!--second-->", + })), + ..MiddlewareAction::pass() + }), + ]); + + let page = page( + &chain, + r#"<html><body><a href="/start">x</a></body></html>"#, + ); + + assert!( + page.contains(r#"<a href="/from-second">"#), + "should hand each processor what the handlers and the processor before it left: {page}" + ); + assert!( + page.ends_with("<!--first--><!--second-->"), + "should run the processors in the order their middleware are named: {page}" + ); +} + +/// What one middleware leaves in the document's state for another. +struct Shared(&'static str); + +#[test] +fn the_middleware_of_one_document_share_its_state_and_see_nothing_a_request_left() { + const KEY: &str = "example"; + let chain = chain_of(vec![ + middleware("example.leaves", |context| { + context + .document_state + .get_or_insert_with(KEY, || Shared("left by the first")); + MiddlewareAction::pass() + }), + middleware("example.reads", |context| { + let shared = context + .document_state + .get::<Shared>(KEY) + .map_or("nothing", |shared| shared.0); + let request_mark = context + .document_state + .get::<request_fixture::Mark>(request_fixture::ID) + .is_some(); + MiddlewareAction { + head_inserts: vec![format!( + "<meta name=\"state\" content=\"{shared}; request mark {request_mark}\">" + )], + ..MiddlewareAction::pass() + } + }), + ]); + // The request left a mark, which a serve middleware may read. + let config = config().with_request_state(request_fixture::marked()); + + let page = page_with(config, &chain, PAGE, 8192); + + assert!( + page.contains("content=\"left by the first; request mark false\""), + "should share the document's state along the chain, with nothing of the request in \ + it: {page}" + ); +} + +#[test] +fn each_document_is_planned_afresh() { + let created = Arc::new(AtomicUsize::new(0)); + let counted = Arc::clone(&created); + let chain = chain_of(vec![middleware("example.counts", move |context| { + let nth = counted.fetch_add(1, Ordering::SeqCst) + 1; + let earlier = context.document_state.get::<Shared>("example").is_some(); + context + .document_state + .get_or_insert_with("example", || Shared("this document")); + MiddlewareAction { + head_inserts: vec![format!( + "<meta name=\"nth\" content=\"{nth}; earlier state {earlier}\">" + )], + ..MiddlewareAction::pass() + } + })]); + + let first = page(&chain, PAGE); + let second = page(&chain, PAGE); + + assert!(first.contains("content=\"1; earlier state false\"")); + assert!( + second.contains("content=\"2; earlier state false\""), + "should ask the middleware again for the next document, with a state of its own: \ + {second}" + ); + assert_eq!(created.load(Ordering::SeqCst), 2); +} + +#[test] +fn a_middleware_is_told_the_phase_the_hosts_and_how_much_it_may_hold() { + let chain = chain_of(vec![middleware("example.context", |context| { + MiddlewareAction { + head_inserts: vec![format!( + "<meta name=\"context\" content=\"{} {}://{} from {} holding {}\">", + context.phase, + context.request_scheme, + context.request_host, + context.origin_host, + context.max_buffered_script_bytes + )], + ..MiddlewareAction::pass() + } + })]); + let most = create_test_settings().publisher.max_buffered_body_bytes; + + let page = page(&chain, PAGE); + + assert!( + page.contains(&format!( + "content=\"fetch https://{REQUEST_HOST} from {ORIGIN_HOST} holding {most}\"" + )), + "should tell the middleware about the document: {page}" + ); +} + +#[test] +fn a_selector_that_does_not_parse_refuses_the_document_naming_the_middleware() { + let chains = [ + chain_of(vec![ + head("example.fine", "<meta name=\"fine\">"), + element("example.broken", "a[", "href", |_| { + AttributeRewriteAction::keep() + }), + ]), + chain_of(vec![text("example.broken", "a[", || { + |_text: &str, _is_last: bool| ScriptRewriteAction::keep() + })]), + ]; + + for chain in chains { + let message = create_html_processor_with_middleware(config(), &chain, None) + .err() + .expect("should refuse to plan the document"); + + assert!( + message.contains("the fetch middleware `example.broken` cannot run") + && message.contains("`a[` is not a usable selector"), + "should name the middleware and the selector: {message}" + ); + } +} + +#[test] +fn a_fetch_middleware_that_writes_a_response_header_is_refused() { + let chain = chain_of(vec![header("example.version", "x-example", "1")]); + + let message = create_html_processor_with_middleware(config(), &chain, None) + .err() + .expect("should refuse to plan the document"); + + assert!( + message.contains("the fetch middleware `example.version` cannot run") + && message.contains("writes a response header"), + "should name the middleware and the reason: {message}" + ); +} + +#[test] +fn a_serve_middleware_s_headers_are_written_on_the_reader_s_response() { + let chain = serve_chain_of(vec![ + header("example.a", "x-example", "a"), + header("example.b", "x-example", "b"), + header("example.c", "x-other", "c"), + ]); + + let pass = create_serve_processor(&chain, &reader(&IntegrationRequestState::default())) + .expect("should plan the reader's middleware"); + let mut response = http::Response::new(()); + write_response_headers(&mut response, pass.response_headers); + + assert!( + pass.processor.is_none(), + "should leave the document as it arrived" + ); + assert_eq!( + response + .headers() + .get("x-example") + .map(HeaderValue::as_bytes), + Some(b"b".as_slice()), + "a later middleware's header should replace an earlier one's" + ); + assert_eq!( + response.headers().get("x-other").map(HeaderValue::as_bytes), + Some(b"c".as_slice()), + "should write a header no other middleware wrote" + ); +} + +#[test] +fn the_inline_path_hands_back_the_serve_chain_s_headers() { + let serve = serve_chain_of(vec![header("example.version", "x-example", "1")]); + + let (_, headers) = + create_html_processor_with_middleware(config(), &chain_of(Vec::new()), Some(&serve)) + .expect("should plan the document"); + + assert_eq!( + headers + .iter() + .map(|(name, value)| (name.as_str(), value.as_bytes())) + .collect::<Vec<_>>(), + [("x-example", b"1".as_slice())], + "should hand back the serve chain's headers for the response" + ); +} + +#[test] +fn an_action_with_only_a_header_is_not_a_pass() { + let action = MiddlewareAction { + response_headers: vec![( + HeaderName::from_static("x-example"), + HeaderValue::from_static("1"), + )], + ..MiddlewareAction::pass() + }; + + assert!(!action.is_pass(), "a header is a change to the response"); + assert!(MiddlewareAction::pass().is_pass()); +} + +#[test] +fn a_middleware_is_told_the_platform_the_deployment_runs_on() { + let platform = PlatformIdentity { + service_version: "39".to_owned(), + ..PlatformIdentity::named("example-platform") + }; + let document_state = IntegrationDocumentState::default(); + + let context = test_support::context_on(MiddlewarePhase::Serve, &document_state, &platform); + + assert_eq!(context.platform.name, "example-platform"); + assert_eq!(context.platform.service_version, "39"); + assert_eq!( + test_support::context(MiddlewarePhase::Serve, &document_state) + .platform + .name, + "test-platform", + "a test context should name the test platform" + ); +} + +#[test] +fn a_chain_reports_its_phase_and_the_names_in_the_order_they_run() { + let chain = chain_of(vec![ + head("example.b", "<meta name=\"b\">"), + head("example.a", "<meta name=\"a\">"), + ]); + + assert_eq!(chain.phase(), MiddlewarePhase::Fetch); + assert_eq!(chain.ids(), ["example.b", "example.a"]); + assert!(!chain.is_empty()); + assert!(chain_of(Vec::new()).is_empty()); +} + +// --------------------------------------------------------------------------- +// What the document pipeline does with a serve chain +// --------------------------------------------------------------------------- + +/// [`PAGE`] as core leaves it for a store, with the script bundle in its +/// head and one link. +fn rewritten_page() -> String { + page( + &chain_of(Vec::new()), + r#"<html><head><title>Pagex"#, + ) +} + +#[test] +fn a_serve_chain_writes_head_markup_straight_before_the_bundle_and_the_rest_straight_after() { + let chain = serve_chain_of(vec![ + middleware("example.first", |_| MiddlewareAction { + head_inserts: vec!["".to_owned()], + after_bundle_inserts: vec!["".to_owned()], + ..MiddlewareAction::pass() + }), + middleware("example.second", |_| MiddlewareAction { + head_inserts: vec!["".to_owned()], + after_bundle_inserts: vec!["".to_owned()], + ..MiddlewareAction::pass() + }), + ]); + let rewritten = rewritten_page(); + + let page = served(&chain, &rewritten); + + assert!( + page.contains("" + ), + "should write the markup for after the bundle straight after it: {page}" + ); + assert_eq!( + page.matches(BUNDLE_TAG).count(), + 1, + "should write no second bundle: {page}" + ); + assert_eq!( + page.replace("", "") + .replace( + "", + "" + ), + rewritten, + "should change nothing else of a document core has already rewritten" + ); +} + +#[test] +fn a_serve_chain_writes_head_markup_once_where_the_page_repeats_the_bundle_s_id() { + let chain = serve_chain_of(vec![head("example.first", "")]); + let rewritten = rewritten_page().replace( + "", + "", + ); + + let page = served(&chain, &rewritten); + + assert_eq!( + page.matches("name=\"first\"").count(), + 1, + "should write head markup at core's tag alone: {page}" + ); + assert!( + index_of(&page, "name=\"first\"") < index_of(&page, ""), + "should write it in the head: {page}" + ); +} + +#[test] +fn a_reader_s_copy_with_no_bundle_gets_no_head_markup() { + let chain = serve_chain_of(vec![head("example.first", "<meta name=\"first\">")]); + + let page = served(&chain, "<p>a fragment</p>"); + + assert_eq!(page, "<p>a fragment</p>"); +} + +#[test] +fn a_serve_chain_s_handlers_work_on_the_page_as_core_left_it() { + let chain = serve_chain_of(vec![ + element("example.link", "a", "href", |matched| { + AttributeRewriteAction::replace(format!("{}?reader=1", matched.value)) + }), + text("example.script", "script", || { + |text: &str, _is_last: bool| { + if text.contains("var a = 1;") { + ScriptRewriteAction::replace(text.replace("var a = 1;", "var a = 2 && 3;")) + } else { + ScriptRewriteAction::keep() + } + } + }), + ]); + + let page = served(&chain, &rewritten_page()); + + assert!( + page.contains(r#"<a href="https://test.example.com/x?reader=1">"#), + "should hand an element handler the address core already moved: {page}" + ); + assert!( + page.contains("<script>var a = 2 && 3;</script>"), + "should write a text handler's replacement as it is: {page}" + ); +} + +#[test] +fn a_serve_chain_s_stream_processor_runs_over_the_reader_s_copy() { + let chain = serve_chain_of(vec![middleware("example.stream", |_| MiddlewareAction { + stream: Some(Box::new(Rewording { + from: "<title>Page", + to: "Reader", + comment: "", + })), + ..MiddlewareAction::pass() + })]); + + let page = served(&chain, &rewritten_page()); + + assert!(page.contains("Reader") && page.ends_with("")); +} + +#[test] +fn a_serve_middleware_reads_what_the_request_left() { + let chain = serve_chain_of(vec![middleware("example.reads", |context| { + let marked = context + .document_state + .get::(request_fixture::ID) + .is_some(); + MiddlewareAction { + head_inserts: vec![format!( + "", + context.phase, context.request_scheme, context.request_host, context.origin_host + )], + ..MiddlewareAction::pass() + } + })]); + let rewritten = rewritten_page(); + + let marked = served_with(&chain, &request_fixture::marked(), &rewritten, 8192); + let unmarked = served(&chain, &rewritten); + + let told = format!("serve https://{REQUEST_HOST} from {ORIGIN_HOST}"); + assert!( + marked.contains(&format!("content=\"{told}; marked true\"")), + "should hand a serve middleware what the request left: {marked}" + ); + assert!( + unmarked.contains(&format!("content=\"{told}; marked false\"")), + "should hand it nothing where the request left nothing: {unmarked}" + ); +} + +#[test] +fn a_serve_chain_with_nothing_to_do_makes_no_processor() { + let state = IntegrationRequestState::default(); + + let none = create_serve_processor(&serve_chain_of(Vec::new()), &reader(&state)) + .expect("should plan an empty chain"); + assert!( + none.processor.is_none(), + "should make no processor for no middleware" + ); + + let passes = serve_chain_of(vec![middleware("example.pass", |_| { + MiddlewareAction::pass() + })]); + let none = + create_serve_processor(&passes, &reader(&state)).expect("should plan a chain that passes"); + assert!( + none.processor.is_none(), + "should make no processor where every middleware leaves the page alone" + ); +} + +#[test] +fn a_serve_selector_that_does_not_parse_refuses_the_reader_s_copy() { + let chain = serve_chain_of(vec![element("example.broken", "a[", "href", |_| { + AttributeRewriteAction::keep() + })]); + + let message = create_serve_processor(&chain, &reader(&IntegrationRequestState::default())) + .err() + .expect("should refuse to plan the reader's copy"); + + assert!( + message.contains("the serve middleware `example.broken` cannot run") + && message.contains("`a[` is not a usable selector"), + "should name the phase, the middleware and the selector: {message}" + ); +} + +#[test] +fn on_a_page_that_is_not_stored_the_serve_chain_follows_the_fetch_chain_on_the_one_pass() { + let fetch = chain_of(vec![ + head("example.fetch-head", ""), + element("example.fetch-link", "a", "href", |_| { + AttributeRewriteAction::replace("/fetch") + }), + ]); + let serve = serve_chain_of(vec![ + head("example.serve-head", ""), + element("example.serve-link", "a", "href", |matched| { + AttributeRewriteAction::replace(format!("{}/serve", matched.value)) + }), + ]); + let html = + r#"Pagex"#; + + // Read whole, and then seven bytes at a time, so the second rewriter is + // fed the first one's output in pieces. + for chunk_size in [8192, 7] { + let (processor, _) = create_html_processor_with_middleware(config(), &fetch, Some(&serve)) + .expect("should plan both chains"); + let mut pipeline = StreamingPipeline::new( + PipelineConfig { + input_compression: Compression::None, + output_compression: Compression::None, + chunk_size, + }, + processor, + ); + let mut output = Vec::new(); + pipeline + .process(Cursor::new(html.as_bytes()), &mut output) + .expect("should process the document"); + let page = String::from_utf8(output).expect("should leave the document UTF-8"); + + assert!( + page.contains(r#""#), + "should hand the serve chain the page as the fetch chain left it: {page}" + ); + let fetch_head = index_of(&page, "name=\"fetch\""); + let serve_head = index_of(&page, "name=\"serve\""); + assert!( + fetch_head < serve_head && serve_head < index_of(&page, BUNDLE_TAG), + "should write the serve chain's head markup after the fetch chain's and ahead of \ + the bundle: {page}" + ); + } +} + +// --------------------------------------------------------------------------- +// The settings +// --------------------------------------------------------------------------- + +fn settings_with_document(extra: &str) -> Result { + let document = format!( + "{}\n{extra}", + crate::test_support::tests::crate_test_settings_str() + ); + crate::settings::Settings::from_toml(&document).map_err(|error| format!("{error:?}")) +} + +#[test] +fn fetch_entries_are_read_from_the_settings_in_the_order_written() { + let settings = settings_with_document( + r#" +[[fetch]] +media_type = "text/html" +path = "/news/" +middleware = ["example.news", "example.all"] + +[[fetch]] +media_type = "text/html" +middleware = ["example.all"] +"#, + ) + .expect("should read settings with fetch entries"); + + assert_eq!( + settings.phase_entries(MiddlewarePhase::Fetch), + &PhaseEntries::new(vec![ + entry(Some("/news/"), &["example.news", "example.all"]), + entry(None, &["example.all"]), + ]) + ); +} + +#[test] +fn serve_entries_are_read_from_the_settings_beside_the_fetch_entries() { + let settings = settings_with_document( + r#" +[[fetch]] +media_type = "text/html" +middleware = ["example.all"] + +[[serve]] +media_type = "text/html" +path = "/news/" +middleware = ["example.reader"] +"#, + ) + .expect("should read settings with an entry in each phase"); + + assert_eq!( + settings.phase_entries(MiddlewarePhase::Fetch), + &PhaseEntries::new(vec![entry(None, &["example.all"])]) + ); + assert_eq!( + settings.phase_entries(MiddlewarePhase::Serve), + &PhaseEntries::new(vec![entry(Some("/news/"), &["example.reader"])]) + ); + let written = serde_json::to_value(&settings).expect("should write the settings"); + assert_eq!( + written.get("serve"), + Some(&json!([ + { "media_type": "text/html", "path": "/news/", "middleware": ["example.reader"] } + ])), + "should write the serve entries a document holds" + ); +} + +#[test] +fn settings_with_a_misshapen_serve_entry_are_refused_naming_the_phase() { + let message = + settings_with_document("\n[[serve]]\nmedia_type = \"text/html\"\nmiddleware = []\n") + .expect_err("should refuse the settings"); + + assert!( + message.contains("[[serve]] entry 1 names no middleware"), + "should name the phase and the entry: {message}" + ); +} + +#[test] +fn settings_without_entries_write_no_fetch_key() { + let settings = settings_with_document("").expect("should read settings with no entries"); + let written = serde_json::to_value(&settings).expect("should write the settings"); + assert!( + written.get("fetch").is_none() && written.get("serve").is_none(), + "should leave both keys out of a stored document that wrote no entries" + ); + + let settings = settings_with_document( + "\n[[fetch]]\nmedia_type = \"text/html\"\nmiddleware = [\"example.all\"]\n", + ) + .expect("should read settings with one entry"); + let written = serde_json::to_value(&settings).expect("should write the settings"); + assert_eq!( + written.get("fetch"), + Some(&json!([{ "media_type": "text/html", "middleware": ["example.all"] }])), + "should write the entries a document holds" + ); +} + +#[test] +fn settings_with_a_misshapen_entry_are_refused() { + let cases = [ + ( + "\n[[fetch]]\nmedia_type = \"image/png\"\nmiddleware = [\"example.all\"]\n", + "[[fetch]] entry 1 covers `image/png`", + ), + ( + "\n[fetch]\nmedia_type = \"text/html\"\nmiddleware = [\"example.all\"]\n", + "A phase is a list of entries, each written as [[fetch]]", + ), + ( + "\n[[fetch]]\nmedia_type = \"text/html\"\nmiddleware = [\"example.all\"]\nenabled = true\n", + "unknown field `enabled`", + ), + ( + "\n[[fetch]]\nmedia_type = \"text/html\"\n", + "missing field `middleware`", + ), + ]; + + for (document, expected) in cases { + let message = settings_with_document(document).expect_err("should refuse the settings"); + assert!( + message.contains(expected), + "should say `{expected}`: {message}" + ); + } +} + +#[test] +fn deploy_validation_refuses_a_misshapen_entry() { + let mut settings = create_test_settings(); + settings.fetch = PhaseEntries::new(vec![ + entry(None, &["example.all"]), + entry(Some("/news/"), &["example.news"]), + ]); + + let error = crate::config::validate_settings_for_deploy(&settings) + .expect_err("should refuse entries with one that is never reached"); + + assert!( + error + .to_string() + .contains("[[fetch]] entry 2 is never reached"), + "should name the entry: {error}" + ); +} + +#[test] +fn the_making_entry_has_a_path_covering_the_request_and_names_a_middleware() { + let entries = PhaseEntries::new(vec![ + PhaseEntry { + media_type: "text/plain".to_owned(), + path: Some("/ads.txt".to_owned()), + middleware: Vec::new(), + }, + PhaseEntry { + media_type: "text/plain".to_owned(), + path: Some("/ads".to_owned()), + middleware: vec!["example.sellers".to_owned()], + }, + PhaseEntry { + media_type: HTML_MEDIA_TYPE.to_owned(), + path: None, + middleware: vec!["example.page".to_owned()], + }, + ]); + + assert_eq!( + entries + .making("/ads.txt") + .map(|entry| entry.middleware.as_slice()), + Some(&["example.sellers".to_owned()][..]), + "should pass the entry naming no middleware and take the first that names one" + ); + assert!( + entries.making("/robots.txt").is_none(), + "should make nothing where no entry with a path covers the request, whatever covers every path" + ); +} + +#[test] +fn what_is_prepared_is_taken_once_by_name_and_as_its_type() { + let prepared = Prepared::default(); + assert!(prepared.is_empty()); + let fetch = MiddlewarePhase::Fetch; + prepared.insert(fetch, "example.one", Box::new(String::from("one"))); + prepared.insert(fetch, "example.two", Box::new(7_u32)); + + assert_eq!( + prepared.take::(fetch, "example.one"), + None, + "should leave a value of another type where it is" + ); + assert_eq!( + prepared.take::(fetch, "example.one").as_deref(), + Some("one") + ); + assert_eq!( + prepared.take::(fetch, "example.one"), + None, + "should give what was prepared once" + ); + assert_eq!(prepared.take::(fetch, "example.two"), Some(7)); + assert!(prepared.is_empty()); +} + +/// Prepares the name it is called as, and writes it into the head. +struct Prepares; + +impl Middleware for Prepares { + fn middleware_id(&self) -> &'static str { + "example.prepares" + } + + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Fetch] + } + + fn prepare<'a>( + &'a self, + call: ModuleCall<'a>, + _context: &'a MiddlewareContext<'a>, + ) -> Option> { + Some(Box::pin(async move { + Ok(Box::new(call.module().to_owned()) as Box) + })) + } + + fn create(&self, context: &MiddlewareContext<'_>) -> MiddlewareAction { + let prepared: String = context + .take_prepared("example.prepares") + .unwrap_or_else(|| "nothing prepared".to_owned()); + MiddlewareAction { + head_inserts: vec![prepared], + ..MiddlewareAction::pass() + } + } +} + +/// Prepares nothing. +struct Plain; + +impl Middleware for Plain { + fn middleware_id(&self) -> &'static str { + "example.plain" + } + + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Fetch] + } + + fn create(&self, _context: &MiddlewareContext<'_>) -> MiddlewareAction { + MiddlewareAction::pass() + } +} + +/// Refuses every request. +struct Refuses; + +impl Middleware for Refuses { + fn middleware_id(&self) -> &'static str { + "example.refuses" + } + + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Fetch] + } + + fn prepare<'a>( + &'a self, + _call: ModuleCall<'a>, + _context: &'a MiddlewareContext<'a>, + ) -> Option> { + Some(Box::pin(async { + Err(Refusal::service_unavailable( + Duration::from_secs(5), + "later", + )) + })) + } + + fn create(&self, _context: &MiddlewareContext<'_>) -> MiddlewareAction { + MiddlewareAction::pass() + } +} + +#[tokio::test] +async fn a_chain_keeps_what_each_middleware_prepares_under_its_name() { + let chain = MiddlewareChain::new( + MiddlewarePhase::Fetch, + HTML_MEDIA_TYPE, + vec![Arc::new(Plain), Arc::new(Prepares)], + ); + let document_state = IntegrationDocumentState::default(); + let prepared = Prepared::default(); + let context = MiddlewareContext { + prepared: &prepared, + ..test_support::context(MiddlewarePhase::Fetch, &document_state) + }; + + chain + .prepare(ModuleContext::empty(), &context) + .await + .expect("should prepare"); + let plan = chain.plan(&context).expect("should plan"); + + assert_eq!( + plan.head_inserts, + vec!["example.prepares".to_owned()], + "should hand create what the middleware prepared, called as its own name" + ); + assert!( + prepared.is_empty(), + "should have given the preparation to create" + ); +} + +/// Prepares the phase it is called in, and writes it into the head. +struct PreparesItsPhase; + +impl Middleware for PreparesItsPhase { + fn middleware_id(&self) -> &'static str { + "example.both" + } + + fn phases(&self) -> &[MiddlewarePhase] { + &MiddlewarePhase::ALL + } + + fn prepare<'a>( + &'a self, + _call: ModuleCall<'a>, + context: &'a MiddlewareContext<'a>, + ) -> Option> { + let phase = context.phase.to_string(); + Some(Box::pin(async move { + Ok(Box::new(phase) as Box) + })) + } + + fn create(&self, context: &MiddlewareContext<'_>) -> MiddlewareAction { + MiddlewareAction { + head_inserts: vec![ + context + .take_prepared::("example.both") + .unwrap_or_else(|| "nothing prepared".to_owned()), + ], + ..MiddlewareAction::pass() + } + } +} + +#[tokio::test] +async fn a_middleware_preparing_in_both_phases_keeps_what_each_phase_prepared() { + let prepared = Prepared::default(); + let document_state = IntegrationDocumentState::default(); + let chains: Vec<(MiddlewarePhase, MiddlewareChain)> = MiddlewarePhase::ALL + .into_iter() + .map(|phase| { + ( + phase, + MiddlewareChain::new(phase, HTML_MEDIA_TYPE, vec![Arc::new(PreparesItsPhase)]), + ) + }) + .collect(); + // A document that is not stored is prepared in both phases before + // either phase is planned, so both preparations share one store. + for (phase, chain) in &chains { + let context = MiddlewareContext { + prepared: &prepared, + ..test_support::context(*phase, &document_state) + }; + chain + .prepare(ModuleContext::empty(), &context) + .await + .expect("should prepare"); + } + let mut plans = Vec::new(); + for (phase, chain) in &chains { + let context = MiddlewareContext { + prepared: &prepared, + ..test_support::context(*phase, &document_state) + }; + plans.push(chain.plan(&context).expect("should plan").head_inserts); + } + + assert_eq!( + plans, + vec![vec!["fetch".to_owned()], vec!["serve".to_owned()]], + "should hand each phase what was prepared in it" + ); + assert!(prepared.is_empty(), "should have given every preparation"); +} + +#[tokio::test] +async fn the_first_refusal_ends_the_preparation() { + let chain = MiddlewareChain::new( + MiddlewarePhase::Fetch, + HTML_MEDIA_TYPE, + vec![Arc::new(Refuses), Arc::new(Prepares)], + ); + let document_state = IntegrationDocumentState::default(); + let prepared = Prepared::default(); + let context = MiddlewareContext { + prepared: &prepared, + ..test_support::context(MiddlewarePhase::Fetch, &document_state) + }; + + let refusal = chain + .prepare(ModuleContext::empty(), &context) + .await + .expect_err("should refuse"); + + assert_eq!(refusal.status, StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(refusal.retry_after, Some(Duration::from_secs(5))); + assert_eq!(refusal.message, "later"); + assert!( + prepared.is_empty(), + "should prepare nothing after the refusal" + ); +} + +#[test] +fn an_html_entry_with_a_path_makes_nothing() { + let entries = PhaseEntries::new(vec![PhaseEntry { + media_type: HTML_MEDIA_TYPE.to_owned(), + path: Some("/news/".to_owned()), + middleware: vec!["example.page".to_owned()], + }]); + + assert!( + entries.making("/news/missing").is_none(), + "should leave a page the origin does not have as the origin's answer" + ); +} + +#[test] +fn the_entry_with_the_longest_covering_path_makes_the_file() { + let entries = PhaseEntries::new(vec![ + PhaseEntry { + media_type: "text/css".to_owned(), + path: Some("/".to_owned()), + middleware: vec!["example.styles".to_owned()], + }, + PhaseEntry { + media_type: "text/plain".to_owned(), + path: Some("/ads.txt".to_owned()), + middleware: vec!["example.sellers".to_owned()], + }, + ]); + + assert_eq!( + entries + .making("/ads.txt") + .map(|entry| entry.media_type.as_str()), + Some("text/plain"), + "should make the file from the entry for its own path, not one written ahead for every path" + ); + assert_eq!( + entries + .making("/site.css") + .map(|entry| entry.media_type.as_str()), + Some("text/css"), + "should still make a file only the shorter path covers" + ); +} + +#[test] +fn an_html_entry_ahead_does_not_shadow_the_entry_that_makes_the_file() { + let entries = PhaseEntries::new(vec![ + PhaseEntry { + media_type: HTML_MEDIA_TYPE.to_owned(), + path: Some("/".to_owned()), + middleware: vec!["example.page".to_owned()], + }, + PhaseEntry { + media_type: "text/plain".to_owned(), + path: Some("/ads.txt".to_owned()), + middleware: vec!["example.sellers".to_owned()], + }, + ]); + + assert_eq!( + entries + .making("/ads.txt") + .map(|entry| entry.media_type.as_str()), + Some("text/plain"), + "should make the file from the text/plain entry, not as HTML" + ); +} diff --git a/crates/trusted-server-core/src/migration_guards.rs b/crates/trusted-server-core/src/migration_guards.rs index 17b3d624f6..59deccec97 100644 --- a/crates/trusted-server-core/src/migration_guards.rs +++ b/crates/trusted-server-core/src/migration_guards.rs @@ -27,206 +27,18 @@ fn fastly_sdk_pattern() -> regex::Regex { .expect("should compile migration guard regex") } +// The list of files this guard checks is generated by `build.rs` from every +// `.rs` file under `src`, so a file added to core joins the guard on its own +// and a file that leaves core (a vendor integration moving to its own crate, +// for instance) leaves the guard without anyone editing this file. The +// generated file defines `CHECKED_SOURCES`. +// +// `migration_guards.rs` itself is left out, because the banned pattern appears +// here as the regex literal the guard matches with. +include!(concat!(env!("OUT_DIR"), "/migration_guard_sources.rs")); + fn checked_sources() -> &'static [(&'static str, &'static str)] { - &[ - ( - "asset_image_optimizer.rs", - include_str!("asset_image_optimizer.rs"), - ), - ("auction/config.rs", include_str!("auction/config.rs")), - ("auction/context.rs", include_str!("auction/context.rs")), - ("auction/endpoints.rs", include_str!("auction/endpoints.rs")), - ("auction/formats.rs", include_str!("auction/formats.rs")), - ("auction/mod.rs", include_str!("auction/mod.rs")), - ( - "auction/orchestrator.rs", - include_str!("auction/orchestrator.rs"), - ), - ("auction/provider.rs", include_str!("auction/provider.rs")), - ( - "auction/test_support.rs", - include_str!("auction/test_support.rs"), - ), - ("auction/types.rs", include_str!("auction/types.rs")), - ( - "auction_config_types.rs", - include_str!("auction_config_types.rs"), - ), - ("auth.rs", include_str!("auth.rs")), - ( - "consent/extraction.rs", - include_str!("consent/extraction.rs"), - ), - ("consent/gpp.rs", include_str!("consent/gpp.rs")), - ( - "consent/jurisdiction.rs", - include_str!("consent/jurisdiction.rs"), - ), - ("consent/mod.rs", include_str!("consent/mod.rs")), - ("consent/tcf.rs", include_str!("consent/tcf.rs")), - ("consent/types.rs", include_str!("consent/types.rs")), - ( - "consent/us_privacy.rs", - include_str!("consent/us_privacy.rs"), - ), - ("consent_config.rs", include_str!("consent_config.rs")), - ("constants.rs", include_str!("constants.rs")), - ("cookies.rs", include_str!("cookies.rs")), - ("creative.rs", include_str!("creative.rs")), - ("ec/auth.rs", include_str!("ec/auth.rs")), - ("ec/batch_sync.rs", include_str!("ec/batch_sync.rs")), - ("ec/consent.rs", include_str!("ec/consent.rs")), - ("ec/cookies.rs", include_str!("ec/cookies.rs")), - ("ec/device.rs", include_str!("ec/device.rs")), - ("ec/eids.rs", include_str!("ec/eids.rs")), - ("ec/finalize.rs", include_str!("ec/finalize.rs")), - ("ec/generation.rs", include_str!("ec/generation.rs")), - ("ec/identify.rs", include_str!("ec/identify.rs")), - ("ec/kv.rs", include_str!("ec/kv.rs")), - ("ec/kv_types.rs", include_str!("ec/kv_types.rs")), - ("ec/mod.rs", include_str!("ec/mod.rs")), - ("ec/partner.rs", include_str!("ec/partner.rs")), - ("ec/prebid_eids.rs", include_str!("ec/prebid_eids.rs")), - ("ec/pull_sync.rs", include_str!("ec/pull_sync.rs")), - ("ec/rate_limiter.rs", include_str!("ec/rate_limiter.rs")), - ("ec/registry.rs", include_str!("ec/registry.rs")), - ("edge_cookie.rs", include_str!("edge_cookie.rs")), - ("error.rs", include_str!("error.rs")), - ("geo.rs", include_str!("geo.rs")), - ("host_header.rs", include_str!("host_header.rs")), - ("host_rewrite.rs", include_str!("host_rewrite.rs")), - ("html_processor.rs", include_str!("html_processor.rs")), - ("http_util.rs", include_str!("http_util.rs")), - ( - "integrations/adserver_mock.rs", - include_str!("integrations/adserver_mock.rs"), - ), - ("integrations/aps.rs", include_str!("integrations/aps.rs")), - ( - "integrations/datadome.rs", - include_str!("integrations/datadome.rs"), - ), - ( - "integrations/didomi.rs", - include_str!("integrations/didomi.rs"), - ), - ( - "integrations/google_tag_manager.rs", - include_str!("integrations/google_tag_manager.rs"), - ), - ("integrations/gpt.rs", include_str!("integrations/gpt.rs")), - ( - "integrations/gpt_diagnostics.rs", - include_str!("integrations/gpt_diagnostics.rs"), - ), - ( - "integrations/lockr.rs", - include_str!("integrations/lockr.rs"), - ), - ("integrations/mod.rs", include_str!("integrations/mod.rs")), - ( - "integrations/nextjs/mod.rs", - include_str!("integrations/nextjs/mod.rs"), - ), - ( - "integrations/nextjs/rsc.rs", - include_str!("integrations/nextjs/rsc.rs"), - ), - ( - "integrations/nextjs/rsc_placeholders.rs", - include_str!("integrations/nextjs/rsc_placeholders.rs"), - ), - ( - "integrations/nextjs/rsc_stream.rs", - include_str!("integrations/nextjs/rsc_stream.rs"), - ), - ( - "integrations/nextjs/script_rewriter.rs", - include_str!("integrations/nextjs/script_rewriter.rs"), - ), - ( - "integrations/nextjs/shared.rs", - include_str!("integrations/nextjs/shared.rs"), - ), - ( - "integrations/permutive.rs", - include_str!("integrations/permutive.rs"), - ), - ( - "integrations/prebid.rs", - include_str!("integrations/prebid.rs"), - ), - ( - "integrations/registry.rs", - include_str!("integrations/registry.rs"), - ), - ( - "integrations/sourcepoint.rs", - include_str!("integrations/sourcepoint.rs"), - ), - ( - "integrations/testlight.rs", - include_str!("integrations/testlight.rs"), - ), - ("lib.rs", include_str!("lib.rs")), - ("models.rs", include_str!("models.rs")), - ("openrtb.rs", include_str!("openrtb.rs")), - ("platform/error.rs", include_str!("platform/error.rs")), - ("platform/http.rs", include_str!("platform/http.rs")), - ( - "platform/image_optimizer.rs", - include_str!("platform/image_optimizer.rs"), - ), - ("platform/kv.rs", include_str!("platform/kv.rs")), - ("platform/mod.rs", include_str!("platform/mod.rs")), - ( - "platform/test_support.rs", - include_str!("platform/test_support.rs"), - ), - ("platform/traits.rs", include_str!("platform/traits.rs")), - ("platform/types.rs", include_str!("platform/types.rs")), - ("proxy.rs", include_str!("proxy.rs")), - ("publisher.rs", include_str!("publisher.rs")), - ("redacted.rs", include_str!("redacted.rs")), - ( - "request_signing/discovery.rs", - include_str!("request_signing/discovery.rs"), - ), - ( - "request_signing/endpoints.rs", - include_str!("request_signing/endpoints.rs"), - ), - ( - "request_signing/jwks.rs", - include_str!("request_signing/jwks.rs"), - ), - ( - "request_signing/mod.rs", - include_str!("request_signing/mod.rs"), - ), - ( - "request_signing/rotation.rs", - include_str!("request_signing/rotation.rs"), - ), - ( - "request_signing/signing.rs", - include_str!("request_signing/signing.rs"), - ), - ("rsc_flight.rs", include_str!("rsc_flight.rs")), - ("s3_sigv4.rs", include_str!("s3_sigv4.rs")), - ("settings.rs", include_str!("settings.rs")), - ("settings_data.rs", include_str!("settings_data.rs")), - ( - "streaming_processor.rs", - include_str!("streaming_processor.rs"), - ), - ( - "streaming_replacer.rs", - include_str!("streaming_replacer.rs"), - ), - ("test_support.rs", include_str!("test_support.rs")), - ("tsjs.rs", include_str!("tsjs.rs")), - ] + CHECKED_SOURCES } #[test] @@ -241,3 +53,33 @@ fn core_modules_do_not_reference_the_fastly_sdk() { ); } } + +/// A generation bug that produced an empty or truncated list would leave the +/// guard above passing while checking nothing, so this pins the list to the +/// tree it is meant to cover. +#[test] +fn the_generated_list_covers_the_crate() { + let sources = checked_sources(); + + assert!( + sources.len() > 100, + "should check every core source file, found {}", + sources.len() + ); + assert!( + sources.iter().any(|&(path, _)| path == "lib.rs"), + "should check the crate root" + ); + assert!( + sources + .iter() + .any(|&(path, _)| path == "integrations/registry.rs"), + "should check files in subdirectories" + ); + assert!( + !sources + .iter() + .any(|&(path, _)| path == "migration_guards.rs"), + "should leave out the guard's own source, which holds the banned pattern" + ); +} diff --git a/crates/trusted-server-core/src/models.rs b/crates/trusted-server-core/src/models.rs deleted file mode 100644 index 87cadcb94f..0000000000 --- a/crates/trusted-server-core/src/models.rs +++ /dev/null @@ -1,285 +0,0 @@ -//! Data models for ad serving and callbacks. -//! -//! This module defines the structures used for communication with ad servers -//! and tracking callbacks. - -use serde::Deserialize; - -/// Response from an ad server containing creative details. -/// -/// Contains all the information needed to display an ad and track -/// its performance through various callbacks. -#[allow( - dead_code, - reason = "model includes fields deserialized for external ad-server compatibility" -)] -#[derive(Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -pub struct AdResponse { - /// Network identifier for the ad network. - pub network_id: String, - /// Site identifier where the ad will be displayed. - pub site_id: String, - /// Page identifier within the site. - pub page_id: String, - /// Format identifier for the ad format. - pub format_id: String, - /// Advertiser identifier. - pub advertiser_id: String, - /// Campaign identifier. - pub campaign_id: String, - /// Insertion order identifier. - pub insertion_id: String, - /// Creative identifier. - pub creative_id: String, - /// URL of the creative asset to display. - pub creative_url: String, - /// List of tracking callbacks for various events. - pub callbacks: Vec, -} - -/// Tracking callback for ad events. -/// -/// Represents a URL that should be called when specific ad events occur, -/// such as impressions, clicks, or viewability milestones. -#[derive(Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -pub struct Callback { - /// Type of callback (e.g., "impression", "click", "viewable"). - #[serde(rename = "type")] - pub callback_type: String, - /// URL to call when the event occurs. - pub url: String, -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - #[test] - fn test_callback_deserialization() { - let json_data = json!({ - "type": "impression", - "url": "https://example.com/track/impression" - }); - - let callback: Callback = - serde_json::from_value(json_data).expect("should deserialize callback"); - assert_eq!(callback.callback_type, "impression"); - assert_eq!(callback.url, "https://example.com/track/impression"); - } - - #[test] - fn test_ad_response_full_deserialization() { - let json_data = json!({ - "networkId": "12345", - "siteId": "67890", - "pageId": "11111", - "formatId": "22222", - "advertiserId": "33333", - "campaignId": "44444", - "insertionId": "55555", - "creativeId": "66666", - "creativeUrl": "https://cdn.example.com/creative/12345.jpg", - "callbacks": [ - { - "type": "impression", - "url": "https://track.example.com/impression/12345" - }, - { - "type": "click", - "url": "https://track.example.com/click/12345" - }, - { - "type": "viewability", - "url": "https://track.example.com/viewability/12345" - } - ] - }); - - let ad_response: AdResponse = - serde_json::from_value(json_data).expect("should deserialize ad response"); - - assert_eq!(ad_response.network_id, "12345"); - assert_eq!(ad_response.site_id, "67890"); - assert_eq!(ad_response.page_id, "11111"); - assert_eq!(ad_response.format_id, "22222"); - assert_eq!(ad_response.advertiser_id, "33333"); - assert_eq!(ad_response.campaign_id, "44444"); - assert_eq!(ad_response.insertion_id, "55555"); - assert_eq!(ad_response.creative_id, "66666"); - assert_eq!( - ad_response.creative_url, - "https://cdn.example.com/creative/12345.jpg" - ); - - assert_eq!(ad_response.callbacks.len(), 3); - assert_eq!(ad_response.callbacks[0].callback_type, "impression"); - assert_eq!( - ad_response.callbacks[0].url, - "https://track.example.com/impression/12345" - ); - assert_eq!(ad_response.callbacks[1].callback_type, "click"); - assert_eq!(ad_response.callbacks[2].callback_type, "viewability"); - } - - #[test] - fn test_ad_response_empty_callbacks() { - let json_data = json!({ - "networkId": "12345", - "siteId": "67890", - "pageId": "11111", - "formatId": "22222", - "advertiserId": "33333", - "campaignId": "44444", - "insertionId": "55555", - "creativeId": "66666", - "creativeUrl": "https://cdn.example.com/creative/12345.jpg", - "callbacks": [] - }); - - let ad_response: AdResponse = serde_json::from_value(json_data) - .expect("should deserialize ad response with empty callbacks"); - assert_eq!(ad_response.callbacks.len(), 0); - } - - #[test] - fn test_ad_response_missing_field() { - // Missing required field should fail - let json_data = json!({ - "networkId": "12345", - "siteId": "67890", - // Missing pageId - "formatId": "22222", - "advertiserId": "33333", - "campaignId": "44444", - "insertionId": "55555", - "creativeId": "66666", - "creativeUrl": "https://cdn.example.com/creative/12345.jpg", - "callbacks": [] - }); - - let result: Result = serde_json::from_value(json_data); - result.unwrap_err(); - } - - #[test] - fn test_ad_response_case_sensitivity() { - // Test camelCase to snake_case conversion - let json_str = r#"{ - "networkId": "net123", - "siteId": "site456", - "pageId": "page789", - "formatId": "format000", - "advertiserId": "adv111", - "campaignId": "camp222", - "insertionId": "ins333", - "creativeId": "cre444", - "creativeUrl": "https://example.com/creative.png", - "callbacks": [] - }"#; - - let ad_response: AdResponse = - serde_json::from_str(json_str).expect("should deserialize ad response from str"); - assert_eq!(ad_response.network_id, "net123"); - assert_eq!(ad_response.site_id, "site456"); - assert_eq!(ad_response.page_id, "page789"); - assert_eq!(ad_response.format_id, "format000"); - } - - #[test] - fn test_callback_missing_field() { - let json_data = json!({ - "type": "impression" - // Missing url field - }); - - let result: Result = serde_json::from_value(json_data); - result.unwrap_err(); - } - - #[test] - fn test_callback_extra_fields() { - // Extra fields should be ignored - let json_data = json!({ - "type": "conversion", - "url": "https://example.com/track/conversion", - "extra": "ignored", - "another": 123 - }); - - let callback: Callback = serde_json::from_value(json_data) - .expect("should deserialize callback with extra fields"); - assert_eq!(callback.callback_type, "conversion"); - assert_eq!(callback.url, "https://example.com/track/conversion"); - } - - #[test] - fn test_ad_response_debug_format() { - let callback = Callback { - callback_type: "test".to_owned(), - url: "https://test.com".to_owned(), - }; - - let ad_response = AdResponse { - network_id: "123".to_owned(), - site_id: "456".to_owned(), - page_id: "789".to_owned(), - format_id: "000".to_owned(), - advertiser_id: "111".to_owned(), - campaign_id: "222".to_owned(), - insertion_id: "333".to_owned(), - creative_id: "444".to_owned(), - creative_url: "https://example.com/ad.jpg".to_owned(), - callbacks: vec![callback], - }; - - let debug_str = format!("{ad_response:?}"); - assert!(debug_str.contains("AdResponse")); - assert!(debug_str.contains("network_id")); - assert!(debug_str.contains("123")); - } - - #[test] - fn test_callback_debug_format() { - let callback = Callback { - callback_type: "debug_test".to_owned(), - url: "https://debug.test.com".to_owned(), - }; - - let debug_str = format!("{callback:?}"); - assert!(debug_str.contains("Callback")); - assert!(debug_str.contains("callback_type")); - assert!(debug_str.contains("debug_test")); - assert!(debug_str.contains("url")); - assert!(debug_str.contains("https://debug.test.com")); - } - - #[test] - fn test_various_callback_types() { - let callback_types = vec![ - "impression", - "click", - "viewability", - "conversion", - "engagement", - "complete", - "firstQuartile", - "midpoint", - "thirdQuartile", - ]; - - for cb_type in callback_types { - let json_data = json!({ - "type": cb_type, - "url": format!("https://example.com/track/{}", cb_type) - }); - - let callback: Callback = - serde_json::from_value(json_data).expect("should deserialize callback type"); - assert_eq!(callback.callback_type, cb_type); - assert_eq!(callback.url, format!("https://example.com/track/{cb_type}")); - } - } -} diff --git a/crates/trusted-server-core/src/module_context.rs b/crates/trusted-server-core/src/module_context.rs new file mode 100644 index 0000000000..e53ef8231e --- /dev/null +++ b/crates/trusted-server-core/src/module_context.rs @@ -0,0 +1,724 @@ +//! One context for every module, and the values a module's function may +//! name as parameters. +//! +//! Core builds one [`ModuleContext`] where it calls modules, from what the +//! request already holds. The context borrows everything it carries, so +//! building it copies no request data, and every module called there borrows +//! the same context. Each module is handed a [`ModuleCall`], being the context +//! together with the permissions that module declares. +//! +//! A module's function names what it needs as parameters, and +//! [`ModuleCall::inject`] passes each one in from the context. A parameter may +//! be any type that implements [`FromModuleContext`], an `Option` of one, which +//! is `None` when the value cannot be passed, or a tuple of them. The function +//! shapes that can be called are in [`inject`]. +//! +//! | Parameter | What it is | Absent where | +//! | --- | --- | --- | +//! | [`ModuleRequest`] | method, path, query, host and scheme | never | +//! | `&dyn RequestInfo` | the reader's evidence, headers and client IP | in a route, which holds the request itself | +//! | `&ClientInfo` | the reader's connection | where the device is classified | +//! | `&PermissionState` | the permissions resolved for the request | before they are resolved, which is where the device is classified and where the signal modules run | +//! | `&ConsentContext` | the decoded consent signals | where the device is classified | +//! | `&Settings` | the deployment's settings | outside any request | +//! | `&RuntimeServices` | stores, caches, backends and the HTTP client | outside any request | +//! | `&GeoInfo` | the reader's location | no location resolved, or withheld | +//! | `&DeviceSignals` | the device classification | not classified, or withheld | +//! | [`EdgeCookie`] | the request's Edge Cookie identifier | none, or withheld | +//! +//! A module asked outside any request, such as a signal module asked on its +//! own, is called in [`ModuleContext::empty`], which carries the request line +//! of the table and nothing else. +//! +//! # Permissions +//! +//! Geo, the device signals and the Edge Cookie identifier each carry the +//! permissions the module that produced them declares. Such a value is passed +//! only to a module that declares every one of those permissions and is granted +//! them on this request. A value whose producer declares none is passed to +//! every module, because evidence is not rationed and its use is what the +//! permissions govern. When a value is withheld, an `Option` parameter is +//! `None`, and a call naming the value itself is skipped, with the reason in +//! the debug log. + +use core::fmt; + +use edgezero_core::body::Body as EdgeBody; +use http::Method; + +use crate::consent::ConsentContext; +use crate::ec::EcContext; +use crate::ec::device::DeviceSignals; +use crate::error::TrustedServerError; +use crate::evidence::RequestInfo; +use crate::geo::GeoInfo; +use crate::permissions::{PermissionSet, PermissionState}; +use crate::platform::{ClientInfo, RuntimeServices}; +use crate::settings::Settings; + +pub mod inject; + +pub use inject::{Process, ProcessWith}; + +/// What a request resolved to: its method and address, and the host and +/// scheme it was made for. +/// +/// Carries nothing about who asked, so it is in every context. +#[derive(Debug, Clone, Copy)] +pub struct ModuleRequest<'r> { + method: &'r Method, + path: &'r str, + query: &'r str, + host: &'r str, + scheme: &'r str, +} + +impl<'r> ModuleRequest<'r> { + /// A request for `path` on `host` over `scheme`, with no query. + #[must_use] + pub const fn new(method: &'r Method, host: &'r str, scheme: &'r str, path: &'r str) -> Self { + Self { + method, + path, + query: "", + host, + scheme, + } + } + + /// What `request` resolved to, borrowed from it, the host and scheme read + /// as the publisher path reads them. + #[must_use] + pub fn of(request: &'r http::Request, client: &ClientInfo) -> Self { + Self::new( + request.method(), + crate::http_util::request_host(request), + crate::http_util::request_scheme(request, client), + request.uri().path(), + ) + .with_query(request.uri().query().unwrap_or_default()) + } + + /// The same request with the query string, without its leading `?`. + #[must_use] + pub fn with_query(self, query: &'r str) -> Self { + Self { query, ..self } + } + + /// The request method. + #[must_use] + pub fn method(&self) -> &'r Method { + self.method + } + + /// The URL path, without the query string. + #[must_use] + pub fn path(&self) -> &'r str { + self.path + } + + /// The query string, without its leading `?`, or `""`. + #[must_use] + pub fn query(&self) -> &'r str { + self.query + } + + /// The publisher-facing host the reader asked for. + #[must_use] + pub fn host(&self) -> &'r str { + self.host + } + + /// The publisher-facing scheme the reader asked for. + #[must_use] + pub fn scheme(&self) -> &'r str { + self.scheme + } +} + +/// A [`ModuleRequest`] that owns its parts, for a caller that must keep them +/// past the borrow of the request, such as one that hands the request on. +/// +/// The default is a `GET` of nothing, which a caller holds where it captured +/// no request. +#[derive(Debug, Clone, Default)] +pub struct ResolvedRequest { + method: Method, + path: String, + query: String, + host: String, + scheme: String, +} + +impl ResolvedRequest { + /// The method, address, host and scheme of `request`, the host and + /// scheme read as the publisher path reads them. + #[must_use] + pub fn of(request: &http::Request, client: &ClientInfo) -> Self { + let view = ModuleRequest::of(request, client); + Self { + method: view.method.clone(), + path: view.path.to_owned(), + query: view.query.to_owned(), + host: view.host.to_owned(), + scheme: view.scheme.to_owned(), + } + } + + /// The method, address, host and scheme `view` carries. + #[must_use] + pub fn from_view(view: &ModuleRequest<'_>) -> Self { + Self { + method: view.method.clone(), + path: view.path.to_owned(), + query: view.query.to_owned(), + host: view.host.to_owned(), + scheme: view.scheme.to_owned(), + } + } + + /// The same request with no query, for an answer that is held and served + /// to every caller, which must not depend on what one caller put in the + /// address. + #[must_use] + pub fn without_query(mut self) -> Self { + self.query.clear(); + self + } + + /// The borrowed view a context carries. + #[must_use] + pub fn view(&self) -> ModuleRequest<'_> { + ModuleRequest::new(&self.method, &self.host, &self.scheme, &self.path) + .with_query(&self.query) + } +} + +/// The request's Edge Cookie identifier, as the module that created it wrote +/// it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct EdgeCookie<'r>(&'r str); + +impl<'r> EdgeCookie<'r> { + /// The identifier. + #[must_use] + pub fn as_str(&self) -> &'r str { + self.0 + } +} + +/// A value whose use needs the permissions its producer declares. +struct Gated<'r, T: ?Sized> { + value: &'r T, + requires: PermissionSet, +} + +impl Clone for Gated<'_, T> { + fn clone(&self) -> Self { + *self + } +} + +impl Copy for Gated<'_, T> {} + +/// Everything a module may be passed for one request. +/// +/// Built once where core calls modules, by borrowing what the request already +/// holds, and borrowed by every module called there. A module never sees it +/// directly. It is handed a [`ModuleCall`] and names what it needs, see +/// [`ModuleCall::inject`]. +#[derive(Clone, Copy)] +pub struct ModuleContext<'r> { + request: ModuleRequest<'r>, + evidence: Option<&'r dyn RequestInfo>, + client: Option<&'r ClientInfo>, + permissions: Option<&'r PermissionState>, + consent: Option<&'r ConsentContext>, + settings: Option<&'r Settings>, + services: Option<&'r RuntimeServices>, + geo: Option>, + device: Option>, + edge_cookie: Option>, +} + +impl<'r> ModuleContext<'r> { + /// A context carrying `request` and nothing else. + /// + /// This is the context of a module asked outside any request, until the + /// builder methods add to it. + #[must_use] + pub const fn new(request: ModuleRequest<'r>) -> Self { + Self { + request, + evidence: None, + client: None, + permissions: None, + consent: None, + settings: None, + services: None, + geo: None, + device: None, + edge_cookie: None, + } + } + + /// The same context carrying the reader's evidence. + #[must_use] + pub fn with_evidence(self, evidence: &'r dyn RequestInfo) -> Self { + Self { + evidence: Some(evidence), + ..self + } + } + + /// The same context carrying the reader's connection. + #[must_use] + pub fn with_client(self, client: &'r ClientInfo) -> Self { + Self { + client: Some(client), + ..self + } + } + + /// The same context carrying the permissions resolved for the request. + #[must_use] + pub fn with_permissions(self, permissions: &'r PermissionState) -> Self { + Self { + permissions: Some(permissions), + ..self + } + } + + /// The same context carrying the request's decoded consent signals. + #[must_use] + pub fn with_consent(self, consent: &'r ConsentContext) -> Self { + Self { + consent: Some(consent), + ..self + } + } + + /// The same context carrying the deployment's settings. + #[must_use] + pub fn with_settings(self, settings: &'r Settings) -> Self { + Self { + settings: Some(settings), + ..self + } + } + + /// The same context carrying the request's services. + #[must_use] + pub fn with_services(self, services: &'r RuntimeServices) -> Self { + Self { + services: Some(services), + ..self + } + } + + /// The same context carrying the reader's location, whose use needs + /// `requires`, being what the geo module that resolved it declares. + #[must_use] + pub fn with_geo(self, geo: &'r GeoInfo, requires: PermissionSet) -> Self { + Self { + geo: Some(Gated { + value: geo, + requires, + }), + ..self + } + } + + /// The same context carrying the device signals, whose use needs + /// `requires`, being what the device module that derived them declares. + #[must_use] + pub fn with_device(self, device: &'r DeviceSignals, requires: PermissionSet) -> Self { + Self { + device: Some(Gated { + value: device, + requires, + }), + ..self + } + } + + /// The same context carrying the request's Edge Cookie identifier, whose + /// use needs `requires`, being what the Edge Cookie module declares. + #[must_use] + pub fn with_edge_cookie(self, id: &'r str, requires: PermissionSet) -> Self { + Self { + edge_cookie: Some(Gated { + value: id, + requires, + }), + ..self + } + } + + /// The same context carrying what the request's Edge Cookie state holds, + /// being its permissions, consent, location, device signals and Edge + /// Cookie identifier, and the request's services and connection. + /// + /// Each gated value needs what its producer declares. The location needs + /// what the geo module in `services` declares, the device signals what the + /// device module in `services` declares, and the identifier what the Edge + /// Cookie module the state was built with declares. Signals from a device + /// module the adapter builds itself, as Fastly does with none registered, + /// are gated by nothing, so such a module must declare no permission. + #[must_use] + pub fn with_request_state(self, state: &'r EcContext, services: &'r RuntimeServices) -> Self { + let mut context = self + .with_services(services) + .with_client(services.client_info()) + .with_permissions(state.permissions()) + .with_consent(state.consent()); + if let Some(geo) = state.geo_info() { + context = context.with_geo(geo, services.geo().required_permissions()); + } + if let Some(device) = state.device_signals() { + let requires = services + .device_module() + .map_or_else(PermissionSet::none, |module| module.required_permissions()); + context = context.with_device(device, requires); + } + if let (Some(id), Some(module)) = (state.ec_value(), state.selected_module()) { + context = context.with_edge_cookie(id, module.required_permissions()); + } + context + } + + /// What the request resolved to. + #[must_use] + pub fn request(&self) -> ModuleRequest<'r> { + self.request + } + + /// The request's decoded consent signals, when the context carries them. + pub(crate) fn consent(&self) -> Option<&'r ConsentContext> { + self.consent + } + + /// The reader's evidence, when the context carries it. + pub(crate) fn evidence(&self) -> Option<&'r dyn RequestInfo> { + self.evidence + } + + /// The call to one module, named `module`, which declares the permissions + /// in `declared`. + #[must_use] + pub fn call(&self, module: &'static str, declared: PermissionSet) -> ModuleCall<'_> { + ModuleCall { + context: self, + module, + declared, + } + } +} + +/// The method of the empty context. +static EMPTY_METHOD: Method = Method::GET; + +/// A context carrying nothing but a request for `/` on no host, for a module +/// asked outside any request. +static EMPTY: ModuleContext<'static> = + ModuleContext::new(ModuleRequest::new(&EMPTY_METHOD, "", "", "/")); + +impl ModuleContext<'static> { + /// The context of a module asked outside any request, which carries + /// nothing a module could name but an empty request. + /// + /// A caller with no request to resolve, such as a tool that evaluates + /// signals on their own, builds on a copy of it with the builder methods. + #[must_use] + pub fn empty() -> &'static Self { + &EMPTY + } +} + +impl ModuleCall<'static> { + /// A call into [`ModuleContext::empty`], by no name and declaring + /// nothing, for a module asked outside any request, such as by a test. + #[must_use] + pub fn empty() -> Self { + ModuleContext::empty().call("", PermissionSet::none()) + } +} + +impl fmt::Debug for ModuleContext<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ModuleContext") + .field("request", &self.request) + .field("evidence", &self.evidence.is_some()) + .field("permissions", &self.permissions.is_some()) + .field("settings", &self.settings.is_some()) + .field("services", &self.services.is_some()) + .field("geo", &self.geo.is_some()) + .field("device", &self.device.is_some()) + .field("edge_cookie", &self.edge_cookie.is_some()) + .finish_non_exhaustive() + } +} + +/// Core's call to one module, being the request's [`ModuleContext`] and the +/// permissions the module declares. +/// +/// A module's function is handed its parameters through +/// [`inject`](Self::inject). +#[derive(Debug, Clone, Copy)] +pub struct ModuleCall<'c> { + context: &'c ModuleContext<'c>, + module: &'static str, + declared: PermissionSet, +} + +impl<'c> ModuleCall<'c> { + /// The module being called. + #[must_use] + pub fn module(&self) -> &'static str { + self.module + } + + /// The permissions the module declares. + #[must_use] + pub fn declared(&self) -> PermissionSet { + self.declared + } + + /// A gated value, when this module may be passed it. + fn gated( + &self, + value: Option>, + name: &'static str, + ) -> Result<&'c T, Withheld> { + let gated = value.ok_or(Withheld::absent(name))?; + if gated.requires.is_empty() { + return Ok(gated.value); + } + let undeclared = gated.requires.difference(self.declared); + if !undeclared.is_empty() { + return Err(Withheld::new(name, WithheldReason::NotDeclared(undeclared))); + } + let state = self + .context + .permissions + .ok_or(Withheld::new(name, WithheldReason::Unresolved))?; + let unset = gated.requires.difference(state.permissions()); + if !unset.is_empty() { + return Err(Withheld::new(name, WithheldReason::NotGranted(unset))); + } + Ok(gated.value) + } +} + +/// Why a value is not passed to a module. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Withheld { + value: &'static str, + reason: WithheldReason, +} + +/// What kept a value from a module. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum WithheldReason { + /// The context carries no such value where this module runs. + Absent, + /// The module does not declare these permissions, which the value's use + /// needs. + NotDeclared(PermissionSet), + /// These permissions, which the value's use needs, are not set on this + /// request. + NotGranted(PermissionSet), + /// The value's use needs permissions, and none are resolved where this + /// module runs. + Unresolved, +} + +impl Withheld { + fn new(value: &'static str, reason: WithheldReason) -> Self { + Self { value, reason } + } + + fn absent(value: &'static str) -> Self { + Self::new(value, WithheldReason::Absent) + } + + /// The value that was withheld, such as `geo`. + #[must_use] + pub fn value(&self) -> &'static str { + self.value + } + + /// Why it was withheld. + #[must_use] + pub fn reason(&self) -> WithheldReason { + self.reason + } +} + +impl fmt::Display for Withheld { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let value = self.value; + match self.reason { + WithheldReason::Absent => write!(f, "there is no {value} where this module runs"), + WithheldReason::NotDeclared(missing) => write!( + f, + "{value} is withheld because the module does not declare {missing}, which its \ + use needs" + ), + WithheldReason::NotGranted(missing) => write!( + f, + "{value} is withheld because {missing} is not set on this request" + ), + WithheldReason::Unresolved => write!( + f, + "{value} is withheld because its use needs permissions and none are resolved \ + where this module runs" + ), + } + } +} + +impl core::error::Error for Withheld {} + +/// A route whose parameter the request's permissions withhold answers `403`. +/// A value absent where the module runs, or needing permissions that are not +/// resolved there, is a fault in how the module was written or placed, and +/// answers `500`. +impl From for error_stack::Report { + fn from(withheld: Withheld) -> Self { + let message = withheld.to_string(); + error_stack::Report::new(match withheld.reason { + WithheldReason::NotDeclared(_) | WithheldReason::NotGranted(_) => { + TrustedServerError::Forbidden { message } + } + WithheldReason::Absent | WithheldReason::Unresolved => { + TrustedServerError::Configuration { message } + } + }) + } +} + +/// A value a module's function can name as a parameter. +/// +/// Implemented for each value a [`ModuleContext`] carries, for an `Option` of +/// one, which is `None` rather than withheld, and for tuples of up to eight. +pub trait FromModuleContext<'c>: Sized { + /// The value for this call. + /// + /// # Errors + /// + /// [`Withheld`] when the context carries no such value here, or when the + /// value's use needs a permission this module does not declare or is not + /// granted. + fn from_module_context(call: &ModuleCall<'c>) -> Result; +} + +impl<'c, T: FromModuleContext<'c>> FromModuleContext<'c> for Option { + fn from_module_context(call: &ModuleCall<'c>) -> Result { + Ok(T::from_module_context(call).ok()) + } +} + +impl<'c> FromModuleContext<'c> for ModuleRequest<'c> { + fn from_module_context(call: &ModuleCall<'c>) -> Result { + Ok(call.context.request) + } +} + +impl<'c> FromModuleContext<'c> for &'c dyn RequestInfo { + fn from_module_context(call: &ModuleCall<'c>) -> Result { + call.context + .evidence + .ok_or(Withheld::absent("request evidence")) + } +} + +impl<'c> FromModuleContext<'c> for &'c ClientInfo { + fn from_module_context(call: &ModuleCall<'c>) -> Result { + call.context.client.ok_or(Withheld::absent("client info")) + } +} + +impl<'c> FromModuleContext<'c> for &'c PermissionState { + fn from_module_context(call: &ModuleCall<'c>) -> Result { + call.context + .permissions + .ok_or(Withheld::absent("permissions")) + } +} + +impl<'c> FromModuleContext<'c> for &'c ConsentContext { + fn from_module_context(call: &ModuleCall<'c>) -> Result { + call.context.consent.ok_or(Withheld::absent("consent")) + } +} + +impl<'c> FromModuleContext<'c> for &'c Settings { + fn from_module_context(call: &ModuleCall<'c>) -> Result { + call.context.settings.ok_or(Withheld::absent("settings")) + } +} + +impl<'c> FromModuleContext<'c> for &'c RuntimeServices { + fn from_module_context(call: &ModuleCall<'c>) -> Result { + call.context.services.ok_or(Withheld::absent("services")) + } +} + +impl<'c> FromModuleContext<'c> for &'c GeoInfo { + fn from_module_context(call: &ModuleCall<'c>) -> Result { + call.gated(call.context.geo, "geo") + } +} + +impl<'c> FromModuleContext<'c> for &'c DeviceSignals { + fn from_module_context(call: &ModuleCall<'c>) -> Result { + call.gated(call.context.device, "device signals") + } +} + +impl<'c> FromModuleContext<'c> for EdgeCookie<'c> { + fn from_module_context(call: &ModuleCall<'c>) -> Result { + call.gated(call.context.edge_cookie, "the Edge Cookie") + .map(EdgeCookie) + } +} + +/// A tuple of values is passed when every one of them is. +macro_rules! tuple_from_module_context { + ($($value:ident),+) => { + impl<'c, $($value: FromModuleContext<'c>),+> FromModuleContext<'c> for ($($value,)+) { + fn from_module_context(call: &ModuleCall<'c>) -> Result { + Ok(($($value::from_module_context(call)?,)+)) + } + } + }; +} + +tuple_from_module_context!(A1); +tuple_from_module_context!(A1, A2); +tuple_from_module_context!(A1, A2, A3); +tuple_from_module_context!(A1, A2, A3, A4); +tuple_from_module_context!(A1, A2, A3, A4, A5); +tuple_from_module_context!(A1, A2, A3, A4, A5, A6); +tuple_from_module_context!(A1, A2, A3, A4, A5, A6, A7); +tuple_from_module_context!(A1, A2, A3, A4, A5, A6, A7, A8); + +/// Requests for a module's own tests, which have no request to resolve. +#[cfg(any(test, feature = "test-utils"))] +pub mod test_support { + use http::Method; + + use super::ModuleRequest; + + static GET: Method = Method::GET; + + /// A `GET` of `path` on `publisher.example` over `https`, with no query. + #[must_use] + pub fn request(path: &'static str) -> ModuleRequest<'static> { + ModuleRequest::new(&GET, "publisher.example", "https", path) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/trusted-server-core/src/module_context/inject.rs b/crates/trusted-server-core/src/module_context/inject.rs new file mode 100644 index 0000000000..ba41fed5f0 --- /dev/null +++ b/crates/trusted-server-core/src/module_context/inject.rs @@ -0,0 +1,160 @@ +//! Calling a module's function with its parameters taken from the request's +//! [`ModuleContext`](super::ModuleContext). +//! +//! A trait method takes a [`ModuleCall`] and the implementation hands its own +//! function to [`ModuleCall::inject`], which is generic where the module type +//! is known. +//! +//! The function takes the module first, then up to eight parameters, each a +//! [`FromModuleContext`] value. [`ModuleCall::inject_with`] passes one argument +//! of the caller's own after the module, for the input that is the module's to +//! act on, such as the request a route answers. The function may be `async`, +//! in which case the future it returns is what comes back. +//! +//! ``` +//! use trusted_server_core::evidence::RequestInfo; +//! use trusted_server_core::geo::GeoInfo; +//! use trusted_server_core::module_context::{ModuleCall, Withheld}; +//! +//! struct Greeter; +//! +//! impl Greeter { +//! fn greet(&self, request: &dyn RequestInfo, geo: Option<&GeoInfo>) -> String { +//! let place = geo.map_or("somewhere", |geo| geo.country.as_str()); +//! format!("{} from {place}", request.user_agent()) +//! } +//! +//! fn process(&self, call: ModuleCall<'_>) -> Result { +//! call.inject(self, Self::greet) +//! } +//! } +//! ``` + +use super::{FromModuleContext, ModuleCall, Withheld}; + +/// A function core can call with the module and its parameters taken from a +/// [`ModuleCall`]. +/// +/// Implemented for every function and closure taking `&M` and up to eight +/// [`FromModuleContext`] parameters, so a module never implements it itself. +pub trait Process<'c, M: ?Sized, Args, O> { + /// Calls the function. + /// + /// # Errors + /// + /// [`Withheld`] for the first parameter that cannot be passed, in which case + /// the function is not called. + fn process(self, module: &'c M, call: &ModuleCall<'c>) -> Result; +} + +/// A function core can call with the module, one argument of the caller's +/// own, and its parameters taken from a [`ModuleCall`]. +/// +/// Implemented for every function and closure taking `&M`, the argument, and +/// up to eight [`FromModuleContext`] parameters. +pub trait ProcessWith<'c, M: ?Sized, X, Args, O> { + /// Calls the function. + /// + /// # Errors + /// + /// [`Withheld`] for the first parameter that cannot be passed, in which case + /// the function is not called and the argument is dropped. + fn process_with(self, module: &'c M, argument: X, call: &ModuleCall<'c>) + -> Result; +} + +macro_rules! process { + ($($parameter:ident),*) => { + impl<'c, M, F, O, $($parameter,)*> Process<'c, M, ($($parameter,)*), O> for F + where + M: ?Sized + 'c, + F: FnOnce(&'c M, $($parameter,)*) -> O, + $($parameter: FromModuleContext<'c>,)* + { + #[allow(non_snake_case, reason = "the parameters are named by their types")] + fn process(self, module: &'c M, call: &ModuleCall<'c>) -> Result { + let _ = call; + $(let $parameter = $parameter::from_module_context(call)?;)* + Ok(self(module, $($parameter,)*)) + } + } + + impl<'c, M, X, F, O, $($parameter,)*> ProcessWith<'c, M, X, ($($parameter,)*), O> for F + where + M: ?Sized + 'c, + F: FnOnce(&'c M, X, $($parameter,)*) -> O, + $($parameter: FromModuleContext<'c>,)* + { + #[allow(non_snake_case, reason = "the parameters are named by their types")] + fn process_with( + self, + module: &'c M, + argument: X, + call: &ModuleCall<'c>, + ) -> Result { + let _ = call; + $(let $parameter = $parameter::from_module_context(call)?;)* + Ok(self(module, argument, $($parameter,)*)) + } + } + }; +} + +process!(); +process!(A1); +process!(A1, A2); +process!(A1, A2, A3); +process!(A1, A2, A3, A4); +process!(A1, A2, A3, A4, A5); +process!(A1, A2, A3, A4, A5, A6); +process!(A1, A2, A3, A4, A5, A6, A7); +process!(A1, A2, A3, A4, A5, A6, A7, A8); + +impl<'c> ModuleCall<'c> { + /// Calls `function` with `module` and each of its other parameters taken + /// from the context. + /// + /// A parameter that is withheld skips the call, which is logged at debug + /// level with the reason, and an `Option` parameter that is withheld is + /// `None` instead. See [`module_context`](super) for what is withheld when. + /// + /// # Errors + /// + /// [`Withheld`] when a parameter cannot be passed, and the function is not + /// called. + pub fn inject(&self, module: &'c M, function: F) -> Result + where + M: ?Sized, + F: Process<'c, M, Args, O>, + { + function + .process(module, self) + .inspect_err(|withheld| self.log_skip(withheld)) + } + + /// Calls `function` with `module`, `argument`, and each of its other + /// parameters taken from the context, as [`inject`](Self::inject) does. + /// + /// # Errors + /// + /// [`Withheld`] when a parameter cannot be passed, and the function is not + /// called. + pub fn inject_with( + &self, + module: &'c M, + argument: X, + function: F, + ) -> Result + where + M: ?Sized, + F: ProcessWith<'c, M, X, Args, O>, + { + function + .process_with(module, argument, self) + .inspect_err(|withheld| self.log_skip(withheld)) + } + + fn log_skip(&self, withheld: &Withheld) { + log::debug!("Skipping module `{}`: {withheld}", self.module()); + } +} diff --git a/crates/trusted-server-core/src/module_context/tests.rs b/crates/trusted-server-core/src/module_context/tests.rs new file mode 100644 index 0000000000..dff4fbc923 --- /dev/null +++ b/crates/trusted-server-core/src/module_context/tests.rs @@ -0,0 +1,456 @@ +use std::cell::Cell; +use std::sync::Arc; + +use http::{HeaderMap, HeaderValue, Method, header}; + +use super::*; +use crate::ec::module::HmacModule; +use crate::evidence::OwnedRequestInfo; +use crate::permissions::{Permission, PermissionSet, PermissionState}; +use crate::platform::test_support::{noop_services, noop_services_with_client_ip}; +use crate::redacted::Redacted; + +/// The permission the gated values of these tests need. +fn storage() -> PermissionSet { + PermissionSet::none().with(Permission::StoreOnDevice) +} + +fn geo() -> GeoInfo { + GeoInfo { + city: "Bristol".to_owned(), + country: "GB".to_owned(), + continent: "Europe".to_owned(), + latitude: 51.45, + longitude: -2.58, + metro_code: 0, + region: Some("BST".to_owned()), + asn: None, + } +} + +fn device() -> DeviceSignals { + DeviceSignals::derive_ua_only("Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/146.0") +} + +fn evidence() -> OwnedRequestInfo { + let mut headers = HeaderMap::new(); + headers.insert( + header::USER_AGENT, + HeaderValue::from_static("ExampleAgent/1.0"), + ); + OwnedRequestInfo::new("203.0.113.7".to_owned(), headers) +} + +/// A module that is no more than the functions these tests hand to it. +struct Module; + +#[test] +fn the_request_reaches_every_function_that_names_it() { + let method = Method::POST; + let request = + ModuleRequest::new(&method, "publisher.example", "https", "/page").with_query("a=1"); + let context = ModuleContext::new(request); + let call = context.call("module", PermissionSet::none()); + + let seen = call + .inject(&Module, |_: &Module, request: ModuleRequest<'_>| { + ( + request.method().clone(), + request.host().to_owned(), + request.scheme().to_owned(), + request.path().to_owned(), + request.query().to_owned(), + ) + }) + .expect("the request is in every context"); + + assert_eq!( + seen, + ( + Method::POST, + "publisher.example".to_owned(), + "https".to_owned(), + "/page".to_owned(), + "a=1".to_owned(), + ) + ); +} + +#[test] +fn each_value_the_context_carries_reaches_a_function_that_names_it() { + let method = Method::GET; + let evidence = evidence(); + let client = ClientInfo { + tls_protocol: Some("TLSv1.3".to_owned()), + ..ClientInfo::default() + }; + let permissions = PermissionState::new(storage()); + let consent = ConsentContext::default(); + let mut settings = Settings::default(); + settings.publisher.domain = "publisher.example".to_owned(); + let services_ip: std::net::IpAddr = "203.0.113.9".parse().expect("should parse an IP"); + let services = noop_services_with_client_ip(services_ip); + let context = ModuleContext::new(ModuleRequest::new(&method, "h", "https", "/")) + .with_evidence(&evidence) + .with_client(&client) + .with_permissions(&permissions) + .with_consent(&consent) + .with_settings(&settings) + .with_services(&services); + let call = context.call("module", PermissionSet::none()); + + let first = call + .inject( + &Module, + |_: &Module, + evidence: &dyn RequestInfo, + client: &ClientInfo, + permissions: &PermissionState, + consent: &ConsentContext| { + ( + evidence.user_agent().to_owned(), + evidence.client_ip().to_owned(), + client.tls_protocol.clone(), + permissions.is_set(Permission::StoreOnDevice), + consent.jurisdiction.clone(), + ) + }, + ) + .expect("every value named is carried"); + assert_eq!( + first, + ( + "ExampleAgent/1.0".to_owned(), + "203.0.113.7".to_owned(), + Some("TLSv1.3".to_owned()), + true, + ConsentContext::default().jurisdiction, + ) + ); + + let second = call + .inject( + &Module, + |_: &Module, settings: &Settings, services: &RuntimeServices| { + ( + settings.publisher.domain.clone(), + services.client_info().client_ip, + ) + }, + ) + .expect("every value named is carried"); + assert_eq!(second, ("publisher.example".to_owned(), Some(services_ip))); +} + +#[test] +fn a_value_the_context_does_not_carry_is_none_or_skips_the_call() { + let method = Method::GET; + let context = ModuleContext::new(ModuleRequest::new(&method, "h", "https", "/")); + let call = context.call("module", PermissionSet::none()); + + let absent = call + .inject( + &Module, + |_: &Module, + evidence: Option<&dyn RequestInfo>, + settings: Option<&Settings>, + permissions: Option<&PermissionState>| { + ( + evidence.is_none(), + settings.is_none(), + permissions.is_none(), + ) + }, + ) + .expect("an optional parameter never skips the call"); + assert_eq!(absent, (true, true, true)); + + let called = Cell::new(false); + let skipped = call + .inject(&Module, |_: &Module, _settings: &Settings| called.set(true)) + .expect_err("a parameter the context does not carry skips the call"); + assert!(!called.get(), "the function is not called"); + assert_eq!(skipped.value(), "settings"); + assert_eq!(skipped.reason(), WithheldReason::Absent); +} + +/// The three gated values, each set on a context that needs `requires` for +/// it, read back through `Option` parameters by a module declaring +/// `declared`. +fn gated_values( + permissions: Option<&PermissionState>, + requires: PermissionSet, + declared: PermissionSet, +) -> (bool, bool, bool) { + let method = Method::GET; + let geo = geo(); + let device = device(); + let mut context = ModuleContext::new(ModuleRequest::new(&method, "h", "https", "/")) + .with_geo(&geo, requires) + .with_device(&device, requires) + .with_edge_cookie("an-id", requires); + if let Some(permissions) = permissions { + context = context.with_permissions(permissions); + } + context + .call("module", declared) + .inject( + &Module, + |_: &Module, + geo: Option<&GeoInfo>, + device: Option<&DeviceSignals>, + edge_cookie: Option>| { + (geo.is_some(), device.is_some(), edge_cookie.is_some()) + }, + ) + .expect("an optional parameter never skips the call") +} + +#[test] +fn a_gated_value_reaches_a_module_that_declares_its_permission_and_is_granted_it() { + let granted = PermissionState::new(storage()); + + assert_eq!( + gated_values(Some(&granted), storage(), storage()), + (true, true, true) + ); +} + +#[test] +fn a_gated_value_is_withheld_from_a_module_that_does_not_declare_its_permission() { + let granted = PermissionState::new(storage()); + + assert_eq!( + gated_values(Some(&granted), storage(), PermissionSet::none()), + (false, false, false), + "granted on the request is not enough without the declaration" + ); +} + +#[test] +fn a_gated_value_is_withheld_when_its_permission_is_not_set() { + let refused = PermissionState::default(); + + assert_eq!( + gated_values(Some(&refused), storage(), storage()), + (false, false, false), + "the declaration is not enough without the permission being set" + ); + assert_eq!( + gated_values(None, storage(), storage()), + (false, false, false), + "and nothing is granted where no permissions are resolved" + ); +} + +#[test] +fn a_value_whose_producer_declares_nothing_reaches_every_module() { + assert_eq!( + gated_values(None, PermissionSet::none(), PermissionSet::none()), + (true, true, true), + "evidence is not rationed" + ); +} + +#[test] +fn a_withheld_value_skips_a_call_that_names_it_and_says_why() { + let method = Method::GET; + let geo = geo(); + let refused = PermissionState::default(); + let context = ModuleContext::new(ModuleRequest::new(&method, "h", "https", "/")) + .with_geo(&geo, storage()) + .with_permissions(&refused); + let called = Cell::new(false); + + let undeclared = context + .call("module", PermissionSet::none()) + .inject(&Module, |_: &Module, _geo: &GeoInfo| called.set(true)) + .expect_err("geo is withheld from a module that does not declare storage"); + let unset = context + .call("module", storage()) + .inject(&Module, |_: &Module, _geo: &GeoInfo| called.set(true)) + .expect_err("geo is withheld while storage is not set"); + + assert!(!called.get(), "a skipped call is never made"); + assert_eq!(undeclared.reason(), WithheldReason::NotDeclared(storage())); + assert_eq!(unset.reason(), WithheldReason::NotGranted(storage())); + assert_eq!( + undeclared.to_string(), + "geo is withheld because the module does not declare necessary.operations.storage, \ + which its use needs" + ); +} + +#[test] +fn an_argument_of_the_callers_own_is_passed_after_the_module() { + let method = Method::GET; + let context = ModuleContext::new(ModuleRequest::new(&method, "h", "https", "/path")); + + let answer = context + .call("module", PermissionSet::none()) + .inject_with( + &Module, + String::from("argument"), + |_: &Module, argument: String, request: ModuleRequest<'_>| { + format!("{argument} {}", request.path()) + }, + ) + .expect("nothing is withheld"); + + assert_eq!(answer, "argument /path"); +} + +impl Module { + async fn classify(&self, evidence: &dyn RequestInfo, services: &RuntimeServices) -> String { + format!( + "{} {}", + evidence.user_agent(), + services.client_info().client_ip.is_some() + ) + } + + #[allow( + clippy::too_many_arguments, + reason = "the most parameters a module function may name" + )] + fn eight( + &self, + _request: ModuleRequest<'_>, + _evidence: &dyn RequestInfo, + _permissions: &PermissionState, + _settings: &Settings, + _services: &RuntimeServices, + _geo: Option<&GeoInfo>, + _device: Option<&DeviceSignals>, + _edge_cookie: Option>, + ) -> usize { + 8 + } +} + +#[test] +fn an_async_function_is_called_and_its_future_comes_back() { + let method = Method::GET; + let evidence = evidence(); + let services = noop_services_with_client_ip("203.0.113.9".parse().expect("should parse an IP")); + let context = ModuleContext::new(ModuleRequest::new(&method, "h", "https", "/")) + .with_evidence(&evidence) + .with_services(&services); + let call = context.call("module", PermissionSet::none()); + + let future = call + .inject(&Module, Module::classify) + .expect("nothing is withheld"); + + assert_eq!(futures::executor::block_on(future), "ExampleAgent/1.0 true"); +} + +#[test] +fn a_function_may_name_eight_values_and_a_tuple_groups_more() { + let method = Method::GET; + let evidence = evidence(); + let permissions = PermissionState::default(); + let deployment = Settings::default(); + let request_services = noop_services(); + let context = ModuleContext::new(ModuleRequest::new(&method, "h", "https", "/")) + .with_evidence(&evidence) + .with_permissions(&permissions) + .with_settings(&deployment) + .with_services(&request_services); + let call = context.call("module", PermissionSet::none()); + + assert_eq!(call.inject(&Module, Module::eight).expect("all carried"), 8); + let grouped = call + .inject( + &Module, + |_: &Module, (settings, services): (&Settings, &RuntimeServices)| { + ( + std::ptr::eq(settings, &deployment), + std::ptr::eq(services, &request_services), + ) + }, + ) + .expect("a tuple is passed when every value in it is"); + assert_eq!( + grouped, + (true, true), + "the tuple holds the context's own values" + ); +} + +#[test] +fn the_request_state_carries_each_value_gated_by_what_its_producer_declares() { + let method = Method::GET; + let mut state = EcContext::new_for_test(Some("an-id".to_owned()), ConsentContext::default()) + .with_module_for_test(Arc::new(HmacModule::new(Redacted::new( + "test-secret-key-32-bytes-minimum".to_owned(), + )))) + .with_geo_for_test(geo()); + state.set_device_signals(device()); + let services = + noop_services().with_device_module(Arc::new(crate::ec::device::BuiltinDeviceModule)); + let context = ModuleContext::new(ModuleRequest::new(&method, "h", "https", "/")) + .with_request_state(&state, &services); + + let undeclared = context + .call("module", PermissionSet::none()) + .inject( + &Module, + |_: &Module, + geo: Option<&GeoInfo>, + device: Option<&DeviceSignals>, + edge_cookie: Option>, + permissions: &PermissionState| { + ( + geo.map(|geo| geo.country.clone()), + device.is_some(), + edge_cookie.map(|id| id.as_str().to_owned()), + permissions.is_set(Permission::StoreOnDevice), + ) + }, + ) + .expect("nothing named is required"); + assert_eq!( + undeclared, + (Some("GB".to_owned()), true, None, true), + "geo and the device signals come from producers declaring nothing, and the \ + identifier needs what the HMAC module declares" + ); + + let declared = context + .call("module", storage()) + .inject(&Module, |_: &Module, edge_cookie: EdgeCookie<'_>| { + edge_cookie.as_str().to_owned() + }) + .expect("a module declaring storage is passed the identifier"); + assert_eq!(declared, "an-id"); +} + +#[test] +fn a_route_answers_forbidden_only_for_a_value_its_permissions_withhold() { + use crate::error::IntoHttpResponse as _; + + let status = |reason| { + error_stack::Report::::from(Withheld::new("geo", reason)) + .current_context() + .status_code() + }; + + assert_eq!( + status(WithheldReason::NotDeclared(storage())), + http::StatusCode::FORBIDDEN + ); + assert_eq!( + status(WithheldReason::NotGranted(storage())), + http::StatusCode::FORBIDDEN + ); + assert_eq!( + status(WithheldReason::Absent), + http::StatusCode::INTERNAL_SERVER_ERROR, + "a value absent where the route runs is the route's fault, not the reader's" + ); + assert_eq!( + status(WithheldReason::Unresolved), + http::StatusCode::INTERNAL_SERVER_ERROR + ); +} diff --git a/crates/trusted-server-core/src/module_name.rs b/crates/trusted-server-core/src/module_name.rs new file mode 100644 index 0000000000..3e25831a82 --- /dev/null +++ b/crates/trusted-server-core/src/module_name.rs @@ -0,0 +1,247 @@ +//! The name a module is selected by, taken from the folder its crate lives in. +//! +//! A crate at `crates/geo/example` supplies the module `geo.example`: the path +//! below the nearest `crates` directory, with `.` between the parts. A crate +//! supplying several modules names each one after itself, as +//! `middleware.example.hide`. A section may leave its own type folder off, so +//! under `[geo]` the name `example` means `geo.example`. Core's own modules +//! take bare names, such as `builtin`. + +use std::collections::HashMap; +use std::sync::{Mutex, OnceLock, PoisonError}; + +/// The name of the module the calling crate supplies, or, given an entry, the +/// name of that entry, as `.`. +/// +/// Derived from Cargo's `CARGO_MANIFEST_DIR` when the calling crate is built, +/// so a name cannot drift from its folder. A crate built from outside a +/// `crates` directory, such as a published package, is named by its package +/// name. +#[macro_export] +macro_rules! module_name { + () => { + $crate::module_name::from_manifest(env!("CARGO_MANIFEST_DIR"), env!("CARGO_PKG_NAME"), None) + }; + ($entry:expr) => { + $crate::module_name::from_manifest( + env!("CARGO_MANIFEST_DIR"), + env!("CARGO_PKG_NAME"), + Some($entry), + ) + }; +} + +/// The module name for a crate built at `manifest_dir`, with `entry` after it +/// when the crate supplies several. +/// +/// Each distinct name is made once and kept for the life of the process, so +/// a module can hand out `&'static str` without its crate holding a copy. +#[must_use] +pub fn from_manifest(manifest_dir: &str, package: &str, entry: Option<&str>) -> &'static str { + static NAMES: OnceLock>> = OnceLock::new(); + let mut name = crate_path(manifest_dir).unwrap_or_else(|| package.to_owned()); + if let Some(entry) = entry { + name.push('.'); + name.push_str(entry); + } + let mut names = NAMES + .get_or_init(Mutex::default) + .lock() + .unwrap_or_else(PoisonError::into_inner); + if let Some(existing) = names.get(&name) { + return existing; + } + let leaked: &'static str = Box::leak(name.clone().into_boxed_str()); + names.insert(name, leaked); + leaked +} + +/// The path below the nearest `crates` directory, joined by `.`, or `None` +/// when the folder is not below one. +fn crate_path(manifest_dir: &str) -> Option { + let parts: Vec<&str> = manifest_dir + .split(['/', '\\']) + .filter(|part| !part.is_empty()) + .collect(); + let crates = parts.iter().rposition(|part| *part == "crates")?; + let below = &parts[crates + 1..]; + (!below.is_empty()).then(|| below.join(".")) +} + +/// The longest module name, in bytes. +const MAX_NAME_BYTES: usize = 255; + +/// Whether `name` can name a module: parts joined by `.`, each of lower case +/// letters, digits, `_` or `-`. +/// +/// The same rule serves a crate's derived name and a name written in a +/// section, so anything a crate can be called can also be written. +#[must_use] +pub fn is_valid(name: &str) -> bool { + !name.is_empty() + && name.len() <= MAX_NAME_BYTES + && name.split('.').all(|part| { + !part.is_empty() + && part.bytes().all(|byte| { + byte.is_ascii_lowercase() + || byte.is_ascii_digit() + || matches!(byte, b'_' | b'-') + }) + }) +} + +/// The offered name that `written`, selected in a section of type +/// `type_folder`, means. +/// +/// The name as written wins, which is how a bare name reaches one of core's +/// own modules and a full path reaches a module under another type. Otherwise +/// the type folder is put in front, because a section is named exactly as its +/// folder. +#[must_use] +pub fn resolve<'a>(type_folder: &str, written: &str, offered: &[&'a str]) -> Option<&'a str> { + if let Some(exact) = offered.iter().find(|name| **name == written) { + return Some(exact); + } + offered.iter().copied().find(|name| { + name.split_once('.') + .is_some_and(|(folder, rest)| rest == written && folder == type_folder) + }) +} + +/// The short form of `name` within a section of type `type_folder`, being the +/// name with the type folder taken off, or the name itself when it lives +/// under another type or is one of core's own. +#[must_use] +pub fn short_form<'a>(type_folder: &str, name: &'a str) -> &'a str { + match name.split_once('.') { + Some((folder, rest)) if folder == type_folder => rest, + _ => name, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_crate_is_named_by_its_path_below_crates() { + for (dir, expected) in [ + ( + "/home/runner/work/trusted-server/crates/geo/example", + "geo.example", + ), + ( + "D:\\Workspace\\trusted-server\\crates\\permission-signal\\example-scheme", + "permission-signal.example-scheme", + ), + ( + "/repo/vendor/trusted-server/crates/middleware/example", + "middleware.example", + ), + ("/repo/crates/demo/example", "demo.example"), + ] { + assert_eq!(crate_path(dir).as_deref(), Some(expected), "{dir}"); + } + } + + #[test] + fn the_nearest_crates_directory_counts() { + assert_eq!( + crate_path("/work/crates/tools/checkout/crates/geo/example").as_deref(), + Some("geo.example") + ); + } + + #[test] + fn a_crate_outside_a_crates_directory_takes_its_package_name() { + assert_eq!( + from_manifest("/registry/src/example-geo-0.1.0", "example-geo", None), + "example-geo" + ); + assert_eq!(crate_path("/work/crates"), None); + } + + #[test] + fn an_entry_follows_the_crate_path_and_each_name_is_kept_once() { + let first = from_manifest("/repo/crates/middleware/example", "x", Some("hide")); + let second = from_manifest("/repo/crates/middleware/example", "x", Some("hide")); + + assert_eq!(first, "middleware.example.hide"); + assert!( + std::ptr::eq(first, second), + "the same name is not made twice" + ); + } + + #[test] + fn this_crate_names_itself() { + assert_eq!(crate::module_name!(), "trusted-server-core"); + } + + #[test] + fn a_name_resolves_as_written_then_within_its_type() { + let offered = [ + "builtin", + "geo.example", + "permission-signal.us-privacy", + "testing.mock", + ]; + + assert_eq!(resolve("device", "builtin", &offered), Some("builtin")); + assert_eq!(resolve("geo", "example", &offered), Some("geo.example")); + assert_eq!(resolve("geo", "geo.example", &offered), Some("geo.example")); + assert_eq!( + resolve("permission-signal", "us-privacy", &offered), + Some("permission-signal.us-privacy") + ); + assert_eq!( + resolve("permission_signal", "us-privacy", &offered), + None, + "a section is named exactly as its folder" + ); + assert_eq!( + resolve("ad-server", "testing.mock", &offered), + Some("testing.mock") + ); + assert_eq!( + resolve("device", "example", &offered), + None, + "another type's module" + ); + assert_eq!(resolve("geo", "missing", &offered), None); + } + + #[test] + fn a_name_is_lower_case_parts_joined_by_dots() { + for good in [ + "4example", + "us-privacy", + "host_signals", + "middleware.example.hide", + ] { + assert!(is_valid(good), "{good}"); + } + for bad in [ + "", + "Geo", + "geo..example", + ".geo", + "geo.", + "a b", + "geo/example", + ] { + assert!(!is_valid(bad), "{bad:?}"); + } + } + + #[test] + fn a_short_form_drops_only_the_sections_own_type() { + assert_eq!(short_form("geo", "geo.example"), "example"); + assert_eq!( + short_form("permission-signal", "permission-signal.example-scheme"), + "example-scheme" + ); + assert_eq!(short_form("ad-server", "testing.mock"), "testing.mock"); + assert_eq!(short_form("geo", "platform"), "platform"); + } +} diff --git a/crates/trusted-server-core/src/module_secrets.rs b/crates/trusted-server-core/src/module_secrets.rs new file mode 100644 index 0000000000..24e9653fc6 --- /dev/null +++ b/crates/trusted-server-core/src/module_secrets.rs @@ -0,0 +1,368 @@ +//! The secret settings a module declares in its own table. +//! +//! A module's builder lists the settings in its table that hold the name of +//! a secret, each with a rule for whether the table as written puts the +//! setting to use. As the settings load, the ones not in use are cleared and +//! the others are looked up in the default secret store, and at deploy time +//! each one in use is checked to name a key. None of that needs to know +//! which module declared them. + +use std::borrow::Cow; + +use edgezero_core::app_config::{SecretField, SecretKind, SecretPathSegment}; +use serde_json::{Map, Value}; + +use crate::integrations::IntegrationBuilder; +use crate::module_name::short_form; + +/// The section a module's table sits under and the names it can be written +/// at there, being the name with the section's type folder left off and the +/// name in full where the two differ, or a vendor's shared table under +/// `shared`, at the vendor's name alone. +fn written_forms(builder: &IntegrationBuilder) -> Option<(&'static str, Vec<&'static str>)> { + if let Some(vendor) = builder.shared_settings() { + return Some((crate::settings::SHARED_TYPE, vec![vendor])); + } + let name = builder.module_name()?; + let section = builder.section()?; + let short = short_form(section, name); + let mut forms = vec![short]; + if short != name { + forms.push(name); + } + Some((section, forms)) +} + +/// The form `section` selects the module by in a serialized configuration, +/// when it selects it. +fn selected_as<'a>(data: &Value, section: &str, forms: &[&'a str]) -> Option<&'a str> { + let section = data.get(section)?; + let named = |value: &Value| { + let written = value.as_str()?; + forms.iter().copied().find(|form| *form == written) + }; + section.get("module").and_then(named).or_else(|| { + section + .get("modules") + .and_then(Value::as_array) + .and_then(|modules| modules.iter().find_map(named)) + }) +} + +/// The table of a name as written in a section, found by the name's parts. +fn table_mut<'a>( + data: &'a mut Value, + section: &str, + written: &str, +) -> Option<&'a mut Map> { + let mut node = data.get_mut(section)?; + for part in written.split('.') { + node = node.get_mut(part)?; + } + node.as_object_mut() +} + +/// The value at `path` inside a module's table. +#[must_use] +pub(crate) fn value_at<'a>(table: &'a Map, path: &[&str]) -> Option<&'a Value> { + let (first, rest) = path.split_first()?; + let mut node = table.get(*first)?; + for part in rest { + node = node.get(*part)?; + } + Some(node) +} + +fn remove_at(table: &mut Map, path: &[&str]) { + let Some((leaf, parents)) = path.split_last() else { + return; + }; + let mut node = table; + for parent in parents { + let Some(next) = node.get_mut(*parent).and_then(Value::as_object_mut) else { + return; + }; + node = next; + } + node.remove(*leaf); +} + +/// Clears every declared secret setting that is not in use, in a serialized +/// configuration whose secrets have not been looked up yet. +/// +/// A setting is in use when its section selects the module and the module's +/// rule for the setting says its table uses it. A table the section does not +/// select is refused once the settings are deserialized, and clearing its +/// references first means that refusal is what an operator sees, rather than +/// a secret lookup failing. +pub(crate) fn clear_unused(data: &mut Value, builders: &[IntegrationBuilder]) { + for builder in builders { + let secrets = builder.secret_settings(); + if secrets.is_empty() { + continue; + } + let Some((section, forms)) = written_forms(builder) else { + continue; + }; + // A vendor's table runs its builder, so a secret in the table counts + // as in use by the table's own rules. + let selected = match builder.shared_settings() { + Some(vendor) => Some(vendor), + None => selected_as(data, section, &forms), + }; + for form in &forms { + let Some(table) = table_mut(data, section, form) else { + continue; + }; + let runs = selected == Some(*form); + let unused = secrets + .iter() + .filter(|secret| !(runs && (secret.in_use)(table))) + .map(|secret| secret.path) + .collect::>(); + for path in unused { + remove_at(table, path); + } + } + } +} + +/// The secret leaves the builders' modules declare, at each name a module's +/// table can be written under. +/// +/// Every level is optional, so a module that is not configured, or a setting +/// [`clear_unused`] removed, is passed over. +#[must_use] +pub(crate) fn secret_fields(builders: &[IntegrationBuilder]) -> Vec { + let optional = |name: &'static str| SecretPathSegment::OptionalField(Cow::Borrowed(name)); + let mut fields = Vec::new(); + for builder in builders { + let Some((section, forms)) = written_forms(builder) else { + continue; + }; + for secret in builder.secret_settings() { + let Some((leaf, parents)) = secret.path.split_last() else { + continue; + }; + for form in &forms { + let mut path = vec![optional(section)]; + path.extend(form.split('.').map(optional)); + path.extend(parents.iter().copied().map(optional)); + path.push(SecretPathSegment::Field(Cow::Borrowed(*leaf))); + fields.push(SecretField { + kind: SecretKind::KeyInDefault, + optional: true, + path, + }); + } + } + } + fields +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + use crate::integrations::ModuleSecretSetting; + use crate::integrations::registry_test_support::{probe_registration, validate_nothing}; + + fn uses_key(table: &Map) -> bool { + table.get("lock").and_then(Value::as_bool) == Some(true) + } + + fn uses_spare(table: &Map) -> bool { + uses_key(table) + && table + .get("spare") + .and_then(|spare| spare.get("enabled")) + .and_then(Value::as_bool) + == Some(true) + } + + const SECRETS: &[ModuleSecretSetting] = &[ + ModuleSecretSetting { + path: &["key_name"], + in_use: uses_key, + }, + ModuleSecretSetting { + path: &["spare", "key_name"], + in_use: uses_spare, + }, + ]; + + fn builder() -> IntegrationBuilder { + IntegrationBuilder::new("probe", "seam-probe", probe_registration, validate_nothing) + .with_module_name("testing.probe") + .with_secret_settings(SECRETS) + } + + const VENDOR: &str = "example"; + + fn shared_builder() -> IntegrationBuilder { + IntegrationBuilder::new( + "shared_probe", + "seam-probe", + probe_registration, + validate_nothing, + ) + .with_shared_settings(VENDOR) + .with_secret_settings(SECRETS) + } + + #[test] + fn a_secret_in_a_shared_table_is_kept_when_the_table_is_present_and_in_use() { + let mut data = json!({ "shared": { VENDOR: table(true, false) } }); + + clear_unused(&mut data, &[shared_builder()]); + + assert_eq!( + data.pointer("/shared/example/key_name"), + Some(&json!("main-key")), + "the table runs the builder, so its setting in use stays" + ); + assert_eq!(data.pointer("/shared/example/spare/key_name"), None); + } + + #[test] + fn a_secret_in_a_shared_table_is_declared_under_the_vendors_name() { + let paths: Vec = secret_fields(&[shared_builder()]) + .iter() + .map(|field| { + field + .path + .iter() + .map(|segment| match segment { + SecretPathSegment::Field(name) | SecretPathSegment::OptionalField(name) => { + name.to_string() + } + _ => "*".to_owned(), + }) + .collect::>() + .join(".") + }) + .collect(); + + assert_eq!( + paths, + vec!["shared.example.key_name", "shared.example.spare.key_name"] + ); + } + + fn table(lock: bool, spare: bool) -> Value { + json!({ + "lock": lock, + "key_name": "main-key", + "spare": { "enabled": spare, "key_name": "spare-key" }, + }) + } + + #[test] + fn a_setting_in_use_is_kept_and_one_not_in_use_is_cleared() { + let mut data = json!({ "testing": { "modules": ["probe"], "probe": table(true, false) } }); + + clear_unused(&mut data, &[builder()]); + + assert_eq!( + data.pointer("/testing/probe/key_name"), + Some(&json!("main-key")), + "should keep the setting the table puts to use" + ); + assert_eq!( + data.pointer("/testing/probe/spare/key_name"), + None, + "should clear the setting the table does not put to use" + ); + assert_eq!( + data.pointer("/testing/probe/spare/enabled"), + Some(&json!(false)), + "should leave the rest of the table as written" + ); + } + + #[test] + fn every_setting_of_a_module_no_section_selects_is_cleared() { + let mut data = json!({ "testing": { "probe": table(true, true) } }); + + clear_unused(&mut data, &[builder()]); + + assert_eq!(data.pointer("/testing/probe/key_name"), None); + assert_eq!(data.pointer("/testing/probe/spare/key_name"), None); + } + + #[test] + fn a_module_selected_by_its_full_name_keeps_the_settings_in_the_table_at_that_name() { + let mut data = json!({ + "testing": { + "module": "testing.probe", + "testing": { "probe": table(true, true) }, + "probe": table(true, true), + } + }); + + clear_unused(&mut data, &[builder()]); + + assert_eq!( + data.pointer("/testing/testing/probe/key_name"), + Some(&json!("main-key")), + "should keep the settings of the table the selection names" + ); + assert_eq!( + data.pointer("/testing/testing/probe/spare/key_name"), + Some(&json!("spare-key")) + ); + assert_eq!( + data.pointer("/testing/probe/key_name"), + None, + "should clear the settings of a table under the name not selected" + ); + } + + #[test] + fn the_declared_leaves_are_listed_under_both_written_names() { + let paths = secret_fields(&[builder()]) + .iter() + .map(|field| (field.dotted_path(), field.optional)) + .collect::>(); + + assert_eq!( + paths, + vec![ + ("testing.probe.key_name".to_owned(), true), + ("testing.testing.probe.key_name".to_owned(), true), + ("testing.probe.spare.key_name".to_owned(), true), + ("testing.testing.probe.spare.key_name".to_owned(), true), + ], + "should list each declared leaf at the short name and at the full one" + ); + } + + #[test] + fn a_builder_that_declares_no_secret_setting_lists_none() { + let plain = + IntegrationBuilder::new("probe", "seam-probe", probe_registration, validate_nothing) + .with_module_name("testing.probe"); + let mut data = json!({ "testing": { "probe": table(false, false) } }); + let before = data.clone(); + + clear_unused(&mut data, &[plain]); + + assert_eq!(data, before, "should leave the table alone"); + assert!(secret_fields(&[plain]).is_empty()); + } + + #[test] + fn value_at_reads_a_nested_setting() { + let Value::Object(table) = table(true, true) else { + panic!("should build a table"); + }; + + assert_eq!(value_at(&table, &["key_name"]), Some(&json!("main-key"))); + assert_eq!( + value_at(&table, &["spare", "key_name"]), + Some(&json!("spare-key")) + ); + assert_eq!(value_at(&table, &["absent", "key_name"]), None); + } +} diff --git a/crates/trusted-server-core/src/openrtb.rs b/crates/trusted-server-core/src/openrtb.rs index 4aded74882..7260d8b134 100644 --- a/crates/trusted-server-core/src/openrtb.rs +++ b/crates/trusted-server-core/src/openrtb.rs @@ -111,24 +111,6 @@ pub struct RegsExt { impl ToExt for RegsExt {} -#[derive(Debug, Serialize)] -pub struct RequestExt { - #[serde(skip_serializing_if = "Option::is_none")] - pub prebid: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub trusted_server: Option, -} - -#[derive(Debug, Serialize, serde::Deserialize)] -pub struct PrebidExt { - #[serde(skip_serializing_if = "Option::is_none")] - pub debug: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub returnallbidstatus: Option, -} - -impl ToExt for RequestExt {} - #[derive(Debug, Serialize)] pub struct TrustedServerExt { /// Version of the signing protocol (e.g., "1.1") @@ -147,30 +129,6 @@ pub struct TrustedServerExt { pub ts: Option, } -#[derive(Debug, Serialize)] -pub struct ImpExt { - pub prebid: PrebidImpExt, -} - -impl ToExt for ImpExt {} - -#[derive(Debug, Default, Serialize)] -pub struct PrebidImpExt { - #[serde(skip_serializing_if = "std::collections::HashMap::is_empty")] - pub bidder: std::collections::HashMap, - #[serde(skip_serializing_if = "Option::is_none")] - pub storedrequest: Option, -} - -/// PBS imp-level stored request reference. -/// -/// PBS merges the stored imp JSON (keyed by `id`) into the outgoing request, -/// populating bidder params that are not sent inline. -#[derive(Debug, Serialize)] -pub struct ImpStoredRequest { - pub id: String, -} - #[derive(Debug, Serialize)] pub struct BidExt<'a> { pub trusted_server: BidTrustedServerExt<'a>, diff --git a/crates/trusted-server-core/src/permission_signal/README.md b/crates/trusted-server-core/src/permission_signal/README.md new file mode 100644 index 0000000000..7c2c8d25aa --- /dev/null +++ b/crates/trusted-server-core/src/permission_signal/README.md @@ -0,0 +1,257 @@ +# Permission signals + +A permission is the primitive. This is the seam that decides whether one is +set, and it is open so that a new way of learning a permission can be added +without changing core. + +"Signal" rather than "consent" because consent is only one of the things +arriving here. Global Privacy Control is a setting a browser sends, not an +answer anyone gave to a question, and a jurisdiction rule is neither. Consent +is one kind of signal, so the seam takes the wider name and the consent +subsystem keeps the narrower one. + +## What core holds + +The reasons the schemes live outside core are on the docs page, +docs/guide/permission-signals.md. + +Core holds the trait, the ordering, the country baseline, and the policy +vocabulary for what a deployment decides about the shipped schemes, being +whether a TCF record answers, which signals count as a US-style opt-out and +what an opt-out takes away. It holds no scheme's wire format and no scheme's +meaning. Every scheme lives in its own crate outside core, including the five +that ship by default, so none of them is privileged by being the one that +happens to be built in. Core does not know what a TCF purpose is, because the +mapping from purpose to Data Use lives in the TCF crate, and a deployment +that runs no TCF carries no table of another scheme's numbers. The purposes +are the IAB TCF Europe purposes and what they grant are IAB Tech Lab Privacy +Taxonomy Data Uses, so a reader checks the mapping against the industry's own +documents rather than against us, and docs/guide/permission-model.md lists it +in full. Two purposes have no Data Use yet, so the crate carries a proposed +key for one and the TCF identifier for the other until the taxonomy adds +them. + +A scheme that is not one of the five is added the same way, as a crate that +reads its own signal from the request, without a change to core. The fifth, +Model Terms for Marketing (MTM), is the first terms scheme, where a publisher +and the parties it passes data to agree to be bound by a published set of +terms, and what the module reads is the visitor's answer to a preference +platform, one of three words in the first party cookie +`__mtm_pref`, a name any platform may set. It is one of many terms schemes +rather than the only one, because a publisher, a trade body or a regulator can +each publish terms and each set becomes a module. The five here are a +starting set and not the list. + +## The hierarchy + +Permissions are resolved in layers, each amending the one before. + +```text + country / region rules the baseline: granted, requires-signal, denied + | + v + module 1 (configured order) may amend + | + v + module 2 may amend + | + v + ... may amend + | + v + the permission state for this request +``` + +The baseline comes from `permissions.yaml`, keyed by country and region, with +the top node of the rules tree standing in for a request whose place is +unknown. A geo module supplies the place. No geo module means no country, +so every request resolves at that top node, and only a lookup that failed +resolves at the requires-signal floor, because a place that could not be +determined must not be treated as the declared default. + +Modules are then asked in order. Each sees what the modules before it +settled on and may amend it. A module with no opinion returns `Neutral` and +leaves the prior value standing, which is different from refusing. + +## The order is the policy + +The last module with an opinion decides, so the order is the policy. It is +a deployment's to set, not this code's to assume. + +```toml +[permission-signal] +modules = ["gpc", "gpp", "us-privacy", "tcf", "mtm"] +``` + +A module not on the list does not run, and there is no separate switch. A +publisher who does not want to act on Global Privacy Control removes `"gpc"` +from the list, and the module that reads the header then does not run. One +caveat: core's consent pipeline can also synthesize a US Privacy opt-out from +that header for a visitor in a US state, when the consent settings say to, +which they do by default, and the `us-privacy` module then acts on the +record it produced. A publisher who wants the header to have no effect at all +turns that setting off as well. Leaving `modules` out, or the section +entirely, runs every module the adapter offers, in the order it offers them, +so a signal is never quietly ignored because someone forgot to list it. An +unknown or repeated name is refused at startup, so a typo cannot silently stop +a scheme being honored. + +A module is named by its crate folder below `crates/permission-signal`, so +`gpp` and `us-privacy` are the folders those crates live in, and a name may +also be written in full, as `permission-signal.gpp`. A module that gains +settings will take them in a `[permission-signal.]` +block named for it. None of the five here has settings, so `modules` is the +only key the section accepts, and a block or any other key is refused as an +unknown field rather than ignored. + +The default order asks the signal with no interface of its own first and the +ones carrying a choice made through an interface after. Global Privacy +Control is a browser setting, so it revokes personalization on arrival, +whereas a GPP sale opt-out, a US Privacy string and a TCF record each carry +an answer a person gave, so they are asked later and amend it. A deployment +wanting the browser setting to stand over a later answer puts `gpc` last. + +Trusted Server takes no view on which scheme should win. That is a question +about a jurisdiction and a publisher. + +## A module can see the others + +Amending well sometimes needs to know who set the prior value. A module is +given the whole ordered list and its own position in it, so it can look up a +peer by name, see whether a peer it cares about is configured at all, and ask +a peer directly what that peer makes of a permission. + +That is what makes a rule like "personalization is off, but only because +Global Privacy Control set it, so my answer supersedes it" expressible. The +rule itself belongs to whichever module wants it. This seam only makes the +information available. + +Consulting a peer goes one level deep. A module answering a consultation +cannot consult in turn, so two modules asking each other cannot loop. + +## The terms the data is available under + +A module may also declare the terms documents the request's data is available +under, through `tdls` on the trait, and core carries what every configured +module declared on the permission state. Whoever receives the data reads them +to decide whether those are terms they accept, and whether they may pass the +data on. No declaration means no terms were declared, which is not the same as +terms permitting anything, so a recipient needing a basis and finding none has +none. + +Each entry is the address of a published document a person can read, and the +document must never be edited once published, which is why a version belongs in +its address. A document that can be rewritten tomorrow means a recipient can +never prove what it agreed to, and one edit silently rewrites the basis of every +transaction already sent under it. That is a property of how the document is +published, so the [`Tdl`](crate::tdl::Tdl) type refuses only an address nothing +could fetch. + +Of the five schemes here only MTM carries terms, declaring the versioned Model +Terms document whenever a PMP answer is present, and it is the first of many +rather than the only one. The name matches the `tdl` member the Data Labels work puts on a +node of an `OpenRTB` request, which is where these travel once a bid request +carries them. + +## Who could still answer + +A permission whose baseline requires a signal, and for which every module +answered `Neutral`, is not set, and it is also not refused. Resolution records +it as awaited, and the page reads the list as `awaiting` beside `set`, so a +prompt that has not run yet can be told from a visitor who said no. + +Waiting is only right for a permission some configured module could grant. +So each module declares, through `grants` on the trait, which permissions it +can ever answer `Grant` for under the policy it is given. An opt-out revokes and +declares nothing. TCF declares every Data Use a purpose maps to, and nothing +when the policy silences the record. The assembly keeps as awaited only what +some module declared, and everything else that requires a signal and got none +is simply unset. + +## The signals that were valid + +Each module also says, through `valid_signal` on the trait, which signal it +read from the request and used, as it was received. Core carries what every +configured module vouched for on the permission state, and the page reads +the list as `signals` beside `set`, `awaiting` and `tdls`. A page, a bid +request or a person reading the state relies on exactly those signals and no +other. + +A signal that was absent, could not be read, has expired, or that no +configured module acts on is not in the list, and nothing says which. What +each of those means for the permissions is the decision of the module for +that scheme, taken in `signal` and taken silently. Nothing is logged, because +an unreadable record is a visitor's preference and not an operational fault. +The shipped modules read their own unreadable record as the refusal it +may have carried, on the permissions their scheme covers, and say nothing +about any other scheme. Core answers nothing ahead of the modules, so the +order decides what a readable record from one scheme means beside an +unreadable one from another, as it decides everything else. + +A signal nobody vouched for goes no further. After assembly the consent +context keeps only the raw strings the valid signals name, so a corrupt or +expired record never reaches a bid request or anything else Trusted Server +sends on. + +## Withdrawal is a separate question + +A module may also say that the request explicitly _withdraws_ a permission, +which is different from not granting it. A withdrawal of storage expires the +browser cookie and writes the authoritative tombstone against the identifier. +A permission that is merely not set strips the response headers and leaves an +already-issued identifier alone, so a returning visitor is not permanently +withdrawn before they get to answer. Most schemes have no such notion, a +browser setting and a sale opt-out included, and only TCF answers it. Core +then scopes the answer to the jurisdiction, so a refusal only withdraws where +the storage baseline did not grant storage outright, because where it did +the identifier never depended on the record. + +## Writing a module + +Implement `PermissionSignalModule` in a crate that depends on core, and give +it its name through `module_name!()`, which is the name configuration uses. Answer +`Neutral` for a permission the module has no opinion on, including when the +signal it reads is absent from the request. Returning `Revoke` for an absent +signal turns silence into refusal and would revoke the permission on every +request not carrying that scheme, which is most of them. + +Read the request through `SignalInput::evidence`, which offers headers, +cookies, the path and the query, so a scheme core has never heard of can read +its own signal. The decoded consent record is offered too, for the schemes +core's consent pipeline already decodes, caches against the identifier and +expires. Prefer the record where it exists, because a module that +re-decodes the wire would skip the cached record on a returning visitor and +answer differently from every other reader of the same request. + +Read the policy for what is a deployment's decision rather than the scheme's, +such as which signals count as an opt-out and what an opt-out takes away, so +that a deployment can change those without changing a module. + +Register the crate at the adapter's composition root, where every adapter +builds its `RuntimeServices`. The adapter lists the modules it links, in the +default order, and `build_permission_signal_modules` selects and orders them +from configuration. + +## The modules supplied + +Five crates ship, under `crates/permission-signal/`, and a deployment +configuring nothing gets all five in this order: + +| Name | Crate | Reads | +| ------------ | ------------ | --------------------------------------------------- | +| `gpc` | `gpc` | The `Sec-GPC` header, Global Privacy Control | +| `gpp` | `gpp` | The US sale opt-out in a GPP string | +| `us-privacy` | `us-privacy` | The sale opt-out in a US Privacy string | +| `tcf` | `tcf` | A TCF v2 record, with its purpose mapping in code | +| `mtm` | `mtm` | The PMP answer, under the Model Terms for Marketing | + +The three opt-outs are separate rather than one so that a publisher who does +not act on Global Privacy Control can remove it and keep the other two. + +## What is not a module + +Nothing. Core reads no scheme and answers for none, an unreadable record +included. A publisher chooses which signals to act on by listing modules, +and each module decides for its own scheme what an absent, unreadable or +expired signal means. A scheme that is not listed does not run, whatever the +request carries for it. diff --git a/crates/trusted-server-core/src/permission_signal/mod.rs b/crates/trusted-server-core/src/permission_signal/mod.rs new file mode 100644 index 0000000000..74d7d33b8e --- /dev/null +++ b/crates/trusted-server-core/src/permission_signal/mod.rs @@ -0,0 +1,1196 @@ +#![doc = include_str!("README.md")] + +use std::sync::Arc; + +use error_stack::Report; + +use crate::consent::ConsentContext; +use crate::error::TrustedServerError; +use crate::evidence::RequestInfo; +use crate::module_context::{ModuleCall, ModuleContext}; +use crate::permissions::{ + Acquisition, ConsentSignal, Permission, PermissionSet, SignalPolicy, ValidSignal, +}; +use crate::settings::Settings; +use crate::tdl::Tdl; + +/// What a signal module may read about a request. +/// +/// A struct rather than a parameter list, so a module needing something new +/// does not change every implementation. The fields are what the signal run +/// is deciding. Anything else in the request's module context, such as what +/// the request resolved to or the settings, a module names through +/// [`call`](Self::call). The permissions are what the run produces, so none +/// are resolved yet and no value whose use needs one is passed here. +pub struct SignalInput<'a> { + /// The decoded consent record for this request. + /// + /// Offered because the schemes that ship by default already decode into + /// it, core keeps it against the Edge Cookie identifier between requests, + /// and the rest of the system reads it. A module is not required to use + /// it, and a scheme core has never heard of will not appear in it. Such a + /// module reads [`evidence`](Self::evidence) instead and decodes whatever + /// its scheme needs. + pub consent: &'a ConsentContext, + /// The request itself, as evidence. + /// + /// This is the open half of the seam, and it is [`RequestInfo`], the same + /// abstraction the Edge Cookie and device modules already read. A + /// module asks for the header, cookie, path or query parameter its own + /// scheme uses, so core holds no list of which evidence a scheme may read. + pub evidence: &'a dyn RequestInfo, + /// The policy from `permissions.yaml`, which decides what a signal means + /// rather than leaving each module to invent its own meaning. + pub policy: &'a SignalPolicy, + /// What the country and region rules say about this permission, before any + /// module is asked. A module amends this rather than deciding alone. + pub baseline: Acquisition, + /// What the modules asked before this one settled on. + /// + /// [`ConsentSignal::Neutral`] means none of them had an opinion, so the + /// baseline still stands. + pub settled: ConsentSignal, + /// Every module in configured order, this one included. + modules: &'a [Arc], + /// Where in that order the module being asked sits. + position: usize, + /// Whether this module may consult a peer. + /// + /// False while answering a consultation, which is what stops two modules + /// that consult each other from looping. + may_ask: bool, + /// The request's module context. + context: &'a ModuleContext<'a>, +} + +impl<'a> SignalInput<'a> { + /// An input for a module asked on its own, outside an ordered run, in a + /// context carrying nothing, see [`ModuleContext::empty`]. + #[must_use] + pub fn new( + consent: &'a ConsentContext, + evidence: &'a dyn RequestInfo, + policy: &'a SignalPolicy, + baseline: Acquisition, + ) -> Self { + Self { + consent, + evidence, + policy, + baseline, + settled: ConsentSignal::Neutral, + modules: &[], + position: 0, + may_ask: true, + context: ModuleContext::empty(), + } + } + + /// The same input in the request's module context. + #[must_use] + pub fn with_context(self, context: &'a ModuleContext<'a>) -> Self { + Self { context, ..self } + } + + /// The module's call into the request's module context, which it hands + /// its own function to, naming what it needs. A module asked on its own + /// is called by no name and declares nothing. + #[must_use] + pub fn call(&self) -> ModuleCall<'a> { + match self.modules.get(self.position) { + Some(module) => self + .context + .call(module.id(), module.required_permissions()), + None => self.context.call("", PermissionSet::none()), + } + } + + /// Every module in configured order, this one included. + /// + /// A module consults the list to decide whether a peer it cares about is + /// configured at all, and where it sits relative to this one. + #[must_use] + pub fn modules(&self) -> &[Arc] { + self.modules + } + + /// Where the module being asked sits in that order. + #[must_use] + pub const fn position(&self) -> usize { + self.position + } + + /// Whether a module with this identifier is configured. + #[must_use] + pub fn has(&self, id: &str) -> bool { + self.modules.iter().any(|module| module.id() == id) + } + + /// What a peer makes of `permission`, asked directly. + /// + /// The peer answers as if it were first, so the reply is that peer's own + /// opinion rather than what the run has settled on so far. That is the + /// useful question, because a module wanting to know whether the prior + /// value came from a particular peer asks that peer what it says. + /// + /// Returns `None` when no module carries the identifier, when the caller + /// names itself, and when this input is itself answering a consultation. + /// A module must handle `None` rather than assume a peer is present. + #[must_use] + pub fn ask(&self, id: &str, permission: Permission) -> Option { + if !self.may_ask { + return None; + } + let names: Vec<&str> = self.modules.iter().map(|module| module.id()).collect(); + let name = crate::module_name::resolve(MODULE_TYPE, id, &names)?; + let (position, module) = self + .modules + .iter() + .enumerate() + .find(|(position, module)| module.id() == name && *position != self.position)?; + let input = Self { + consent: self.consent, + evidence: self.evidence, + policy: self.policy, + baseline: self.baseline, + settled: ConsentSignal::Neutral, + modules: self.modules, + position, + may_ask: false, + context: self.context, + }; + Some(module.signal(permission, &input)) + } +} + +/// The type folder of every permission signal crate, which a name written in +/// `[permission-signal] modules` may leave off. +pub const MODULE_TYPE: &str = "permission-signal"; + +/// The name a module reports on the page and in logs, being its name without +/// the type folder, such as `example` for `permission-signal.example`. +#[must_use] +pub fn short_name(module: &dyn PermissionSignalModule) -> &'static str { + crate::module_name::short_form(MODULE_TYPE, module.id()) +} + +/// A module of permission signals. +/// +/// An implementation answers for one signaling scheme. It reads the request, +/// applies whatever the policy says about its own scheme, and returns how it +/// would amend one permission. +/// +/// # Contract +/// +/// Answer [`ConsentSignal::Neutral`] for a permission this module has no +/// opinion on, including when the signal it reads is absent from the request. +/// Returning [`ConsentSignal::Revoke`] for an absent signal would turn silence +/// into refusal and revoke the permission on every request that did not carry +/// this scheme. +pub trait PermissionSignalModule: Send + Sync { + /// The module's name, used in configuration, in logs, and by a peer + /// looking this module up through [`SignalInput::ask`]. + /// + /// A module in a crate takes its crate folder's name through + /// [`crate::module_name!`], such as `permission-signal.example`, which + /// `[permission-signal] modules` may write as `example`. + fn id(&self) -> &'static str; + + /// The permissions this module declares, which decide whether a gated + /// value is passed to it. None by default. Signal modules run before the + /// permissions are resolved, so a declaration here passes nothing yet. + fn required_permissions(&self) -> PermissionSet { + PermissionSet::none() + } + + /// How this module would amend `permission` for this request. + fn signal(&self, permission: Permission, input: &SignalInput<'_>) -> ConsentSignal; + + /// The permissions this module can ever answer [`ConsentSignal::Grant`] + /// for under `policy`, so a permission a signal could still set can be + /// told from one nothing in this deployment will ever set. + /// + /// Resolution records a permission that requires a signal and got none as + /// awaited, and a page holds what depends on an awaited permission until + /// an answer arrives. That is right only for a permission some configured + /// module could grant. For any other, waiting is waiting for ever, so + /// the assembly narrows the awaited list to the union of these + /// declarations. + /// + /// A scheme that only ever revokes, which is every opt-out, leaves this + /// at its default of nothing. A scheme that grants declares exactly what + /// it maps, under the policy it is given, so a record the policy does not + /// let answer declares nothing either. + fn grants(&self, _policy: &SignalPolicy) -> PermissionSet { + PermissionSet::none() + } + + /// Whether the request carries an explicit withdrawal of `permission` + /// under this scheme, as opposed to merely not granting it. + /// + /// The difference is destructive. A withdrawal of storage expires the + /// browser cookie and writes the authoritative tombstone against the + /// identifier, where a permission that is simply not set for this request + /// strips the response headers and leaves the identifier alone, so that a + /// returning visitor is not permanently withdrawn before they ever get to + /// answer. Most schemes have no such notion, a browser setting and a sale + /// opt-out included, and leave this at its default of `false`. Whether a + /// withdrawal is acted on at all is core's decision from the jurisdiction's + /// baseline, so a refusal only withdraws where the baseline did not grant + /// the permission outright, because where it did the permission never + /// depended on the record. + fn withdraws(&self, _permission: Permission, _input: &SignalInput<'_>) -> bool { + false + } + + /// The signal this module read from the request and used, as it was + /// received, or `None` when there is none it can use. + /// + /// Absent, unreadable, expired and not-acted-on all answer `None`, and + /// nothing here says which. What each of those means for the + /// permissions is this module's decision in [`signal`](Self::signal), + /// taken silently. Core carries what every module vouched for on the + /// permission state, and a signal nobody vouched for is dropped from + /// everything Trusted Server sends on, so a corrupt record never reaches + /// a page or a bid request. See [`crate::module_context`] for what an + /// implementation may name. + fn valid_signal(&self, _call: ModuleCall<'_>) -> Option { + None + } + + /// The terms documents this module says the request's data is + /// available under, empty when it declares none. + /// + /// A locator tells whoever receives the data what terms cover it, so + /// they can decide whether those are terms they accept and whether they + /// may pass the data on. Most schemes carry no terms of their own and + /// leave this at its default, and a module for a terms scheme returns the + /// document that applies to this request. Model Terms for Marketing (MTM) + /// is the first such scheme and one of many rather than the only one. Core + /// does not read the documents, it carries the locators, so what a document + /// says stays between the parties bound by it. + /// + /// A locator must point at a document that is never edited once + /// published, which [`Tdl`] documents and cannot enforce. + fn tdls(&self, _call: ModuleCall<'_>) -> Vec { + Vec::new() + } +} + +/// Asks every module in order and returns what they settle on together. +/// +/// See the module documentation for the layering and why the order is the +/// configuration. +#[must_use] +pub(crate) fn combine( + modules: &[Arc], + permission: Permission, + context: &ModuleContext<'_>, + consent: &ConsentContext, + evidence: &dyn RequestInfo, + policy: &SignalPolicy, + baseline: Acquisition, +) -> ConsentSignal { + let mut settled = ConsentSignal::Neutral; + for (position, module) in modules.iter().enumerate() { + let input = SignalInput { + consent, + evidence, + policy, + baseline, + settled, + modules, + position, + may_ask: true, + context, + }; + // Every module is asked, because a later one may amend what an + // earlier one settled. Stopping at the first answer would make the + // order mean the opposite of what it says. + match module.signal(permission, &input) { + ConsentSignal::Neutral => {} + answer => settled = answer, + } + } + settled +} + +/// Whether the request explicitly withdraws `permission`, scoped to the +/// jurisdiction's baseline for it. +/// +/// A withdrawal only counts where the baseline did not grant the permission +/// outright. Under a `requires_signal` baseline a refusal is the visitor +/// declining the very signal the permission depended on, so it is +/// destructive. Under a `granted` baseline the permission never depended on +/// the record, so the same refusal suppresses use for the request without +/// destroying anything. Any one module answering [`withdraws`] is enough, +/// and no module answering it, or none configured, is never a withdrawal. +/// +/// [`withdraws`]: PermissionSignalModule::withdraws +#[must_use] +pub(crate) fn withdrawn( + modules: &[Arc], + permission: Permission, + context: &ModuleContext<'_>, + consent: &ConsentContext, + evidence: &dyn RequestInfo, + policy: &SignalPolicy, + baseline: Acquisition, +) -> bool { + if matches!(baseline, Acquisition::Granted) { + return false; + } + modules.iter().enumerate().any(|(position, module)| { + let input = SignalInput { + consent, + evidence, + policy, + baseline, + settled: ConsentSignal::Neutral, + modules, + position, + may_ask: true, + context, + }; + module.withdraws(permission, &input) + }) +} + +/// The terms documents the configured modules declare for this request. +/// +/// Asked in the same order the modules answer in, so the list reads the +/// way the deployment is configured, and a document named by two modules +/// is carried once. No module declaring anything leaves the list empty, +/// which says no terms were declared rather than that any terms apply. +#[must_use] +pub(crate) fn tdls( + modules: &[Arc], + context: &ModuleContext<'_>, +) -> Arc<[Tdl]> { + let mut declared: Vec = Vec::new(); + for module in modules { + for tdl in module.tdls(context.call(module.id(), module.required_permissions())) { + if !declared.contains(&tdl) { + declared.push(tdl); + } + } + } + Arc::from(declared) +} + +/// Every permission some module in `modules` declares it can grant under +/// `policy`, which is the most a signal arriving later could still set. +#[must_use] +pub fn answerable( + modules: &[Arc], + policy: &SignalPolicy, +) -> PermissionSet { + modules + .iter() + .map(|module| module.grants(policy)) + .fold(PermissionSet::none(), PermissionSet::union) +} + +/// The signals the configured modules read and found valid, in the order +/// the modules are asked. +/// +/// Each module vouches for at most one signal, its own scheme's, so the +/// list reads the way the deployment is configured. No module vouching +/// for anything leaves the list empty, which says the request carried +/// nothing a configured module could use. +#[must_use] +pub(crate) fn signals( + modules: &[Arc], + context: &ModuleContext<'_>, +) -> Arc<[ValidSignal]> { + modules + .iter() + .filter_map(|module| { + module.valid_signal(context.call(module.id(), module.required_permissions())) + }) + .collect() +} + +/// The modules a deployment named, in the order it named them, drawn from +/// the ones the build makes available. +/// +/// `None` means nothing was configured, which runs every available module +/// in the order the adapter offered them. That is deliberate, so a publisher +/// gets every scheme the build knows about until they say otherwise and a +/// signal is never quietly ignored because someone forgot to list it. An +/// empty list is a publisher acting on no signal at all, and is accepted. +/// +/// # Errors +/// +/// A name matching no available module, or a name given twice, is refused +/// rather than ignored, so a typo cannot silently stop a scheme being honored +/// and a scheme cannot run twice at two places in the order. +pub(crate) fn select( + available: &[Arc], + configured: Option<&[String]>, +) -> Result>, Report> { + let Some(names) = configured else { + return Ok(available.to_vec()); + }; + let mut selected = Vec::with_capacity(names.len()); + for (position, name) in names.iter().enumerate() { + if names[..position].contains(name) { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "Permission signal module `{name}` is named more than once in \ + [permission-signal] modules. Each module runs once, at one place \ + in the order" + ), + })); + } + let offered = ids(available); + let Some(module) = crate::module_name::resolve(MODULE_TYPE, name, &offered) + .and_then(|resolved| available.iter().find(|module| module.id() == resolved)) + else { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "Permission signal module `{name}` is not available in this build. \ + Available modules are {}", + offered + .iter() + .map(|id| crate::module_name::short_form(MODULE_TYPE, id)) + .collect::>() + .join(", ") + ), + })); + }; + selected.push(Arc::clone(module)); + } + Ok(selected) +} + +/// The available modules a configured list leaves out, for the startup log. +#[must_use] +pub(crate) fn omitted<'a>( + available: &'a [Arc], + configured: Option<&[String]>, +) -> Vec<&'a str> { + let Some(names) = configured else { + return Vec::new(); + }; + available + .iter() + .map(|module| module.id()) + .filter(|id| !names.iter().any(|name| name == id)) + .collect() +} + +/// The identifiers of `modules`, in order. +#[must_use] +pub(crate) fn ids(modules: &[Arc]) -> Vec<&str> { + modules.iter().map(|module| module.id()).collect() +} + +/// Selects the modules a deployment runs from the ones an adapter makes +/// available, and says so in the log once at startup. +/// +/// This is the adapter's composition point. Core supplies no module of its +/// own, so an adapter hands in every scheme crate it links, in the order that +/// stands when configuration names none, and receives back the ordered list +/// the request path asks. The list is shared rather than owned, so handing it +/// to every request's services is one reference count and no copy. +/// +/// # Errors +/// +/// A configured name that matches nothing available, or a name given twice, +/// fails startup with a message naming what is available, so a typo cannot +/// silently stop a scheme being honored. +pub fn build_permission_signal_modules( + settings: &Settings, + available: &[Arc], +) -> Result]>, Report> { + let configured = settings.permission_signal.modules.as_deref(); + let selected = select(available, configured)?; + match configured { + None => log::info!( + "Permission signals: acting on every module this build offers, [{}], no \ + [permission-signal] modules configured", + ids(&selected).join(", ") + ), + Some([]) => log::info!( + "Permission signals: acting on no module, [permission-signal] modules is \ + empty, so every permission stays at its country and region baseline" + ), + Some(_) => log::info!( + "Permission signals: acting on [{}], asked in that order", + ids(&selected).join(", ") + ), + } + let left_out = omitted(available, configured); + if !left_out.is_empty() { + log::warn!( + "Permission signals: not acting on [{}], which are not in [permission-signal] \ + modules. A signal this deployment does not act on is read from the request \ + and then ignored", + left_out.join(", ") + ); + } + Ok(Arc::from(selected)) +} + +#[cfg(test)] +mod tests { + use http::HeaderMap; + + use super::*; + use crate::evidence::OwnedRequestInfo; + use crate::module_context::test_support; + + /// A module that always answers the same thing, for testing the rule + /// rather than any particular scheme. + struct Fixed(&'static str, ConsentSignal); + + impl PermissionSignalModule for Fixed { + fn id(&self) -> &'static str { + self.0 + } + + fn signal(&self, _permission: Permission, _input: &SignalInput<'_>) -> ConsentSignal { + self.1 + } + } + + /// A module that answers by consulting a peer, which is the behavior the + /// peer visibility exists for. + struct Consulting { + id: &'static str, + peer: &'static str, + } + + impl PermissionSignalModule for Consulting { + fn id(&self) -> &'static str { + self.id + } + + fn signal(&self, permission: Permission, input: &SignalInput<'_>) -> ConsentSignal { + match input.ask(self.peer, permission) { + // The peer refused, and this module takes the opposite view + // of the same request, which is the override the seam allows. + Some(ConsentSignal::Revoke) => ConsentSignal::Grant, + _ => ConsentSignal::Neutral, + } + } + } + + /// A module that declares a terms document, which is what a scheme like + /// Model Terms for Marketing does and most schemes do not. + struct Declaring(&'static str, &'static str); + + impl PermissionSignalModule for Declaring { + fn id(&self) -> &'static str { + self.0 + } + + fn signal(&self, _permission: Permission, _input: &SignalInput<'_>) -> ConsentSignal { + ConsentSignal::Neutral + } + + fn tdls(&self, _call: ModuleCall<'_>) -> Vec { + vec![Tdl::new(self.1).expect("should accept the test locator")] + } + } + + /// A module that withdraws storage, for testing the scoping rule. + struct Withdrawing; + + impl PermissionSignalModule for Withdrawing { + fn id(&self) -> &'static str { + "withdrawing" + } + + fn signal(&self, _permission: Permission, _input: &SignalInput<'_>) -> ConsentSignal { + ConsentSignal::Revoke + } + + fn withdraws(&self, permission: Permission, _input: &SignalInput<'_>) -> bool { + permission == Permission::StoreOnDevice + } + } + + fn no_evidence() -> OwnedRequestInfo { + OwnedRequestInfo::new(String::new(), HeaderMap::new()) + } + + /// The signals `modules` vouch for, asked in a context carrying `consent` + /// and no evidence. + fn signals_for( + modules: &[Arc], + consent: &ConsentContext, + ) -> Arc<[ValidSignal]> { + let evidence = no_evidence(); + let context = ModuleContext::new(test_support::request("/")) + .with_consent(consent) + .with_evidence(&evidence); + signals(modules, &context) + } + + /// The terms `modules` declare, asked as [`signals_for`] asks. + fn tdls_for( + modules: &[Arc], + consent: &ConsentContext, + ) -> Arc<[Tdl]> { + let evidence = no_evidence(); + let context = ModuleContext::new(test_support::request("/")) + .with_consent(consent) + .with_evidence(&evidence); + tdls(modules, &context) + } + + fn fixed(id: &'static str, signal: ConsentSignal) -> Arc { + Arc::new(Fixed(id, signal)) + } + + /// A module that declares it can grant one permission, whatever it then + /// answers, standing in for a scheme with a mapping. + struct Granting(&'static str, Permission); + + impl PermissionSignalModule for Granting { + fn id(&self) -> &'static str { + self.0 + } + + fn signal(&self, _permission: Permission, _input: &SignalInput<'_>) -> ConsentSignal { + ConsentSignal::Neutral + } + + fn grants(&self, _policy: &SignalPolicy) -> PermissionSet { + PermissionSet::none().with(self.1) + } + } + + #[test] + fn a_module_declares_nothing_grantable_unless_it_says_otherwise() { + // An opt-out only ever revokes, so the default declaration is empty + // and a deployment of opt-outs alone can grant nothing. + let policy = SignalPolicy::default(); + assert!( + fixed("opt_out", ConsentSignal::Revoke) + .grants(&policy) + .is_empty() + ); + assert!(answerable(&[fixed("a", ConsentSignal::Grant)], &policy).is_empty()); + } + + #[test] + fn what_is_answerable_is_the_union_of_every_declaration() { + let policy = SignalPolicy::default(); + let modules: Vec> = vec![ + Arc::new(Granting("storage", Permission::StoreOnDevice)), + fixed("opt_out", ConsentSignal::Revoke), + Arc::new(Granting("profiling", Permission::CreateAdsProfile)), + ]; + assert_eq!( + answerable(&modules, &policy), + PermissionSet::none() + .with(Permission::StoreOnDevice) + .with(Permission::CreateAdsProfile), + "should collect what every module declares, in any order" + ); + } + + /// A module that vouches for a signal, standing in for a scheme that + /// read its string and could decode it. + struct Vouching(&'static str, &'static str); + + impl PermissionSignalModule for Vouching { + fn id(&self) -> &'static str { + self.0 + } + + fn signal(&self, _permission: Permission, _input: &SignalInput<'_>) -> ConsentSignal { + ConsentSignal::Neutral + } + + fn valid_signal(&self, _call: ModuleCall<'_>) -> Option { + Some(ValidSignal::new(self.0, self.0, self.1)) + } + } + + #[test] + fn the_valid_signals_are_what_each_module_vouched_for_in_order() { + let consent = ConsentContext::default(); + let modules: Vec> = vec![ + Arc::new(Vouching("first", "one")), + fixed("silent", ConsentSignal::Revoke), + Arc::new(Vouching("second", "two")), + ]; + let valid = signals_for(&modules, &consent); + assert_eq!( + &*valid, + &[ + ValidSignal::new("first", "first", "one"), + ValidSignal::new("second", "second", "two"), + ], + "should carry what was vouched for, in configured order, and nothing else" + ); + assert!( + signals_for(&[fixed("silent", ConsentSignal::Grant)], &consent).is_empty(), + "a module vouches for nothing unless it says otherwise" + ); + } + + #[test] + fn a_module_declaring_no_terms_leaves_the_list_empty() { + let consent = ConsentContext::default(); + let declared = tdls_for(&[fixed("quiet", ConsentSignal::Grant)], &consent); + assert!( + declared.is_empty(), + "should declare nothing, because the IAB schemes carry no terms" + ); + } + + #[test] + fn terms_are_collected_in_the_order_the_modules_are_asked() { + let consent = ConsentContext::default(); + let modules: Vec> = vec![ + Arc::new(Declaring("first", "https://terms.example.com/a/1.txt")), + fixed("quiet", ConsentSignal::Neutral), + Arc::new(Declaring("second", "https://terms.example.com/b/1.txt")), + ]; + let declared = tdls_for(&modules, &consent); + let addresses: Vec<&str> = declared.iter().map(Tdl::as_str).collect(); + assert_eq!( + addresses, + vec![ + "https://terms.example.com/a/1.txt", + "https://terms.example.com/b/1.txt" + ], + "should read in the configured order, so the list matches the deployment" + ); + } + + #[test] + fn one_document_named_by_two_modules_is_carried_once() { + let consent = ConsentContext::default(); + let modules: Vec> = vec![ + Arc::new(Declaring("first", "https://terms.example.com/a/1.txt")), + Arc::new(Declaring("second", "https://terms.example.com/a/1.txt")), + ]; + let declared = tdls_for(&modules, &consent); + assert_eq!( + declared.len(), + 1, + "should carry the same document once, not once per module naming it" + ); + } + + #[test] + fn versions_of_one_document_are_both_carried() { + let consent = ConsentContext::default(); + let modules: Vec> = vec![ + Arc::new(Declaring("first", "https://terms.example.com/a/1.txt")), + Arc::new(Declaring("second", "https://terms.example.com/a/2.txt")), + ]; + let declared = tdls_for(&modules, &consent); + assert_eq!( + declared.len(), + 2, + "should keep both, because a recipient agreed to one version and not the other" + ); + } + + #[test] + fn no_modules_declare_nothing() { + let consent = ConsentContext::default(); + assert!( + tdls_for(&[], &consent).is_empty(), + "should declare nothing when no module runs, rather than implying terms" + ); + } + + fn combined(modules: &[Arc]) -> ConsentSignal { + let consent = ConsentContext::default(); + let policy = SignalPolicy::default(); + combine( + modules, + Permission::StoreOnDevice, + ModuleContext::empty(), + &consent, + &no_evidence(), + &policy, + Acquisition::RequiresSignal, + ) + } + + /// The same, for a run whose modules differ only in what they answer. + fn combined_signals(signals: &[ConsentSignal]) -> ConsentSignal { + let modules: Vec> = signals + .iter() + .map(|signal| fixed("test", *signal)) + .collect(); + combined(&modules) + } + + fn withdrawn_under(modules: &[Arc], baseline: Acquisition) -> bool { + let consent = ConsentContext::default(); + let policy = SignalPolicy::default(); + withdrawn( + modules, + Permission::StoreOnDevice, + ModuleContext::empty(), + &consent, + &no_evidence(), + &policy, + baseline, + ) + } + + fn names(names: &[&str]) -> Vec { + names.iter().map(|name| (*name).to_owned()).collect() + } + + // ------------------------------------------------------------------ + // Combining answers in order. + // ------------------------------------------------------------------ + + #[test] + fn no_modules_leaves_the_place_baseline_alone() { + assert_eq!( + combined_signals(&[]), + ConsentSignal::Neutral, + "with no module configured nothing amends the place baseline" + ); + } + + #[test] + fn the_last_module_with_an_opinion_decides() { + assert_eq!( + combined_signals(&[ConsentSignal::Revoke, ConsentSignal::Grant]), + ConsentSignal::Grant, + "a visitor who answers a prompt after arriving with an opt-out header has \ + their answer applied over it, which is why the order is the policy" + ); + assert_eq!( + combined_signals(&[ConsentSignal::Grant, ConsentSignal::Revoke]), + ConsentSignal::Revoke, + "and a deployment that wants the opt-out to win puts it last" + ); + } + + #[test] + fn silence_leaves_an_earlier_answer_standing() { + // The failure this guards is a later module overwriting a settled + // answer with its own absence, which would let adding a module nobody + // uses undo the one that was working. + assert_eq!( + combined_signals(&[ConsentSignal::Grant, ConsentSignal::Neutral]), + ConsentSignal::Grant, + "a later module with no opinion leaves an earlier grant standing" + ); + assert_eq!( + combined_signals(&[ConsentSignal::Revoke, ConsentSignal::Neutral]), + ConsentSignal::Revoke, + "and a later silence leaves an earlier refusal standing too" + ); + } + + #[test] + fn silence_from_every_module_is_not_a_refusal() { + // The failure this guards is a module that reads an absent signal as + // a refusal. It would revoke the permission on every request that did + // not carry that scheme, which is most of them. + assert_eq!( + combined_signals(&[ConsentSignal::Neutral, ConsentSignal::Neutral]), + ConsentSignal::Neutral, + "no module having an opinion is not a refusal" + ); + } + + #[test] + fn a_module_sees_what_the_earlier_ones_settled() { + struct Recording; + + impl PermissionSignalModule for Recording { + fn id(&self) -> &'static str { + "recording" + } + + fn signal(&self, _permission: Permission, input: &SignalInput<'_>) -> ConsentSignal { + assert_eq!( + input.settled, + ConsentSignal::Revoke, + "a module is asked with the value the modules before it settled on" + ); + assert_eq!(input.position(), 1, "and with its own place in the order"); + assert_eq!(input.modules().len(), 2, "and with the whole list"); + assert_eq!( + input.baseline, + Acquisition::RequiresSignal, + "and with what the place rules said before anyone was asked" + ); + ConsentSignal::Neutral + } + } + + let modules: Vec> = + vec![fixed("opt-out", ConsentSignal::Revoke), Arc::new(Recording)]; + assert_eq!( + combined(&modules), + ConsentSignal::Revoke, + "the recording module has no opinion, so the opt-out stands" + ); + } + + #[test] + fn a_module_can_override_a_peer_by_consulting_it() { + // The worked example from the README: a later module overrides a + // refusal because of who made it, not merely that one was made. + let modules: Vec> = vec![ + fixed("gpc", ConsentSignal::Revoke), + Arc::new(Consulting { + id: "prompt", + peer: "gpc", + }), + ]; + assert_eq!( + combined(&modules), + ConsentSignal::Grant, + "a later module overrides a refusal made by the peer it consulted" + ); + + // The same module leaves the refusal alone when it came from a peer + // it was not told to override. + let modules: Vec> = vec![ + fixed("other", ConsentSignal::Revoke), + Arc::new(Consulting { + id: "prompt", + peer: "gpc", + }), + ]; + assert_eq!( + combined(&modules), + ConsentSignal::Revoke, + "and leaves a refusal from any other peer standing" + ); + } + + #[test] + fn asking_reaches_a_peer_wherever_it_sits_in_the_order() { + // A module may consult one configured after it, not only before, so + // a reordering does not silently change what a module can see. + let modules: Vec> = vec![ + Arc::new(Consulting { + id: "prompt", + peer: "gpc", + }), + fixed("gpc", ConsentSignal::Revoke), + ]; + assert_eq!( + combined(&modules), + ConsentSignal::Revoke, + "the consultation succeeded, and the later opt-out then settled it" + ); + } + + #[test] + fn asking_for_a_module_that_is_not_configured_answers_nothing() { + struct Absent; + + impl PermissionSignalModule for Absent { + fn id(&self) -> &'static str { + "absent" + } + + fn signal(&self, permission: Permission, input: &SignalInput<'_>) -> ConsentSignal { + assert!( + input.ask("not-configured", permission).is_none(), + "a module must be able to tell a missing peer from a silent one" + ); + assert!( + !input.has("not-configured"), + "and sees that the peer is not in the list" + ); + assert!(input.has("absent"), "and can see itself in the list"); + ConsentSignal::Neutral + } + } + + let modules: Vec> = vec![Arc::new(Absent)]; + assert_eq!( + combined(&modules), + ConsentSignal::Neutral, + "a module that finds its peer missing leaves the permission unsettled" + ); + } + + #[test] + fn a_module_cannot_consult_itself() { + struct SelfAsking; + + impl PermissionSignalModule for SelfAsking { + fn id(&self) -> &'static str { + "self-asking" + } + + fn signal(&self, permission: Permission, input: &SignalInput<'_>) -> ConsentSignal { + // Without the guard this recurses until the stack is gone. + assert!( + input.ask("self-asking", permission).is_none(), + "a module asking itself gets no answer" + ); + ConsentSignal::Grant + } + } + + let modules: Vec> = vec![Arc::new(SelfAsking)]; + assert_eq!( + combined(&modules), + ConsentSignal::Grant, + "a module refused its own consultation still answers for itself" + ); + } + + #[test] + fn two_modules_that_consult_each_other_do_not_loop() { + // Each consults the other, and the one answering a consultation is + // refused a consultation of its own, so the pair settles instead of + // recursing. + let modules: Vec> = vec![ + Arc::new(Consulting { + id: "first", + peer: "second", + }), + Arc::new(Consulting { + id: "second", + peer: "first", + }), + ]; + assert_eq!( + combined(&modules), + ConsentSignal::Neutral, + "two modules consulting each other settle instead of looping" + ); + } + + // ------------------------------------------------------------------ + // Withdrawal scoping. + // ------------------------------------------------------------------ + + #[test] + fn a_withdrawal_counts_only_where_the_baseline_did_not_grant() { + let modules: Vec> = vec![Arc::new(Withdrawing)]; + assert!( + withdrawn_under(&modules, Acquisition::RequiresSignal), + "refusing the signal the permission depended on is destructive" + ); + assert!( + withdrawn_under(&modules, Acquisition::Denied), + "and so is refusing under a baseline that never allowed it" + ); + assert!( + !withdrawn_under(&modules, Acquisition::Granted), + "where the permission never depended on the record, the refusal suppresses \ + without destroying" + ); + } + + #[test] + fn a_module_that_merely_revokes_does_not_withdraw() { + // Revoke and withdraw are different questions. A sale opt-out revokes + // and must never destroy an identifier. + let modules: Vec> = + vec![fixed("opt-out", ConsentSignal::Revoke)]; + assert!( + !withdrawn_under(&modules, Acquisition::RequiresSignal), + "a module that only revokes never withdraws" + ); + } + + #[test] + fn no_modules_never_withdraw() { + assert!( + !withdrawn_under(&[], Acquisition::RequiresSignal), + "with no module configured nothing can withdraw" + ); + } + + // ------------------------------------------------------------------ + // Selecting which modules run. + // ------------------------------------------------------------------ + + fn four() -> Vec> { + vec![ + fixed("gpc", ConsentSignal::Neutral), + fixed("gpp", ConsentSignal::Neutral), + fixed("us-privacy", ConsentSignal::Neutral), + fixed("tcf", ConsentSignal::Neutral), + ] + } + + #[test] + fn naming_nothing_runs_every_available_module_in_the_offered_order() { + let selected = select(&four(), None).expect("should accept no configuration"); + assert_eq!( + ids(&selected), + vec!["gpc", "gpp", "us-privacy", "tcf"], + "a publisher who configures nothing acts on every scheme the build knows, so \ + one is never ignored because they forgot to list it" + ); + } + + #[test] + fn the_configured_order_is_the_order_they_run_in() { + let reversed = names(&["tcf", "us-privacy", "gpp", "gpc"]); + let selected = select(&four(), Some(&reversed)).expect("should accept known names"); + assert_eq!( + ids(&selected), + vec!["tcf", "us-privacy", "gpp", "gpc"], + "the list is the order, not merely the membership" + ); + } + + #[test] + fn an_empty_list_is_acting_on_no_signal_and_is_accepted() { + let selected = select(&four(), Some(&[])).expect("should accept an empty list"); + assert!(selected.is_empty(), "an empty list selects no module"); + assert_eq!( + omitted(&four(), Some(&[])).len(), + 4, + "and every module is reported left out" + ); + } + + #[test] + fn a_name_matching_no_available_module_is_refused() { + // Matched rather than `expect_err`, because the success type holds + // trait objects that are deliberately not `Debug`. + let Err(error) = select(&four(), Some(&names(&["gpc", "not-a-module"]))) else { + panic!("should refuse a name this build does not offer"); + }; + let message = format!("{error:?}"); + assert!( + message.contains("not-a-module") && message.contains("gpc, gpp"), + "the refusal names the bad entry and what is available: {message}" + ); + } + + #[test] + fn naming_a_module_twice_is_refused() { + let Err(error) = select(&four(), Some(&names(&["gpc", "tcf", "gpc"]))) else { + panic!("should refuse a module named twice"); + }; + assert!( + format!("{error:?}").contains("more than once"), + "a module runs once, at one place in the order" + ); + } + + #[test] + fn a_left_out_module_is_reported_as_omitted() { + let configured = names(&["gpc", "tcf"]); + assert_eq!( + omitted(&four(), Some(&configured)), + vec!["gpp", "us-privacy"], + "the modules the list leaves out are reported in the offered order" + ); + assert!( + omitted(&four(), None).is_empty(), + "configuring nothing omits nothing" + ); + } +} diff --git a/crates/trusted-server-core/src/permissions.rs b/crates/trusted-server-core/src/permissions.rs new file mode 100644 index 0000000000..77d3dc5867 --- /dev/null +++ b/crates/trusted-server-core/src/permissions.rs @@ -0,0 +1,2552 @@ +//! Module permissions: a technical permission model gating module execution. +//! +//! A module advertises the [`Permission`]s its data use *requires*. Trusted +//! Server resolves which permissions are currently *set* from the session's +//! signals and the country it resolves to, and refuses to run the Edge Cookie +//! module when its required permissions are not set. The device and geo +//! modules declare their requirements through the same method, and the +//! built-in and host ones require none. A device module supplied by an +//! integration module that declares any permission is refused at startup, +//! because no per-request device gate exists yet to honor it. A geo +//! declaration is not consulted, because geo runs before the permissions are +//! assembled and supplies the country they depend on. +//! +//! The vocabulary is the IAB Privacy Taxonomy Data Uses, mapped from the IAB TCF +//! Europe purposes and used **only** as a technical identifier for a permission. +//! Two purposes have no Data Use yet: TCF purpose 1 (device storage) uses a +//! proposed `necessary.operations.storage` key and TCF purpose 11 keeps its TCF +//! identifier. No CMP or TCF *policy* is implemented here. Every TCF purpose +//! is resolved against the session's signals (a present TCF record grants or +//! revokes each purpose, and a US-style opt-out revokes), and the Data Uses +//! with no TCF purpose keep their configured baseline. +//! +//! How a permission is acquired varies by place, so the policy is written as a +//! tree of places: the top of the tree stands for the whole world, countries +//! sit under it keyed by ISO 3166-1 alpha-2 code, and a country's regions sit +//! under that country keyed by ISO 3166-2 subdivision code. +//! [`PermissionMaps::standard`] loads that tree from the embedded +//! `permissions.yaml` (see `DEFAULT_PERMISSION_RULES`). +//! +//! Resolution takes the most specific match and falls back to the node above: +//! the request's region when it is listed, otherwise its country, otherwise +//! the top of the tree. So a request with no country at all (no geo module, +//! or a lookup that resolved nothing), and a request whose country has no rule, +//! both resolve to the top node's group. The top node also declares the +//! `jurisdiction` the consent gates use for a visitor whose place could not be +//! resolved (see [`PermissionMaps::default_jurisdiction`]). A geo module that +//! reports an outright lookup failure is the exception, resolving every +//! permission to the requires-signal floor rather than the top node (see +//! [`PermissionMaps::floor_with`]). + +use std::collections::BTreeMap; +use std::sync::{Arc, OnceLock}; + +use serde::Deserialize; +use serde_yaml_ng::Value; + +use crate::consent::jurisdiction::Jurisdiction; +use crate::tdl::Tdl; + +/// A technical permission a module may require, labeled with its IAB Privacy +/// Taxonomy Data Use, or its IAB TCF Europe purpose where no Data Use exists yet. +/// +/// Only the identifier is used, with no TCF or taxonomy policy implemented. Every +/// named variant with a TCF purpose in `permissions.yaml` is resolved against +/// the session's signals. Only [`Permission::StoreOnDevice`] (and +/// [`Permission::SelectPersonalisedAds`] for sharing) gates a shipped module. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum Permission { + /// TCF Purpose 1, store and/or access information on a device. It gates + /// every shipped Edge Cookie module. No IAB Privacy Taxonomy Data Use exists + /// for device storage yet, so this uses a proposed `necessary.operations` + /// key pending an upstream addition. + StoreOnDevice, + /// TCF Purpose 2, use limited data to select advertising. + SelectBasicAds, + /// TCF Purpose 3, create profiles for personalised advertising. + CreateAdsProfile, + /// TCF Purpose 4, use profiles to select personalised advertising. + SelectPersonalisedAds, + /// TCF Purpose 5, create profiles to personalise content. + CreateContentProfile, + /// TCF Purpose 6, use profiles to select personalised content. + SelectPersonalisedContent, + /// TCF Purpose 7, measure advertising performance. + MeasureAdPerformance, + /// TCF Purpose 8, measure content performance. + MeasureContentPerformance, + /// TCF Purpose 9, understand audiences through statistics. + MarketResearch, + /// TCF Purpose 10, develop and improve services. + DevelopServices, + /// TCF Purpose 11, use limited data to select content. No IAB Privacy + /// Taxonomy Data Use exists for limited-data content selection yet, so this + /// keeps its TCF identifier and is proposed upstream. No shipped module + /// gates on it. + SelectBasicContent, + /// An IAB Privacy Taxonomy Data Use with no dedicated variant, identified by + /// its index into [`EXTRA_DATA_USES`]. These carry a policy flag in + /// `permissions.yaml` for completeness, and no shipped module gates on them. + Extra(u8), +} + +/// The Data Use identifiers for the named [`Permission`] variants, in variant +/// order (bit index 0..11). +const NAMED_DATA_USES: [&str; 11] = [ + "necessary.operations.storage", + "advertising_marketing.first_party.contextual", + "advertising_marketing.profiling", + "advertising_marketing.first_party.targeted", + "advertising_marketing.personalize.profiling", + "advertising_marketing.personalize.content", + "analytics.ad_reporting.measure_ad_performance", + "analytics.ad_reporting.content_performance", + "analytics.ad_reporting.market_research", + "necessary.operations.improve", + "select-basic-content", +]; + +/// Every other IAB Privacy Taxonomy Data Use, carried so `permissions.yaml` can +/// set a policy flag for the whole taxonomy (bit index 11..). No shipped module +/// gates on these. They exist for completeness, testing, and demonstration. +const EXTRA_DATA_USES: [&str; 53] = [ + "advertising_marketing", + "advertising_marketing.communications", + "advertising_marketing.communications.email", + "advertising_marketing.communications.sms", + "advertising_marketing.first_party", + "advertising_marketing.frequency_capping", + "advertising_marketing.negative_targeting", + "advertising_marketing.personalize", + "advertising_marketing.personalize.system", + "advertising_marketing.serving", + "advertising_marketing.third_party", + "advertising_marketing.third_party.targeted", + "analytics", + "analytics.ad_reporting", + "analytics.ad_reporting.ad_delivery_and_targeting", + "analytics.ad_reporting.ad_fraud_detection", + "analytics.ad_reporting.ad_viewability", + "analytics.ad_reporting.campaign_insights", + "analytics.reporting", + "analytics.reporting.system", + "disclosure", + "disclosure.law_enforcement", + "disclosure.outside_counsel", + "disclosure.sale", + "disclosure.share", + "disclosure.third_party_sale", + "functional", + "functional.performance", + "functional.personalization", + "functional.security", + "necessary", + "necessary.employment", + "necessary.employment.hr", + "necessary.employment.hr.hiring", + "necessary.fraud_detection", + "necessary.legal_obligation", + "necessary.legal_obligation.age_verification", + "necessary.legal_obligation.content_moderation", + "necessary.legal_obligation.dsr", + "necessary.legal_obligation.hold", + "necessary.operations", + "necessary.operations.authentication", + "necessary.operations.debugging", + "necessary.operations.notifications", + "necessary.operations.notifications.email", + "necessary.operations.notifications.sms", + "necessary.operations.payment_processing", + "necessary.operations.quality_assurance", + "necessary.operations.security", + "necessary.operations.support", + "necessary.operations.survey", + "necessary.operations.upgrades", + "necessary.operations.website_use", +]; + +impl Permission { + /// The named permission variants, in bit-index order. + const NAMED: [Permission; 11] = [ + Permission::StoreOnDevice, + Permission::SelectBasicAds, + Permission::CreateAdsProfile, + Permission::SelectPersonalisedAds, + Permission::CreateContentProfile, + Permission::SelectPersonalisedContent, + Permission::MeasureAdPerformance, + Permission::MeasureContentPerformance, + Permission::MarketResearch, + Permission::DevelopServices, + Permission::SelectBasicContent, + ]; + + /// Every modeled permission: the named variants, then every other Privacy + /// Taxonomy Data Use. + pub fn all() -> impl Iterator { + Self::NAMED + .into_iter() + .chain((0..EXTRA_DATA_USES.len() as u8).map(Permission::Extra)) + } + + /// The Data Use identifier for this permission. + #[must_use] + pub fn as_str(self) -> &'static str { + match self { + Permission::Extra(index) => EXTRA_DATA_USES[index as usize], + named => NAMED_DATA_USES[named.index() as usize], + } + } + + /// The stable bit position for this permission within a [`PermissionSet`]. + #[must_use] + const fn index(self) -> u8 { + match self { + Permission::StoreOnDevice => 0, + Permission::SelectBasicAds => 1, + Permission::CreateAdsProfile => 2, + Permission::SelectPersonalisedAds => 3, + Permission::CreateContentProfile => 4, + Permission::SelectPersonalisedContent => 5, + Permission::MeasureAdPerformance => 6, + Permission::MeasureContentPerformance => 7, + Permission::MarketResearch => 8, + Permission::DevelopServices => 9, + Permission::SelectBasicContent => 10, + Permission::Extra(index) => 11 + index, + } + } + + /// The single-bit mask for this permission within a [`PermissionSet`]. + const fn bit(self) -> u128 { + 1 << self.index() + } + + /// Returns the permission whose Data Use identifier matches `id` (for + /// example `"necessary.operations.storage"`), or `None` when it is unknown. + /// + /// Used to parse permission names from `permissions.yaml`. + #[must_use] + pub fn from_identifier(id: &str) -> Option { + Permission::all().find(|p| p.as_str() == id) + } +} + +impl core::fmt::Display for Permission { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// A set of [`Permission`]s, stored as a bitset keyed by each permission's bit +/// index. +/// +/// Used both for what a module requires and for what Trusted Server has set. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct PermissionSet(u128); + +impl PermissionSet { + /// The empty set, requiring or containing nothing. + #[must_use] + pub const fn none() -> Self { + Self(0) + } + + /// Returns this set with `permission` added. + #[must_use] + pub const fn with(self, permission: Permission) -> Self { + Self(self.0 | permission.bit()) + } + + /// Whether `permission` is in the set. + #[must_use] + pub const fn contains(self, permission: Permission) -> bool { + self.0 & permission.bit() != 0 + } + + /// Whether the set is empty. + #[must_use] + pub const fn is_empty(self) -> bool { + self.0 == 0 + } + + /// Whether every permission in `other` is also in this set. + #[must_use] + pub const fn contains_all(self, other: PermissionSet) -> bool { + self.0 & other.0 == other.0 + } + + /// The permissions in both this set and `other`. + #[must_use] + pub const fn intersect(self, other: PermissionSet) -> Self { + Self(self.0 & other.0) + } + + /// The permissions in either this set or `other`. + #[must_use] + pub const fn union(self, other: PermissionSet) -> Self { + Self(self.0 | other.0) + } + + /// The permissions in this set and not in `other`. + #[must_use] + pub const fn difference(self, other: PermissionSet) -> Self { + Self(self.0 & !other.0) + } + + /// Iterates the permissions in the set, in bit-index order. + /// + /// The built-ins read nothing from the full set; this serves a module or + /// diagnostic path that enumerates what is present. + pub fn iter(self) -> impl Iterator { + Permission::all().filter(move |p| self.contains(*p)) + } +} + +/// The Data Use identifiers in the set, in bit-index order, separated by +/// `, `, or `nothing` for the empty set. +impl core::fmt::Display for PermissionSet { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + if self.is_empty() { + return f.write_str("nothing"); + } + for (index, permission) in self.iter().enumerate() { + if index > 0 { + f.write_str(", ")?; + } + f.write_str(permission.as_str())?; + } + Ok(()) + } +} + +impl FromIterator for PermissionSet { + fn from_iter>(iter: I) -> Self { + iter.into_iter() + .fold(PermissionSet::none(), PermissionSet::with) + } +} + +/// How a permission is acquired in a given country. +/// +/// This is intentionally country-keyed, not module-keyed: a module only +/// advertises *which* permissions it needs, and the country's rules decide *how* +/// each is obtained. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum Acquisition { + /// Set without any signal, exempt or strictly necessary here. + Granted, + /// Set only when the incoming signals grant the matching TCF purpose. + /// The default, matching the floor an unresolved location falls to. + #[default] + RequiresSignal, + /// Never set in this country. + Denied, +} + +/// What a session signal says about a permission, layered on top of the +/// country/region baseline by the consent mapping. +/// +/// This module never reads consent directly. A caller maps its consent model (or +/// any other signal source) to a [`ConsentSignal`] per permission, and the +/// permission model applies it: a [`Grant`](Self::Grant) sets a +/// `RequiresSignal` permission, a [`Revoke`](Self::Revoke) drops a `Granted` one +/// (an opt-out), and [`Neutral`](Self::Neutral) leaves the baseline unchanged. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ConsentSignal { + /// The signal grants this permission, so a `RequiresSignal` baseline is set. + Grant, + /// The signal withdraws this permission, dropping a `Granted` baseline. + Revoke, + /// The signal says nothing, so the baseline stands. + Neutral, +} + +/// The acquisition rule for each permission in one country or region. +/// +/// A `default` applies to any permission not explicitly overridden, so a rule +/// table stays compact. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CountryRules { + default: Acquisition, + overrides: BTreeMap, +} + +impl CountryRules { + /// Rules with `default` for every permission and no per-permission override. + /// Groups in `permissions.yaml` are built from this plus [`with_rule`]. + /// + /// [`with_rule`]: Self::with_rule + #[must_use] + pub fn with_default(default: Acquisition) -> Self { + Self { + default, + overrides: BTreeMap::new(), + } + } + + /// Sets the acquisition rule for a single permission, overriding the default. + #[must_use] + pub fn with_rule(mut self, permission: Permission, acquisition: Acquisition) -> Self { + self.overrides.insert(permission.index(), acquisition); + self + } + + /// The acquisition rule for `permission`. + #[must_use] + pub fn rule_for(&self, permission: Permission) -> Acquisition { + self.overrides + .get(&permission.index()) + .copied() + .unwrap_or(self.default) + } +} + +/// Which Data Uses a signal revokes. +#[derive(Debug, Clone, Default)] +enum RevokeSet { + /// The signal revokes nothing. + #[default] + None, + /// The signal revokes every Data Use (the map bounds what a revoke drops). + All, + /// The signal revokes only the listed Data Uses. + Set(PermissionSet), +} + +/// What a deployment decides about the shipped signal schemes, parsed from the +/// `signals` section of `permissions.yaml`. +/// +/// The permission model holds this as data so a deployment changes it without +/// changing a module. It is jurisdiction-free, and it carries only the +/// decisions that are a deployment's to make: whether a TCF record answers at +/// all, which signals count as a US-style opt-out, and what an opt-out takes +/// away. What each scheme's own signal means, such as which TCF purpose grants +/// which Data Use, is that scheme's module crate's, not this policy's. The +/// country/region baseline decides the rest. +#[derive(Debug, Clone, Default)] +pub struct SignalPolicy { + /// Whether a present TCF record's grants and revokes apply. Whether a + /// consenting record then stands over an opt-out, or the opt-out over it, + /// is decided by the order the modules are asked in, which is + /// `[permission-signal] modules`, not by this flag. + tcf_authoritative: bool, + /// The signals that constitute a US-style opt-out. + opt_out_sources: Vec, + /// Which Data Uses a US-style opt-out revokes. + opt_out_revokes: RevokeSet, +} + +impl SignalPolicy { + /// Whether a present TCF record's grants and revokes apply. + #[must_use] + pub fn tcf_authoritative(&self) -> bool { + self.tcf_authoritative + } + + /// The signals that constitute a US-style opt-out. + #[must_use] + pub fn opt_out_sources(&self) -> &[OptOutSource] { + &self.opt_out_sources + } + + /// Whether a US-style opt-out revokes `permission`. + #[must_use] + pub fn opt_out_revokes(&self, permission: Permission) -> bool { + match &self.opt_out_revokes { + RevokeSet::None => false, + RevokeSet::All => true, + RevokeSet::Set(set) => set.contains(permission), + } + } +} + +/// Builds a validated [`SignalPolicy`] from the parsed `signals` section, +/// erroring when it names an unknown Data Use or revoke rule. +fn build_signal_policy(spec: &SignalsSpec) -> Result { + let mut policy = SignalPolicy::default(); + if let Some(tcf) = &spec.tcf { + policy.tcf_authoritative = tcf.authoritative; + } + if let Some(opt_out) = &spec.us_opt_out { + policy.opt_out_sources = opt_out.sources.clone(); + policy.opt_out_revokes = match &opt_out.revokes { + RevokeSpec::Keyword(keyword) if keyword == "all" => RevokeSet::All, + RevokeSpec::Keyword(other) => { + return Err(PermissionsError::UnknownRevoke { + value: other.clone(), + }); + } + RevokeSpec::List(names) => { + let mut set = PermissionSet::none(); + for name in names { + let permission = Permission::from_identifier(name).ok_or_else(|| { + PermissionsError::UnknownPermission { name: name.clone() } + })?; + set = set.with(permission); + } + RevokeSet::Set(set) + } + }; + } + Ok(policy) +} + +/// The place tree from `permissions.yaml`, flattened for lookup. +/// +/// `by_country` is keyed on the ISO 3166-1 alpha-2 code a geo module returns +/// (upper-cased). `by_region` keeps the finer rules written under a country, +/// keyed by country and region (for example a US state), which take precedence +/// over the country entry. `default_rules` is the top node's group, the answer +/// for a request whose country and region are both unlisted, and for a request +/// with no place at all. +#[derive(Debug, Clone, Default)] +pub struct PermissionMaps { + by_country: BTreeMap, + by_region: BTreeMap, + jurisdiction_by_country: BTreeMap, + jurisdiction_by_region: BTreeMap, + default_rules: Option, + default_jurisdiction: Option, + signals: SignalPolicy, +} + +/// The default permission rules, compiled into the build from the repository's +/// sample in `config/permissions`. +/// +/// That sample is for testing and evaluation only and is not a production +/// policy. A deployer chooses or replaces the compiled-in file to set the +/// default policy, which is not read at runtime. +const DEFAULT_PERMISSION_RULES: &str = include_str!("../../../config/permissions/sample.yaml"); + +/// Builds the upper-cased `COUNTRY:REGION` key for [`PermissionMaps::by_region`]. +fn region_key(country: &str, region: &str) -> String { + format!( + "{}:{}", + country.to_ascii_uppercase(), + region.to_ascii_uppercase() + ) +} + +impl PermissionMaps { + /// Builds an empty map set with no country or region entries. + #[must_use] + pub fn empty() -> Self { + Self::default() + } + + /// The signal-to-permission policy parsed from the `signals` section of + /// `permissions.yaml`. The consent mapping reads this rather than encoding + /// any signal policy in the code. + #[must_use] + // Public because the permission signal module crates live outside core + // and read the deployment's policy, in their tests and where a module + // needs the shipped decisions rather than a policy built by hand. + pub fn signals(&self) -> &SignalPolicy { + &self.signals + } + + /// Registers explicit rules for an ISO 3166-1 alpha-2 country code. + #[must_use] + pub fn with_country(mut self, iso_code: &str, rules: CountryRules) -> Self { + self.by_country.insert(iso_code.to_ascii_uppercase(), rules); + self + } + + /// Registers explicit rules for a region within a country, keyed by the ISO + /// 3166-1 alpha-2 country and the geo module's region code (for example + /// `US` and `CA`). + /// + /// A region entry takes precedence over the country entry, so a deployer can + /// vary a single state or province on top of the country baseline. + #[must_use] + pub fn with_region(mut self, iso_country: &str, region: &str, rules: CountryRules) -> Self { + self.by_region + .insert(region_key(iso_country, region), rules); + self + } + + /// Registers the top node's rules, which apply to a request whose country + /// and region are both unlisted, and to a request with no place at all. + /// + /// A map built without one resolves such a request at the requires-signal + /// floor, which is what [`PermissionMaps::empty`] does. Every map parsed + /// from a `permissions.yaml` has one, because the top node's `group` is + /// required. + #[must_use] + pub fn with_default_rules(mut self, rules: CountryRules) -> Self { + self.default_rules = Some(rules); + self + } + + /// Registers the top node's jurisdiction, the consent handling for a + /// visitor whose place could not be resolved. + #[must_use] + pub fn with_default_jurisdiction(mut self, jurisdiction: Jurisdiction) -> Self { + self.default_jurisdiction = Some(jurisdiction); + self + } + + /// The jurisdiction the policy declares for a visitor whose place the geo + /// module could not resolve, taken from the top of the `rules:` tree. + /// + /// The consent gates resolve a jurisdiction from the request's place, so + /// with no place they would resolve [`Jurisdiction::Unknown`] and fail + /// closed even where the policy has declared what to do. This is that + /// declaration. A map with no top node (see [`PermissionMaps::empty`]) + /// reports [`Jurisdiction::Unknown`]. + #[must_use] + pub fn default_jurisdiction(&self) -> Jurisdiction { + self.default_jurisdiction.clone().unwrap_or_default() + } + + /// The jurisdiction that applies to `country` and `region`. + /// + /// Walks the tree the same way [`rules_or_default`](Self::rules_or_default) + /// does: the region when it is listed, otherwise the country, otherwise the + /// top node. Inheritance is settled when the file is parsed, so every + /// listed place already carries the jurisdiction it inherits. + #[must_use] + pub fn jurisdiction_for(&self, country: Option<&str>, region: Option<&str>) -> Jurisdiction { + if let (Some(country), Some(region)) = (country, region) + && let Some(jurisdiction) = self + .jurisdiction_by_region + .get(®ion_key(country, region)) + { + return jurisdiction.clone(); + } + if let Some(jurisdiction) = country + .map(str::to_ascii_uppercase) + .and_then(|code| self.jurisdiction_by_country.get(&code)) + { + return jurisdiction.clone(); + } + self.default_jurisdiction() + } + + /// The built-in default rules, parsed from the embedded `permissions.yaml` + /// (see `DEFAULT_PERMISSION_RULES`). + /// + /// The parse runs once per instance and the result is cached. + /// + /// # Panics + /// + /// Panics if the embedded `permissions.yaml` fails to parse. The file is a + /// build-time constant covered by tests, so a panic means the committed file + /// is malformed, not a runtime condition. + #[must_use] + pub fn standard() -> &'static Self { + static CACHE: OnceLock = OnceLock::new(); + CACHE.get_or_init(|| { + Self::from_yaml(DEFAULT_PERMISSION_RULES) + .expect("should parse the embedded default permissions.yaml") + }) + } + + /// Builds the maps from a `permissions.yaml` document: named `groups`, the + /// `rules` tree of places, and the `signals` section that maps each session + /// signal onto Data Uses. + /// + /// # Errors + /// + /// Returns [`PermissionsError`] when the YAML is malformed, the top of the + /// `rules` tree omits `group` or `jurisdiction`, a node below the top is + /// neither a group name nor a block carrying `group`, a node names an + /// unknown group, permission, or acquisition flag, two sibling place codes + /// name the same place, or a signal's revoke list names an unknown Data + /// Use. + pub fn from_yaml(yaml: &str) -> Result { + let file: RulesFile = + serde_yaml_ng::from_str(yaml).map_err(|error| PermissionsError::Parse { + message: error.to_string(), + })?; + + // Build every named group into its CountryRules. + let mut groups: BTreeMap = BTreeMap::new(); + for (name, flags) in &file.groups { + groups.insert(name.clone(), group_rules(name, flags)?); + } + + let mut maps = Self::empty(); + build_rules_tree(&mut maps, &groups, &file.rules)?; + maps.signals = build_signal_policy(&file.signals)?; + Ok(maps) + } + + /// Returns the rules written for `country` and `region` exactly, preferring + /// a region entry, then the country entry, or `None` when neither is listed. + /// + /// This is the literal tree lookup with no fallback to the top node. Use + /// [`rules_or_default`](Self::rules_or_default) for the resolution a + /// request actually gets. + #[must_use] + pub fn rules_for(&self, country: Option<&str>, region: Option<&str>) -> Option<&CountryRules> { + if let (Some(country), Some(region)) = (country, region) + && let Some(rules) = self.by_region.get(®ion_key(country, region)) + { + return Some(rules); + } + country + .map(str::to_ascii_uppercase) + .and_then(|code| self.by_country.get(&code)) + } + + /// The rules a request resolves to: its region, else its country, else the + /// top node of the tree. + /// + /// Returns `None` only for a map with no top node, which the caller treats + /// as the requires-signal floor. Every map parsed from a `permissions.yaml` + /// has one, because the top node's `group` is required, so this is + /// unreachable in a deployment and exists for maps built by hand in tests. + pub(crate) fn rules_or_default( + &self, + country: Option<&str>, + region: Option<&str>, + ) -> Option<&CountryRules> { + self.rules_for(country, region) + .or(self.default_rules.as_ref()) + } + + /// Resolves the permission state for a request: the place baseline + /// augmented by a session signal. + /// + /// `country` and `region` are what a geo module returns (`region` may be + /// `None`). Whatever the tree does not answer falls back to the node above, + /// ending at the top node, so an unlisted country and a request with no + /// country at all both resolve to the top node's group. `signal` maps each + /// permission to a [`ConsentSignal`]; the caller derives it from its consent + /// model so this module stays independent of how a signal is decoded. A + /// `Granted` baseline is set unless the signal is `Revoke`, a + /// `RequiresSignal` baseline is set only on `Grant`, and `Denied` is never + /// set. + #[must_use] + pub fn resolve_with( + &self, + country: Option<&str>, + region: Option<&str>, + signal: impl Fn(Permission, Acquisition) -> ConsentSignal, + ) -> PermissionState { + Self::resolve_rules(self.rules_or_default(country, region), signal) + } + + /// Resolves every permission at the requires-signal floor, whatever the + /// policy tree says. + /// + /// This is the state for a geo module that reported an outright lookup + /// failure. The request's place is unknown in a way the policy's top node + /// must not paper over, so nothing is set unless the session's signals + /// grant it. + #[must_use] + pub fn floor_with( + signal: impl Fn(Permission, Acquisition) -> ConsentSignal, + ) -> PermissionState { + Self::resolve_rules(None, signal) + } + + /// Applies `signal` against `rules`, or against the requires-signal floor + /// when no rules resolved. + fn resolve_rules( + rules: Option<&CountryRules>, + signal: impl Fn(Permission, Acquisition) -> ConsentSignal, + ) -> PermissionState { + let acquisition = + |permission| rules.map_or(Acquisition::RequiresSignal, |r| r.rule_for(permission)); + let mut set = PermissionSet::none(); + let mut awaiting = PermissionSet::none(); + for permission in Permission::all() { + // The baseline is passed to the signal as well as applied to its + // answer, because a source amends the place rules and cannot + // amend what it cannot see. + let baseline = acquisition(permission); + match (baseline, signal(permission, baseline)) { + (Acquisition::Denied, _) => {} + (Acquisition::Granted, ConsentSignal::Revoke) => {} + (Acquisition::Granted, _) => set = set.with(permission), + (Acquisition::RequiresSignal, ConsentSignal::Grant) => set = set.with(permission), + (Acquisition::RequiresSignal, ConsentSignal::Revoke) => {} + // Requires a signal and nobody gave one. Not set, and not + // refused either, which the page has to be able to tell apart. + (Acquisition::RequiresSignal, ConsentSignal::Neutral) => { + awaiting = awaiting.with(permission); + } + } + } + PermissionState::new(set).with_awaiting(awaiting) + } + + /// The baseline permission state for a country and region with no session + /// signal. + /// + /// Permissions exist without a consent model, so this is the set of + /// `Granted` permissions for the place (or the top node it falls back to), + /// and is what a request resolves to when no signal is present. + #[must_use] + pub fn baseline(&self, country: Option<&str>, region: Option<&str>) -> PermissionState { + self.resolve_with(country, region, |_, _| ConsentSignal::Neutral) + } + + /// Convenience over [`resolve_with`](Self::resolve_with) for a boolean + /// signal with no region and no revocation: a `true` grants the permission + /// and a `false` is neutral. + #[must_use] + pub fn resolve( + &self, + country: Option<&str>, + signal: impl Fn(Permission) -> bool, + ) -> PermissionState { + self.resolve_with(country, None, |permission, _| { + if signal(permission) { + ConsentSignal::Grant + } else { + ConsentSignal::Neutral + } + }) + } +} + +/// A signal a module read from the request and found valid, as it was +/// received. +/// +/// The permission state carries one of these for every signal a configured +/// module used, so whatever reads the state can rely on exactly those +/// signals and no other. A signal that was absent, could not be read, or +/// that no configured module acts on is not here, and nothing says why, +/// because the module for that scheme has already decided what its +/// absence means for the permissions. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] +pub struct ValidSignal { + /// The module that read and used the signal, by its configured id. + pub module: &'static str, + /// The scheme the signal belongs to, for example `tcf` or `gpp`, so a + /// reader can find a scheme without knowing which module read it. + pub scheme: &'static str, + /// The signal as it was received. + pub value: String, +} + +impl ValidSignal { + /// A valid signal of `scheme`, read by `module`, as received. + #[must_use] + pub fn new(module: &'static str, scheme: &'static str, value: impl Into) -> Self { + Self { + module, + scheme, + value: value.into(), + } + } +} + +/// The permissions Trusted Server currently has set for a request. +/// +/// A module executes only when [`all_set`](Self::all_set) of its required +/// permissions returns `true`. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct PermissionState { + set: PermissionSet, + /// The permissions whose baseline requires a signal and for which no + /// configured module gave one. See [`awaiting`](Self::awaiting). + awaiting: PermissionSet, + /// Whether the request explicitly withdrew device storage, as opposed to + /// storage merely not being set. See + /// [`storage_withdrawn`](Self::storage_withdrawn). + storage_withdrawn: bool, + /// The terms documents the data for this request is available under. See + /// [`tdls`](Self::tdls). Shared rather than owned because the state is + /// cloned along the request path and the list is the same list. + tdls: Arc<[Tdl]>, + /// The signals the configured modules read and found valid. See + /// [`signals`](Self::signals). + signals: Arc<[ValidSignal]>, +} + +impl PermissionState { + /// Builds a state in which exactly the permissions in `set` are set, and + /// nothing is withdrawn, for tests and callers that compute the set + /// directly. + #[must_use] + pub fn new(set: PermissionSet) -> Self { + Self { + set, + awaiting: PermissionSet::none(), + storage_withdrawn: false, + tdls: Arc::default(), + signals: Arc::default(), + } + } + + /// The same state, recording which permissions are still waiting for a + /// signal. Set by resolution from the baseline and the modules' answers. + #[must_use] + pub fn with_awaiting(self, awaiting: PermissionSet) -> Self { + Self { awaiting, ..self } + } + + /// The same state, awaiting only the permissions in `answerable`. + /// + /// Resolution records every permission that requires a signal and got + /// none. Some of those no configured module could ever grant, for + /// example a marketing channel opt-in in a deployment whose modules can + /// grant none, and a page told to wait for them would wait for ever. So + /// the assembly narrows the list to what some module declares it can + /// grant, and the rest are simply not set. + #[must_use] + pub fn awaiting_only(self, answerable: PermissionSet) -> Self { + Self { + awaiting: self.awaiting.intersect(answerable), + ..self + } + } + + /// The permissions whose baseline requires a signal and for which every + /// configured module answered neutral, so nobody has answered yet. + /// + /// This is different from a permission that is not set. A refused + /// permission and one nobody has answered look the same in + /// [`permissions`](Self::permissions), and a page that cannot tell them + /// apart treats a prompt that never ran as a visitor who said no. A + /// permission is never both set and awaited. + #[must_use] + pub const fn awaiting(&self) -> PermissionSet { + self.awaiting + } + + /// Whether a permission is still waiting for a signal. + #[must_use] + pub const fn is_awaited(&self, permission: Permission) -> bool { + self.awaiting.contains(permission) + } + + /// The same state, recording whether device storage was explicitly + /// withdrawn. Set by assembly from what the signal modules answered, + /// scoped to the jurisdiction's storage baseline. + #[must_use] + pub fn with_storage_withdrawn(self, storage_withdrawn: bool) -> Self { + Self { + storage_withdrawn, + ..self + } + } + + /// The same state, carrying the terms documents the data for this request + /// is available under. Set by assembly from what the signal modules + /// declared, in the order they are asked. + #[must_use] + pub fn with_tdls(self, tdls: Arc<[Tdl]>) -> Self { + Self { tdls, ..self } + } + + /// The same state, carrying the signals the modules read and found + /// valid. Set by assembly, in the order the modules are asked. + #[must_use] + pub fn with_signals(self, signals: Arc<[ValidSignal]>) -> Self { + Self { signals, ..self } + } + + /// The signals the configured modules read from the request and + /// found valid, each as it was received. + /// + /// This is the list a page, a bid request or a person reading the state + /// relies on. A signal that is not here was absent, could not be read, or + /// is one no configured module acts on, and the module for its + /// scheme has already decided what that means for the permissions, so + /// nothing downstream needs to reason about the signal itself. + #[must_use] + pub fn signals(&self) -> &[ValidSignal] { + &self.signals + } + + /// The terms documents the data for this request is available under, in + /// the order the modules were asked. + /// + /// Whoever receives the data reads these to decide whether the terms are + /// ones they accept, and whether they may pass the data on. An empty list + /// says no terms were declared, which is not the same as terms that permit + /// anything, so a recipient that needs a basis and finds none has none. + #[must_use] + pub fn tdls(&self) -> &[Tdl] { + &self.tdls + } + + /// Whether the request carries an explicit signal withdrawing device + /// storage, rather than merely lacking the permission. + /// + /// The difference is destructive. A withdrawal expires the browser cookie + /// and writes the authoritative identity-graph tombstone, where a + /// permission that is simply not set strips the Edge Cookie response + /// headers and leaves an already-issued identifier alone, so a returning + /// visitor is not permanently withdrawn before they ever get to answer. + /// Which scheme can withdraw is each module's to say, and only where the + /// jurisdiction's storage baseline did not grant storage outright. + #[must_use] + pub const fn storage_withdrawn(&self) -> bool { + self.storage_withdrawn + } + + /// Whether a single permission is set. + #[must_use] + pub const fn is_set(&self, permission: Permission) -> bool { + self.set.contains(permission) + } + + /// Whether every permission in `required` is set. An empty requirement is + /// always satisfied, so a module that requires nothing always runs. + #[must_use] + pub const fn all_set(&self, required: PermissionSet) -> bool { + self.set.contains_all(required) + } + + /// The full set of permissions that are set, for a module that adapts its + /// behavior to whatever is present. + #[must_use] + pub const fn permissions(&self) -> PermissionSet { + self.set + } + + /// The resolved state as the JSON the page receives in + /// `window.tsjs.permissions`. + /// + /// Names are the [`Permission::as_str`] Data Use identifiers, sorted so the + /// same state always serializes to the same bytes whatever order the set + /// was built in. `awaiting` names the permissions still waiting for a + /// signal, so a page can hold what depends on them rather than read + /// "nobody has answered" as "refused". `tdls` carries the terms documents + /// the data is available under, in the order the modules were asked, so + /// a page module reads the terms alongside the permissions. `signals` + /// carries the signals the modules read and found valid, each as it was + /// received, so a page relies on exactly what the state was built from. + /// An empty state renders as + /// `{"awaiting":[],"set":[],"signals":[],"tdls":[]}`, and all four are + /// answers (nothing is set, nothing is awaited, no signal was valid, no + /// terms were declared) rather than missing values, so page code never + /// has to tell them apart. + /// + /// # Examples + /// + /// ``` + /// use trusted_server_core::permissions::{ + /// Permission, PermissionSet, PermissionState, + /// }; + /// + /// let state = PermissionState::new( + /// PermissionSet::none().with(Permission::StoreOnDevice), + /// ); + /// assert_eq!( + /// state.page_json(), + /// r#"{"awaiting":[],"set":["necessary.operations.storage"],"signals":[],"tdls":[]}"# + /// ); + /// + /// assert_eq!( + /// PermissionState::default().page_json(), + /// r#"{"awaiting":[],"set":[],"signals":[],"tdls":[]}"# + /// ); + /// ``` + #[must_use] + pub fn page_json(&self) -> String { + self.page_value().to_string() + } + + /// The resolved state in the page shape, as a value, for whatever shows + /// the same resolution somewhere other than a page. + /// + /// See [`page_json`](Self::page_json) for what each key means. This is + /// the only place the page shape is spelled, so no caller writes the JSON + /// by hand. + #[must_use] + pub fn page_value(&self) -> serde_json::Value { + let mut names: Vec<&'static str> = self.set.iter().map(Permission::as_str).collect(); + names.sort_unstable(); + let mut awaiting: Vec<&'static str> = + self.awaiting.iter().map(Permission::as_str).collect(); + awaiting.sort_unstable(); + let tdls: Vec<&str> = self.tdls.iter().map(Tdl::as_str).collect(); + serde_json::json!({ + "set": names, + "awaiting": awaiting, + "signals": &*self.signals, + "tdls": tdls, + }) + } +} + +// --------------------------------------------------------------------------- +// permissions.yaml parsing +// --------------------------------------------------------------------------- + +/// The shape of a `permissions.yaml` document. +#[derive(Debug, Deserialize)] +struct RulesFile { + /// Named permission baselines, keyed by group name. Each group is a flat map + /// of per-permission flags, with an optional `default` shorthand for any + /// permission it omits. + #[serde(default)] + groups: BTreeMap>, + /// The `rules` tree of places. Walked by [`build_rules_tree`] rather than + /// deserialized into a fixed shape, because every key below the reserved + /// words is a place code chosen by the policy owner. + #[serde(default)] + rules: Value, + /// How each session signal maps onto Data Uses. + #[serde(default)] + signals: SignalsSpec, +} + +/// The `signals` section: how each signal source maps onto Data Uses. Parsed +/// into a [`SignalPolicy`] by [`build_signal_policy`]. +#[derive(Debug, Default, Deserialize)] +struct SignalsSpec { + /// The TCF record mapping, or `None` when the file declares no TCF policy. + #[serde(default)] + tcf: Option, + /// The US-style opt-out mapping, or `None` when none is declared. + #[serde(default)] + us_opt_out: Option, +} + +/// The `signals.tcf` block. +/// +/// Only whether a TCF record answers for this deployment. Which purpose grants +/// which Data Use is the TCF scheme's own knowledge and lives in the TCF +/// permission signal module crate, so this file carries no table of another +/// scheme's numbers and a deployment running no TCF configures none. +#[derive(Debug, Deserialize)] +struct TcfSignalSpec { + /// Whether a present TCF record's grants and revokes apply. + #[serde(default = "default_true")] + authoritative: bool, +} + +/// The `signals.us_opt_out` block. +#[derive(Debug, Deserialize)] +struct OptOutSpec { + /// The signals that constitute a US-style opt-out. + #[serde(default)] + sources: Vec, + /// Which Data Uses the opt-out revokes. + #[serde(default)] + revokes: RevokeSpec, +} + +/// A `revokes` value: the keyword `all`, or an explicit list of Data Uses. +#[derive(Debug, Deserialize)] +#[serde(untagged)] +enum RevokeSpec { + /// A bare keyword, expected to be `all`. + Keyword(String), + /// An explicit list of Data Use identifiers. + List(Vec), +} + +impl Default for RevokeSpec { + fn default() -> Self { + RevokeSpec::List(Vec::new()) + } +} + +/// A single US-style opt-out signal source. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum OptOutSource { + /// The `Sec-GPC` request header (Global Privacy Control). + Gpc, + /// A GPP US sale opt-out. + GppSaleOptOut, + /// A US Privacy string sale opt-out. + UsPrivacyOptOut, +} + +/// Default for `#[serde(default = ...)]` on a `bool` field that should be `true`. +fn default_true() -> bool { + true +} + +/// The reserved key naming a node's permission group. +const KEY_GROUP: &str = "group"; + +/// The reserved key naming the top node's jurisdiction. +const KEY_JURISDICTION: &str = "jurisdiction"; + +/// The reserved key holding a node's per-permission overrides. +const KEY_PERMISSIONS: &str = "permissions"; + +/// Resolves an acquisition rule name to its [`Acquisition`]. +fn parse_acquisition(value: &str) -> Result { + match value { + "granted" => Ok(Acquisition::Granted), + "requires_signal" => Ok(Acquisition::RequiresSignal), + "denied" => Ok(Acquisition::Denied), + other => Err(PermissionsError::UnknownAcquisition { + value: other.to_owned(), + }), + } +} + +/// Builds a group's [`CountryRules`] from its flag map. Each key names a +/// permission and its flag; an optional `default` key sets any permission the +/// group omits. A group without a `default` must list every permission, so its +/// meaning is fully explicit (this is how the shipped groups are written). +fn group_rules( + name: &str, + flags: &BTreeMap, +) -> Result { + let default = flags + .get("default") + .map(|value| parse_acquisition(value)) + .transpose()?; + // With no `default`, every permission must be listed, so this placeholder is + // never consulted once completeness is checked below. + let mut rules = CountryRules::with_default(default.unwrap_or(Acquisition::Denied)); + let mut listed = PermissionSet::none(); + for (key, value) in flags { + if key == "default" { + continue; + } + let permission = Permission::from_identifier(key) + .ok_or_else(|| PermissionsError::UnknownPermission { name: key.clone() })?; + rules = rules.with_rule(permission, parse_acquisition(value)?); + listed = listed.with(permission); + } + if default.is_none() { + for permission in Permission::all() { + if !listed.contains(permission) { + return Err(PermissionsError::IncompleteGroup { + group: name.to_owned(), + permission: permission.to_string(), + }); + } + } + } + Ok(rules) +} + +/// Looks up a group by name, erroring when a rule references a group that is not +/// defined. +fn resolve_group( + groups: &BTreeMap, + name: &str, +) -> Result { + groups + .get(name) + .cloned() + .ok_or_else(|| PermissionsError::UnknownGroup { + name: name.to_owned(), + }) +} + +/// Applies a rule's per-permission acquisition overrides on top of its group's +/// rules. Each entry maps a Data Use to `granted`, `requires_signal`, or +/// `denied`, overriding the group's baseline for that Data Use, so any +/// acquisition (not just grant or deny) is expressible per rule. +fn apply_modifications( + mut rules: CountryRules, + modifications: &BTreeMap, +) -> Result { + for (name, value) in modifications { + let permission = Permission::from_identifier(name).ok_or_else(|| { + PermissionsError::UnknownPermission { + name: name.to_owned(), + } + })?; + rules = rules.with_rule(permission, parse_acquisition(value)?); + } + Ok(rules) +} + +/// One node of the `rules` tree, resolved into its rules, the jurisdiction it +/// names (when it names one), and the child place codes written beside them. +struct RuleNode<'a> { + /// The node's group, with any `permissions` overrides applied. + rules: CountryRules, + /// The `jurisdiction:` value written on this node, or `None` when the node + /// inherits the one above it. + jurisdiction: Option<&'a str>, + /// The child place codes, in document order, each still unparsed. + children: Vec<(&'a str, &'a Value)>, +} + +/// Parses one node of the `rules` tree. +/// +/// A node written as a plain string is the shorthand: the string is its group, +/// it names no jurisdiction, and it has no children. A node written as a block +/// must carry a `group:` line, may carry a `jurisdiction:` line and a +/// `permissions:` map of overrides, and may carry child place codes beside +/// them. `path` names the node for error messages (`"US"`, `"US/CA"`, or the +/// top of the tree). +fn parse_rule_node<'a>( + groups: &BTreeMap, + path: &str, + value: &'a Value, +) -> Result, PermissionsError> { + match value { + Value::String(name) => Ok(RuleNode { + rules: resolve_group(groups, name)?, + jurisdiction: None, + children: Vec::new(), + }), + Value::Mapping(map) => { + let group = map.get(KEY_GROUP).and_then(Value::as_str).ok_or_else(|| { + PermissionsError::MissingGroup { + path: path.to_owned(), + } + })?; + let mut rules = resolve_group(groups, group)?; + if let Some(overrides) = map.get(KEY_PERMISSIONS) { + let overrides: BTreeMap = + serde_yaml_ng::from_value(overrides.clone()).map_err(|error| { + PermissionsError::Parse { + message: format!("`{KEY_PERMISSIONS}` under `{path}`: {error}"), + } + })?; + rules = apply_modifications(rules, &overrides)?; + } + let jurisdiction = + match map.get(KEY_JURISDICTION) { + Some(value) => Some(value.as_str().ok_or_else(|| { + PermissionsError::UnknownJurisdiction { + value: format!("{value:?}"), + } + })?), + None => None, + }; + let mut children = Vec::new(); + for (key, child) in map { + let key = key.as_str().ok_or_else(|| PermissionsError::InvalidRule { + path: path.to_owned(), + })?; + if key == KEY_GROUP || key == KEY_PERMISSIONS || key == KEY_JURISDICTION { + continue; + } + children.push((key, child)); + } + Ok(RuleNode { + rules, + jurisdiction, + children, + }) + } + _ => Err(PermissionsError::InvalidRule { + path: path.to_owned(), + }), + } +} + +/// Resolves a `jurisdiction:` value written on the node at `path`. +/// +/// `region` is the node's own ISO 3166-2 code, or `None` for the top of the +/// tree and for a country. `us-state` carries no code of its own, because the +/// node naming it is the state, so it is rejected wherever there is no region +/// to name. +fn parse_jurisdiction( + path: &str, + region: Option<&str>, + value: &str, +) -> Result { + if value == "us-state" && region.is_none() { + return Err(PermissionsError::MisplacedUsState { + path: path.to_owned(), + }); + } + Jurisdiction::from_policy_name(value, region).ok_or_else(|| { + PermissionsError::UnknownJurisdiction { + value: value.to_owned(), + } + }) +} + +/// Walks the `rules` tree into `maps`: the top node, then the country nodes +/// under it, then the region nodes under each country. +/// +/// Each node's group and jurisdiction are settled here, so a node that names +/// neither is stored carrying what it inherits from the node above it. The top +/// node must name both, which is what makes inheritance always terminate. +/// +/// The tree is three levels deep, because a geo module returns a country and +/// a region and nothing finer, so a place written under a region is rejected +/// rather than silently ignored. +fn build_rules_tree( + maps: &mut PermissionMaps, + groups: &BTreeMap, + rules: &Value, +) -> Result<(), PermissionsError> { + const TOP: &str = "the top of the tree"; + + if !matches!(rules, Value::Mapping(_)) { + return Err(PermissionsError::MissingGroup { + path: TOP.to_owned(), + }); + } + let top = parse_rule_node(groups, TOP, rules)?; + let name = top + .jurisdiction + .ok_or(PermissionsError::MissingJurisdiction)?; + let top_jurisdiction = parse_jurisdiction(TOP, None, name)?; + maps.default_rules = Some(top.rules); + maps.default_jurisdiction = Some(top_jurisdiction.clone()); + + // Place codes are matched without regard to case at lookup, so two + // spellings of one place would silently overwrite each other. Reject the + // collision instead, among the siblings at each level. + let mut seen_countries: BTreeMap = BTreeMap::new(); + for (country, node) in top.children { + check_duplicate(&mut seen_countries, country)?; + let node = parse_rule_node(groups, country, node)?; + let jurisdiction = match node.jurisdiction { + Some(name) => parse_jurisdiction(country, None, name)?, + None => top_jurisdiction.clone(), + }; + let code = country.to_ascii_uppercase(); + maps.by_country.insert(code.clone(), node.rules); + maps.jurisdiction_by_country + .insert(code, jurisdiction.clone()); + + let mut seen_regions: BTreeMap = BTreeMap::new(); + for (region, child) in node.children { + let path = format!("{country}/{region}"); + check_duplicate(&mut seen_regions, region)?; + let child = parse_rule_node(groups, &path, child)?; + if !child.children.is_empty() { + return Err(PermissionsError::NestedTooDeep { path }); + } + let child_jurisdiction = match child.jurisdiction { + Some(name) => parse_jurisdiction(&path, Some(region), name)?, + None => jurisdiction.clone(), + }; + let key = region_key(country, region); + maps.by_region.insert(key.clone(), child.rules); + maps.jurisdiction_by_region.insert(key, child_jurisdiction); + } + } + Ok(()) +} + +/// Records a place code among its siblings, erroring when another spelling of +/// the same code is already there. +fn check_duplicate<'a>( + seen: &mut BTreeMap, + key: &'a str, +) -> Result<(), PermissionsError> { + if let Some(first) = seen.insert(key.to_ascii_uppercase(), key) { + return Err(PermissionsError::DuplicateRule { + first: first.to_owned(), + second: key.to_owned(), + }); + } + Ok(()) +} + +/// Parses the compiled `permissions.yaml`, so startup validation can reject a +/// malformed policy with a configuration error instead of panicking on the +/// first lookup. +/// +/// # Errors +/// +/// Returns the same [`PermissionsError`] values [`PermissionMaps::from_yaml`] +/// does, most usefully a top node that omits `group` or `jurisdiction`. +pub fn validate_default_policy() -> Result<(), PermissionsError> { + PermissionMaps::from_yaml(DEFAULT_PERMISSION_RULES).map(|_| ()) +} + +/// An error parsing a `permissions.yaml` document. +#[derive(Debug, derive_more::Display)] +pub enum PermissionsError { + /// Two sibling place codes are the same place spelled differently. + #[display("place codes `{first}` and `{second}` name the same place, so keep one")] + DuplicateRule { first: String, second: String }, + /// A node of the `rules` tree carried no `group`. + #[display( + "the rule for `{path}` needs a `group:` line (or write the group name on its own, as `GB: gdpr-uk`)" + )] + MissingGroup { path: String }, + /// The top of the `rules` tree carried no `jurisdiction`. + #[display( + "the top of the `rules` tree needs a `jurisdiction:` line, naming the consent handling for a visitor whose place cannot be resolved" + )] + MissingJurisdiction, + /// A node with no region of its own named `us-state`. + #[display( + "the rule for `{path}` names `jurisdiction: us-state`, which only a region may name, because the region is the state" + )] + MisplacedUsState { path: String }, + /// A `jurisdiction` value named something the consent model cannot + /// represent. + #[display( + "unknown jurisdiction `{value}` (expected gdpr, us-state, non-regulated, or unknown)" + )] + UnknownJurisdiction { value: String }, + /// A node of the `rules` tree was neither a group name nor a block. + #[display("the rule for `{path}` must be a group name or a block with a `group:` line")] + InvalidRule { path: String }, + /// A place was written under a region, deeper than a geo module resolves. + #[display( + "the rule for `{path}` has places written under it; the tree stops at a region, because that is the finest place a geo module returns" + )] + NestedTooDeep { path: String }, + /// The YAML was malformed or did not match the expected shape. + #[display("failed to parse permission rules: {message}")] + Parse { message: String }, + /// A group without a `default` did not list every permission. + #[display( + "permission group `{group}` has no `default` and is missing a flag for `{permission}` (list every permission, or add a `default`)" + )] + IncompleteGroup { group: String, permission: String }, + /// A rule referenced a group that is not defined. + #[display("unknown permission group `{name}`")] + UnknownGroup { name: String }, + /// A permission flag or modification named an unknown permission. + #[display("unknown permission `{name}`")] + UnknownPermission { name: String }, + /// An acquisition rule was not `granted`, `requires_signal`, or `denied`. + #[display("unknown acquisition rule `{value}` (expected granted, requires_signal, or denied)")] + UnknownAcquisition { value: String }, + /// A `signals` opt-out `revokes` value was neither `all` nor a list. + #[display("unknown revoke rule `{value}` (expected `all` or a list of Data Uses)")] + UnknownRevoke { value: String }, +} + +impl core::error::Error for PermissionsError {} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn permission_set_membership_and_subset() { + let set = PermissionSet::none() + .with(Permission::StoreOnDevice) + .with(Permission::SelectBasicAds); + + assert!(set.contains(Permission::StoreOnDevice)); + assert!(set.contains(Permission::SelectBasicAds)); + assert!( + !set.contains(Permission::SelectPersonalisedAds), + "an absent permission should not be reported as present" + ); + + let required = PermissionSet::none().with(Permission::StoreOnDevice); + assert!(set.contains_all(required), "a subset should be contained"); + assert!( + !required.contains_all(set), + "a superset is not contained in a subset" + ); + assert!( + set.contains_all(PermissionSet::none()), + "the empty requirement is always satisfied" + ); + } + + #[test] + fn permission_set_iterates_in_bit_index_order() { + let set = PermissionSet::none() + .with(Permission::SelectBasicAds) + .with(Permission::StoreOnDevice); + let order: Vec<&str> = set.iter().map(Permission::as_str).collect(); + assert_eq!( + order, + vec![ + "necessary.operations.storage", + "advertising_marketing.first_party.contextual" + ], + "iteration should be in stable bit-index order" + ); + } + + #[test] + fn page_json_lists_set_permissions_sorted_by_name() { + // Arrange: a state whose bit-index order is the reverse of its name + // order, so the sort is what the assertion sees. + let state = PermissionState::new( + PermissionSet::none() + .with(Permission::StoreOnDevice) + .with(Permission::SelectBasicAds), + ); + + // Act + let json = state.page_json(); + + // Assert + assert_eq!( + json, + json!({ + "set": [ + "advertising_marketing.first_party.contextual", + "necessary.operations.storage", + ], + "awaiting": [], + "signals": [], + "tdls": [], + }) + .to_string(), + "should list every set permission by Data Use name, sorted" + ); + } + + #[test] + fn page_json_carries_the_terms_the_data_is_available_under() { + // Arrange: the state a request resolves to when a terms scheme + // declared the document its data is offered under. + let state = PermissionState::new(PermissionSet::none().with(Permission::StoreOnDevice)) + .with_tdls(Arc::from(vec![ + Tdl::new("https://terms.example.com/marketing/2.txt") + .expect("should accept the test locator"), + ])); + + // Act + let json = state.page_json(); + + // Assert + assert_eq!( + json, + json!({ + "set": ["necessary.operations.storage"], + "awaiting": [], + "signals": [], + "tdls": ["https://terms.example.com/marketing/2.txt"], + }) + .to_string(), + "page code should read the terms alongside the permissions" + ); + } + + #[test] + fn page_json_of_an_empty_state_is_an_empty_set() { + // Arrange + let state = PermissionState::default(); + + // Act + let json = state.page_json(); + + // Assert + assert_eq!( + json, + json!({ "set": [], "awaiting": [], "signals": [], "tdls": [] }).to_string(), + "an empty state should render as an empty set and no declared terms, not as nothing" + ); + } + + #[test] + fn a_permission_requiring_a_signal_that_nobody_gave_is_awaited() { + // Arrange: the requires-signal floor, where every permission needs a + // signal, and modules that answer nothing. + let state = PermissionMaps::floor_with(|_, _| ConsentSignal::Neutral); + + // Assert: nothing is set and everything is awaited, which is the + // state a page must hold on rather than read as a refusal. + assert!( + state.permissions().is_empty(), + "should set nothing without a signal" + ); + assert!( + state.is_awaited(Permission::StoreOnDevice), + "should record that storage is still waiting for a signal" + ); + assert_eq!( + state.page_json(), + json!({ + "set": [], + "awaiting": Permission::all().map(Permission::as_str).collect::>(), + "signals": [], + "tdls": [], + }) + .to_string(), + "the page should be told which permissions are still awaited" + ); + } + + #[test] + fn awaiting_is_narrowed_to_what_a_module_could_still_grant() { + // Arrange: the floor, where everything is awaited, and a deployment + // whose modules can only ever grant storage. + let state = PermissionMaps::floor_with(|_, _| ConsentSignal::Neutral) + .awaiting_only(PermissionSet::none().with(Permission::StoreOnDevice)); + + // Assert: storage is still awaited, and a permission nobody could + // grant is not, because no page should wait for it. + assert!(state.is_awaited(Permission::StoreOnDevice)); + assert!( + !state.is_awaited(Permission::SelectPersonalisedAds), + "should not await a permission no configured module can grant" + ); + assert!( + state.permissions().is_empty(), + "narrowing what is awaited should set nothing" + ); + } + + #[test] + fn an_answered_or_baselined_permission_is_never_awaited() { + // Arrange: the floor again, with a module that grants storage and + // revokes personalised advertising. + let state = PermissionMaps::floor_with(|permission, _| match permission { + Permission::StoreOnDevice => ConsentSignal::Grant, + Permission::SelectPersonalisedAds => ConsentSignal::Revoke, + _ => ConsentSignal::Neutral, + }); + + // Assert: a grant is set and not awaited, a refusal is neither set + // nor awaited, and a permission nobody answered is awaited. + assert!(state.is_set(Permission::StoreOnDevice)); + assert!( + !state.is_awaited(Permission::StoreOnDevice), + "a grant is an answer" + ); + assert!(!state.is_set(Permission::SelectPersonalisedAds)); + assert!( + !state.is_awaited(Permission::SelectPersonalisedAds), + "a refusal is an answer too, and must not look like no answer" + ); + assert!(state.is_awaited(Permission::SelectBasicAds)); + } + + #[test] + fn the_floor_sets_a_permission_only_when_a_signal_grants_it() { + // Empty maps and no top node: every permission is the requires-signal + // floor, set only when a signal grants it. + let maps = PermissionMaps::default(); + + let denied = maps.resolve(Some("GB"), |_| false); + assert!( + !denied.is_set(Permission::StoreOnDevice), + "the floor should not set necessary.operations.storage without a signal" + ); + + let granted = maps.resolve(Some("GB"), |p| p == Permission::StoreOnDevice); + assert!( + granted.is_set(Permission::StoreOnDevice), + "the floor should set necessary.operations.storage once a signal grants it" + ); + } + + #[test] + fn the_floor_ignores_the_top_node_even_where_one_grants() { + // A policy whose top node grants everything, so the only way the floor + // can leave the permission unset is by never consulting the tree. This + // is the geo lookup failure case. + let maps = PermissionMaps::empty() + .with_default_rules(CountryRules::with_default(Acquisition::Granted)); + assert!( + maps.baseline(None, None).is_set(Permission::StoreOnDevice), + "the top node grants storage, or this test proves nothing" + ); + assert!( + !PermissionMaps::floor_with(|_, _| ConsentSignal::Neutral) + .is_set(Permission::StoreOnDevice), + "the floor must not fall back to the top node" + ); + } + + #[test] + fn an_unlisted_country_uses_the_top_node() { + // A map whose top node grants, so an unlisted country is answered by + // it rather than by the floor. + let maps = PermissionMaps::empty() + .with_default_rules(CountryRules::with_default(Acquisition::Granted)); + assert!( + maps.resolve(Some("ZZ"), |_| false) + .is_set(Permission::StoreOnDevice), + "an unlisted country should resolve at the top node" + ); + // No country at all resolves the same way. + assert!( + maps.resolve(None, |_| false) + .is_set(Permission::StoreOnDevice), + "no country should resolve at the top node too" + ); + // With no top node the requires-signal floor sets nothing. + assert!( + !PermissionMaps::empty() + .resolve(None, |_| false) + .is_set(Permission::StoreOnDevice), + "with no top node, an unlisted country sets nothing without a signal" + ); + } + + #[test] + fn a_matching_country_is_used_over_the_top_node() { + // US grants; the top node requires a signal. + let maps = PermissionMaps::empty() + .with_country("us", CountryRules::with_default(Acquisition::Granted)) + .with_default_rules(CountryRules::with_default(Acquisition::RequiresSignal)); + assert!( + maps.resolve(Some("US"), |_| false) + .is_set(Permission::StoreOnDevice), + "a country with its own rule uses it, not the top node" + ); + assert!( + !maps + .resolve(Some("ZZ"), |_| false) + .is_set(Permission::StoreOnDevice), + "an unlisted country falls back to the top node" + ); + } + + #[test] + fn per_permission_override_beats_the_country_default() { + // Granted by default, but deny necessary.operations.storage specifically. + let rules = CountryRules::with_default(Acquisition::Granted) + .with_rule(Permission::StoreOnDevice, Acquisition::Denied); + let maps = PermissionMaps::empty().with_country("zz", rules); + let state = maps.resolve(Some("ZZ"), |_| true); + + assert!( + !state.is_set(Permission::StoreOnDevice), + "an explicit Denied override should beat the granted default" + ); + assert!( + state.is_set(Permission::SelectBasicAds), + "other permissions should still follow the granted default" + ); + } + + #[test] + fn all_set_gates_on_the_required_set() { + let state = PermissionState::new(PermissionSet::none().with(Permission::StoreOnDevice)); + + assert!( + state.all_set(PermissionSet::none()), + "a module requiring nothing always runs" + ); + assert!( + state.all_set(PermissionSet::none().with(Permission::StoreOnDevice)), + "a set requirement is satisfied" + ); + assert!( + !state.all_set(PermissionSet::none().with(Permission::SelectPersonalisedAds)), + "an unset requirement is not satisfied" + ); + } + + #[test] + fn with_default_sets_the_baseline_acquisition() { + // A granted default is set with no signal required. + let granted = PermissionMaps::empty() + .with_country("zz", CountryRules::with_default(Acquisition::Granted)); + assert!( + granted + .resolve(Some("ZZ"), |_| false) + .is_set(Permission::StoreOnDevice), + "a granted default should set with no signal" + ); + // A requires-signal default is set only once a signal grants it. + let opt_in = PermissionMaps::empty().with_country( + "zz", + CountryRules::with_default(Acquisition::RequiresSignal), + ); + assert!( + !opt_in + .resolve(Some("ZZ"), |_| false) + .is_set(Permission::StoreOnDevice), + "a requires-signal default should not be set without a signal" + ); + assert!( + opt_in + .resolve(Some("ZZ"), |p| p == Permission::StoreOnDevice) + .is_set(Permission::StoreOnDevice), + "a requires-signal default should set once a signal grants it" + ); + } + + #[test] + fn granted_and_denied_rules_ignore_signals() { + // A Granted rule is set even when no signal is present. + let granted = CountryRules::with_default(Acquisition::RequiresSignal) + .with_rule(Permission::StoreOnDevice, Acquisition::Granted); + assert!( + PermissionMaps::empty() + .with_country("zz", granted) + .resolve(Some("ZZ"), |_| false) + .is_set(Permission::StoreOnDevice), + "a Granted rule is set with no signal" + ); + + // A Denied rule is never set, even when every signal grants. + let denied = CountryRules::with_default(Acquisition::Granted) + .with_rule(Permission::StoreOnDevice, Acquisition::Denied); + assert!( + !PermissionMaps::empty() + .with_country("zz", denied) + .resolve(Some("ZZ"), |_| true) + .is_set(Permission::StoreOnDevice), + "a Denied rule is never set even with a signal" + ); + } + + #[test] + fn standard_maps_eu_requires_signal_and_uk_grants_storage() { + let maps = PermissionMaps::standard(); + assert!( + !maps + .resolve(Some("DE"), |_| false) + .is_set(Permission::StoreOnDevice), + "an EU country should not set necessary.operations.storage without a signal" + ); + assert!( + maps.resolve(Some("DE"), |p| p == Permission::StoreOnDevice) + .is_set(Permission::StoreOnDevice), + "an EU country should set necessary.operations.storage once a signal grants it" + ); + assert!( + maps.resolve(Some("GB"), |_| false) + .is_set(Permission::StoreOnDevice), + "the UK should grant necessary.operations.storage without a signal" + ); + } + + #[test] + fn standard_maps_us_and_australia_grant_storage_by_default() { + let maps = PermissionMaps::standard(); + for code in ["US", "AU"] { + assert!( + maps.resolve(Some(code), |_| false) + .is_set(Permission::StoreOnDevice), + "{code} should grant necessary.operations.storage by default" + ); + } + // No default configured, so an unmapped country hits the requires-signal + // floor and sets nothing without a signal. + assert!( + !maps + .resolve(Some("ZZ"), |_| false) + .is_set(Permission::StoreOnDevice), + "an unmapped country with no default sets nothing without a signal" + ); + } + + #[test] + fn resolve_with_revokes_a_granted_permission_on_opt_out() { + // US grants necessary.operations.storage by default; an opt-out signal revokes it. + let maps = PermissionMaps::standard(); + assert!( + maps.baseline(Some("US"), None) + .is_set(Permission::StoreOnDevice), + "the US baseline should set necessary.operations.storage" + ); + let revoked = maps.resolve_with(Some("US"), None, |p, _| { + if p == Permission::StoreOnDevice { + ConsentSignal::Revoke + } else { + ConsentSignal::Neutral + } + }); + assert!( + !revoked.is_set(Permission::StoreOnDevice), + "an opt-out signal should revoke a granted permission" + ); + } + + #[test] + fn a_region_entry_overrides_the_country_baseline() { + // US grants by default; a state can require a signal instead. + let maps = PermissionMaps::standard().clone().with_region( + "US", + "CA", + CountryRules::with_default(Acquisition::RequiresSignal), + ); + assert!( + !maps + .baseline(Some("US"), Some("CA")) + .is_set(Permission::StoreOnDevice), + "the CA region rule should require a signal, overriding the US baseline" + ); + assert!( + maps.baseline(Some("US"), Some("NY")) + .is_set(Permission::StoreOnDevice), + "a state with no region entry should follow the US baseline" + ); + } + + #[test] + fn from_yaml_parses_groups_rules_and_modifications() { + let yaml = r#" +groups: + eu: + default: requires_signal + us: + default: granted +rules: + group: eu + jurisdiction: gdpr + FR: eu + US: + group: us + jurisdiction: non-regulated + CA: + group: eu + jurisdiction: us-state + permissions: + necessary.operations.storage: granted + advertising_marketing.first_party.contextual: denied +"#; + let maps = PermissionMaps::from_yaml(yaml).expect("should parse the rules"); + + // Bare group references. + assert!( + !maps + .baseline(Some("FR"), None) + .is_set(Permission::StoreOnDevice), + "FR (eu) requires a signal for device storage" + ); + assert!( + maps.baseline(Some("US"), None) + .is_set(Permission::StoreOnDevice), + "US (us) grants device storage" + ); + + // CA references the eu group, but its permissions map grants + // necessary.operations.storage, overriding the eu baseline. + assert!( + maps.baseline(Some("US"), Some("CA")) + .is_set(Permission::StoreOnDevice), + "the permissions map grants necessary.operations.storage for CA, overriding the eu baseline" + ); + // The permissions map denies advertising_marketing.first_party.contextual, + // so it is not set even when a signal grants it. + assert!( + !maps + .resolve_with(Some("US"), Some("CA"), |_, _| ConsentSignal::Grant) + .is_set(Permission::SelectBasicAds), + "the permissions map denies advertising_marketing.first_party.contextual even when a signal grants it" + ); + + // An unlisted country falls back to the top node (eu, requires signal). + assert!( + !maps + .baseline(Some("ZZ"), None) + .is_set(Permission::StoreOnDevice), + "an unlisted country should use the top node" + ); + } + + /// A two-group policy the tree tests below vary, with `g` granting + /// everything and `strict` requiring a signal for everything. + const TEST_GROUPS: &str = "groups:\n g:\n default: granted\n \ + strict:\n default: requires_signal\n"; + + #[test] + fn a_shorthand_node_is_a_group_with_nothing_below_it() { + let yaml = format!( + "{TEST_GROUPS}\ +rules: + group: strict + jurisdiction: gdpr + GB: g +" + ); + let maps = PermissionMaps::from_yaml(&yaml).expect("a shorthand node should parse"); + assert_eq!( + maps.rules_for(Some("GB"), None) + .expect("GB should have a rule") + .rule_for(Permission::StoreOnDevice), + Acquisition::Granted, + "the shorthand string should name the node's group" + ); + assert_eq!( + maps.jurisdiction_for(Some("GB"), None), + Jurisdiction::Gdpr, + "a shorthand node should inherit the top node's jurisdiction" + ); + } + + #[test] + fn a_block_node_carries_children_beside_its_group() { + let yaml = format!( + "{TEST_GROUPS}\ +rules: + group: g + jurisdiction: gdpr + US: + group: g + jurisdiction: non-regulated + CA: + group: strict + jurisdiction: us-state +" + ); + let maps = PermissionMaps::from_yaml(&yaml).expect("a block node should parse"); + assert_eq!( + maps.rules_for(Some("US"), Some("CA")) + .expect("US/CA should have a rule") + .rule_for(Permission::StoreOnDevice), + Acquisition::RequiresSignal, + "the region child should carry its own group" + ); + assert_eq!( + maps.rules_or_default(Some("US"), Some("NV")) + .expect("an unlisted region should fall back to its country") + .rule_for(Permission::StoreOnDevice), + Acquisition::Granted, + "an unlisted region should resolve at its country node" + ); + } + + /// Every way `PermissionMaps::from_yaml` refuses a policy, each with the + /// error it must report. + #[test] + fn from_yaml_rejects_each_malformed_policy() { + type Refusal = fn(&PermissionsError) -> bool; + // A policy given one YAML line to an element, with or without the test + // groups ahead of it. + let tree = |lines: &[&str]| format!("{TEST_GROUPS}{}\n", lines.join("\n")); + let policy = |lines: &[&str]| format!("{}\n", lines.join("\n")); + let cases: [(&str, String, Refusal); 15] = [ + ( + "a block without a group", + tree(&[ + "rules:", + " group: g", + " jurisdiction: gdpr", + " US:", + " jurisdiction: non-regulated", + ]), + |err| matches!(err, PermissionsError::MissingGroup { path } if path == "US"), + ), + ( + "a top node without a group", + tree(&["rules:", " jurisdiction: gdpr", " US: g"]), + |err| matches!(err, PermissionsError::MissingGroup { .. }), + ), + ( + "a top node without a jurisdiction", + tree(&["rules:", " group: g", " US: g"]), + |err| matches!(err, PermissionsError::MissingJurisdiction), + ), + ( + "an unknown jurisdiction", + tree(&["rules:", " group: g", " jurisdiction: ccpa"]), + |err| matches!(err, PermissionsError::UnknownJurisdiction { .. }), + ), + ( + // `us-state` names no code of its own, so only a region can + // carry it. + "us-state at the top of the tree", + tree(&["rules:", " group: g", " jurisdiction: us-state"]), + |err| matches!(err, PermissionsError::MisplacedUsState { .. }), + ), + ( + "us-state on a country", + tree(&[ + "rules:", + " group: g", + " jurisdiction: gdpr", + " US:", + " group: g", + " jurisdiction: us-state", + ]), + |err| matches!(err, PermissionsError::MisplacedUsState { .. }), + ), + ( + "a place written under a region", + tree(&[ + "rules:", + " group: g", + " jurisdiction: gdpr", + " US:", + " group: g", + " CA:", + " group: g", + " LA:", + " group: g", + ]), + |err| matches!(err, PermissionsError::NestedTooDeep { .. }), + ), + ( + "a rule naming a group that is not defined", + tree(&["rules:", " group: missing", " jurisdiction: gdpr"]), + |err| matches!(err, PermissionsError::UnknownGroup { name } if name == "missing"), + ), + ( + "a rule that is neither a group name nor a block", + tree(&[ + "rules:", + " group: g", + " jurisdiction: gdpr", + " US: [1, 2]", + ]), + |err| matches!(err, PermissionsError::InvalidRule { path } if path == "US"), + ), + ( + // A group with no `default` must list every permission, so this + // one, naming only storage, is rejected rather than silently + // leaving every other permission unset. + "an incomplete group without a default", + policy(&[ + "groups:", + " g:", + " necessary.operations.storage: granted", + "rules: {}", + ]), + |err| matches!(err, PermissionsError::IncompleteGroup { .. }), + ), + ( + "an unknown permission", + policy(&[ + "groups:", + " g:", + " default: granted", + " not-a-permission: denied", + "rules: {}", + ]), + |err| matches!(err, PermissionsError::UnknownPermission { .. }), + ), + ( + "an unknown acquisition", + policy(&["groups:", " g:", " default: maybe", "rules: {}"]), + |err| matches!(err, PermissionsError::UnknownAcquisition { .. }), + ), + ( + "an override that is not an acquisition", + policy(&[ + "groups:", + " g:", + " default: granted", + "rules:", + " group: g", + " jurisdiction: unknown", + " US:", + " group: g", + " permissions:", + " necessary.operations.storage: enabled", + ]), + |err| matches!(err, PermissionsError::UnknownAcquisition { .. }), + ), + ( + "two spellings of one country", + policy(&[ + "groups:", + " g:", + " default: granted", + "rules:", + " group: g", + " jurisdiction: unknown", + " us: g", + " US: g", + ]), + |err| matches!(err, PermissionsError::DuplicateRule { .. }), + ), + ( + "an unknown revoke keyword", + policy(&[ + "groups:", + " g:", + " default: requires_signal", + "rules:", + " group: g", + " jurisdiction: gdpr", + " FR: g", + "signals:", + " us_opt_out:", + " sources: [gpc]", + " revokes: everything", + ]), + |err| matches!(err, PermissionsError::UnknownRevoke { .. }), + ), + ]; + for (case, yaml, refusal) in cases { + let Err(err) = PermissionMaps::from_yaml(&yaml) else { + panic!("{case} should be rejected"); + }; + assert!(refusal(&err), "{case}: wrong refusal, got {err:?}"); + } + } + + #[test] + fn a_jurisdiction_is_inherited_and_can_be_overridden_at_each_level() { + let yaml = format!( + "{TEST_GROUPS}\ +rules: + group: g + jurisdiction: gdpr + GB: g + US: + group: g + jurisdiction: non-regulated + CA: + group: g + jurisdiction: us-state + NY: g +" + ); + let maps = PermissionMaps::from_yaml(&yaml).expect("should parse the tree"); + assert_eq!( + maps.jurisdiction_for(Some("GB"), None), + Jurisdiction::Gdpr, + "a country naming none should inherit the top node" + ); + assert_eq!( + maps.jurisdiction_for(Some("US"), None), + Jurisdiction::NonRegulated, + "a country naming its own should override the top node" + ); + assert_eq!( + maps.jurisdiction_for(Some("US"), Some("NY")), + Jurisdiction::NonRegulated, + "a region naming none should inherit its country" + ); + assert_eq!( + maps.jurisdiction_for(Some("US"), Some("CA")), + Jurisdiction::UsState("CA".to_owned()), + "a region naming us-state should name itself as the state" + ); + assert_eq!( + maps.jurisdiction_for(Some("ZZ"), None), + Jurisdiction::Gdpr, + "an unlisted country should inherit the top node" + ); + assert_eq!( + maps.default_jurisdiction(), + Jurisdiction::Gdpr, + "no place at all should resolve the top node" + ); + } + + #[test] + fn place_codes_are_matched_without_regard_to_case() { + let yaml = format!( + "{TEST_GROUPS}\ +rules: + group: strict + jurisdiction: gdpr + us: + group: strict + jurisdiction: non-regulated + ca: + group: g + jurisdiction: us-state +" + ); + let maps = PermissionMaps::from_yaml(&yaml).expect("should parse lower-case place codes"); + assert!( + maps.baseline(Some("US"), Some("CA")) + .is_set(Permission::StoreOnDevice), + "an upper-case request should match a lower-case rule" + ); + assert_eq!( + maps.jurisdiction_for(Some("US"), Some("CA")), + Jurisdiction::UsState("CA".to_owned()), + "the state code should be upper-cased whatever case the file uses" + ); + } + + #[test] + fn the_settled_example_resolves_as_documented() { + // The example in the permissions.yaml header, kept here so the file's + // own teaching example is proven rather than asserted. + let yaml = r#" +groups: + gdpr-eu: + default: requires_signal + gdpr-uk: + default: requires_signal + us-notice: + default: granted + us-opt-out: + default: granted +rules: + group: gdpr-eu + jurisdiction: gdpr + GB: gdpr-uk + US: + group: us-notice + jurisdiction: non-regulated + CA: + group: us-opt-out + jurisdiction: us-state + NY: us-notice +"#; + let maps = PermissionMaps::from_yaml(yaml).expect("the header example should parse"); + for (country, region, jurisdiction) in [ + (Some("GB"), None, Jurisdiction::Gdpr), + (Some("US"), None, Jurisdiction::NonRegulated), + (Some("US"), Some("NY"), Jurisdiction::NonRegulated), + ( + Some("US"), + Some("CA"), + Jurisdiction::UsState("CA".to_owned()), + ), + (Some("JP"), None, Jurisdiction::Gdpr), + (None, None, Jurisdiction::Gdpr), + ] { + assert_eq!( + maps.jurisdiction_for(country, region), + jurisdiction, + "{country:?}/{region:?} should resolve as the header documents" + ); + } + assert!( + !maps + .baseline(Some("GB"), None) + .is_set(Permission::StoreOnDevice), + "GB takes the gdpr-uk group, which requires a signal in this example" + ); + assert!( + maps.baseline(Some("US"), Some("NY")) + .is_set(Permission::StoreOnDevice), + "NY takes us-notice, which grants" + ); + } + + #[test] + fn the_shipped_policy_reproduces_the_retired_consent_lists() { + // The policy tree carries the 31 GDPR countries and the 20 US + // privacy states, in place of compiled defaults in the consent + // configuration, so the shipped file must resolve each to its regime. + let maps = PermissionMaps::standard(); + let gdpr_countries = [ + "AT", "BE", "BG", "HR", "CY", "CZ", "DK", "EE", "FI", "FR", "DE", "GR", "HU", "IE", + "IT", "LV", "LT", "LU", "MT", "NL", "PL", "PT", "RO", "SK", "SI", "ES", "SE", "IS", + "LI", "NO", "GB", + ]; + assert_eq!( + gdpr_countries.len(), + 31, + "the retired list held 27 EU, 3 EEA and the UK" + ); + for country in gdpr_countries { + assert_eq!( + maps.jurisdiction_for(Some(country), None), + Jurisdiction::Gdpr, + "`{country}` should resolve the GDPR jurisdiction" + ); + } + + let privacy_states = [ + "CA", "VA", "CO", "CT", "UT", "MT", "OR", "TX", "FL", "DE", "IA", "NE", "NH", "NJ", + "TN", "MN", "MD", "IN", "KY", "RI", + ]; + assert_eq!( + privacy_states.len(), + 20, + "the retired list held 20 states with a comprehensive privacy law" + ); + for state in privacy_states { + assert_eq!( + maps.jurisdiction_for(Some("US"), Some(state)), + Jurisdiction::UsState(state.to_owned()), + "`US/{state}` should resolve its own state jurisdiction" + ); + } + + // A state with no law of its own inherits the US node, and the US with + // no region does the same. + for region in [Some("WY"), Some("NY"), None] { + assert_eq!( + maps.jurisdiction_for(Some("US"), region), + Jurisdiction::NonRegulated, + "`US/{region:?}` should inherit the country node" + ); + } + assert_eq!( + maps.jurisdiction_for(Some("AU"), None), + Jurisdiction::NonRegulated, + "Australia should carry its own non-regulated jurisdiction" + ); + assert_eq!( + maps.default_jurisdiction(), + Jurisdiction::Gdpr, + "an unresolved place should take the top node's GDPR jurisdiction" + ); + } + + #[test] + fn a_us_state_keeps_the_country_permission_baseline() { + // Each listed state carries the same group as the country node, so + // adding the states for the jurisdiction changed no permission. + let maps = PermissionMaps::standard(); + for state in ["CA", "TX", "DE", "RI", "WY"] { + assert!( + maps.baseline(Some("US"), Some(state)) + .is_set(Permission::StoreOnDevice), + "`US/{state}` should keep the US opt-out storage baseline" + ); + } + } + + #[test] + fn from_yaml_accepts_an_explicit_group_listing_every_permission() { + // The shipped style: no `default`, every permission spelled out. + let mut group = String::from("groups:\n everything:\n"); + for permission in Permission::all() { + group.push_str(&format!(" {permission}: granted\n")); + } + let yaml = format!( + "{group}rules:\n group: everything\n jurisdiction: unknown\n US: everything\n" + ); + let maps = PermissionMaps::from_yaml(&yaml).expect("an explicit group should parse"); + assert!( + maps.baseline(Some("US"), None) + .is_set(Permission::MarketResearch), + "every listed permission should take its flag" + ); + } + + #[test] + fn a_detailed_rule_can_set_requires_signal_per_permission() { + let yaml = "groups:\n g:\n default: granted\nrules:\n group: g\n jurisdiction: unknown\n US:\n group: g\n permissions:\n necessary.operations.storage: requires_signal\n"; + let maps = PermissionMaps::from_yaml(yaml).expect("should parse the override map"); + let rules = maps + .rules_for(Some("US"), None) + .expect("should resolve the US rule"); + assert_eq!( + rules.rule_for(Permission::StoreOnDevice), + Acquisition::RequiresSignal, + "the per-permission map should express requires_signal" + ); + assert_eq!( + rules.rule_for(Permission::SelectPersonalisedAds), + Acquisition::Granted, + "an unlisted permission should keep the group default" + ); + } + + #[test] + fn every_eu_and_eea_member_requires_a_signal_for_storage() { + // The shipped permissions.yaml must cover all 27 EU member states and + // the three EEA members, each resolving storage as requires-signal, so + // no member state silently falls to the top of the tree. + let maps = PermissionMaps::standard(); + for country in [ + "AT", "BE", "BG", "HR", "CY", "CZ", "DK", "EE", "FI", "FR", "DE", "GR", "HU", "IE", + "IT", "LV", "LT", "LU", "MT", "NL", "PL", "PT", "RO", "SK", "SI", "ES", "SE", "IS", + "LI", "NO", + ] { + let rules = maps + .rules_for(Some(country), None) + .unwrap_or_else(|| panic!("`{country}` should have a rule")); + assert_eq!( + rules.rule_for(Permission::StoreOnDevice), + Acquisition::RequiresSignal, + "storage in `{country}` should require a signal" + ); + } + } + + #[test] + fn from_yaml_parses_the_signals_section_into_a_policy() { + let yaml = "\ +groups: + g: + default: requires_signal +rules: + group: g + jurisdiction: gdpr + FR: g +signals: + tcf: + authoritative: true + us_opt_out: + sources: [gpc] + revokes: [advertising_marketing.first_party.targeted] +"; + let maps = PermissionMaps::from_yaml(yaml).expect("should parse the signals section"); + let signals = maps.signals(); + assert!(signals.tcf_authoritative(), "tcf should be authoritative"); + assert!( + signals.opt_out_revokes(Permission::SelectPersonalisedAds), + "a listed Data Use is revoked by the opt-out" + ); + assert!( + !signals.opt_out_revokes(Permission::StoreOnDevice), + "an unlisted Data Use is not revoked by the opt-out" + ); + } +} diff --git a/crates/trusted-server-core/src/platform/geo.rs b/crates/trusted-server-core/src/platform/geo.rs new file mode 100644 index 0000000000..c8077215dd --- /dev/null +++ b/crates/trusted-server-core/src/platform/geo.rs @@ -0,0 +1,30 @@ +use std::net::IpAddr; + +use error_stack::Report; + +use super::{GeoInfo, PlatformError, PlatformGeo, RuntimeServices}; + +/// A geo module that resolves nothing. +/// +/// Installed when the `[geo] module` selector is unset, and when it is set to +/// `"none"`, which spells the same choice explicitly. A client IP is then never +/// sent to any host geo service, so a default deployment is not tied to any host +/// geo capability. Every geo consumer already treats [`GeoInfo`] as optional, so +/// a `None` result degrades gracefully: the permission baseline falls back to +/// the top of the `permissions.yaml` rules tree, the auction omits geo, and so on. +/// +/// Adapter crates should use this type rather than defining their own stub so +/// the behavior is the same on every platform. +#[derive(Debug, Default, Clone, Copy)] +pub struct DisabledGeo; + +#[async_trait::async_trait(?Send)] +impl PlatformGeo for DisabledGeo { + async fn lookup( + &self, + _client_ip: Option, + _services: &RuntimeServices, + ) -> Result, Report> { + Ok(None) + } +} diff --git a/crates/trusted-server-core/src/platform/loading_cache.rs b/crates/trusted-server-core/src/platform/loading_cache.rs new file mode 100644 index 0000000000..8677381e1e --- /dev/null +++ b/crates/trusted-server-core/src/platform/loading_cache.rs @@ -0,0 +1,354 @@ +//! A cache that loads a missing value once, however many callers ask for it +//! at the same time. +//! +//! The first caller for a key that no layer holds runs the [`Loader`]. Every +//! caller that asks for the key while it runs waits for that load, and all of +//! them continue with its value or its error. A failed load is not stored, so +//! the next caller loads again. +//! +//! Core declares the interface and [`NoLoadingCache`], which holds nothing. A +//! module outside core supplies the implementation through +//! [`IntegrationBuilder::with_loading_cache`](crate::integrations::IntegrationBuilder::with_loading_cache), +//! keeping a brief copy in memory over the host's shared store, being its +//! value cache where it has one and its key-value store otherwise. +//! +//! Every key is a [`StoreKey`], which starts with the publisher's +//! [`ServerKeyPrefix`], so two publishers sharing a store never share an +//! entry. + +use core::fmt; +use std::collections::HashMap; +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; +use std::time::Duration; + +use bytes::Bytes; + +use super::{RuntimeServices, ServerKeyPrefix}; + +/// A value a [`Loader`] produced. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LoadedValue { + /// The value. + pub bytes: Bytes, + /// How long the value may be used from now, or `None` for as long as the + /// cache's own lifetimes allow. Every copy of it ends by then. + pub lifetime: Option, +} + +impl LoadedValue { + /// A value with no lifetime of its own. + #[must_use] + pub fn new(bytes: impl Into) -> Self { + Self { + bytes: bytes.into(), + lifetime: None, + } + } + + /// Sets how long the value may be used from now. + #[must_use] + pub fn with_lifetime(mut self, lifetime: Duration) -> Self { + self.lifetime = Some(lifetime); + self + } +} + +/// Why a load produced no value. +/// +/// Every caller waiting on the load receives it, and nothing is stored, so the +/// next caller loads again. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LoadError { + message: std::sync::Arc, +} + +impl LoadError { + /// An error saying `message`. + #[must_use] + pub fn new(message: impl Into) -> Self { + Self { + message: message.into().into(), + } + } + + /// What went wrong. + #[must_use] + pub fn message(&self) -> &str { + &self.message + } +} + +impl fmt::Display for LoadError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.message) + } +} + +impl core::error::Error for LoadError {} + +/// The work a [`Loader`] does. +/// +/// Not `Send`, because the platform's own futures are not. It is made on the +/// thread that runs it. +pub type LoadFuture = Pin>>>; + +/// What a cache runs when no layer holds a key. +/// +/// It owns everything it uses, and is `Send`, so a host that can run work on +/// its own can start it on another thread and keep it running after the +/// caller that started it has gone. +pub type Loader = Box LoadFuture + Send>; + +/// How long one cache's copies last. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CacheLifetimes { + /// How long the brief copy in memory is used. + pub memory: Duration, + /// How long the copy in the shared store is used, at most. A value with a + /// shorter lifetime of its own ends sooner. + pub store: Duration, + /// How long the copy in the shared store lasts unused. + pub store_idle: Duration, +} + +/// One cache: its name, the lifetimes its copies get, and how its loads may +/// run. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CacheSpec { + /// The cache's name, which is part of every key it writes. + pub name: &'static str, + /// How long its copies last. + pub lifetimes: CacheLifetimes, + /// The most values the brief copy in memory holds. + pub memory_entries: usize, + /// Whether a load may run apart from the caller that started it, on a + /// host that can run work on its own. False for a load that blocks the + /// thread it runs on, which only the caller's own thread can afford. + pub detachable: bool, +} + +/// A key in a shared store, being the publisher's prefix and the cache's name +/// as a namespace, and the caller's own key below it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StoreKey { + namespace: String, + key: String, +} + +impl StoreKey { + /// The key `key` in the cache `cache`, for the publisher `prefix` names. + /// + /// The caller's key should be short, such as a digest, because a store + /// refuses a long key. + #[must_use] + pub fn new(prefix: &ServerKeyPrefix, cache: &str, key: impl Into) -> Self { + Self { + namespace: format!("{}/{cache}", prefix.as_str()), + key: key.into(), + } + } + + /// The publisher's prefix and the cache's name, as `/`. + #[must_use] + pub fn namespace(&self) -> &str { + &self.namespace + } + + /// The caller's own key. + #[must_use] + pub fn key(&self) -> &str { + &self.key + } +} + +impl fmt::Display for StoreKey { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}/{}", self.namespace, self.key) + } +} + +/// A cache that loads a missing value once per key at a time. +/// +/// `Send + Sync` on the trait and `?Send` on the futures, as for the other +/// platform services, because [`RuntimeServices`] crosses threads while the +/// platform layer's futures never do. +#[async_trait::async_trait(?Send)] +pub trait PlatformLoadingCache: Send + Sync { + /// The value for `key` in the cache `spec` describes, from a copy that is + /// still fresh or from `load`. + /// + /// `load` runs at most once per key at a time, and a caller arriving while + /// it runs waits for it. The copies are kept in the stores `services` + /// supplies. + /// + /// # Errors + /// + /// The error of the load this caller waited on, which every caller waiting + /// on that load receives. + async fn get_or_load( + &self, + services: &RuntimeServices, + spec: &CacheSpec, + key: &StoreKey, + load: Loader, + ) -> Result; + + /// Removes `key` from the memory copy and the shared store. + async fn remove(&self, services: &RuntimeServices, spec: &CacheSpec, key: &StoreKey); + + /// Whether this cache keeps what it loads. A caller whose load pays off only + /// when its value is kept does without it on a cache that keeps nothing. + fn keeps_values(&self) -> bool { + true + } +} + +impl fmt::Debug for dyn PlatformLoadingCache { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("PlatformLoadingCache") + } +} + +/// The loading cache of a deployment that links no module supplying one. +/// +/// It holds nothing and runs every load, so each caller asks the source +/// itself. +#[derive(Debug, Default, Clone, Copy)] +pub struct NoLoadingCache; + +#[async_trait::async_trait(?Send)] +impl PlatformLoadingCache for NoLoadingCache { + async fn get_or_load( + &self, + _services: &RuntimeServices, + _spec: &CacheSpec, + _key: &StoreKey, + load: Loader, + ) -> Result { + load().await.map(|loaded| loaded.bytes) + } + + async fn remove(&self, _services: &RuntimeServices, _spec: &CacheSpec, _key: &StoreKey) {} + + fn keeps_values(&self) -> bool { + false + } +} + +/// The loading cache each `[cache.]` table selected, by the name a +/// [`CacheSpec`] carries. A cache whose table selected none loads through +/// [`NoLoadingCache`]. +#[derive(Default)] +pub struct LoadingCaches { + by_feature: HashMap<&'static str, Arc>, + /// What a cache whose table selected nothing loads through, when a test + /// gives every caller one cache. + unselected: Option>, +} + +impl LoadingCaches { + /// Every caller loading through `cache`, for a test of one cache. + #[must_use] + pub fn every(cache: Arc) -> Self { + Self { + by_feature: HashMap::new(), + unselected: Some(cache), + } + } + + /// Keeps `cache` as the one the table `name` selected. + pub fn insert(&mut self, name: &'static str, cache: Arc) { + self.by_feature.insert(name, cache); + } + + /// The cache named `name` loads through. + #[must_use] + pub fn for_feature(&self, name: &str) -> &dyn PlatformLoadingCache { + match self.by_feature.get(name).or(self.unselected.as_ref()) { + Some(cache) => &**cache, + None => &NoLoadingCache, + } + } + + /// Whether no table selected a cache. + #[must_use] + pub fn is_empty(&self) -> bool { + self.by_feature.is_empty() && self.unselected.is_none() + } +} + +impl fmt::Debug for LoadingCaches { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut selected: Vec<&&str> = self.by_feature.keys().collect(); + selected.sort(); + f.debug_struct("LoadingCaches") + .field("selected", &selected) + .field("every", &self.unselected.is_some()) + .finish() + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + + use super::*; + + #[test] + fn two_publishers_never_share_a_key() { + crate::prefix::test_support::assert_keys_follow_the_publisher(|domain| { + StoreKey::new(&ServerKeyPrefix::new("ts", domain), "page", "abc").to_string() + }); + } + + #[test] + fn a_key_names_the_publisher_then_the_cache() { + let key = StoreKey::new( + &ServerKeyPrefix::new("ts", "www.example.com"), + "robots-txt", + "k1", + ); + + assert_eq!(key.namespace(), "ts.example.com/robots-txt"); + assert_eq!(key.key(), "k1"); + assert_eq!(key.to_string(), "ts.example.com/robots-txt/k1"); + } + + #[test] + fn the_default_cache_runs_every_load() { + let services = super::super::test_support::noop_services(); + let spec = CacheSpec { + name: "test", + lifetimes: CacheLifetimes { + memory: Duration::from_secs(1), + store: Duration::from_secs(1), + store_idle: Duration::from_secs(1), + }, + memory_entries: 1, + detachable: false, + }; + let key = StoreKey::new(&ServerKeyPrefix::new("ts", "example.com"), "test", "k"); + let runs = Arc::new(AtomicUsize::new(0)); + for _ in 0..2 { + let counted = Arc::clone(&runs); + let value = futures::executor::block_on(NoLoadingCache.get_or_load( + &services, + &spec, + &key, + Box::new(move || { + Box::pin(async move { + counted.fetch_add(1, Ordering::SeqCst); + Ok(LoadedValue::new(Bytes::from_static(b"v"))) + }) + }), + )) + .expect("the load should succeed"); + assert_eq!(value, Bytes::from_static(b"v")); + } + + assert_eq!(runs.load(Ordering::SeqCst), 2); + } +} diff --git a/crates/trusted-server-core/src/platform/mod.rs b/crates/trusted-server-core/src/platform/mod.rs index 63f5c2a94e..36231dfc6c 100644 --- a/crates/trusted-server-core/src/platform/mod.rs +++ b/crates/trusted-server-core/src/platform/mod.rs @@ -14,6 +14,8 @@ //! - [`PlatformGeo`] — geographic information lookup //! - [`PlatformTemplateAssembler`] — cold-response shared-template assembly //! - [`PlatformTemplateCache`] — shared transformed-template caching +//! - [`PlatformValueCache`], a shared cache of small values with lifetimes +//! - [`PlatformLoadingCache`], a cache that loads a missing value once per key //! //! ## Platform-Agnostic Components //! @@ -36,15 +38,26 @@ use std::time::Duration; mod backend_naming; mod error; +mod geo; mod http; mod image_optimizer; mod kv; +mod loading_cache; +mod server_key; mod template_assembly; mod template_cache; -#[cfg(test)] -pub(crate) mod test_support; +#[cfg(any(test, feature = "test-utils"))] +#[allow( + clippy::must_use_candidate, + clippy::missing_panics_doc, + clippy::missing_errors_doc, + clippy::new_without_default, + reason = "test doubles, offered to other crates' tests under the test-utils feature" +)] +pub mod test_support; mod traits; mod types; +mod value_cache; pub use backend_naming::{ AuctionTargetCapabilities, AuctionTargetDescriptor, AuctionTargetId, BackendNamingError, @@ -52,6 +65,7 @@ pub use backend_naming::{ }; pub use edgezero_core::key_value_store::{KvError, KvHandle, KvStore as PlatformKvStore}; pub use error::PlatformError; +pub use geo::DisabledGeo; pub use http::{ PlatformCacheIntent, PlatformHttpClient, PlatformHttpRequest, PlatformPendingRequest, PlatformResponse, PlatformSelectResult, UnavailableHttpClient, @@ -61,6 +75,11 @@ pub use image_optimizer::{ PlatformImageOptimizerParams, PlatformImageOptimizerRegion, }; pub use kv::UnavailableKvStore; +pub use loading_cache::{ + CacheLifetimes, CacheSpec, LoadError, LoadFuture, LoadedValue, Loader, LoadingCaches, + NoLoadingCache, PlatformLoadingCache, StoreKey, +}; +pub use server_key::ServerKeyPrefix; pub use template_assembly::{ PlatformTemplateAssembler, TemplateAssemblyError, UnavailableTemplateAssembler, contains_publisher_esi_directive, @@ -77,10 +96,42 @@ pub use types::{ ClientInfo, GeoInfo, PlatformBackendSpec, RuntimeServices, RuntimeServicesBuilder, StoreId, StoreName, }; +pub use value_cache::{PlatformValueCache, ValueLookup, ValueReservation}; /// Default first-byte timeout for platform backends. pub(crate) const DEFAULT_FIRST_BYTE_TIMEOUT: Duration = Duration::from_secs(15); +use std::sync::Arc; + +use crate::settings::Settings; + +/// Selects the geo module named by the `[geo] module` selector. +/// +/// Returns [`DisabledGeo`] when no module is selected, so a default +/// deployment makes no host geo call and the permission baseline comes from +/// the top of the `permissions.yaml` rules tree. `module = "none"` spells the +/// same choice explicitly. The host platform's own geo lookup is opt-in: +/// `module = "platform"` returns `host_default`, which the adapter passes +/// as its platform geo implementation. +/// +/// Any other value names an integration module that declares a geo module. +/// This function returns [`DisabledGeo`] for one, because it cannot see the +/// registry, and the adapter then replaces it with the module's module from +/// `IntegrationRegistry::geo_module`. So the value returned here is the base +/// the adapter starts from, not necessarily what serves the request. A selector +/// naming a module that supplies no geo module is rejected when the registry +/// is built, which is the only layer that can tell. +#[must_use] +pub fn build_geo_module( + settings: &Settings, + host_default: Arc, +) -> Arc { + match settings.geo.module.as_deref() { + Some("platform") => host_default, + _ => Arc::new(DisabledGeo), + } +} + #[cfg(test)] mod tests { use std::net::{IpAddr, Ipv4Addr}; @@ -148,6 +199,14 @@ mod tests { { } + #[test] + fn disabled_geo_requires_no_permissions() { + assert!( + DisabledGeo.required_permissions().is_empty(), + "the default disabled geo module requires no permissions" + ); + } + #[test] fn runtime_services_can_be_constructed_and_cloned() { let services = noop_services(); @@ -163,16 +222,63 @@ mod tests { ); } - #[test] - fn runtime_services_geo_lookup_returns_none_for_no_ip() { + #[tokio::test] + async fn runtime_services_geo_lookup_returns_none_for_no_ip() { let services = noop_services(); let result = services .geo() - .lookup(services.client_info().client_ip) + .lookup(services.client_info().client_ip, &noop_services()) + .await .expect("should not fail for noop geo with no ip"); assert!(result.is_none(), "should return None when no IP is present"); } + #[tokio::test] + async fn build_geo_module_defaults_to_no_geo() { + let settings = Settings::default(); + let host: Arc = Arc::new(test_support::NoopGeo); + let selected = build_geo_module(&settings, Arc::clone(&host)); + assert!( + !Arc::ptr_eq(&host, &selected), + "default settings should not use the host geo" + ); + assert!( + selected + .lookup( + Some(IpAddr::V4(Ipv4Addr::new(203, 0, 113, 7))), + &noop_services() + ) + .await + .expect("disabled geo lookup should not fail") + .is_none(), + "the default geo module should resolve nothing" + ); + } + + #[test] + fn build_geo_module_none_selects_no_geo_explicitly() { + let mut settings = Settings::default(); + settings.geo.module = Some("none".to_owned()); + let host: Arc = Arc::new(test_support::NoopGeo); + let selected = build_geo_module(&settings, Arc::clone(&host)); + assert!( + !Arc::ptr_eq(&host, &selected), + "module none should not use the host geo" + ); + } + + #[test] + fn build_geo_module_uses_host_geo_when_platform_is_selected() { + let mut settings = Settings::default(); + settings.geo.module = Some("platform".to_owned()); + let host: Arc = Arc::new(test_support::NoopGeo); + let selected = build_geo_module(&settings, Arc::clone(&host)); + assert!( + Arc::ptr_eq(&host, &selected), + "the platform selector should use the host geo" + ); + } + #[test] fn runtime_services_with_kv_store_replaces_only_the_new_clone() { let services = noop_services_with_client_ip(IpAddr::V4(Ipv4Addr::new(198, 51, 100, 7))); diff --git a/crates/trusted-server-core/src/platform/server_key.rs b/crates/trusted-server-core/src/platform/server_key.rs new file mode 100644 index 0000000000..435dc36dc5 --- /dev/null +++ b/crates/trusted-server-core/src/platform/server_key.rs @@ -0,0 +1,88 @@ +//! The start of every key a publisher's entries take in a shared store. + +use core::fmt; + +/// The start of every key a publisher's entries take in a shared store, so +/// two publishers sharing a store never share a key. +/// +/// Written `.`, where the head names who keeps the store, such +/// as `ts`, and the domain is the publisher's. The selected prefix module +/// gives both, see [`crate::prefix`], taking the domain from the publisher the +/// request resolved to and never from the raw request, because a `Host` +/// header can say anything and a prefix built from one would let any request +/// start a key space of its own. +/// +/// A single leading `www.` is folded, so `www.example.com` and `example.com` +/// share keys, and every other label is kept, so `clients.example.com` and +/// `public.example.com` do not. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ServerKeyPrefix { + text: String, + domain_start: usize, +} + +impl ServerKeyPrefix { + /// The prefix `head` and the publisher's `domain` make. Every shared store + /// key's prefix is made here. + #[must_use] + pub fn new(head: &str, domain: &str) -> Self { + let head = head.trim(); + let domain = domain.trim().trim_end_matches('.').to_ascii_lowercase(); + let folded = domain.strip_prefix("www.").unwrap_or(&domain); + Self { + text: format!("{head}.{folded}"), + domain_start: head.len() + 1, + } + } + + /// The prefix as text. + #[must_use] + pub fn as_str(&self) -> &str { + &self.text + } + + /// The publisher's domain the prefix carries, lowercased and with a + /// leading `www.` folded. + #[must_use] + pub fn domain(&self) -> &str { + &self.text[self.domain_start..] + } +} + +impl fmt::Display for ServerKeyPrefix { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.text) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_leading_www_is_folded_and_every_other_label_kept() { + for (domain, prefix) in [ + ("example.com", "ts.example.com"), + ("www.example.com", "ts.example.com"), + ("WWW.Example.COM", "ts.example.com"), + ("example.com.", "ts.example.com"), + ("clients.example.com", "ts.clients.example.com"), + ("public.example.com", "ts.public.example.com"), + ("www.www.example.com", "ts.www.example.com"), + ] { + assert_eq!( + ServerKeyPrefix::new("ts", domain).as_str(), + prefix, + "{domain}" + ); + } + } + + #[test] + fn the_domain_is_the_folded_one() { + let prefix = ServerKeyPrefix::new("ts", "www.example.com"); + + assert_eq!(prefix.domain(), "example.com"); + assert_eq!(prefix.to_string(), "ts.example.com"); + } +} diff --git a/crates/trusted-server-core/src/platform/template_cache.rs b/crates/trusted-server-core/src/platform/template_cache.rs index 203bba1625..e50bfe5032 100644 --- a/crates/trusted-server-core/src/platform/template_cache.rs +++ b/crates/trusted-server-core/src/platform/template_cache.rs @@ -59,7 +59,7 @@ pub struct TemplateCacheKey { /// mutate first. pub url: String, /// Host and scheme. The post-processed output is host-dependent by construction: - /// both reach `IntegrationHtmlContext` and drive URL rewriting. + /// both are handed to every middleware and drive URL rewriting. pub request_host: String, /// See [`Self::request_host`]. pub request_scheme: String, @@ -819,17 +819,6 @@ pub trait PlatformTemplateCache: Send + Sync { /// Purge every cached variant for one publisher URL. async fn purge_url(&self, key: &TemplateCacheKey) -> Result<(), TemplateCacheError>; - - /// Purge one already-derived surrogate key. - /// - /// Exists because [`Self::purge_url`] needs a whole [`TemplateCacheKey`], which a - /// purge caller cannot build: an operator or a CMS webhook has a URL, not an origin - /// identity, a template fingerprint, or the origin's `Vary` values. Pair it with - /// [`reader_url_surrogate_key`]. - async fn purge_url_surrogate_key(&self, key: &str) -> Result<(), TemplateCacheError>; - - /// Purge every stored template. The rollback lever. - async fn purge_all(&self) -> Result<(), TemplateCacheError>; } /// A template read from the cache. @@ -874,17 +863,6 @@ impl PlatformTemplateCache for UnavailableTemplateCache { async fn purge_url(&self, _key: &TemplateCacheKey) -> Result<(), TemplateCacheError> { Err(TemplateCacheError::Unsupported) } - - /// Reports unsupported rather than succeeding. A purge surface that silently does - /// nothing is worse than one that refuses: an operator mid-incident would read the - /// success and stop looking. - async fn purge_url_surrogate_key(&self, _key: &str) -> Result<(), TemplateCacheError> { - Err(TemplateCacheError::Unsupported) - } - - async fn purge_all(&self) -> Result<(), TemplateCacheError> { - Err(TemplateCacheError::Unsupported) - } } #[cfg(test)] @@ -1197,19 +1175,6 @@ mod tests { } } - #[test] - fn an_adapter_without_a_template_cache_refuses_a_surrogate_purge() { - let outcome = futures::executor::block_on( - UnavailableTemplateCache.purge_url_surrogate_key("ts-template-readerurl-abc"), - ); - - assert!( - matches!(outcome, Err(TemplateCacheError::Unsupported)), - "a purge surface that silently does nothing is worse than one that refuses: \ - an operator mid-incident would read the success and stop looking" - ); - } - #[test] fn the_reader_facing_key_ignores_origin_rewriting() { // `url` is the origin-rewritten target URI, not what an operator types. Two diff --git a/crates/trusted-server-core/src/platform/test_support.rs b/crates/trusted-server-core/src/platform/test_support.rs index 17813c77dc..90348e16a0 100644 --- a/crates/trusted-server-core/src/platform/test_support.rs +++ b/crates/trusted-server-core/src/platform/test_support.rs @@ -11,13 +11,11 @@ use rand::rngs::OsRng; use super::{ ClientInfo, GeoInfo, PlatformBackend, PlatformBackendSpec, PlatformCacheIntent, PlatformConfigStore, PlatformError, PlatformGeo, PlatformHttpClient, PlatformHttpRequest, - PlatformImageOptimizerOptions, PlatformImageOptimizerParams, PlatformPendingRequest, - PlatformResponse, PlatformSecretStore, PlatformSelectResult, RuntimeServices, StoreId, - StoreName, + PlatformImageOptimizerOptions, PlatformPendingRequest, PlatformResponse, PlatformSecretStore, + PlatformSelectResult, RuntimeServices, StoreId, StoreName, }; -use crate::request_signing::{JWKS_STORE_NAME, SIGNING_STORE_NAME}; -pub(crate) struct NoopConfigStore; +pub struct NoopConfigStore; impl PlatformConfigStore for NoopConfigStore { fn get(&self, _store_name: &StoreName, _key: &str) -> Result> { @@ -38,7 +36,7 @@ impl PlatformConfigStore for NoopConfigStore { } } -pub(crate) struct NoopSecretStore; +pub struct NoopSecretStore; impl PlatformSecretStore for NoopSecretStore { fn get_bytes( @@ -63,12 +61,12 @@ impl PlatformSecretStore for NoopSecretStore { } } -pub(crate) struct HashMapConfigStore { +pub struct HashMapConfigStore { data: HashMap, } impl HashMapConfigStore { - pub(crate) fn new(data: HashMap) -> Self { + pub fn new(data: HashMap) -> Self { Self { data } } } @@ -95,12 +93,12 @@ impl PlatformConfigStore for HashMapConfigStore { } } -pub(crate) struct HashMapSecretStore { +pub struct HashMapSecretStore { data: HashMap>, } impl HashMapSecretStore { - pub(crate) fn new(data: HashMap>) -> Self { + pub fn new(data: HashMap>) -> Self { Self { data } } } @@ -131,7 +129,7 @@ impl PlatformSecretStore for HashMapSecretStore { } } -pub(crate) struct NoopBackend; +pub struct NoopBackend; impl PlatformBackend for NoopBackend { fn naming_policy(&self) -> super::BackendNamingPolicy { @@ -147,7 +145,7 @@ impl PlatformBackend for NoopBackend { } } -pub(crate) struct NoopHttpClient; +pub struct NoopHttpClient; // ?Send matches PlatformHttpClient. Body wraps LocalBoxStream which is !Send // by design; see http.rs for the full rationale. @@ -181,7 +179,7 @@ impl PlatformHttpClient for NoopHttpClient { /// Test stub for [`PlatformBackend`] that returns `"stub-backend"` for any /// spec, allowing callers to proceed past backend registration. -pub(crate) struct StubBackend; +pub struct StubBackend; impl PlatformBackend for StubBackend { fn naming_policy(&self) -> super::BackendNamingPolicy { @@ -197,6 +195,82 @@ impl PlatformBackend for StubBackend { } } +// --------------------------------------------------------------------------- +// NamingBackend +// --------------------------------------------------------------------------- + +/// Test stub for [`PlatformBackend`] that names a backend the way the given +/// policy does, so two demand sources get two names, and records what it was +/// asked. +pub struct NamingBackend { + pub policy: super::BackendNamingPolicy, + /// How many names were predicted. + pub predicted: std::sync::atomic::AtomicUsize, + /// How many backends were registered or refused. + pub ensured: std::sync::atomic::AtomicUsize, + /// The specification of every backend registered, in order. + pub specs: Mutex>, + /// The discriminators whose registration fails. + pub fail_ensure_for: Mutex>, +} + +impl NamingBackend { + pub fn new(policy: super::BackendNamingPolicy) -> Self { + Self { + policy, + predicted: std::sync::atomic::AtomicUsize::new(0), + ensured: std::sync::atomic::AtomicUsize::new(0), + specs: Mutex::new(Vec::new()), + fail_ensure_for: Mutex::new(std::collections::HashSet::new()), + } + } + + /// Makes registration fail for the source `provider_id`. + pub fn fail_ensure_for(&self, provider_id: &str) { + self.fail_ensure_for + .lock() + .expect("should lock failing provider IDs") + .insert(provider_id.to_string()); + } + + fn name(&self, spec: &PlatformBackendSpec) -> Result> { + self.policy + .predict(spec) + .map(|prediction| prediction.name) + .change_context(PlatformError::Backend) + } +} + +impl PlatformBackend for NamingBackend { + fn naming_policy(&self) -> super::BackendNamingPolicy { + self.policy + } + + fn predict_name(&self, spec: &PlatformBackendSpec) -> Result> { + self.predicted + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + self.name(spec) + } + + fn ensure(&self, spec: &PlatformBackendSpec) -> Result> { + self.ensured + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + if spec.discriminator.as_deref().is_some_and(|provider_id| { + self.fail_ensure_for + .lock() + .expect("should lock failing provider IDs") + .contains(provider_id) + }) { + return Err(Report::new(PlatformError::Backend)); + } + self.specs + .lock() + .expect("should lock planned backend specs") + .push(spec.clone()); + self.name(spec) + } +} + // --------------------------------------------------------------------------- // StubHttpClient // --------------------------------------------------------------------------- @@ -226,7 +300,7 @@ struct StubPendingResponse { const MAX_RECORDED_BODY_BYTES: usize = 64 * 1024 * 1024; type RecordedHeaderBytes = Vec)>>; -pub(crate) struct StubHttpClient { +pub struct StubHttpClient { calls: Mutex>, responses: Mutex>, // Headers captured per send call, stored as (name, value) string pairs. @@ -302,7 +376,7 @@ impl StubHttpClient { } /// Make `has_enforceable_total_request_deadline()` report the given value. - pub(crate) fn set_enforceable_total_request_deadline(&self, supported: bool) { + pub fn set_enforceable_total_request_deadline(&self, supported: bool) { self.enforceable_total_request_deadline .store(supported, std::sync::atomic::Ordering::Relaxed); } @@ -370,7 +444,7 @@ impl StubHttpClient { } /// Make the next `select()` complete successfully before a later queued error. - pub(crate) fn push_select_success(&self) { + pub fn push_select_success(&self) { self.select_errors .lock() .expect("should lock select_errors") @@ -379,7 +453,7 @@ impl StubHttpClient { /// Override backend metadata on the next pending handle returned by /// [`Self::send_async`]. `None` removes the metadata entirely. - pub(crate) fn push_pending_backend_name_override(&self, backend_name: Option<&str>) { + pub fn push_pending_backend_name_override(&self, backend_name: Option<&str>) { self.pending_backend_name_overrides .lock() .expect("should lock pending backend name overrides") @@ -391,7 +465,7 @@ impl StubHttpClient { /// This is test-only timing control for deadline behavior. It deliberately /// uses a caller-selected, generous contrast with the tested budget rather /// than relying on scheduler races. - pub(crate) fn push_select_delay(&self, delay: Duration) { + pub fn push_select_delay(&self, delay: Duration) { self.select_delays .lock() .expect("should lock select_delays") @@ -399,7 +473,7 @@ impl StubHttpClient { } /// Queue a wall-clock delay before the next direct pending-request `wait()`. - pub(crate) fn push_wait_delay(&self, delay: Duration) { + pub fn push_wait_delay(&self, delay: Duration) { self.wait_delays .lock() .expect("should lock wait_delays") @@ -432,7 +506,7 @@ impl StubHttpClient { /// Return raw request header values captured per request, in order. /// /// Unlike [`Self::recorded_request_headers`], this includes malformed bytes. - pub(crate) fn recorded_request_header_bytes(&self) -> Vec)>> { + pub fn recorded_request_header_bytes(&self) -> Vec)>> { self.request_header_bytes .lock() .expect("should lock request_header_bytes") @@ -448,7 +522,7 @@ impl StubHttpClient { } /// Return the cache intent captured per `send` or `send_async` call, in order. - pub(crate) fn recorded_cache_intents(&self) -> Vec { + pub fn recorded_cache_intents(&self) -> Vec { self.cache_intents .lock() .expect("should lock cache intents") @@ -857,10 +931,15 @@ fn build_stub_pending_response( builder.body(body).change_context(PlatformError::HttpClient) } -pub(crate) struct NoopGeo; +pub struct NoopGeo; +#[async_trait::async_trait(?Send)] impl PlatformGeo for NoopGeo { - fn lookup(&self, _client_ip: Option) -> Result, Report> { + async fn lookup( + &self, + _client_ip: Option, + _services: &crate::platform::RuntimeServices, + ) -> Result, Report> { Ok(None) } } @@ -869,7 +948,7 @@ impl PlatformGeo for NoopGeo { /// /// Use this when a test exercises code that reads from config AND secret stores, /// such as `request_signing::signing` and `request_signing::rotation`. -pub(crate) fn build_services_with_config_and_secret( +pub fn build_services_with_config_and_secret( config_store: impl PlatformConfigStore + 'static, secret_store: impl PlatformSecretStore + 'static, ) -> RuntimeServices { @@ -884,7 +963,7 @@ pub(crate) fn build_services_with_config_and_secret( .build() } -pub(crate) fn build_services_with_config_and_secret_and_client_ip( +pub fn build_services_with_config_and_secret_and_client_ip( config_store: impl PlatformConfigStore + 'static, secret_store: impl PlatformSecretStore + 'static, client_ip: IpAddr, @@ -903,7 +982,7 @@ pub(crate) fn build_services_with_config_and_secret_and_client_ip( .build() } -pub(crate) fn build_request_signing_services() -> RuntimeServices { +pub fn build_request_signing_services() -> RuntimeServices { let signing_key = SigningKey::generate(&mut OsRng); let key_b64 = general_purpose::STANDARD.encode(signing_key.as_bytes()); let x_b64 = general_purpose::URL_SAFE_NO_PAD.encode(signing_key.verifying_key().as_bytes()); @@ -923,7 +1002,7 @@ pub(crate) fn build_request_signing_services() -> RuntimeServices { ) } -pub(crate) fn build_services_with_config( +pub fn build_services_with_config( config_store: impl PlatformConfigStore + 'static, ) -> RuntimeServices { RuntimeServices::builder() @@ -937,14 +1016,132 @@ pub(crate) fn build_services_with_config( .build() } -pub(crate) fn noop_services() -> RuntimeServices { +pub fn noop_services() -> RuntimeServices { build_services_with_config(NoopConfigStore) } +/// Build a [`RuntimeServices`] with an injected geo module, so a test can +/// drive a geo outcome through the [`PlatformGeo`] seam rather than +/// constructing the resolved status by hand. +/// +/// This is the only way to reach the lookup-failure path, because the seam is +/// what turns an `Err` into the requires-signal floor. +pub fn build_services_with_geo(geo: Arc) -> RuntimeServices { + RuntimeServices::builder() + .config_store(Arc::new(NoopConfigStore)) + .secret_store(Arc::new(NoopSecretStore)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::new(NoopBackend)) + .http_client(Arc::new(NoopHttpClient)) + .geo(geo) + .client_info(ClientInfo::default()) + .build() +} + +/// Build a [`RuntimeServices`] carrying an Edge Cookie module, threaded the +/// way a composition root threads the module it resolved, so a test can +/// exercise the seam a vendor module reaches core through and check that the +/// request path reuses that instance. +pub fn noop_services_with_ec_module( + ec_module: Arc, +) -> RuntimeServices { + // A fixed client IP, so a module that reads one (the built-in HMAC + // module does) can run. + noop_services_with_ec_module_and_ip( + ec_module, + Some("203.0.113.10".parse().expect("should parse test client IP")), + ) +} + +/// Build a [`RuntimeServices`] with an injected Edge Cookie module and no +/// client IP, modeling a host that cannot determine one. +/// +/// Whether that matters is the module's decision, so this exists to test +/// both answers: a module reading other evidence still creates an +/// identifier, and one that needs the IP refuses. +/// A config store that answers one known key, so a test can prove a module +/// reached the config store it was handed rather than a value it already held. +#[derive(Debug)] +pub struct FixedConfigStore { + pub store: &'static str, + pub key: &'static str, + pub value: &'static str, +} + +impl PlatformConfigStore for FixedConfigStore { + fn get(&self, store_name: &StoreName, key: &str) -> Result> { + if store_name.as_ref() == self.store && key == self.key { + Ok(self.value.to_owned()) + } else { + Err(Report::new(PlatformError::ConfigStore)) + } + } + + fn put( + &self, + _store_id: &StoreId, + _key: &str, + _value: &str, + ) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } + + fn delete(&self, _store_id: &StoreId, _key: &str) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } +} + +/// Build a [`RuntimeServices`] carrying both an Edge Cookie module and a +/// config store the module is expected to read through, so a test can prove +/// the services reaching the module are the ones the caller supplied. +pub fn services_with_ec_module_and_config_store( + ec_module: Arc, + config_store: Arc, +) -> RuntimeServices { + RuntimeServices::builder() + .config_store(config_store) + .secret_store(Arc::new(NoopSecretStore)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::new(NoopBackend)) + .http_client(Arc::new(NoopHttpClient)) + .geo(Arc::new(NoopGeo)) + .client_info(ClientInfo { + client_ip: Some("203.0.113.10".parse().expect("should parse test client IP")), + ..ClientInfo::default() + }) + .resolved_ec_module(ec_module) + .build() +} + +pub fn noop_services_with_ec_module_without_client_ip( + ec_module: Arc, +) -> RuntimeServices { + noop_services_with_ec_module_and_ip(ec_module, None) +} + +fn noop_services_with_ec_module_and_ip( + ec_module: Arc, + client_ip: Option, +) -> RuntimeServices { + RuntimeServices::builder() + .config_store(Arc::new(NoopConfigStore)) + .secret_store(Arc::new(NoopSecretStore)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::new(NoopBackend)) + .http_client(Arc::new(NoopHttpClient)) + .geo(Arc::new(NoopGeo)) + .client_info(ClientInfo { + client_ip, + ..ClientInfo::default() + }) + .resolved_ec_module(ec_module) + .build() +} + /// Build a [`RuntimeServices`] whose auction telemetry sink is the supplied /// recording (or otherwise custom) sink, so tests can assert which terminal /// auction events were emitted. -pub(crate) fn noop_services_with_telemetry_sink( +pub fn noop_services_with_telemetry_sink( auction_telemetry_sink: Arc, ) -> RuntimeServices { RuntimeServices::builder() @@ -966,14 +1163,14 @@ pub(crate) fn noop_services_with_telemetry_sink( /// both make HTTP calls and resolve backends don't need two separate service /// setups. If your test must verify that a missing backend returns an error, /// use [`noop_services`] directly. -pub(crate) fn build_services_with_http_client( +pub fn build_services_with_http_client( http_client: Arc, ) -> RuntimeServices { build_services_with_secret_and_http_client(NoopSecretStore, http_client) } /// Build test services that dispatch HTTP requests with an attested client IP. -pub(crate) fn build_services_with_http_client_and_client_ip( +pub fn build_services_with_http_client_and_client_ip( http_client: Arc, client_ip: IpAddr, ) -> RuntimeServices { @@ -991,7 +1188,7 @@ pub(crate) fn build_services_with_http_client_and_client_ip( .build() } -pub(crate) fn noop_services_with_client_ip(ip: IpAddr) -> RuntimeServices { +pub fn noop_services_with_client_ip(ip: IpAddr) -> RuntimeServices { RuntimeServices::builder() .config_store(Arc::new(NoopConfigStore)) .secret_store(Arc::new(NoopSecretStore)) @@ -1017,7 +1214,7 @@ pub(crate) fn noop_services_with_client_ip(ip: IpAddr) -> RuntimeServices { dead_code, reason = "retained for target-specific transport-timeout tests" )] -pub(crate) fn build_services_with_backend_and_http_client( +pub fn build_services_with_backend_and_http_client( backend: Arc, http_client: Arc, ) -> RuntimeServices { @@ -1038,7 +1235,7 @@ pub(crate) fn build_services_with_backend_and_http_client( } /// Build a [`RuntimeServices`] with a custom secret store, [`StubBackend`], and HTTP client. -pub(crate) fn build_services_with_config_secret_and_http_client( +pub fn build_services_with_config_secret_and_http_client( config_store: impl PlatformConfigStore + 'static, secret_store: impl PlatformSecretStore + 'static, http_client: Arc, @@ -1054,7 +1251,7 @@ pub(crate) fn build_services_with_config_secret_and_http_client( .build() } -pub(crate) fn build_services_with_secret_http_client_and_client_ip( +pub fn build_services_with_secret_http_client_and_client_ip( secret_store: impl PlatformSecretStore + 'static, http_client: Arc, client_ip: Option, @@ -1076,16 +1273,100 @@ pub(crate) fn build_services_with_secret_http_client_and_client_ip( } /// Build test services with a custom secret store and the standard test config store. -pub(crate) fn build_services_with_secret_and_http_client( +pub fn build_services_with_secret_and_http_client( secret_store: impl PlatformSecretStore + 'static, http_client: Arc, ) -> RuntimeServices { build_services_with_config_secret_and_http_client(NoopConfigStore, secret_store, http_client) } +/// A loading cache for tests that cannot link the module supplying the real +/// one. +/// +/// It keeps each value until the lifetime its load gave runs out, records the +/// key and cache of every call, and runs a load for every key it does not +/// hold. Loads are not joined, because these tests ask one at a time, and +/// the module's own tests prove the joining. +#[derive(Default)] +pub struct MemoryLoadingCache { + entries: Mutex)>>, + calls: Mutex>, +} + +impl MemoryLoadingCache { + /// The cache name and key of every call, in order. + pub fn calls(&self) -> Vec<(&'static str, String)> { + self.calls.lock().expect("should lock the calls").clone() + } + + /// Every key held, sorted. + pub fn keys(&self) -> Vec { + let mut keys: Vec = self + .entries + .lock() + .expect("should lock the entries") + .keys() + .cloned() + .collect(); + keys.sort(); + keys + } +} + +#[async_trait::async_trait(?Send)] +impl super::PlatformLoadingCache for MemoryLoadingCache { + async fn get_or_load( + &self, + _services: &RuntimeServices, + spec: &super::CacheSpec, + key: &super::StoreKey, + load: super::Loader, + ) -> Result { + let key = key.to_string(); + self.calls + .lock() + .expect("should lock the calls") + .push((spec.name, key.clone())); + let held = self + .entries + .lock() + .expect("should lock the entries") + .get(&key) + .filter(|(_, end)| end.is_none_or(|end| web_time::Instant::now() < end)) + .map(|(bytes, _)| bytes.clone()); + if let Some(bytes) = held { + return Ok(bytes); + } + let loaded = load().await?; + let end = loaded + .lifetime + .map(|lifetime| web_time::Instant::now() + lifetime); + if loaded.lifetime.is_none_or(|lifetime| !lifetime.is_zero()) { + self.entries + .lock() + .expect("should lock the entries") + .insert(key, (loaded.bytes.clone(), end)); + } + Ok(loaded.bytes) + } + + async fn remove( + &self, + _services: &RuntimeServices, + _spec: &super::CacheSpec, + key: &super::StoreKey, + ) { + self.entries + .lock() + .expect("should lock the entries") + .remove(&key.to_string()); + } +} + #[cfg(test)] mod tests { - use crate::platform::DEFAULT_FIRST_BYTE_TIMEOUT; + use crate::platform::{DEFAULT_FIRST_BYTE_TIMEOUT, PlatformImageOptimizerParams}; + use crate::request_signing::{JWKS_STORE_NAME, SIGNING_STORE_NAME}; use edgezero_core::body::Body; use edgezero_core::http::request_builder; diff --git a/crates/trusted-server-core/src/platform/traits.rs b/crates/trusted-server-core/src/platform/traits.rs index 6c31cd2796..47d7d9cf57 100644 --- a/crates/trusted-server-core/src/platform/traits.rs +++ b/crates/trusted-server-core/src/platform/traits.rs @@ -2,7 +2,10 @@ use std::net::IpAddr; use error_stack::Report; -use super::{BackendNamingPolicy, GeoInfo, PlatformBackendSpec, PlatformError, StoreId, StoreName}; +use super::{ + BackendNamingPolicy, GeoInfo, PlatformBackendSpec, PlatformError, RuntimeServices, StoreId, + StoreName, +}; /// Synchronous, object-safe access to a key-value config store. /// @@ -136,13 +139,44 @@ pub trait PlatformBackend: Send + Sync { } } -/// Synchronous, object-safe geo lookup. +/// Object-safe geo lookup. +/// +/// Asynchronous because a geo module may reach a backend, a key-value store +/// or a secret to resolve a location, and a module that cannot make those +/// calls cannot be written at all. A module that resolves from data already +/// in hand still declares an async method and returns immediately. +/// +/// Uses `#[async_trait(?Send)]` for the same reason as +/// [`PlatformHttpClient`](super::PlatformHttpClient): the trait object stays +/// `Send + Sync` so it can be shared and run multi-threaded, while the future +/// it returns is pinned to one thread because the host SDKs produce `!Send` +/// futures on wasm32. +#[async_trait::async_trait(?Send)] pub trait PlatformGeo: Send + Sync { /// Look up geographic information for the given client IP address. /// + /// An implementation must return [`GeoInfo`] with the country as an + /// ISO 3166-1 alpha-2 code (for example `US`) and the region as the + /// ISO 3166-2 subdivision code without the country prefix (for example + /// `CA`). The permission model keys its country and region rules on these + /// codes, matched case-insensitively, so the Fastly and other geo + /// modules feed the same rules without translation. + /// /// # Errors /// /// Returns [`PlatformError::Geo`] when the platform geo lookup fails /// unexpectedly. Returns `Ok(None)` when no data is available for the IP. - fn lookup(&self, client_ip: Option) -> Result, Report>; + async fn lookup( + &self, + client_ip: Option, + services: &RuntimeServices, + ) -> Result, Report>; + + /// The permissions this module's data use requires. + /// + /// The default is empty, so the default (disabled) geo module requires no + /// permission. + fn required_permissions(&self) -> crate::permissions::PermissionSet { + crate::permissions::PermissionSet::none() + } } diff --git a/crates/trusted-server-core/src/platform/types.rs b/crates/trusted-server-core/src/platform/types.rs index 3570bdc102..8b2102fa0f 100644 --- a/crates/trusted-server-core/src/platform/types.rs +++ b/crates/trusted-server-core/src/platform/types.rs @@ -9,6 +9,10 @@ use super::{ PlatformBackend, PlatformConfigStore, PlatformGeo, PlatformHttpClient, PlatformKvStore, PlatformSecretStore, }; +use crate::ec::device::DeviceModule; +use crate::ec::module::EdgeCookieModule; +use crate::evidence::HostSignals; +use crate::permission_signal::PermissionSignalModule; /// Geographic information extracted from a request. /// @@ -18,7 +22,7 @@ use super::{ pub struct GeoInfo { /// City name. pub city: String, - /// Two-letter country code. + /// ISO 3166-1 alpha-2 country code, for example `US` or `GB`. pub country: String, /// Continent name. pub continent: String, @@ -28,7 +32,8 @@ pub struct GeoInfo { pub longitude: f64, /// DMA (Designated Market Area) / metro code. pub metro_code: i64, - /// Region code. + /// ISO 3166-2 subdivision code without the country prefix, for example `CA` + /// for California, or `None` when no region resolves. pub region: Option, /// Autonomous System Number (e.g. `7922` = Comcast). /// Used to distinguish home ISP vs. corporate VPN. @@ -168,6 +173,16 @@ pub struct RuntimeServices { /// per-request basis by cloning [`RuntimeServices`] with /// [`RuntimeServices::with_kv_store`]. pub(crate) kv_store: Arc, + /// The host's shared cache of small values, or `None` where the host has + /// none, in which case a caller holds what it pays for in the key-value + /// store instead. + pub(crate) value_cache: Option>, + /// The loading cache each `[cache.]` table selected, which loads a + /// missing value once per key and keeps its copies in + /// [`value_cache`](Self::value_cache) or the key-value store. A cache whose + /// table selected none loads through [`super::NoLoadingCache`], which holds + /// nothing. + pub(crate) loading_caches: Arc, /// Shared transformed-template cache. Defaults to /// [`UnavailableTemplateCache`], so adapters without one degrade to transforming /// per request rather than failing. See @@ -188,6 +203,29 @@ pub struct RuntimeServices { pub(crate) auction_telemetry_sink: Arc, /// Per-request client metadata extracted at the entry point. pub(crate) client_info: ClientInfo, + /// Host-computed client signals (TLS JA4, HTTP/2), when the host + /// supplies them. `None` on a host that exposes none, so a module that + /// requires them cannot be built and the request stops. + pub(crate) host_signals: Option>, + /// The Edge Cookie module this deployment already resolved from + /// `[ec] module` while it built application state. + /// + /// `None` when the adapter resolved nothing here, in which case the request + /// path resolves the selection itself, which is what a deployment that + /// selects no module, the Axum adapter, and the core tests all do. + pub(crate) resolved_ec_module: Option>, + pub(crate) device_module: Option>, + /// The permission signal modules this deployment runs, in the order + /// they are asked, selected at the composition root from the scheme + /// crates the adapter links. Empty when the adapter offers none, in which + /// case every permission stays at its country and region baseline. Shared, + /// so building the services for a request bumps a reference count rather + /// than copying the list, and cloning the services does the same. + pub(crate) permission_signal_modules: Arc<[Arc]>, + /// The prefixes of the publisher this request was resolved to, from the + /// selected prefix module, which start every name the browser holds and + /// every key in a shared store. See [`crate::prefix`]. + pub(crate) prefixes: crate::prefix::Prefixes, } impl RuntimeServices { @@ -233,6 +271,37 @@ impl RuntimeServices { &*self.kv_store } + /// The host's shared cache of small values, or `None` where it has none. + #[must_use] + pub fn value_cache(&self) -> Option<&dyn super::PlatformValueCache> { + self.value_cache.as_deref() + } + + /// The loading cache the table `[cache.]` selected, by the name a + /// [`super::CacheSpec`] carries. + #[must_use] + pub fn loading_cache_for(&self, name: &str) -> &dyn super::PlatformLoadingCache { + self.loading_caches.for_feature(name) + } + + /// The value for `key` in the cache `spec` describes, from a fresh copy + /// or from `load`, kept in this request's stores. See + /// [`super::PlatformLoadingCache::get_or_load`]. + /// + /// # Errors + /// + /// The error of the load this caller waited on. + pub async fn load_cached( + &self, + spec: &super::CacheSpec, + key: &super::StoreKey, + load: super::Loader, + ) -> Result { + self.loading_cache_for(spec.name) + .get_or_load(self, spec, key, load) + .await + } + /// The shared transformed-template cache. #[must_use] pub fn template_cache(&self) -> &dyn super::PlatformTemplateCache { @@ -275,6 +344,66 @@ impl RuntimeServices { &self.client_info } + /// Returns the host-computed client signals, when the host supplies + /// them. + /// + /// A module that derives identity from the TLS JA4 or HTTP/2 signals + /// takes these as an injected service. The result is `None` on a host that + /// exposes none, so such a module cannot be built there and the request + /// stops rather than creating a degraded identifier. + #[must_use] + pub fn host_signals(&self) -> Option> { + self.host_signals.clone() + } + + /// Returns the Edge Cookie module the composition root already resolved, + /// when the adapter threaded one through. + /// + /// Resolving `[ec] module` reads no request data, so the answer is the + /// same for every request and an adapter that resolves it once while it + /// builds application state can hand the result here instead of the + /// request path resolving the same settings again. `None` means nothing was + /// threaded, so the request path resolves for itself. Read this through + /// [`request_module`](crate::ec::module::request_module) rather than + /// directly, so both answers are handled in one place. + #[must_use] + pub fn resolved_ec_module(&self) -> Option> { + self.resolved_ec_module.clone() + } + + /// The permission signal modules this deployment runs, in order. + #[must_use] + pub fn permission_signal_modules(&self) -> &[Arc] { + &self.permission_signal_modules + } + + /// The device module a module supplied, when `[device] module` selected + /// one. `None` leaves core's own device classification in place. + #[must_use] + pub fn device_module(&self) -> Option> { + self.device_module.clone() + } + + /// The prefixes of the publisher this request was resolved to, being the + /// start of every name the browser holds and of every key in a shared + /// store. See [`crate::prefix`]. + #[must_use] + pub fn prefixes(&self) -> &crate::prefix::Prefixes { + &self.prefixes + } + + /// The start of every key the publisher's entries take in a shared store. + #[must_use] + pub fn server_key_prefix(&self) -> &super::ServerKeyPrefix { + self.prefixes.server_key() + } + + /// The start of every name the browser holds for the publisher. + #[must_use] + pub fn public_prefix(&self) -> &crate::prefix::PublicPrefix { + self.prefixes.public() + } + /// Wrap the KV store in a [`super::KvHandle`] for ergonomic access to /// JSON helpers, pagination, and validation. #[must_use] @@ -295,6 +424,108 @@ impl RuntimeServices { } } + /// Returns a clone of this instance with the resolved Edge Cookie module + /// replaced. + /// + /// Adapters that build their per-request services through a shared helper + /// with no application state in hand use this to thread the module the + /// composition root resolved. `None` leaves the request path to resolve + /// `[ec] module` for itself, which is what the Axum adapter and a + /// deployment selecting no module both do. + #[must_use] + pub fn with_resolved_ec_module( + self, + resolved_ec_module: Option>, + ) -> Self { + Self { + resolved_ec_module, + ..self + } + } + + /// Returns a clone of this instance with the geo module replaced by + /// `geo`. + /// + /// Adapters use this to apply the module-supplied geo module selected by + /// `[geo] module`, so a vendor module can resolve location in place of + /// the host's own lookup without the rest of the runtime services graph + /// being rebuilt. + #[must_use] + pub fn with_geo(self, geo: Arc) -> Self { + Self { geo, ..self } + } + + /// Returns a clone of this instance with the per-request client metadata + /// replaced. + /// + /// A response-side path that builds the rest of the services graph once and + /// varies only the client metadata per request uses this to apply the real + /// client data without rebuilding the graph. A module called from such a + /// path still needs that data. + #[must_use] + pub fn with_client_info(self, client_info: ClientInfo) -> Self { + Self { + client_info, + ..self + } + } + + /// Returns a clone of this instance with the Edge Cookie module replaced. + /// + /// Adapters use this to apply the module-supplied module selected by + /// `[ec] module`, the same way they apply a module's geo module, so a + /// vendor declares identity on its registration rather than through a + /// second extension mechanism. The registry already matched the selector + /// against the registered modules, so the module's module is the + /// resolved answer and lands on the one seam the request path reads. + #[must_use] + pub fn with_ec_module(self, ec_module: Arc) -> Self { + Self { + resolved_ec_module: Some(ec_module), + ..self + } + } + + /// Returns a clone of this instance with the device module replaced. + /// + /// Adapters use this to apply the module-supplied module selected by + /// `[device] module`. + #[must_use] + pub fn with_device_module(self, device_module: Arc) -> Self { + Self { + device_module: Some(device_module), + ..self + } + } + + /// Returns a clone of this instance carrying the prefixes of the publisher + /// the request was resolved to. + /// + /// Adapters apply what the selected prefix module gives, through + /// [`IntegrationRegistry::prefixes`](crate::integrations::IntegrationRegistry::prefixes), + /// alongside the other modules a request's services carry. + #[must_use] + pub fn with_prefixes(self, prefixes: crate::prefix::Prefixes) -> Self { + Self { prefixes, ..self } + } + + /// Returns a clone of this instance with the loading caches replaced. + #[must_use] + pub fn with_loading_caches(self, caches: Arc) -> Self { + Self { + loading_caches: caches, + ..self + } + } + + /// Returns a clone of this instance with every caller loading through + /// `cache`, for a test of one cache. + #[cfg(any(test, feature = "test-utils"))] + #[must_use] + pub fn with_loading_cache(self, cache: Arc) -> Self { + self.with_loading_caches(Arc::new(super::LoadingCaches::every(cache))) + } + /// Returns a clone of this instance with the template cache replaced. #[must_use] pub fn with_template_cache(self, cache: Arc) -> Self { @@ -333,6 +564,8 @@ pub struct RuntimeServicesBuilder { config_store: Option>, secret_store: Option>, kv_store: Option>, + value_cache: Option>, + loading_caches: Option>, template_cache: Option>, template_assembler: Option>, backend: Option>, @@ -340,6 +573,11 @@ pub struct RuntimeServicesBuilder { geo: Option>, auction_telemetry_sink: Option>, client_info: Option, + host_signals: Option>, + resolved_ec_module: Option>, + device_module: Option>, + prefixes: Option, + permission_signal_modules: Arc<[Arc]>, } impl RuntimeServicesBuilder { @@ -348,6 +586,8 @@ impl RuntimeServicesBuilder { config_store: None, secret_store: None, kv_store: None, + value_cache: None, + loading_caches: None, template_cache: None, template_assembler: None, backend: None, @@ -355,6 +595,11 @@ impl RuntimeServicesBuilder { geo: None, auction_telemetry_sink: None, client_info: None, + host_signals: None, + resolved_ec_module: None, + device_module: None, + prefixes: None, + permission_signal_modules: Arc::default(), } } @@ -372,6 +617,20 @@ impl RuntimeServicesBuilder { self } + /// Set the host's shared cache of small values. + #[must_use] + pub fn value_cache(mut self, cache: Arc) -> Self { + self.value_cache = Some(cache); + self + } + + /// Set the loading cache each `[cache.]` table selected. + #[must_use] + pub fn loading_caches(mut self, caches: Arc) -> Self { + self.loading_caches = Some(caches); + self + } + /// Set the shared transformed-template cache. #[must_use] pub fn template_cache(mut self, cache: Arc) -> Self { @@ -434,6 +693,61 @@ impl RuntimeServicesBuilder { self } + /// Set the host-computed client signals service. + /// + /// Optional: a host that exposes no TLS or HTTP/2 signals leaves this + /// unset, so a module that requires them cannot be built and the request + /// stops. + #[must_use] + pub fn host_signals(mut self, host_signals: Arc) -> Self { + self.host_signals = Some(host_signals); + self + } + + /// Set the Edge Cookie module the composition root already resolved. + /// + /// Optional. This is the module the selector actually chose, so setting + /// it keeps the request path from resolving the same settings a second + /// time. It is the single seam through which a vendor or host Edge Cookie + /// module reaches the request path. + #[must_use] + pub fn resolved_ec_module(mut self, module: Arc) -> Self { + self.resolved_ec_module = Some(module); + self + } + + /// Set the permission signal modules this deployment runs, in the order + /// they are asked. + /// + /// Optional, and empty when unset. An adapter hands in the shared list + /// [`build_permission_signal_modules`] selected from the scheme crates it + /// links, so the request path asks exactly the modules configuration + /// named, in that order, and core supplies none of its own. + /// + /// [`build_permission_signal_modules`]: + /// crate::permission_signal::build_permission_signal_modules + #[must_use] + pub fn permission_signal_modules( + mut self, + modules: Arc<[Arc]>, + ) -> Self { + self.permission_signal_modules = modules; + self + } + + /// Set the prefixes of the publisher the request was resolved to. + /// + /// Optional in the builder, because services built for no request, such + /// as a test's, have no publisher to resolve, and + /// [`Prefixes::default`](crate::prefix::Prefixes::default), carrying no + /// publisher domain, stands when it is not set. Every adapter sets it on + /// the services of a request. + #[must_use] + pub fn prefixes(mut self, prefixes: crate::prefix::Prefixes) -> Self { + self.prefixes = Some(prefixes); + self + } + /// Construct [`RuntimeServices`] from the accumulated configuration. /// /// # Panics @@ -451,6 +765,10 @@ impl RuntimeServicesBuilder { kv_store: self .kv_store .expect("should set kv_store before building RuntimeServices"), + value_cache: self.value_cache, + // Defaulted rather than required: a deployment that links no + // module supplying one asks each source itself. + loading_caches: self.loading_caches.unwrap_or_default(), // Defaulted rather than required: an adapter with no template cache // should degrade to transforming per request, not fail to build. template_cache: self @@ -474,6 +792,11 @@ impl RuntimeServicesBuilder { client_info: self .client_info .expect("should set client_info before building RuntimeServices"), + host_signals: self.host_signals, + resolved_ec_module: self.resolved_ec_module, + device_module: self.device_module, + prefixes: self.prefixes.unwrap_or_default(), + permission_signal_modules: self.permission_signal_modules, } } } diff --git a/crates/trusted-server-core/src/platform/value_cache.rs b/crates/trusted-server-core/src/platform/value_cache.rs new file mode 100644 index 0000000000..0e617b77fb --- /dev/null +++ b/crates/trusted-server-core/src/platform/value_cache.rs @@ -0,0 +1,80 @@ +//! A shared cache of small values with lifetimes, which a host supplies when it +//! has one. +//! +//! A module that pays for a remote answer holds it here, so the next +//! equivalent request, or a second lookup in the same request, is answered +//! without paying again. An entry can go at any time, which costs the caller one +//! fetch and never an error, so nothing durable belongs here. + +use std::time::Duration; + +use bytes::Bytes; + +/// Values keyed by text, shared by every request the host serves from the same +/// place. The caller owns its key space, so a key starts with a prefix naming +/// the module that wrote it. +/// +/// `Send + Sync` on the trait and `?Send` on the futures, as for the other +/// platform services, because [`super::RuntimeServices`] crosses threads while +/// the platform layer's futures never do. +#[async_trait::async_trait(?Send)] +pub trait PlatformValueCache: Send + Sync { + /// The value held under `key`, or `None` when there is none or it cannot be + /// read. + async fn get(&self, key: &str) -> Option; + + /// Holds `value` under `key` for `lifetime`. A failure is the host's to + /// log and is not returned, because the caller's only cost is fetching + /// again. + async fn put(&self, key: &str, value: Bytes, lifetime: Duration); + + /// The value held under `key`, or, when there is none and the host can + /// make other callers for the key wait, a [`ValueReservation`] obliging + /// this caller to fill it. + /// + /// The default never reserves, for a host whose cache cannot make callers + /// wait. + async fn lookup_or_reserve(&self, key: &str) -> ValueLookup { + match self.get(key).await { + Some(value) => ValueLookup::Hit(value), + None => ValueLookup::Miss, + } + } + + /// Removes the value held under `key`. The default does nothing, for a + /// host whose cache cannot remove an entry before its lifetime ends. + async fn remove(&self, _key: &str) {} +} + +/// What [`PlatformValueCache::lookup_or_reserve`] found. +pub enum ValueLookup { + /// The value held for the key. + Hit(Bytes), + /// Nothing held, and this caller is the one to fill the key. The host + /// makes other callers for the key wait until the reservation is filled + /// or dropped. + Reserved(Box), + /// Nothing held, and no reservation. + Miss, +} + +impl std::fmt::Debug for ValueLookup { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Hit(value) => f.debug_tuple("Hit").field(&value.len()).finish(), + Self::Reserved(_) => f.write_str("Reserved"), + Self::Miss => f.write_str("Miss"), + } + } +} + +/// The obligation to fill a key that other callers are waiting for. +/// +/// Dropping it unfilled releases the callers waiting on it, and one of them +/// is given the obligation, so a failed or canceled load never leaves them +/// waiting. +pub trait ValueReservation { + /// Holds `value` for `lifetime` and releases the callers waiting on it. A + /// failure is the host's to log, as for [`PlatformValueCache::put`]. + fn fill(self: Box, value: Bytes, lifetime: Duration); +} diff --git a/crates/trusted-server-core/src/prefix.rs b/crates/trusted-server-core/src/prefix.rs new file mode 100644 index 0000000000..357b442c85 --- /dev/null +++ b/crates/trusted-server-core/src/prefix.rs @@ -0,0 +1,456 @@ +//! The start of every name the browser holds for a publisher, and of every key +//! the publisher's entries take in a shared store. +//! +//! A deployment selects the module that gives both with `[prefix] module`. Core +//! holds the extension point, [`PrefixModule`], and its own module, +//! [`TrustedServerPrefix`], which gives `ts` and `ts.` and runs when +//! nothing is selected. +//! +//! The two prefixes answer different questions. A public prefix starts the +//! names the browser holds, being cookies and first-party storage keys, so it +//! carries no domain, because a browser already keeps each site's apart from +//! every other site's. A server key prefix starts every key in a store +//! publishers share, so it carries the publisher's domain, see +//! [`ServerKeyPrefix`]. +//! +//! The template cache is the exception. Its surrogate keys, `ts-template` and +//! `ts-template-url-`, and its store key, +//! `ts-template-cache-v-`, keep `ts` under every module, so +//! `ts cache purge` names one key whatever the deployment. The store key's +//! digest covers the request's host and the public prefix, so no two +//! publishers share an entry. +//! +//! A module gives the parts, and each use writes its own form from them, +//! because what is allowed differs by use: +//! +//! | Use | Form | `ts` | `_ex`, `www.example.com` | +//! | --- | --- | --- | --- | +//! | cookie | [`PublicPrefix::cookie_name`] | `ts-ec` | `_ex.www.example.com-ec` | +//! | store key | [`ServerKeyPrefix::as_str`] | `ts.example.com` | as its module gives it | +//! +//! The page's client script is told the public prefix on its bundle tag rather +//! than holding a copy, see [`PUBLIC_PREFIX_ATTRIBUTE`]. Its own globals, such +//! as `window.tsjs`, are what page code calls it by and keep their names under +//! every module. + +use core::fmt; +use std::sync::Arc; + +use crate::module_context::{ModuleCall, ModuleContext, ModuleRequest, Withheld}; +use crate::permissions::PermissionSet; +use crate::platform::ServerKeyPrefix; +use crate::settings::Settings; + +/// The type folder, and the section, that selects the prefix module. +pub const MODULE_TYPE: &str = "prefix"; + +/// The name core's own module is called by when it runs. +pub const TRUSTED_SERVER_MODULE: &str = "ts"; + +/// The head of every name core's own module gives. +const TRUSTED_SERVER_HEAD: &str = "ts"; + +/// The bundle tag attribute that tells the client script the public prefix, +/// so the script writes the deployment's names and holds no copy of its own. +pub const PUBLIC_PREFIX_ATTRIBUTE: &str = "data-ts-public-prefix"; + +/// The start of every name the browser holds for a publisher, such as `_ex`. +/// +/// Made of parts, such as `_ex` and `www.example.com`, joined by `.`. Every +/// part holds only ASCII letters, digits, `_`, `-` and `.`, so the prefix is a +/// valid cookie name and attribute value. The first part starts with a letter +/// or a single `_`, so no cookie name starts with `__`, where browsers give +/// names such as `__Host-` rules of their own and drop a cookie that breaks +/// them. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PublicPrefix { + dotted: String, +} + +impl PublicPrefix { + /// The prefix made of `parts`, in order, or `None` when there are none, + /// when a part is empty or holds anything else, or when the first part + /// starts with anything but a letter or a single `_`. + #[must_use] + pub fn new(parts: I) -> Option + where + I: IntoIterator, + S: AsRef, + { + let parts: Vec = parts + .into_iter() + .map(|part| part.as_ref().to_owned()) + .collect(); + let well_formed = parts.iter().all(|part| { + !part.is_empty() + && part + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-' | b'.')) + }); + let first_is_a_name = parts.first().is_some_and(|first| { + first.starts_with(|c: char| c.is_ascii_alphabetic() || c == '_') + && !first.starts_with("__") + }); + if !well_formed || !first_is_a_name { + return None; + } + Some(Self { + dotted: parts.join("."), + }) + } + + /// The parts joined by `.`, such as `_ex.www.example.com`, which every + /// cookie name and storage key starts with. + #[must_use] + pub fn as_str(&self) -> &str { + &self.dotted + } + + /// The name of the cookie `name`, written `-`, + /// such as `ts-ec` or `_ex-ec`. + #[must_use] + pub fn cookie_name(&self, name: &str) -> String { + format!("{}-{name}", self.dotted) + } + + /// Whether `cookie` is in the namespace this prefix starts, which is every + /// name [`Self::cookie_name`] gives, compared without regard to case. + #[must_use] + pub fn owns_cookie_name(&self, cookie: &[u8]) -> bool { + let length = self.dotted.len(); + cookie + .get(..length) + .is_some_and(|head| head.eq_ignore_ascii_case(self.dotted.as_bytes())) + && cookie.get(length) == Some(&b'-') + } +} + +/// Core's own, `ts`, for a value no request resolved, such as a default +/// Edge Cookie state. +impl Default for PublicPrefix { + fn default() -> Self { + TrustedServerPrefix::public() + } +} + +impl fmt::Display for PublicPrefix { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.dotted) + } +} + +/// Gives the start of the names the browser holds for a publisher and of the +/// keys its entries take in shared stores. +/// +/// Selected with `[prefix] module`, and core's [`TrustedServerPrefix`] runs +/// when nothing is selected. Each method is handed a [`ModuleCall`] carrying +/// the request and the deployment's settings, and +/// nothing else, and an implementation hands its own function to +/// [`ModuleCall::inject`] naming what it needs. +/// +/// The domain a server key prefix carries comes from the publisher the request +/// resolved to, being the loaded settings' `publisher.domain`, and never from +/// the raw request. +pub trait PrefixModule: Send + Sync + fmt::Debug { + /// The start of every cookie name and first-party storage key for the + /// publisher this call was resolved to. + /// + /// # Errors + /// + /// [`Withheld`] when the module names a value the call does not carry. + fn public_prefix(&self, call: ModuleCall<'_>) -> Result; + + /// The start of every key the publisher's entries take in a shared store. + /// + /// # Errors + /// + /// [`Withheld`] when the module names a value the call does not carry. + fn server_key_prefix(&self, call: ModuleCall<'_>) -> Result; +} + +/// Core's own prefix module, which gives `ts`, the prefix of the cookie names +/// Trusted Server writes, and `ts.`. It runs when `[prefix]` +/// selects nothing. +#[derive(Debug, Default, Clone, Copy)] +pub struct TrustedServerPrefix; + +impl TrustedServerPrefix { + /// `ts`. + #[must_use] + pub fn public() -> PublicPrefix { + PublicPrefix { + dotted: TRUSTED_SERVER_HEAD.to_owned(), + } + } + + fn server_key(&self, settings: &Settings) -> ServerKeyPrefix { + ServerKeyPrefix::new(TRUSTED_SERVER_HEAD, &settings.publisher.domain) + } +} + +impl PrefixModule for TrustedServerPrefix { + fn public_prefix(&self, _call: ModuleCall<'_>) -> Result { + Ok(Self::public()) + } + + fn server_key_prefix(&self, call: ModuleCall<'_>) -> Result { + call.inject(self, Self::server_key) + } +} + +/// Both prefixes for one request, as its services carry them. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Prefixes { + public: PublicPrefix, + server_key: ServerKeyPrefix, +} + +impl Prefixes { + /// The two prefixes. + #[must_use] + pub fn new(public: PublicPrefix, server_key: ServerKeyPrefix) -> Self { + Self { public, server_key } + } + + /// What `module`, called `name`, gives for the publisher `request` was + /// resolved to under `settings`. + /// + /// # Errors + /// + /// [`Withheld`] when the module names a value other than the request and + /// the settings. + pub fn resolve( + module: &dyn PrefixModule, + name: &'static str, + settings: &Settings, + request: ModuleRequest<'_>, + ) -> Result { + let context = ModuleContext::new(request).with_settings(settings); + let call = context.call(name, PermissionSet::none()); + Ok(Self { + public: module.public_prefix(call)?, + server_key: module.server_key_prefix(call)?, + }) + } + + /// What core's own module gives for the publisher `settings` describe. + #[must_use] + pub fn trusted_server(settings: &Settings) -> Self { + Self::new( + TrustedServerPrefix::public(), + TrustedServerPrefix.server_key(settings), + ) + } + + /// The start of every name the browser holds. + #[must_use] + pub fn public(&self) -> &PublicPrefix { + &self.public + } + + /// The start of every key in a shared store. + #[must_use] + pub fn server_key(&self) -> &ServerKeyPrefix { + &self.server_key + } +} + +/// The prefixes of services built for no request, such as a test's or the +/// operator command line's, being core's own names with no publisher domain. +impl Default for Prefixes { + fn default() -> Self { + Self::new( + TrustedServerPrefix::public(), + ServerKeyPrefix::new(TRUSTED_SERVER_HEAD, ""), + ) + } +} + +/// The selected prefix module and the name it runs under. +#[derive(Debug, Clone)] +pub struct SelectedPrefixModule { + name: &'static str, + module: Arc, +} + +impl SelectedPrefixModule { + /// `module`, which runs under `name`. + #[must_use] + pub fn new(name: &'static str, module: Arc) -> Self { + Self { name, module } + } + + /// Core's own module, which runs when nothing is selected. + #[must_use] + pub fn trusted_server() -> Self { + Self::new(TRUSTED_SERVER_MODULE, Arc::new(TrustedServerPrefix)) + } + + /// The name the module runs under. + #[must_use] + pub fn name(&self) -> &'static str { + self.name + } + + /// What the module gives for the publisher `request` was resolved to. + /// + /// # Errors + /// + /// [`Withheld`] when the module names a value other than the request and + /// the settings. + pub fn prefixes( + &self, + settings: &Settings, + request: ModuleRequest<'_>, + ) -> Result { + Prefixes::resolve(&*self.module, self.name, settings, request) + } +} + +impl Default for SelectedPrefixModule { + fn default() -> Self { + Self::trusted_server() + } +} + +/// Prefixes and checks for a module's own tests of its store keys. +#[cfg(any(test, feature = "test-utils"))] +pub mod test_support { + use super::{Prefixes, ServerKeyPrefix, TrustedServerPrefix}; + + /// The prefixes core's own module gives a publisher on `domain`. + #[must_use] + pub fn trusted_server_for(domain: &str) -> Prefixes { + Prefixes::new( + TrustedServerPrefix::public(), + ServerKeyPrefix::new(super::TRUSTED_SERVER_HEAD, domain), + ) + } + + /// Checks that the keys `key_for` makes for a publisher's domain follow + /// the publisher: two publishers never share one, `www.example.com` and + /// `example.com` do, and `clients.example.com` and `public.example.com` + /// do not. + /// + /// # Panics + /// + /// When any of the three does not hold, naming it. + pub fn assert_keys_follow_the_publisher(key_for: impl Fn(&str) -> String) { + assert_ne!( + key_for("first.example"), + key_for("second.example"), + "two publishers should never share a key" + ); + assert_eq!( + key_for("www.example.com"), + key_for("example.com"), + "a publisher's www. and bare names should share a key" + ); + assert_ne!( + key_for("clients.example.com"), + key_for("public.example.com"), + "two subdomains other than www. should not share a key" + ); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::module_context::test_support::request; + + fn settings_for(domain: &str) -> Settings { + let mut settings = Settings::default(); + settings.publisher.domain = domain.to_owned(); + settings + } + + #[test] + fn no_module_selected_gives_exactly_ts() { + let prefixes = SelectedPrefixModule::default() + .prefixes(&settings_for("www.example.com"), request("/")) + .expect("core's module should name only what the call carries"); + + assert_eq!(prefixes.public().as_str(), "ts"); + assert_eq!(prefixes.server_key().as_str(), "ts.example.com"); + for (name, cookie) in [ + ("ec", "ts-ec"), + ("ecr", "ts-ecr"), + ("eids", "ts-eids"), + ("tester", "ts-tester"), + ] { + assert_eq!(prefixes.public().cookie_name(name), cookie); + } + } + + #[test] + fn each_use_writes_its_own_form_from_the_parts() { + let prefix = + PublicPrefix::new(["_ex", "www.example.com"]).expect("should accept the parts"); + + assert_eq!(prefix.as_str(), "_ex.www.example.com"); + assert_eq!(prefix.cookie_name("ec"), "_ex.www.example.com-ec"); + } + + #[test] + fn a_part_that_could_leave_a_name_is_refused() { + for parts in [ + vec![], + vec![""], + vec!["9ex"], + vec!["-ex"], + vec![".ex"], + vec!["__ex"], + vec!["__Host"], + vec!["_ex", ""], + vec!["_ex", "a b"], + vec!["_ex", "a;b"], + vec!["_ex", "a=b"], + vec!["_ex", "a\"b"], + vec!["_ex", "a/b"], + ] { + assert!(PublicPrefix::new(parts.clone()).is_none(), "{parts:?}"); + } + } + + #[test] + fn a_prefix_owns_the_cookie_names_it_gives_and_no_others() { + let prefix = TrustedServerPrefix::public(); + + for owned in ["ts-ec", "TS-EIDS", "ts-anything", "ts-"] { + assert!(prefix.owns_cookie_name(owned.as_bytes()), "{owned}"); + } + for other in ["ts", "tsx-ec", "_ex-ec", "sharedId"] { + assert!(!prefix.owns_cookie_name(other.as_bytes()), "{other}"); + } + } + + /// The client script reads the prefix from the attribute the server + /// writes, on the bundle tag, so both names are the same in the two + /// places. + #[test] + fn the_client_script_reads_the_attribute_the_server_writes() { + let script = include_str!("../../trusted-server-js/lib/src/shared/public_prefix.ts"); + + assert!( + script.contains(&format!( + "export const PUBLIC_PREFIX_ATTRIBUTE = '{PUBLIC_PREFIX_ATTRIBUTE}'" + )), + "the client script should read the attribute the server writes" + ); + assert!( + script.contains("const BUNDLE_TAG_ID = 'trustedserver-js'") + && crate::tsjs::tsjs_script_tag_with_attributes(&[], &[]) + .contains(r#"id="trustedserver-js""#), + "the client script should read the tag the server writes" + ); + } + + #[test] + fn the_default_carries_no_publisher_domain() { + let prefixes = Prefixes::default(); + + assert_eq!(prefixes.public().as_str(), "ts"); + assert_eq!(prefixes.server_key().domain(), ""); + } +} diff --git a/crates/trusted-server-core/src/provider_table.rs b/crates/trusted-server-core/src/provider_table.rs new file mode 100644 index 0000000000..9715accdc7 --- /dev/null +++ b/crates/trusted-server-core/src/provider_table.rs @@ -0,0 +1,886 @@ +//! The table the auction's `[demand]` and `[ad-server]` share. +//! +//! Each is one top-level table. Its `modules` key, or `module` where one +//! runs, selects what runs, and every other key is one named provider's +//! settings table: +//! +//! ```toml +//! [demand] +//! modules = ["pbs_demo"] +//! +//! [demand.pbs_demo] +//! implementation = "auction.prebid-server" +//! endpoint = "https://prebid.example.com/openrtb2/auction" +//! ``` +//! +//! A table's `implementation` line names the implementation it configures by +//! its module path, which lets several providers share one implementation +//! under names of their own. Without the line the table's name stands for +//! the implementation, which only a section named by the implementation's +//! type can match, as `mock` in `[ad-server]` is `ad-server.mock`. A provider +//! with nothing to set needs no table. [`ProviderTable::validate`] checks the +//! rules that hold for every type, and the code that reads a type checks what +//! only that type knows, such as which implementations exist. + +use std::collections::BTreeMap; +use std::fmt; + +use serde::de::{self, MapAccess, Visitor}; +use serde::ser::SerializeMap as _; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use serde_json::{Map, Value}; + +/// The line naming the implementation a table configures when the table's +/// name is a label rather than the implementation. +pub const IMPLEMENTATION_KEY: &str = "implementation"; + +/// The longest provider name, in bytes. +const MAX_NAME_BYTES: usize = 63; + +/// A provider type whose `modules` names any number of providers. +pub type ProviderList = ProviderTable>; + +/// A provider type whose `module` names at most one provider. +pub type ProviderChoice = ProviderTable>; + +/// How a type's selection is written. +pub trait ProviderSelection: Default + Clone + PartialEq + fmt::Debug { + /// The key the selection is written under, `module` for a type that runs + /// one and `modules` for a type that runs several. + const KEY: &'static str; + + /// The selected names, in the order they were written. + fn names(&self) -> Vec<&str>; + + /// Reads the selection. + /// + /// # Errors + /// + /// Returns a message when the value has the wrong shape for this type. + fn from_value(value: Value) -> Result; + + /// Writes the selection, or `None` when nothing is selected. + fn to_value(&self) -> Option; +} + +impl ProviderSelection for Vec { + const KEY: &'static str = "modules"; + + fn names(&self) -> Vec<&str> { + self.iter().map(String::as_str).collect() + } + + fn from_value(value: Value) -> Result { + match value { + Value::Array(items) => items + .into_iter() + .map(|item| match item { + Value::String(name) => Ok(name), + other => Err(format!("`modules` must list names, found `{other}`")), + }) + .collect(), + Value::String(name) => Err(format!( + "`modules` is a list for this type, so write [\"{name}\"]" + )), + other => Err(format!( + "`modules` must be a list of names, found `{other}`" + )), + } + } + + fn to_value(&self) -> Option { + (!self.is_empty()).then(|| Value::Array(self.iter().cloned().map(Value::String).collect())) + } +} + +impl ProviderSelection for Option { + const KEY: &'static str = "module"; + + fn names(&self) -> Vec<&str> { + self.iter().map(String::as_str).collect() + } + + fn from_value(value: Value) -> Result { + match value { + Value::String(name) => Ok(Some(name)), + Value::Array(_) => { + Err("`module` names one provider for this type, not a list".to_owned()) + } + other => Err(format!("`module` must be a name, found `{other}`")), + } + } + + fn to_value(&self) -> Option { + self.clone().map(Value::String) + } +} + +/// One provider type's table. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct ProviderTable { + selected: S, + tables: BTreeMap>, +} + +impl ProviderTable { + /// Builds a table from its selection and its named settings tables. + #[must_use] + pub fn new(selected: S, tables: BTreeMap>) -> Self { + Self { selected, tables } + } + + /// The selected provider names, in the order they were written. + #[must_use] + pub fn selected(&self) -> Vec<&str> { + self.selected.names() + } + + /// Whether no provider is selected. + #[must_use] + pub fn is_empty(&self) -> bool { + self.selected.names().is_empty() + } + + /// Whether the table sets nothing at all, neither a selection nor any named + /// table, so a serialized configuration can leave it out. + #[must_use] + pub fn is_unset(&self) -> bool { + self.is_empty() && self.tables.is_empty() + } + + /// Every named settings table, selected or not. + #[must_use] + pub fn tables(&self) -> &BTreeMap> { + &self.tables + } + + /// The implementation a selected name uses, being its `implementation` + /// line or else the name itself. + #[must_use] + pub fn implementation_of<'a>(&'a self, name: &'a str) -> &'a str { + self.tables + .get(name) + .and_then(|table| table.get(IMPLEMENTATION_KEY)) + .and_then(Value::as_str) + .unwrap_or(name) + } + + /// A name's settings, without its `implementation` line. A name with no + /// table has no settings. + #[must_use] + pub fn settings_of(&self, name: &str) -> Map { + let mut settings = self.tables.get(name).cloned().unwrap_or_default(); + settings.remove(IMPLEMENTATION_KEY); + settings + } + + /// Checks the rules every provider type shares, naming the type in each + /// message. + /// + /// # Errors + /// + /// Returns a message for a name that is not `snake_case`, a name selected + /// twice, an `implementation` line that is not a module name, or a table + /// the selection does not name. + pub fn validate(&self, type_name: &str) -> Result<(), String> { + let selected = self.selected.names(); + for (position, name) in selected.iter().enumerate() { + check_name(type_name, "module name", name)?; + if selected[..position].contains(name) { + return Err(format!( + "[{type_name}] {} names `{name}` more than once", + S::KEY + )); + } + } + for (name, table) in &self.tables { + check_name(type_name, "table name", name)?; + if let Some(implementation) = table.get(IMPLEMENTATION_KEY) { + let valid = implementation + .as_str() + .is_some_and(crate::module_name::is_valid); + if !valid { + return Err(format!( + "[{type_name}.{name}] implementation must be a module name, its \ + crate's parts joined by `.`, each in lower case letters, digits, \ + `_` or `-`, such as `auction.example`" + )); + } + } + if !selected.contains(&name.as_str()) { + return Err(format!( + "[{type_name}.{name}] is configured, but [{type_name}] {key} does not select `{name}`. Add it to {key}, or remove the table", + key = S::KEY + )); + } + } + Ok(()) + } +} + +/// Checks that a module's name within its type is `snake_case`. +fn check_name(type_name: &str, what: &str, name: &str) -> Result<(), String> { + let mut bytes = name.bytes(); + let valid = name.len() <= MAX_NAME_BYTES + && bytes.next().is_some_and(|first| first.is_ascii_lowercase()) + && bytes.all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_'); + if valid { + Ok(()) + } else { + Err(format!( + "[{type_name}] {what} `{name}` must be snake_case: a lowercase letter followed by up to {} lowercase letters, digits or underscores", + MAX_NAME_BYTES - 1 + )) + } +} + +impl<'de, S: ProviderSelection> Deserialize<'de> for ProviderTable { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct TableVisitor(core::marker::PhantomData); + + impl<'de, S: ProviderSelection> Visitor<'de> for TableVisitor { + type Value = ProviderTable; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a provider table with its selection key and named tables") + } + + fn visit_map(self, mut map: A) -> Result + where + A: MapAccess<'de>, + { + let mut table = ProviderTable::::default(); + while let Some(key) = map.next_key::()? { + let value = map.next_value::()?; + if key == S::KEY { + table.selected = S::from_value(value).map_err(de::Error::custom)?; + continue; + } + if key == "provider" { + return Err(de::Error::custom(format!( + "`provider` is `{}` here. Name what runs with `{}`", + S::KEY, + S::KEY + ))); + } + match value { + Value::Object(settings) => { + table.tables.insert(key, settings); + } + _ => { + return Err(de::Error::custom(format!( + "`{key}` is not a setting of this table. Only `{}` and named provider tables belong here", + S::KEY + ))); + } + } + } + Ok(table) + } + } + + deserializer.deserialize_map(TableVisitor(core::marker::PhantomData)) + } +} + +impl Serialize for ProviderTable { + fn serialize(&self, serializer: Z) -> Result + where + Z: Serializer, + { + let selection = self.selected.to_value(); + let mut map = + serializer.serialize_map(Some(self.tables.len() + usize::from(selection.is_some())))?; + if let Some(selection) = selection { + map.serialize_entry(S::KEY, &selection)?; + } + for (name, settings) in &self.tables { + map.serialize_entry(name, settings)?; + } + map.end() + } +} + +/// Which key a section's selection was written under. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SelectionKey { + Module, + Modules, +} + +impl SelectionKey { + const fn as_str(self) -> &'static str { + match self { + Self::Module => "module", + Self::Modules => "modules", + } + } +} + +/// The modules one section selects, and each one's settings table. +/// +/// `modules` selects several, in the order they run, and `module` one, and a +/// section carries at most one of the two. Every other table in the section is +/// the settings of a selected module, at the name written in the selection +/// with `.` between its parts, so `modules = ["testing.example"]` reads its +/// settings from `[
.testing.example]`. +#[derive(Clone, Default, PartialEq)] +pub struct SectionModules { + key: Option, + selected: Vec, + tables: Map, +} + +impl fmt::Debug for SectionModules { + // The tables can hold secrets, so only the selection and the tables' + // names are shown. + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("SectionModules") + .field("selected", &self.selected) + .field("tables", &self.tables.keys().collect::>()) + .finish() + } +} + +impl SectionModules { + /// Reads a section's entries, in a section that holds nothing else. + /// + /// # Errors + /// + /// Returns a message for a selection of the wrong shape, for both keys at + /// once, and for an entry that is neither a selection nor a table. + pub fn from_entries(entries: Map) -> Result { + let mut section = Self::default(); + for (key, value) in entries { + match key.as_str() { + "module" | "modules" => { + let key = if key == "module" { + SelectionKey::Module + } else { + SelectionKey::Modules + }; + if section.key.is_some_and(|written| written != key) { + return Err( + "carries both `module` and `modules`. Write `module` for one or \ + `modules` for several" + .to_owned(), + ); + } + section.key = Some(key); + section.selected = match key { + SelectionKey::Module => { + as ProviderSelection>::from_value(value)? + .into_iter() + .collect() + } + SelectionKey::Modules => { + as ProviderSelection>::from_value(value)? + } + }; + } + _ => match value { + Value::Object(_) => { + section.tables.insert(key, value); + } + _ => { + return Err(format!( + "has `{key}`, which is not a setting it reads. Only `module` or \ + `modules` and the settings tables of the modules they select \ + belong there" + )); + } + }, + } + } + Ok(section) + } + + /// The selected names, as written, in the order they run. + #[must_use] + pub fn selected(&self) -> &[String] { + &self.selected + } + + /// Whether the section selects nothing and holds no table. + #[must_use] + pub fn is_unset(&self) -> bool { + self.selected.is_empty() && self.tables.is_empty() + } + + /// The settings table of a name as written in the selection, found by + /// its parts. + #[must_use] + pub fn settings_of(&self, written: &str) -> Option<&Map> { + let mut parts = written.split('.'); + let mut table = self.tables.get(parts.next()?)?.as_object()?; + for part in parts { + table = table.get(part)?.as_object()?; + } + Some(table) + } + + /// The settings table of a name as written in the selection, to change. + pub fn settings_of_mut(&mut self, written: &str) -> Option<&mut Map> { + let mut parts = written.split('.'); + let mut table = self.tables.get_mut(parts.next()?)?.as_object_mut()?; + for part in parts { + table = table.get_mut(part)?.as_object_mut()?; + } + Some(table) + } + + /// Selects `written` and stores its settings table, which is what writing + /// both in a document does. + pub fn insert(&mut self, written: &str, settings: Map) { + if !self.selected.iter().any(|name| name == written) { + self.selected.push(written.to_owned()); + } + if self.key.is_none() { + self.key = Some(SelectionKey::Modules); + } + let mut parts: Vec<&str> = written.split('.').collect(); + let last = parts.pop().unwrap_or(written); + let mut table = &mut self.tables; + for part in parts { + let entry = table + .entry(part.to_owned()) + .or_insert_with(|| Value::Object(Map::new())); + if !entry.is_object() { + *entry = Value::Object(Map::new()); + } + let Value::Object(next) = entry else { + return; + }; + table = next; + } + table.insert(last.to_owned(), Value::Object(settings)); + } + + /// Selects `written` with no settings table of its own. + pub fn select(&mut self, written: &str) { + if !self.selected.iter().any(|name| name == written) { + self.selected.push(written.to_owned()); + } + if self.key.is_none() { + self.key = Some(SelectionKey::Modules); + } + } + + /// Stops selecting `written` and drops its table. + pub fn remove(&mut self, written: &str) { + self.selected.retain(|name| name != written); + let mut parts: Vec<&str> = written.split('.').collect(); + let Some(last) = parts.pop() else { + return; + }; + let mut table = &mut self.tables; + for part in parts { + let Some(Value::Object(next)) = table.get_mut(part) else { + return; + }; + table = next; + } + table.remove(last); + } + + /// Stops selecting everything and drops every table. + pub fn clear(&mut self) { + self.selected.clear(); + self.tables.clear(); + } + + /// Checks the rules every section's selection follows, naming the section. + /// + /// # Errors + /// + /// Returns a message for a name that is not a module name, a name selected + /// twice, and a table that is not the settings of a selected name. + pub fn validate(&self, section: &str) -> Result<(), String> { + let key = self.key.unwrap_or(SelectionKey::Modules).as_str(); + for (position, name) in self.selected.iter().enumerate() { + if !crate::module_name::is_valid(name) { + return Err(format!( + "[{section}] {key} names `{name}`, which is not a module name, being parts \ + joined by `.`, each of lower case letters, digits, `_` or `-`" + )); + } + if self.selected[..position].contains(name) { + return Err(format!("[{section}] {key} names `{name}` more than once")); + } + } + check_tables(section, key, &self.selected, &self.tables, "") + } +} + +/// Checks that every table under `prefix` is a selected name's settings, or +/// lies on the way to one. +fn check_tables( + section: &str, + key: &str, + selected: &[String], + tables: &Map, + prefix: &str, +) -> Result<(), String> { + for (name, value) in tables { + let path = if prefix.is_empty() { + name.clone() + } else { + format!("{prefix}.{name}") + }; + if selected.contains(&path) { + continue; + } + let leads_to_one = selected + .iter() + .any(|written| written.starts_with(&format!("{path}."))); + match value { + Value::Object(inner) if leads_to_one => { + check_tables(section, key, selected, inner, &path)?; + } + _ => { + return Err(format!( + "[{section}.{path}] is configured, but [{section}] {key} does not select \ + `{path}`. Add it to {key}, or remove the table" + )); + } + } + } + Ok(()) +} + +impl<'de> Deserialize<'de> for SectionModules { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let entries = Map::::deserialize(deserializer)?; + Self::from_entries(entries) + .map_err(|message| serde::de::Error::custom(format!("the section {message}"))) + } +} + +impl Serialize for SectionModules { + fn serialize(&self, serializer: Z) -> Result + where + Z: Serializer, + { + let selection = + (!self.selected.is_empty()).then(|| match self.key.unwrap_or(SelectionKey::Modules) { + SelectionKey::Module if self.selected.len() == 1 => ( + SelectionKey::Module.as_str(), + Value::String(self.selected[0].clone()), + ), + _ => ( + SelectionKey::Modules.as_str(), + Value::Array(self.selected.iter().cloned().map(Value::String).collect()), + ), + }); + let mut map = + serializer.serialize_map(Some(self.tables.len() + usize::from(selection.is_some())))?; + if let Some((key, value)) = selection { + map.serialize_entry(key, &value)?; + } + for (name, table) in &self.tables { + map.serialize_entry(name, table)?; + } + map.end() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn list(json: Value) -> Result { + serde_json::from_value(json) + } + + #[test] + fn reads_the_selection_and_named_tables() { + let table = list(serde_json::json!({ + "modules": ["main_source", "second_source"], + "main_source": { "implementation": "exchange", "endpoint": "https://exchange.example" } + })) + .expect("should read a module list"); + + assert_eq!(table.selected(), vec!["main_source", "second_source"]); + assert_eq!(table.implementation_of("main_source"), "exchange"); + assert_eq!( + table.implementation_of("second_source"), + "second_source", + "a name with no table should be its own implementation" + ); + assert!( + !table + .settings_of("main_source") + .contains_key(IMPLEMENTATION_KEY), + "settings should not carry the implementation line" + ); + table.validate("demand").expect("should accept the table"); + } + + #[test] + fn rejects_a_table_that_is_not_selected() { + let table = list(serde_json::json!({ + "modules": ["second_source"], + "main_source": { "endpoint": "https://exchange.example" } + })) + .expect("should read the table"); + + let error = table + .validate("demand") + .expect_err("should reject an unselected table"); + assert!( + error.contains("[demand.main_source]") && error.contains("does not select"), + "should name the table and the fix: {error}" + ); + } + + #[test] + fn rejects_a_name_selected_twice() { + let table = list(serde_json::json!({ "modules": ["second_source", "second_source"] })) + .expect("should read the table"); + + let error = table + .validate("demand") + .expect_err("should reject a duplicate"); + assert!(error.contains("more than once"), "should say why: {error}"); + } + + #[test] + fn rejects_names_that_are_not_snake_case() { + for name in ["main-source", "MainSource", "1source", ""] { + let table = list(serde_json::json!({ "modules": [name] })).expect("should read"); + let error = table + .validate("demand") + .expect_err("should reject a name that is not snake_case"); + assert!( + error.contains("snake_case"), + "should say why for `{name}`: {error}" + ); + } + } + + #[test] + fn an_implementation_is_named_by_its_module_path() { + let table = list(serde_json::json!({ + "modules": ["main_source"], + "main_source": { "implementation": "auction.example-exchange" } + })) + .expect("should read the table"); + table + .validate("demand") + .expect("a module path should name an implementation"); + + for implementation in [ + serde_json::json!("Auction.Exchange"), + serde_json::json!("auction/exchange"), + serde_json::json!("auction..exchange"), + serde_json::json!(7), + ] { + let table = list(serde_json::json!({ + "modules": ["main_source"], + "main_source": { "implementation": implementation } + })) + .expect("should read the table"); + let error = table + .validate("demand") + .expect_err("should refuse an implementation that is not a module name"); + assert!( + error.contains("[demand.main_source] implementation must be a module name"), + "should say why for {implementation}: {error}" + ); + } + } + + #[test] + fn rejects_a_value_that_is_not_a_table() { + let error = list(serde_json::json!({ "modules": ["second_source"], "timeout_ms": 500 })) + .expect_err("should reject a stray value"); + assert!( + error.to_string().contains("`timeout_ms` is not a setting"), + "should name the stray key: {error}" + ); + } + + #[test] + fn a_choice_takes_one_name_and_a_list_takes_a_list() { + let choice: ProviderChoice = + serde_json::from_value(serde_json::json!({ "module": "picker" })) + .expect("should read a single name"); + assert_eq!(choice.selected(), vec!["picker"]); + + serde_json::from_value::(serde_json::json!({ "module": ["a"] })) + .expect_err("a single-module type should refuse a list"); + serde_json::from_value::(serde_json::json!({ "modules": "a" })) + .expect_err("a list type should refuse a single name"); + } + + #[test] + fn refuses_the_previous_selection_key() { + let error = list(serde_json::json!({ "provider": ["main_source"] })) + .expect_err("should refuse the key the selection was written under before"); + assert!( + error.to_string().contains("`provider` is `modules` here"), + "the refusal names the key to use: {error}" + ); + + let error = + serde_json::from_value::(serde_json::json!({ "provider": "a" })) + .expect_err("should refuse the previous key for a type that runs one"); + assert!( + error.to_string().contains("`provider` is `module` here"), + "the refusal names the key to use: {error}" + ); + } + + #[test] + fn round_trips_through_serialization() { + let table = list(serde_json::json!({ + "modules": ["main_source"], + "main_source": { "implementation": "exchange", "timeout_ms": 900 } + })) + .expect("should read the table"); + + let value = serde_json::to_value(&table).expect("should serialize"); + let again: ProviderList = serde_json::from_value(value).expect("should read it back"); + assert_eq!(table, again); + } +} + +#[cfg(test)] +mod section_tests { + use super::SectionModules; + use serde_json::{Map, Value, json}; + + fn read(json: Value) -> Result { + serde_json::from_value(json) + } + + #[test] + fn reads_the_selection_and_the_selected_tables() { + let section = read(json!({ + "modules": ["didomi", "sourcepoint"], + "didomi": { "api_key": "k" } + })) + .expect("should read a section"); + assert_eq!(section.selected(), ["didomi", "sourcepoint"]); + assert_eq!( + section.settings_of("didomi").and_then(|t| t.get("api_key")), + Some(&json!("k")) + ); + assert!(section.settings_of("sourcepoint").is_none()); + section.validate("cmp").expect("should pass validation"); + } + + #[test] + fn module_selects_one_and_a_list_there_is_refused() { + let section = read(json!({ "module": "didomi" })).expect("should read one"); + assert_eq!(section.selected(), ["didomi"]); + let error = read(json!({ "module": ["didomi"] })).expect_err("should refuse a list"); + assert!(error.to_string().contains("`module` names one"), "{error}"); + } + + #[test] + fn both_keys_at_once_are_refused() { + let error = + read(json!({ "module": "a", "modules": ["b"] })).expect_err("should refuse both keys"); + assert!( + error + .to_string() + .contains("carries both `module` and `modules`"), + "{error}" + ); + } + + #[test] + fn a_value_that_is_neither_selection_nor_table_is_refused() { + let error = read(json!({ "modules": ["a"], "enabled": true })) + .expect_err("should refuse a stray value"); + assert!( + error + .to_string() + .contains("has `enabled`, which is not a setting it reads"), + "{error}" + ); + } + + #[test] + fn a_table_no_selection_names_is_refused_by_validation() { + let section = read(json!({ "modules": ["a"], "b": {} })).expect("should read"); + let error = section + .validate("cmp") + .expect_err("should refuse the stray table"); + assert_eq!( + error, + "[cmp.b] is configured, but [cmp] modules does not select `b`. Add it to modules, or remove the table" + ); + } + + #[test] + fn a_name_written_with_dots_reads_a_nested_table() { + let section = read(json!({ + "modules": ["google", "google.diagnostics"], + "google": { "timeout_ms": 5, "diagnostics": { "overlay": true } } + })) + .expect("should read"); + section + .validate("ad-tag") + .expect("the nested table is a selected name's"); + assert_eq!( + section + .settings_of("google.diagnostics") + .and_then(|t| t.get("overlay")), + Some(&json!(true)) + ); + } + + #[test] + fn a_name_that_is_not_a_module_name_or_is_repeated_is_refused() { + let section = read(json!({ "modules": ["Didomi"] })).expect("should read"); + let error = section.validate("cmp").expect_err("should refuse the name"); + assert!(error.contains("is not a module name"), "{error}"); + let section = read(json!({ "modules": ["a", "a"] })).expect("should read"); + let error = section + .validate("cmp") + .expect_err("should refuse the repeat"); + assert_eq!(error, "[cmp] modules names `a` more than once"); + } + + #[test] + fn insert_select_remove_and_clear_keep_the_selection_and_tables_together() { + let mut section = SectionModules::default(); + assert!(section.is_unset()); + let mut settings = Map::new(); + settings.insert("api_key".to_owned(), json!("k")); + section.insert("didomi", settings); + section.select("osano"); + assert_eq!(section.selected(), ["didomi", "osano"]); + assert!(section.settings_of("didomi").is_some()); + section.remove("didomi"); + assert_eq!(section.selected(), ["osano"]); + assert!(section.settings_of("didomi").is_none()); + section.clear(); + assert!(section.is_unset()); + } + + #[test] + fn round_trips_through_serialization() { + let section = + read(json!({ "module": "didomi", "didomi": { "api_key": "k" } })).expect("should read"); + let value = serde_json::to_value(§ion).expect("should serialize"); + assert_eq!( + value, + json!({ "module": "didomi", "didomi": { "api_key": "k" } }) + ); + let several = read(json!({ "modules": ["a", "b"] })).expect("should read"); + assert_eq!( + serde_json::to_value(&several).expect("should serialize"), + json!({ "modules": ["a", "b"] }) + ); + } +} diff --git a/crates/trusted-server-core/src/proxy.rs b/crates/trusted-server-core/src/proxy.rs index 6a30645aba..e7160087a3 100644 --- a/crates/trusted-server-core/src/proxy.rs +++ b/crates/trusted-server-core/src/proxy.rs @@ -21,7 +21,7 @@ use crate::constants::{ HEADER_USER_AGENT, HEADER_X_FORWARDED_FOR, }; use crate::creative::{CreativeCssProcessor, CreativeHtmlProcessor}; -use crate::edge_cookie::get_ec_id; +use crate::edge_cookie::recognized_ec_id; use crate::error::TrustedServerError; use crate::platform::{ DEFAULT_FIRST_BYTE_TIMEOUT, PlatformBackendSpec, PlatformHttpRequest, PlatformResponse, @@ -107,7 +107,10 @@ const ASSET_PROXY_FORWARD_HEADERS: [header::HeaderName; 11] = [ /// All three take effect against the serving origin, so leaving them in place /// would let an upstream write first-party cookies, pin HSTS, or clear the /// publisher's site data. -pub(crate) const FIRST_PARTY_PASSTHROUGH_STRIP_HEADERS: [&str; 3] = +/// +/// Public so an integration crate that passes an upstream body through strips +/// the same three. +pub const FIRST_PARTY_PASSTHROUGH_STRIP_HEADERS: [&str; 3] = ["set-cookie", "strict-transport-security", "clear-site-data"]; /// Cache-control value used when asset proxy responses must not be stored. @@ -818,7 +821,7 @@ pub async fn proxy_request( })?; if forward_ec_id { - append_ec_id(&req, &mut target_url_parsed); + append_ec_id(settings, services, &req, &mut target_url_parsed); } proxy_with_redirects( @@ -1221,8 +1224,19 @@ fn upsert_ec_query_param(url: &mut url::Url, ec_id: &str) { url.set_query(Some(&serializer.finish())); } -fn append_ec_id(req: &Request, target_url_parsed: &mut url::Url) { - let ec_id_param = match get_ec_id(req) { +/// Forwards the request's Edge Cookie identifier to the outbound target URL. +/// +/// Only an identifier the selected module recognizes is forwarded. A value +/// carrying another deployment's module code, and any value at all in a +/// stateless deployment, is withheld, so nothing this deployment did not issue +/// reaches the origin. +fn append_ec_id( + settings: &Settings, + services: &RuntimeServices, + req: &Request, + target_url_parsed: &mut url::Url, +) { + let ec_id_param = match recognized_ec_id(settings, services, req) { Ok(id) => id, Err(e) => { log::warn!("failed to extract EC ID for forwarding: {:?}", e); @@ -1259,7 +1273,11 @@ fn is_host_permitted>(allowed_domains: &[S], host: &str) -> bool { /// /// Comparison is case-insensitive. The wildcard check requires a dot boundary, /// so `"*.example.com"` does **not** match `"evil-example.com"`. -pub(crate) fn is_host_allowed(host: &str, pattern: &str) -> bool { +/// +/// Public so an integration crate matches a host against an operator's +/// allowed domains the way the proxy does. +#[must_use] +pub fn is_host_allowed(host: &str, pattern: &str) -> bool { let host = host.to_ascii_lowercase(); let pattern = pattern.to_ascii_lowercase(); @@ -1615,7 +1633,7 @@ pub async fn handle_first_party_proxy( /// Returns an error if the signed target cannot be reconstructed or validation fails. pub async fn handle_first_party_click( settings: &Settings, - _services: &RuntimeServices, + services: &RuntimeServices, req: Request, ) -> Result, Report> { let SignedTarget { @@ -1624,7 +1642,10 @@ pub async fn handle_first_party_click( had_params, } = reconstruct_and_validate_signed_target(settings, &req.uri().to_string())?; - let ec_id = match get_ec_id(&req) { + // The redirect target is a third party's URL, so only an identifier the + // selected module recognizes is added to it. A stateless deployment adds + // nothing. + let ec_id = match recognized_ec_id(settings, services, &req) { Ok(id) => id, Err(e) => { log::warn!("failed to extract EC ID for forwarding: {:?}", e); @@ -2238,16 +2259,17 @@ mod tests { use super::{ AssetProxyCachePolicy, IMAGE_FALLBACK_CONTENT_TYPE, ProxyRequestConfig, - SUPPORTED_ENCODINGS, asset_origin_host_header, asset_path_skips_image_optimizer, - build_asset_proxy_target_url, handle_asset_proxy_request, handle_first_party_click, - handle_first_party_proxy, handle_first_party_proxy_rebuild, handle_first_party_proxy_sign, - is_host_allowed, is_host_permitted, proxy_request, rebuild_response_with_body, - reconstruct_and_validate_signed_target, stream_asset_body, + SUPPORTED_ENCODINGS, append_ec_id, asset_origin_host_header, + asset_path_skips_image_optimizer, build_asset_proxy_target_url, handle_asset_proxy_request, + handle_first_party_click, handle_first_party_proxy, handle_first_party_proxy_rebuild, + handle_first_party_proxy_sign, is_host_allowed, is_host_permitted, proxy_request, + rebuild_response_with_body, reconstruct_and_validate_signed_target, stream_asset_body, }; use crate::cache_policy::{CachePolicy, EdgeCacheHeader}; use crate::constants::{HEADER_ACCEPT, HEADER_X_FORWARDED_FOR}; use crate::creative; use crate::error::{IntoHttpResponse, TrustedServerError}; + use crate::platform::RuntimeServices; use crate::platform::test_support::{ HashMapSecretStore, StubHttpClient, build_services_with_http_client, build_services_with_secret_and_http_client, noop_services, @@ -2267,6 +2289,7 @@ mod tests { use edgezero_core::body::Body as EdgeBody; use edgezero_core::http::response_builder as edge_response_builder; use error_stack::Report; + use http::Request; use http::{HeaderValue, Method, Request as HttpRequest, Response, StatusCode, header}; #[test] @@ -2989,7 +3012,8 @@ mod tests { ); req.headers_mut().insert( crate::constants::HEADER_X_TS_EC, - HeaderValue::from_static("ec-123"), + HeaderValue::from_str(&recognized_hmac_ec_id()) + .expect("should build EC header value"), ); let resp = handle_first_party_click(&settings, &noop_services(), req) @@ -3007,11 +3031,166 @@ mod tests { .map(|(k, v)| (k.into_owned(), v.into_owned())) .collect(); assert_eq!(pairs.remove("foo").as_deref(), Some("1")); - assert_eq!(pairs.remove("ts-ec").as_deref(), Some("ec-123")); + assert_eq!( + pairs.remove("ts-ec").as_deref(), + Some(recognized_hmac_ec_id().as_str()) + ); assert!(pairs.is_empty()); }); } + /// An identifier the built-in HMAC module, the module + /// `create_test_settings` selects, recognizes as its own. + fn recognized_hmac_ec_id() -> String { + format!("hmac~{}", crate::test_support::tests::VALID_SYNTHETIC_ID) + } + + /// A well-formed identifier carrying a module code no deployment here + /// reads, the shape a partner or another deployment would hand back. + const FOREIGN_CODED_EC_ID: &str = "zz00~someone-elses-identifier"; + + fn stateless_settings() -> Settings { + let mut settings = create_test_settings(); + settings.ec.module = None; + settings.ec.module_blocks.clear(); + settings + } + + fn signed_click_request(settings: &Settings, ec_id: &str) -> Request { + let tsurl = "https://cdn.example/a.png"; + let full = format!("{tsurl}?foo=1"); + let sig = crate::http_util::compute_encrypted_sha256_token(settings, &full); + let mut req = build_http_request( + Method::GET, + format!( + "https://edge.example/first-party/click?tsurl={}&foo=1&tstoken={}", + url::form_urlencoded::byte_serialize(tsurl.as_bytes()).collect::(), + sig + ), + ); + req.headers_mut().insert( + crate::constants::HEADER_X_TS_EC, + HeaderValue::from_str(ec_id).expect("should build EC header value"), + ); + req + } + + fn click_ts_ec_param( + settings: &Settings, + services: &RuntimeServices, + req: Request, + ) -> Option { + let resp = futures::executor::block_on(handle_first_party_click(settings, services, req)) + .expect("should redirect"); + let loc = resp + .headers() + .get(header::LOCATION) + .and_then(|h| h.to_str().ok()) + .expect("Location header should be present and valid") + .to_owned(); + url::Url::parse(&loc) + .expect("Location should be a valid URL") + .query_pairs() + .find(|(k, _)| k == "ts-ec") + .map(|(_, v)| v.into_owned()) + } + + #[test] + fn click_withholds_an_ec_id_the_module_does_not_recognize() { + let settings = create_test_settings(); + let param = click_ts_ec_param( + &settings, + &noop_services(), + signed_click_request(&settings, FOREIGN_CODED_EC_ID), + ); + + assert_eq!( + param, None, + "a value carrying another deployment's module code should not reach the click target" + ); + } + + #[test] + fn click_withholds_every_ec_id_in_a_stateless_deployment() { + let settings = stateless_settings(); + // The value is one the built-in module would recognize, so only the + // absence of a selected module can withhold it. + let param = click_ts_ec_param( + &settings, + &noop_services(), + signed_click_request(&settings, &recognized_hmac_ec_id()), + ); + + assert_eq!( + param, None, + "a deployment that creates no identifier should hand none to the click target" + ); + } + + #[test] + fn append_ec_id_forwards_only_what_the_module_recognizes() { + let settings = create_test_settings(); + let services = noop_services(); + let recognized = recognized_hmac_ec_id(); + + let mut url = + url::Url::parse("https://origin.example/page?foo=1").expect("should parse origin URL"); + append_ec_id( + &settings, + &services, + &request_with_ec_cookie(&recognized), + &mut url, + ); + assert_eq!( + ts_ec_param(&url), + Some(recognized.clone()), + "the deployment's own identifier should still reach the origin" + ); + + let mut url = + url::Url::parse("https://origin.example/page?foo=1").expect("should parse origin URL"); + append_ec_id( + &settings, + &services, + &request_with_ec_cookie(FOREIGN_CODED_EC_ID), + &mut url, + ); + assert_eq!( + ts_ec_param(&url), + None, + "a foreign module code should not reach the origin" + ); + + let mut url = + url::Url::parse("https://origin.example/page?foo=1").expect("should parse origin URL"); + append_ec_id( + &stateless_settings(), + &services, + &request_with_ec_cookie(&recognized), + &mut url, + ); + assert_eq!( + ts_ec_param(&url), + None, + "a stateless deployment should forward nothing to the origin" + ); + } + + fn request_with_ec_cookie(ec_id: &str) -> Request { + let mut req = build_http_request(Method::GET, "https://edge.example/page"); + req.headers_mut().insert( + http::header::COOKIE, + HeaderValue::from_str(&format!("ts-ec={ec_id}")).expect("should build cookie header"), + ); + req + } + + fn ts_ec_param(url: &url::Url) -> Option { + url.query_pairs() + .find(|(k, _)| k == "ts-ec") + .map(|(_, v)| v.into_owned()) + } + #[test] fn proxy_rebuild_adds_and_removes_params() { futures::executor::block_on(async { diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 682ecc0f15..3168b96421 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -51,13 +51,13 @@ use crate::auction::telemetry::{ emit_auction_events_best_effort_lazy, }; use crate::auction::types::{ - AuctionContext, AuctionRequest, Bid, DeviceInfo, PublisherInfo, SiteInfo, UserInfo, + AuctionContext, AuctionRequest, Bid, BidRenderer, DeviceInfo, PublisherInfo, SiteInfo, UserInfo, }; use crate::cache_policy::{ CachePolicy, EdgeCacheHeader, cache_control_headers_are_private_or_no_store, }; -use crate::consent::{consent_allows_server_side_auction, gate_eids_by_consent}; -use crate::constants::{COOKIE_SHAREDID, COOKIE_TS_EIDS, HEADER_X_COMPRESS_HINT}; +use crate::consent::{consent_allows_server_side_auction, gate_eids_by_permissions}; +use crate::constants::{COOKIE_EIDS, COOKIE_SHAREDID, HEADER_X_COMPRESS_HINT}; use crate::cookies::handle_request_cookies; use crate::cookies::template_cache_policy::{TemplateCookieDecision, evaluate_cookie_policy}; use crate::creative_opportunities::{ @@ -69,8 +69,16 @@ use crate::ec::registry::PartnerRegistry; use crate::ec::{EcContext, EidSyncSource}; use crate::error::TrustedServerError; use crate::html_processor::BodyCloseInjection; -use crate::http_util::{RequestInfo, is_navigation_request, serve_static_with_etag}; -use crate::integrations::IntegrationRegistry; +use crate::html_processor::ServePass; +use crate::http_util::{ + RequestInfo, is_navigation_request, remove_deployment_headers, serve_static_with_etag, +}; +use crate::integrations::{IntegrationDocumentState, IntegrationRegistry, IntegrationRequestState}; +use crate::middleware::{ + MiddlewarePhase, Prepared, Refusal, ResponseHeader, media_type_of, write_response_headers, +}; +use crate::module_context::{ModuleContext, ModuleRequest}; +use crate::permissions::PermissionState; use crate::platform::{ GeoInfo, PlatformBackendSpec, PlatformHttpRequest, RuntimeServices, TEMPLATE_CACHE_PURGE_ALL_SURROGATE_KEY, VarySpec, contains_publisher_esi_directive, @@ -526,8 +534,11 @@ fn encode_complete_body( /// Serves two types of bundles: /// - **Unified bundle** (`tsjs-unified.min.js`): core + immediate (non-deferred) /// integration modules. -/// - **Deferred module** (`tsjs-{id}.min.js`): a single self-contained IIFE for -/// modules loaded with `defer` (e.g., prebid). +/// - **Single module** (`tsjs-{id}.min.js`): a single self-contained IIFE for +/// modules loaded with `defer` (e.g., prebid) or registered as standalone. +/// +/// Every module comes from the registry's parts, so a module a registration +/// carries is served and versioned like a compile-time one. /// /// # Errors /// @@ -548,28 +559,28 @@ pub fn handle_tsjs_dynamic( if UNIFIED_FILENAMES.contains(&filename) { // Serve core + immediate modules (excludes deferred like prebid). - let module_ids = integration_registry.js_module_ids_immediate(); - let body = trusted_server_js::concatenate_modules(&module_ids); - let hash = trusted_server_js::concatenated_hash(&module_ids); + let parts = integration_registry.js_parts_immediate(); + let body = crate::tsjs_bundle::compose(&parts); + let hash = crate::tsjs_bundle::compose_hash(&parts); return Ok(serve_tsjs_static(req, &body, &hash, edge_header)); } - if let Some(module_id) = parse_single_module_filename(filename) { - // Deferred modules and the conditionally injected diagnostics module - // are served as content-addressed standalone assets. Delivery remains - // cookie-independent so the static response can stay publicly cached. - let deferred_ids = integration_registry.js_module_ids_deferred(); - let diagnostics_standalone = module_id - == crate::integrations::gpt_diagnostics::GPT_DIAGNOSTICS_INTEGRATION_ID - && integration_registry.integration_enabled(module_id); - if !deferred_ids.contains(&module_id) && !diagnostics_standalone { + if let Some(module_id) = parse_single_module_filename(filename, integration_registry) { + // Deferred modules and standalone modules are served as + // content-addressed single files. Delivery remains cookie-independent + // so the static response can stay publicly cached. + let deferred = integration_registry.js_module_ids_deferred(); + let standalone = integration_registry.js_standalone_ids(); + if !deferred.contains(&module_id) && !standalone.contains(&module_id) { return Ok(not_found_response()); } - if let (Some(content), Some(hash)) = ( - trusted_server_js::module_bundle(module_id), - trusted_server_js::single_module_hash(module_id), - ) { - return Ok(serve_tsjs_static(req, content, hash, edge_header)); + if let Some(part) = integration_registry.js_part(module_id) { + return Ok(serve_tsjs_static( + req, + part.source, + part.sha256, + edge_header, + )); } } @@ -605,21 +616,22 @@ fn request_version_hash(req: &Request) -> Option<&str> { }) } -/// Extract a module ID from a deferred-module filename like `tsjs-sourcepoint.min.js`. +/// Extract a module ID from a single-module filename like `tsjs-sourcepoint.min.js`. /// -/// Returns `Some(&'static str)` if the filename matches a known JS module ID, -/// `None` otherwise. The caller must additionally verify that the module is -/// both deferred and enabled via the [`IntegrationRegistry`]. +/// Returns `Some(&'static str)` if the filename names a module the registry +/// serves somewhere (bundle, deferred or standalone), resolved through the +/// registry so a carried module is found. `None` otherwise. The caller must +/// additionally verify that the module is deferred or standalone. #[must_use] -fn parse_single_module_filename(filename: &str) -> Option<&'static str> { +fn parse_single_module_filename( + filename: &str, + registry: &IntegrationRegistry, +) -> Option<&'static str> { let stem = filename .strip_prefix("tsjs-") .and_then(|s| s.strip_suffix(".min.js").or_else(|| s.strip_suffix(".js")))?; - trusted_server_js::all_module_ids() - .iter() - .copied() - .find(|&id| id == stem) + registry.js_module_id(stem) } /// Parameters for processing response streaming. @@ -629,22 +641,256 @@ struct ProcessResponseParams<'a> { origin_url: &'a str, request_host: &'a str, request_scheme: &'a str, + /// See [`OwnedProcessResponseParams::request_path`]. + request_path: &'a str, settings: &'a Settings, content_type: &'a str, integration_registry: &'a IntegrationRegistry, + /// Head script carrying this request's permission state, or [`None`] under a + /// shared-template mode. See [`template_permissions_script`]. + permissions_script: Option<&'a str>, ad_slots_script: Option<&'a str>, ad_bids_state: &'a Arc>>, - suppress_datadome_client_side_tag: bool, - gpt_diagnostics: - Option<&'a crate::integrations::gpt_diagnostics::GptDiagnosticsRequestDecision>, + /// See [`HtmlStreamProcessorParams::request_state`]. + request_state: &'a IntegrationRequestState, /// See [`HtmlStreamProcessorParams::shared_template_authorized`]. shared_template_authorized: bool, /// See [`HtmlStreamProcessorParams::csp_nonce_observed`]. csp_nonce_observed: Option<&'a Arc>, + prepared: &'a Arc, + /// See [`OwnedProcessResponseParams::public_prefix`]. + public_prefix: &'a crate::prefix::PublicPrefix, } struct PublisherBodyProcessor { inner: Box, + /// What the serve middleware write on the response, taken by the + /// caller before the response leaves. + response_headers: Vec, +} + +/// A response as the middleware the entries covering it are told of it: +/// any document for its preparation, and one that is not HTML for the +/// stream processors run on its body. +struct TextResponse<'a> { + /// The response's `Content-Type`, parameters and all. + content_type: &'a str, + request_path: &'a str, + request_host: &'a str, + request_scheme: &'a str, + origin_host: &'a str, + request_state: &'a IntegrationRequestState, + max_buffered_script_bytes: usize, + /// What the middleware prepared for this document, see + /// [`Middleware::prepare`](crate::middleware::Middleware::prepare). + prepared: &'a Prepared, +} + +/// The stream processors the entries covering a response that is not HTML +/// run on its body, the fetch entry's and then the serve entry's, with the +/// headers the serve entry's middleware write on the response, or none +/// when its media type is not one a middleware runs on, its charset is not +/// UTF-8 or no entry covers it. Both phases run here because the template +/// cache stores HTML alone. +/// +/// # Errors +/// +/// When a middleware decides about HTML on a response that is not HTML, or +/// asks for a selector that does not parse. +fn middleware_stream_processors( + response: &TextResponse<'_>, + settings: &Settings, + integration_registry: &IntegrationRegistry, +) -> Result> { + use crate::middleware::{ + MiddlewareContext, is_dispatched_media_type, is_utf8_or_undeclared_charset, + }; + + let media_type = media_type_of(response.content_type); + if !is_dispatched_media_type(&media_type) + || !is_utf8_or_undeclared_charset(response.content_type) + { + return Ok(TextPass::default()); + } + let mut processors = Vec::new(); + let mut response_headers = Vec::new(); + for phase in &MiddlewarePhase::ALL { + let chain = integration_registry.middleware_chain( + settings.phase_entries(*phase), + *phase, + &media_type, + response.request_path, + ); + if chain.is_empty() { + continue; + } + let document_state = IntegrationDocumentState::default(); + if *phase == MiddlewarePhase::Serve { + response.request_state.seed(&document_state); + } + let plan = chain + .plan(&MiddlewareContext { + phase: *phase, + request_host: response.request_host, + request_scheme: response.request_scheme, + origin_host: response.origin_host, + platform: integration_registry.platform(), + document_state: &document_state, + max_buffered_script_bytes: response.max_buffered_script_bytes, + prepared: response.prepared, + }) + .map_err(|message| Report::new(TrustedServerError::Configuration { message }))?; + processors.extend(plan.processors); + response_headers.extend(plan.response_headers); + } + Ok(TextPass { + processors, + response_headers, + }) +} + +/// What the entries covering a text response do to it: the processors over +/// its body and the headers the serve entry's middleware write. +#[derive(Default)] +struct TextPass { + processors: Vec>, + response_headers: Vec, +} + +/// Runs the preparation of the middleware the entries of each phase in +/// `phases` select for `document`, each as its module with the context the +/// phase is given, keeping what they prepare in the document's [`Prepared`]. +/// Each phase's preparations are handed a fresh document state, not the one +/// the plan is handed, so anything a preparation writes there is dropped. +/// +/// # Errors +/// +/// The first refusal, which ends the request. +async fn prepare_document( + document: &TextResponse<'_>, + phases: &[(MiddlewarePhase, &ModuleContext<'_>)], + settings: &Settings, + integration_registry: &IntegrationRegistry, +) -> Result<(), Refusal> { + use crate::middleware::{MiddlewareContext, is_dispatched_media_type}; + + let media_type = media_type_of(document.content_type); + if !is_dispatched_media_type(&media_type) { + return Ok(()); + } + for (phase, module_context) in phases { + let chain = integration_registry.middleware_chain( + settings.phase_entries(*phase), + *phase, + &media_type, + document.request_path, + ); + if chain.is_empty() { + continue; + } + let document_state = IntegrationDocumentState::default(); + if *phase == MiddlewarePhase::Serve { + document.request_state.seed(&document_state); + } + chain + .prepare( + module_context, + &MiddlewareContext { + phase: *phase, + request_host: document.request_host, + request_scheme: document.request_scheme, + origin_host: document.origin_host, + platform: integration_registry.platform(), + document_state: &document_state, + max_buffered_script_bytes: document.max_buffered_script_bytes, + prepared: document.prepared, + }, + ) + .await?; + } + Ok(()) +} + +/// The context a phase's preparations of a document are called with. The +/// fetch phase of a document that may be stored is handed the request's +/// services and nothing of the reader, because what its preparations answer +/// is stored with the document. Every other preparation, the serve phase's +/// on every document included, is handed the reader's state. +fn preparation_context<'r>( + request: ModuleRequest<'r>, + stored: bool, + settings: &'r Settings, + ec_context: &'r EcContext, + services: &'r RuntimeServices, +) -> ModuleContext<'r> { + let context = ModuleContext::new(request).with_settings(settings); + if stored { + context.with_services(services) + } else { + context.with_request_state(ec_context, services) + } +} + +/// The contexts the two phases of a fetched document are prepared with. The +/// fetch phase's carries the services alone when the document may be stored, +/// because what it answers is stored with the document, and the serve +/// phase's carries the reader's state on every document, because what it +/// answers is this reader's, so the reader a document is fetched for is +/// prepared as every later reader of the stored copy is. +fn preparation_contexts<'r>( + request: impl Fn() -> ModuleRequest<'r>, + stored: bool, + settings: &'r Settings, + ec_context: &'r EcContext, + services: &'r RuntimeServices, +) -> (ModuleContext<'r>, ModuleContext<'r>) { + ( + preparation_context(request(), stored, settings, ec_context, services), + preparation_context(request(), false, settings, ec_context, services), + ) +} + +/// The response a refusal answers the request with: its status, +/// `Retry-After` when it gives one, the message as plain text, and no +/// store. +fn refusal_response(refusal: &Refusal) -> Result, Report> { + let mut builder = Response::builder() + .status(refusal.status) + .header(header::CONTENT_TYPE, "text/plain; charset=utf-8") + .header(header::CACHE_CONTROL, "no-store"); + if let Some(after) = refusal.retry_after { + let seconds = after.as_secs() + u64::from(after.subsec_nanos() > 0); + builder = builder.header(header::RETRY_AFTER, seconds.to_string()); + } + builder + .body(EdgeBody::from(refusal.message.clone().into_bytes())) + .change_context(TrustedServerError::Proxy { + message: "failed to build a middleware's refusal".to_string(), + }) +} + +/// A processor followed by the ones the entries add, each handed what the +/// one before produced. +struct WithStreamProcessors { + inner: Box, + processors: Vec>, +} + +impl StreamProcessor for WithStreamProcessors { + fn process_chunk(&mut self, chunk: &[u8], is_last: bool) -> Result, std::io::Error> { + let mut output = self.inner.process_chunk(chunk, is_last)?; + for processor in &mut self.processors { + output = processor.process_chunk(&output, is_last)?; + } + Ok(output) + } + + fn reset(&mut self) { + self.inner.reset(); + for processor in &mut self.processors { + processor.reset(); + } + } } impl PublisherBodyProcessor { @@ -668,23 +914,29 @@ impl PublisherBodyProcessor { params.ad_slots_script.is_some(), is_html && params.dispatched_auction.is_some(), ); + let mut response_headers = Vec::new(); let inner: Box = if is_html { - Box::new(create_html_stream_processor(HtmlStreamProcessorParams { + let (processor, headers) = create_html_stream_processor(HtmlStreamProcessorParams { origin_host: ¶ms.origin_host, request_host: ¶ms.request_host, request_scheme: ¶ms.request_scheme, + request_path: ¶ms.request_path, settings, integration_registry, + permissions_script: permissions_script_for(params, settings), ad_slots_script: params.ad_slots_script.as_deref().map(str::to_string), ad_bids_state: Arc::clone(params.ad_bids_state.script_cell()), - suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, - gpt_diagnostics: params.gpt_diagnostics.clone(), + request_state: params.request_state.clone(), shared_template_authorized: params.template_cache_key.is_some(), csp_nonce_observed: params.csp_nonce_observed.clone(), + prepared: Arc::clone(¶ms.prepared), + public_prefix: ¶ms.public_prefix, deferred_inline_marker: inline_seam_token .as_ref() .map(|token| String::from_utf8_lossy(token).into_owned()), - })?) + })?; + response_headers = headers; + Box::new(processor) } else if is_rsc_flight { Box::new(RscFlightUrlRewriter::new( ¶ms.origin_host, @@ -693,15 +945,46 @@ impl PublisherBodyProcessor { ¶ms.request_scheme, )) } else { - Box::new(create_url_replacer( - ¶ms.origin_host, - ¶ms.origin_url, - ¶ms.request_host, - ¶ms.request_scheme, - )) + let pass = middleware_stream_processors( + &TextResponse { + content_type: ¶ms.content_type, + request_path: ¶ms.request_path, + request_host: ¶ms.request_host, + request_scheme: ¶ms.request_scheme, + origin_host: ¶ms.origin_host, + request_state: ¶ms.request_state, + max_buffered_script_bytes: settings.publisher.max_buffered_body_bytes, + prepared: ¶ms.prepared, + }, + settings, + integration_registry, + )?; + response_headers = pass.response_headers; + Box::new(WithStreamProcessors { + inner: Box::new(create_url_replacer( + ¶ms.origin_host, + ¶ms.origin_url, + ¶ms.request_host, + ¶ms.request_scheme, + )), + processors: pass.processors, + }) }; - Ok((Self { inner }, inline_seam_token)) + Ok(( + Self { + inner, + response_headers, + }, + inline_seam_token, + )) + } +} + +impl PublisherBodyProcessor { + /// The headers the serve middleware write on the response, once. + fn take_response_headers(&mut self) -> Vec { + std::mem::take(&mut self.response_headers) } } @@ -726,7 +1009,7 @@ fn process_response_streaming( output: &mut W, params: &ProcessResponseParams, output_compression: Compression, -) -> Result<(), Report> { +) -> Result, Report> { let is_html = is_html_content_type(params.content_type); let is_rsc_flight = content_type_contains_ascii_case_insensitive(params.content_type, "text/x-component"); @@ -752,21 +1035,26 @@ fn process_response_streaming( // gzip would reject bodies the identity, deflate and brotli paths accept. let max_pending_decoded_bytes = params.settings.publisher.max_buffered_body_bytes; + let mut response_headers = Vec::new(); if is_html { - let processor = create_html_stream_processor(HtmlStreamProcessorParams { + let (processor, headers) = create_html_stream_processor(HtmlStreamProcessorParams { origin_host: params.origin_host, request_host: params.request_host, request_scheme: params.request_scheme, + request_path: params.request_path, settings: params.settings, integration_registry: params.integration_registry, + permissions_script: params.permissions_script.map(str::to_string), ad_slots_script: params.ad_slots_script.map(str::to_string), ad_bids_state: params.ad_bids_state.clone(), - suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, - gpt_diagnostics: params.gpt_diagnostics.cloned(), + request_state: params.request_state.clone(), shared_template_authorized: params.shared_template_authorized, csp_nonce_observed: params.csp_nonce_observed.cloned(), + prepared: Arc::clone(params.prepared), + public_prefix: params.public_prefix, deferred_inline_marker: None, })?; + response_headers = headers; StreamingPipeline::new(config, processor) .with_max_pending_decoded_bytes(max_pending_decoded_bytes) .process(body_as_reader(body)?, output)?; @@ -783,18 +1071,36 @@ fn process_response_streaming( .with_max_pending_decoded_bytes(max_pending_decoded_bytes) .process(body_as_reader(body)?, output)?; } else { - let replacer = create_url_replacer( - params.origin_host, - params.origin_url, - params.request_host, - params.request_scheme, - ); - StreamingPipeline::new(config, replacer) + let pass = middleware_stream_processors( + &TextResponse { + content_type: params.content_type, + request_path: params.request_path, + request_host: params.request_host, + request_scheme: params.request_scheme, + origin_host: params.origin_host, + request_state: params.request_state, + max_buffered_script_bytes: params.settings.publisher.max_buffered_body_bytes, + prepared: params.prepared, + }, + params.settings, + params.integration_registry, + )?; + response_headers = pass.response_headers; + let processor = WithStreamProcessors { + inner: Box::new(create_url_replacer( + params.origin_host, + params.origin_url, + params.request_host, + params.request_scheme, + )), + processors: pass.processors, + }; + StreamingPipeline::new(config, processor) .with_max_pending_decoded_bytes(max_pending_decoded_bytes) .process(body_as_reader(body)?, output)?; } - Ok(()) + Ok(response_headers) } async fn process_response_streaming_async( @@ -803,7 +1109,7 @@ async fn process_response_streaming_async( params: &OwnedProcessResponseParams, settings: &Settings, integration_registry: &IntegrationRegistry, -) -> Result<(), Report> { +) -> Result, Report> { log::debug!( "process_response_streaming_async: content_type={}, content_encoding={}", params.content_type, @@ -822,6 +1128,7 @@ async fn process_response_streaming_async( // pending auction; `deferred_inline_seam_token` returns `None` for it. let (mut processor, inline_seam_token) = PublisherBodyProcessor::new(params, settings, integration_registry)?; + let response_headers = processor.take_response_headers(); if inline_seam_token.is_some() { // A `debug_assert!` would be compiled out of the release wasm builds that // actually ship, which is exactly where an unresolved token would reach a @@ -839,7 +1146,8 @@ async fn process_response_streaming_async( output_compression, settings.publisher.max_buffered_body_bytes, ) - .await + .await?; + Ok(response_headers) } /// Pull, decode, process, and encode the next chunk of a no-hold pipeline. @@ -1314,12 +1622,18 @@ struct HtmlStreamProcessorParams<'a> { origin_host: &'a str, request_host: &'a str, request_scheme: &'a str, + /// See [`OwnedProcessResponseParams::request_path`]. + request_path: &'a str, settings: &'a Settings, integration_registry: &'a IntegrationRegistry, + /// Head script carrying this request's permission state, or [`None`] under a + /// shared-template mode. See [`template_permissions_script`]. + permissions_script: Option, ad_slots_script: Option, ad_bids_state: Arc>>, - suppress_datadome_client_side_tag: bool, - gpt_diagnostics: Option, + /// What modules left on the request, which the `[[serve]]` entry's + /// middleware read where that entry runs on this pass. + request_state: IntegrationRequestState, /// Whether a shared template was authorized for this response. /// /// Carried rather than re-derived so both seams see the same answer. See @@ -1329,29 +1643,9 @@ struct HtmlStreamProcessorParams<'a> { csp_nonce_observed: Option>, /// Request-specific parser marker used by a pending inline auction. deferred_inline_marker: Option, -} - -/// The diagnostics decision the template may carry. -/// -/// Diagnostics is request-scoped — activated by a cookie or query parameter, and -/// documented as an immutable per-request decision — so it must not reach a shared -/// template. -/// -/// It does not leak today even without this gate, but only by coincidence: -/// `requires_private_no_store()` is a strict superset of the conditions under which -/// a script is emitted, and that stamp lands before the template cache gate reads response -/// headers, so the gate refuses. Two independent conditions that happen to align, -/// with nothing enforcing the relationship. This makes the guarantee explicit; -/// `requires_private_no_store_is_a_superset_of_injection` keeps the coincidence as a -/// backstop if this gate is ever removed. -pub(crate) fn template_gpt_diagnostics( - mode: AssemblyMode, - decision: Option, -) -> Option { - match mode { - AssemblyMode::Inline => decision, - AssemblyMode::Esi => None, - } + prepared: Arc, + /// See [`OwnedProcessResponseParams::public_prefix`]. + public_prefix: &'a crate::prefix::PublicPrefix, } /// The marker emitted at the `` seam under [`AssemblyMode::Esi`], reserving the @@ -1472,6 +1766,20 @@ pub(crate) fn body_close_injection( } } +/// The head script this response's permission state belongs in, if any. +/// +/// Derived here rather than carried on [`OwnedProcessResponseParams`] so the state +/// has one representation on the request (the JSON) and the head-or-seam decision is +/// taken from the same effective mode both seams use. Under a shared-template mode +/// the answer is [`None`] and the seam carries the state instead. +fn permissions_script_for( + params: &OwnedProcessResponseParams, + settings: &Settings, +) -> Option { + let mode = effective_assembly_mode(settings, params.template_cache_key.is_some()); + template_permissions_script(mode, ¶ms.permissions_json) +} + fn deferred_inline_seam_token( settings: &Settings, shared_template_authorized: bool, @@ -1495,8 +1803,9 @@ fn deferred_inline_seam_token( fn create_html_stream_processor( params: HtmlStreamProcessorParams<'_>, -) -> Result, Report> { - use crate::html_processor::{HtmlProcessorConfig, create_html_processor}; +) -> Result<(impl StreamProcessor + use<>, Vec), Report> { + use crate::html_processor::{HtmlProcessorConfig, create_html_processor_with_middleware}; + use crate::middleware::{HTML_MEDIA_TYPE, MiddlewarePhase}; let config = HtmlProcessorConfig::from_settings( params.settings, @@ -1514,8 +1823,6 @@ fn create_html_stream_processor( _ => body_close_injection(assembly_mode, params.ad_slots_script.is_some()), }; - let gpt_diagnostics = template_gpt_diagnostics(assembly_mode, params.gpt_diagnostics); - // Only a response that can be stored has a consumer for the observation, so the // handlers are not registered for ordinary inline traffic. let csp_nonce_observed = params @@ -1524,13 +1831,109 @@ fn create_html_stream_processor( .flatten(); let config = config + .with_permissions_script(params.permissions_script) .with_ad_state(params.ad_slots_script, params.ad_bids_state) - .with_gpt_diagnostics(gpt_diagnostics) + .with_request_state(params.request_state) .with_body_close(body_close) .with_csp_nonce_observer(csp_nonce_observed) - .with_datadome_client_tag_suppression(params.suppress_datadome_client_side_tag); + .with_public_prefix(params.public_prefix.clone()) + .with_prepared(params.prepared); + + // The middleware of the first [[fetch]] entry covering this path. + let fetch = params.integration_registry.middleware_chain( + ¶ms.settings.fetch, + MiddlewarePhase::Fetch, + HTML_MEDIA_TYPE, + params.request_path, + ); + // A response that may be stored leaves this processor as the template + // every reader shares, so its [[serve]] entry runs on each reader's copy + // once that copy is assembled. See `run_serve_pass`. + let serve = (!params.shared_template_authorized).then(|| { + params.integration_registry.middleware_chain( + ¶ms.settings.serve, + MiddlewarePhase::Serve, + HTML_MEDIA_TYPE, + params.request_path, + ) + }); + create_html_processor_with_middleware(config, &fetch, serve.as_ref()) + .map_err(|message| Report::new(TrustedServerError::Configuration { message })) +} + +/// What the `[[serve]]` entry covering this request does to a reader's copy +/// that is assembled and not yet encoded, and to the response, which is +/// nothing when no entry covers the request. +/// +/// The origin host is read from the settings. A page served from the store +/// keeps no record of it, and a serve middleware is told the same about a +/// page from the store as about one fetched for this request. +fn reader_serve_processor( + params: &OwnedProcessResponseParams, + settings: &Settings, + integration_registry: &IntegrationRegistry, +) -> Result> { + use crate::html_processor::{ReaderDocument, create_serve_processor}; + use crate::middleware::{HTML_MEDIA_TYPE, MiddlewarePhase}; + + if !is_html_content_type(¶ms.content_type) { + return Ok(ServePass::default()); + } + let chain = integration_registry.middleware_chain( + &settings.serve, + MiddlewarePhase::Serve, + HTML_MEDIA_TYPE, + ¶ms.request_path, + ); + let origin_host = settings.publisher.origin_host(); + create_serve_processor( + &chain, + &ReaderDocument { + request_host: ¶ms.request_host, + request_scheme: ¶ms.request_scheme, + origin_host: &origin_host, + platform: integration_registry.platform(), + request_state: ¶ms.request_state, + max_buffered_script_bytes: settings.publisher.max_buffered_body_bytes, + prepared: ¶ms.prepared, + }, + ) + .map_err(|message| Report::new(TrustedServerError::Configuration { message })) +} + +/// Runs the `[[serve]]` entry covering this request over a reader's whole +/// copy, assembled and not yet encoded, and writes its headers on the +/// response. +fn run_serve_pass( + bytes: Vec, + params: &OwnedProcessResponseParams, + settings: &Settings, + integration_registry: &IntegrationRegistry, + response: &mut Response, +) -> Result, Report> { + let pass = reader_serve_processor(params, settings, integration_registry)?; + write_response_headers(response, pass.response_headers); + let Some(mut processor) = pass.processor else { + return Ok(bytes); + }; + processor + .process_chunk(&bytes, true) + .change_context(TrustedServerError::Proxy { + message: "a serve middleware could not rewrite the page".to_string(), + }) +} - Ok(create_html_processor(config)) +/// One piece of a streamed reader's copy through the `[[serve]]` entry, when +/// one covers the request. +fn serve_chunk( + serve: &mut Option>, + bytes: Vec, + is_last: bool, +) -> Result, std::io::Error> { + match serve { + Some(processor) => processor.process_chunk(&bytes, is_last), + None => Ok(bytes), + } } /// Result of publisher request handling, indicating whether the response body @@ -1649,6 +2052,43 @@ pub(crate) fn classify_response_route( ResponseRoute::Stream } +/// Whether the origin answered with a page where `entry` places core's +/// robots.txt middleware, as a single-page application's catch-all does. A +/// crawler reads a page there as no rules at all, so the file is made as it is +/// where the origin has none. +fn answers_robots_txt_with_a_page( + response: &Response, + entry: &crate::middleware::PhaseEntry, +) -> bool { + response.status() == StatusCode::OK + && entry + .middleware + .iter() + .any(|name| name == crate::robots_txt::middleware::MODULE) + && response + .headers() + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .is_some_and(is_html_content_type) +} + +/// An empty document of `media_type`, which the middleware a `[[fetch]]` +/// entry names for a path the origin has nothing at is run on. +fn empty_document(media_type: &str) -> Result, Report> { + let content_type = if media_type.starts_with("text/") { + format!("{media_type}; charset=utf-8") + } else { + media_type.to_owned() + }; + Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, content_type) + .body(EdgeBody::empty()) + .change_context(TrustedServerError::Proxy { + message: format!("failed to build an empty {media_type} document"), + }) +} + fn response_cache_control_is_private_or_no_store(response: &Response) -> bool { cache_control_headers_are_private_or_no_store(response.headers()) } @@ -1699,6 +2139,14 @@ pub struct OwnedProcessResponseParams { /// /// Request-scoped, so it travels with the request rather than into the template. pub(crate) seam_ad_slots: Option, + /// This request's resolved permission state as page JSON, from + /// [`PermissionState::page_json`]. + /// + /// Carried as the JSON rather than as a rendered script because it is delivered in + /// two different places: the head under an inline response, and the `` seam + /// under a shared-template one. An empty string means no caller filled it in and + /// renders as the empty state. + pub(crate) permissions_json: String, /// Origin policy headers to store with the template and replay on a hit. pub(crate) policy_headers: Vec<(String, String)>, pub(crate) content_encoding: String, @@ -1706,6 +2154,9 @@ pub struct OwnedProcessResponseParams { pub(crate) origin_url: String, pub(crate) request_host: String, pub(crate) request_scheme: String, + /// The path the reader asked for, which decides the `[[fetch]]` and + /// `[[serve]]` entries that cover the document. + pub(crate) request_path: String, pub(crate) content_type: String, pub(crate) ad_slots_script: Option, pub(crate) ad_bids_state: AdBidsState, @@ -1719,17 +2170,21 @@ pub struct OwnedProcessResponseParams { pub(crate) dispatched_auction: Option, /// Price granularity used to bucket bids when building `tsjs.bids`. pub(crate) price_granularity: PriceGranularity, - /// Whether to omit Trusted Server's automatic `DataDome` client-side tag. - pub(crate) suppress_datadome_client_side_tag: bool, - /// Request-scoped conditional diagnostics delivery decision. - pub(crate) gpt_diagnostics: - Option, + /// What modules left on the request, for their serve middleware and + /// their response finalizers. + pub(crate) request_state: IntegrationRequestState, /// Set by the transform when the document carries a response-bound CSP nonce. /// /// `None` wherever no transform runs. Recorded by the HTML parser rather than /// rescanned from the output, which cannot tell a `nonce` attribute from the same /// word inside a script. pub(crate) csp_nonce_observed: Option>, + /// What the middleware prepared for this document, see + /// [`Middleware::prepare`](crate::middleware::Middleware::prepare). + pub(crate) prepared: Arc, + /// The request's public prefix, from its services, which the page's + /// bundle tag tells the client script. + pub(crate) public_prefix: crate::prefix::PublicPrefix, } /// Response-authorized template cache insert inputs. The key is built before origin lookup; the @@ -1824,7 +2279,7 @@ pub async fn buffer_publisher_response_async( return Ok(response); } let mut output = BoundedWriter::new(settings.publisher.max_buffered_body_bytes); - stream_publisher_body_async( + let response_headers = stream_publisher_body_async( body, &mut output, &mut params, @@ -1834,6 +2289,7 @@ pub async fn buffer_publisher_response_async( services, ) .await?; + write_response_headers(&mut response, response_headers); // Authorized template cache transforms are emitted as identity by // `process_response_streaming_async`; inline transforms retain the origin // coding. This avoids recompressing and immediately decoding a full document. @@ -1938,6 +2394,15 @@ pub async fn buffer_publisher_response_async( set_assembly_response_state(&mut response, state); } let bytes = if was_authorized { + // The [[serve]] entry runs here on a shared template, and + // inside the transform on a page that was never authorized. + let bytes = run_serve_pass( + bytes, + ¶ms, + settings, + integration_registry, + &mut response, + )?; encode_complete_body(bytes, response_compression(&response))? } else { bytes @@ -1987,6 +2452,13 @@ pub async fn buffer_publisher_response_async( assembled.extend_from_slice(head); assembled.extend_from_slice(seam.as_bytes()); assembled.extend_from_slice(tail); + let assembled = run_serve_pass( + assembled, + ¶ms, + settings, + integration_registry, + &mut response, + )?; let assembled = encode_complete_body(assembled, response_compression(&response))?; response.headers_mut().insert( http::header::CONTENT_LENGTH, @@ -2064,7 +2536,7 @@ fn assemble_if_shared( Ok((out, Some(AssemblyResponseState::ByteSeamFallback))) } -/// Fingerprint of settings, browser bundles, and the core implementation. +/// Fingerprint of settings, the browser modules a registry serves, and the core implementation. /// /// This intentionally over-invalidates. Trying to maintain a hand-written list already /// omitted publisher origin identity and creative-opportunity shaping fields. A digest of @@ -2074,20 +2546,44 @@ fn assemble_if_shared( /// files (including embedded head scripts), build logic, manifests, and the workspace /// lockfile when present, so implementation edits automatically invalidate templates. /// +/// The module digest covers the parts this registry can actually serve, carried modules +/// included, so a vendor crate that rebuilds its browser module moves the hash, and +/// templates keyed under the previous hash are no longer read. It also narrows the +/// module set from every module compiled into the binary to the ones this deployment can +/// serve. That is safe because a module this deployment never serves appears in no bundle +/// hash and no injected URL, so it cannot change what a template renders. +/// /// # Panics /// /// Does not panic: serializing the already-deserialized typed settings to a JSON value is /// infallible for this schema. -fn template_fingerprint(settings: &Settings) -> String { - template_fingerprint_with_build_digest(settings, TEMPLATE_BUILD_DIGEST) +fn template_fingerprint( + settings: &Settings, + integration_registry: &IntegrationRegistry, + public_prefix: &crate::prefix::PublicPrefix, +) -> String { + template_fingerprint_with_build_digest( + settings, + integration_registry, + public_prefix, + TEMPLATE_BUILD_DIGEST, + ) } -fn template_fingerprint_with_build_digest(settings: &Settings, build_digest: &str) -> String { +fn template_fingerprint_with_build_digest( + settings: &Settings, + integration_registry: &IntegrationRegistry, + public_prefix: &crate::prefix::PublicPrefix, + build_digest: &str, +) -> String { let mut hasher = sha2::Sha256::new(); hasher.update(build_digest.as_bytes()); - hasher.update( - trusted_server_js::concatenated_hash(trusted_server_js::all_module_ids()).as_bytes(), - ); + hasher + .update(crate::tsjs_bundle::compose_hash(&integration_registry.js_parts_all()).as_bytes()); + // The bundle tag carries the public prefix, and a change to the selected + // module can change it with nothing in the settings changing. + hasher.update(public_prefix.as_str().as_bytes()); + hasher.update([0]); // EdgeZero's canonical form sorts object keys itself, so independently deserialized // HashMaps hash identically even when a dependency enables `serde_json/preserve_order`. // Array order is preserved: set-valued settings must serialize deterministically @@ -2117,21 +2613,28 @@ fn response_carries_a_seam_marker(was_authorized: bool, settings: &Settings) -> /// calls `scheduleInitialAdInit`, which schedules `adInit` for precisely the traffic /// that opted out. Absent is not the same as empty here. /// +/// It is never nothing at all, because the permission state has to reach the +/// page whether or not the ad stack ran, and a shared template's head cannot carry it. +/// The no-ad-stack answer is [`build_permissions_seam_script`], which sets the state and +/// schedules no ad init. +/// /// Shared by the miss path and by **both** hit finalizers. They previously each spelled /// the decision out, and the two hit paths spelled it `unwrap_or("[]")` — so the gate /// held on a cache miss and was ignored on every cache hit. fn seam_script_for(params: &OwnedProcessResponseParams) -> String { - params - .seam_ad_slots - .as_deref() - .map(|slots| params.ad_bids_state.build_seam_script(slots)) - .unwrap_or_default() + match params.seam_ad_slots.as_deref() { + Some(slots) => params + .ad_bids_state + .build_seam_script(slots, ¶ms.permissions_json), + None => build_permissions_seam_script(¶ms.permissions_json), + } } /// Builds the injection state a cached template needs on the way out. /// -/// The template carries no auction state — that is what makes it shareable — so the -/// per-reader parts are attached here, from this request. +/// The template carries no auction state and no permission state, which is what makes +/// it shareable, so the per-reader parts are attached here, from this request. Both +/// leave through the seam, never through the cached head. fn build_template_assembly_params( entry: &crate::platform::TemplateEntry, settings: &Settings, @@ -2139,9 +2642,11 @@ fn build_template_assembly_params( request_scheme: &str, price_granularity: PriceGranularity, ad_bids_state: AdBidsState, + permissions_json: String, ) -> OwnedProcessResponseParams { OwnedProcessResponseParams { csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), // Already stored; storing again on a hit would be pointless work. template_cache_key: None, seam_ad_slots: None, @@ -2151,16 +2656,22 @@ fn build_template_assembly_params( origin_url: settings.publisher.origin_url.clone(), request_host: request_host.to_string(), request_scheme: request_scheme.to_string(), + // Set by the caller, with the rest of what belongs to the request. + request_path: String::new(), content_type: entry.metadata.content_type.clone(), // The template already carries the head seam; re-injecting would duplicate it. + permissions_json, ad_slots_script: None, ad_bids_state, auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity, - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + // A request that carries any keeps to the origin path, so none reaches a + // document served from a template. + request_state: IntegrationRequestState::default(), + // Set by the caller, from the request's services. + public_prefix: crate::prefix::PublicPrefix::default(), } } @@ -2457,6 +2968,26 @@ pub async fn publisher_response_into_streaming_response( return Ok(response); } + // Planned before the lazy body is built, so a serve entry that + // cannot run fails the response while it still can be failed. + let serve_pass = match reader_serve_processor(¶ms, &settings, integration_registry) + { + Ok(serve_pass) => serve_pass, + Err(err) => { + if let Some(dispatched) = params.dispatched_auction.take() { + emit_abandoned_auction( + &services, + params.auction_observation.take(), + dispatched, + "processor_init_error", + ) + .await; + } + return Err(err); + } + }; + write_response_headers(&mut response, serve_pass.response_headers); + let mut serve = serve_pass.processor; let services = services.clone(); let settings = Arc::clone(&settings); let orchestrator = Arc::clone(&orchestrator); @@ -2480,8 +3011,10 @@ pub async fn publisher_response_into_streaming_response( let (head, tail) = split_template_at_seam(&template) .map_err(|e| std::io::Error::other(e.to_string()))?; let mut encoder = BodyStreamEncoder::new(compression); + // Each piece goes through the serve entry as it leaves, so + // the head is still written before the auction is awaited. let encoded_head = encoder - .encode_chunk(head.to_vec()) + .encode_chunk(serve_chunk(&mut serve, head.to_vec(), false)?) .map_err(publisher_stream_error)?; if !encoded_head.is_empty() { yield bytes::Bytes::from(encoded_head); @@ -2512,14 +3045,14 @@ pub async fn publisher_response_into_streaming_response( let seam = seam_script_for(¶ms); if !seam.is_empty() { let encoded_seam = encoder - .encode_chunk(seam.into_bytes()) + .encode_chunk(serve_chunk(&mut serve, seam.into_bytes(), false)?) .map_err(publisher_stream_error)?; if !encoded_seam.is_empty() { yield bytes::Bytes::from(encoded_seam); } } let encoded_tail = encoder - .encode_chunk(tail.to_vec()) + .encode_chunk(serve_chunk(&mut serve, tail.to_vec(), true)?) .map_err(publisher_stream_error)?; if !encoded_tail.is_empty() { yield bytes::Bytes::from(encoded_tail); @@ -2589,6 +3122,7 @@ pub async fn publisher_response_into_streaming_response( return Err(err); } }; + write_response_headers(&mut response, processor.take_response_headers()); // The guard is created before the lazy stream so an auction whose // response body is dropped unpolled still logs the loss. let dispatched_auction = params.dispatched_auction.take().map(|dispatched| { @@ -2761,6 +3295,13 @@ fn is_html_document_request(req: &Request) -> bool { is_navigation_request(req) } +/// Whether an integration marked this request's response as personalized. +fn request_requires_personalized_delivery(req: &Request) -> bool { + req.extensions() + .get::() + .is_some() +} + /// Removes request headers that can produce a bodyless or partial origin response. fn strip_conditional_and_range_headers(req: &mut Request) { req.headers_mut().remove(header::IF_NONE_MATCH); @@ -2784,14 +3325,14 @@ fn response_carries_body(method: &Method, status: StatusCode) -> bool { && status != StatusCode::NOT_MODIFIED } -/// Prevent shared caches from replaying tag-suppressed HTML to other clients. -fn apply_datadome_client_tag_cache_privacy( +/// Prevent shared caches from replaying personalized HTML to other clients. +fn apply_personalized_response_cache_privacy( response: &mut Response, method: &Method, - suppress_datadome_client_side_tag: bool, + response_is_personalized: bool, content_type: &str, ) { - if suppress_datadome_client_side_tag + if response_is_personalized && response_carries_body(method, response.status()) && is_html_content_type(content_type) { @@ -2903,22 +3444,26 @@ pub fn stream_publisher_body( params: &OwnedProcessResponseParams, settings: &Settings, integration_registry: &IntegrationRegistry, -) -> Result<(), Report> { +) -> Result, Report> { + let permissions_script = permissions_script_for(params, settings); let borrowed = ProcessResponseParams { content_encoding: ¶ms.content_encoding, origin_host: ¶ms.origin_host, origin_url: ¶ms.origin_url, request_host: ¶ms.request_host, request_scheme: ¶ms.request_scheme, + request_path: ¶ms.request_path, settings, content_type: ¶ms.content_type, integration_registry, + permissions_script: permissions_script.as_deref(), ad_slots_script: params.ad_slots_script.as_deref(), ad_bids_state: params.ad_bids_state.script_cell(), - suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, - gpt_diagnostics: params.gpt_diagnostics.as_ref(), + request_state: ¶ms.request_state, shared_template_authorized: params.template_cache_key.is_some(), csp_nonce_observed: params.csp_nonce_observed.as_ref(), + prepared: ¶ms.prepared, + public_prefix: ¶ms.public_prefix, }; let input_compression = Compression::from_content_encoding(¶ms.content_encoding); let output_compression = if params.template_cache_key.is_some() { @@ -2957,7 +3502,7 @@ pub async fn stream_publisher_body_async( integration_registry: &IntegrationRegistry, orchestrator: &AuctionOrchestrator, services: &RuntimeServices, -) -> Result<(), Report> { +) -> Result, Report> { let Some(dispatched) = params.dispatched_auction.take() else { if body.is_stream() { return process_response_streaming_async( @@ -3012,34 +3557,38 @@ pub async fn stream_publisher_body_async( params.ad_slots_script.is_some(), true, ); - let mut processor = match create_html_stream_processor(HtmlStreamProcessorParams { - origin_host: ¶ms.origin_host, - request_host: ¶ms.request_host, - request_scheme: ¶ms.request_scheme, - settings, - integration_registry, - ad_slots_script: params.ad_slots_script.as_deref().map(str::to_string), - ad_bids_state: Arc::clone(params.ad_bids_state.script_cell()), - suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, - gpt_diagnostics: params.gpt_diagnostics.clone(), - shared_template_authorized: params.template_cache_key.is_some(), - csp_nonce_observed: params.csp_nonce_observed.clone(), - deferred_inline_marker: inline_seam_token - .as_ref() - .map(|token| String::from_utf8_lossy(token).into_owned()), - }) { - Ok(processor) => processor, - Err(err) => { - emit_abandoned_auction( - services, - telemetry.observation, - dispatched, - "processor_init_error", - ) - .await; - return Err(err); - } - }; + let (mut processor, response_headers) = + match create_html_stream_processor(HtmlStreamProcessorParams { + prepared: Arc::clone(¶ms.prepared), + public_prefix: ¶ms.public_prefix, + origin_host: ¶ms.origin_host, + request_host: ¶ms.request_host, + request_scheme: ¶ms.request_scheme, + request_path: ¶ms.request_path, + settings, + integration_registry, + permissions_script: permissions_script_for(params, settings), + ad_slots_script: params.ad_slots_script.as_deref().map(str::to_string), + ad_bids_state: Arc::clone(params.ad_bids_state.script_cell()), + request_state: params.request_state.clone(), + shared_template_authorized: params.template_cache_key.is_some(), + csp_nonce_observed: params.csp_nonce_observed.clone(), + deferred_inline_marker: inline_seam_token + .as_ref() + .map(|token| String::from_utf8_lossy(token).into_owned()), + }) { + Ok(built) => built, + Err(err) => { + emit_abandoned_auction( + services, + telemetry.observation, + dispatched, + "processor_init_error", + ) + .await; + return Err(err); + } + }; let input_compression = Compression::from_content_encoding(¶ms.content_encoding); let output_compression = if params.template_cache_key.is_some() { @@ -3067,17 +3616,18 @@ pub async fn stream_publisher_body_async( }, }, ) - .await + .await?; + Ok(response_headers) } -/// Builds the canonical mediator placeholder [`Request`] passed to the collect +/// Builds the canonical ad server placeholder [`Request`] passed to the collect /// phase via [`make_collect_context`]. /// /// The URI is the compile-time constant /// [`MEDIATOR_PLACEHOLDER_URL`](crate::auction::types::MEDIATOR_PLACEHOLDER_URL), /// so the builder is infallible; a default-URI fallback would trip /// [`make_collect_context`]'s `debug_assert_eq!`. -fn mediator_placeholder_request() -> Request { +fn adserver_placeholder_request() -> Request { Request::builder() .uri(crate::auction::types::MEDIATOR_PLACEHOLDER_URL) .body(EdgeBody::empty()) @@ -3150,7 +3700,7 @@ fn rewrite_origin_request( /// See [`AuctionContext::request`]: the orchestrator's collect path runs /// after `send_async` has already consumed the real client request, so this /// context carries a synthetic placeholder. The orchestrator itself -/// instantiates a fresh placeholder when it actually invokes a mediator — +/// instantiates a fresh placeholder when it actually invokes an ad server, /// this argument is plumbing for the (presently unused) case where the /// orchestrator needs the caller's request shape. fn make_collect_context<'a>( @@ -3183,7 +3733,11 @@ pub(crate) const BOT_USER_AGENT_FRAGMENTS: &[&str] = /// Returns true when the request's User-Agent matches any well-known crawler /// fragment in [`BOT_USER_AGENT_FRAGMENTS`]. -pub(crate) fn is_bot_user_agent(req: &Request) -> bool { +/// +/// Public so a module outside this crate treats a crawler the way the page +/// path does. +#[must_use] +pub fn is_bot_user_agent(req: &Request) -> bool { let ua = req .headers() .get("user-agent") @@ -3196,7 +3750,11 @@ pub(crate) fn is_bot_user_agent(req: &Request) -> bool { /// Returns true when the request advertises itself as a prefetch via either /// the standard `Sec-Purpose` or the legacy `Purpose` header. -pub(crate) fn is_prefetch_request(req: &Request) -> bool { +/// +/// Public so a module outside this crate treats a prefetch the way the page +/// path does. +#[must_use] +pub fn is_prefetch_request(req: &Request) -> bool { let header = |name: &str| { req.headers() .get(name) @@ -3295,8 +3853,8 @@ impl AdBidsState { } /// Build the shared-template seam, retaining the same debug prefix as inline. - fn build_seam_script(&self, slots_json: &str) -> String { - let seam = build_seam_script(slots_json, &self.bids()); + fn build_seam_script(&self, slots_json: &str, permissions_json: &str) -> String { + let seam = build_seam_script(slots_json, &self.bids(), permissions_json); let prefix = self .debug_prefix .lock() @@ -3374,11 +3932,10 @@ pub(crate) fn write_bids_to_state( /// enabled cannot bloat every page render without bound. const MAX_AUCTION_DEBUG_DUMP_BYTES: usize = 256 * 1024; -/// Per-bid creative preview length (in bytes) in the `ts-debug` dump. Mirrors -/// the 512-byte upstream-body preview the prebid provider logs on an HTTP error -/// (`integrations/prebid.rs`): enough to identify a creative without copying -/// megabytes of `adm` markup into every page render. The full creative still -/// renders via the injected bids `", + html_escape_for_script(&permissions_json_or_empty(permissions_json)), html_escape_for_script(slots_json), html_escape_for_script(&bids) ) } +/// Build the `` seam script for a request whose ad stack did not run. +/// +/// The head of a shared template carries nothing request-scoped, so the seam is +/// the only place this reader's permission state can be delivered. Before this +/// existed the seam was empty whenever the ad stack was skipped, which left a +/// bot-classified or permission-denied visitor with no state on the page at all. +/// +/// Carries the state and nothing else. It deliberately does not set `adSlots` or +/// `bids` and does not call `scheduleInitialAdInit`, because scheduling `adInit` +/// for traffic that opted out is what the gate in [`seam_script_for`] exists to +/// prevent. +pub(crate) fn build_permissions_seam_script(permissions_json: &str) -> String { + format!( + "", + html_escape_for_script(&permissions_json_or_empty(permissions_json)) + ) +} + /// The slot definitions a shared-mode seam must carry, as JSON. /// /// Mirrors [`template_ad_slots_script`]'s gating: same `should_run_ad_stack` condition, @@ -6148,13 +6888,12 @@ pub(crate) enum TemplateCacheBypassReason { /// still-positive origin lifetime rather than inventing one. #[display("origin response has no positive shared freshness")] NoPositiveFreshness, - /// The request was authenticated. #1009 describes a Basic-Auth-gated - /// deployment, so an authorized response entering a shared cache is a live - /// concern rather than a hypothetical one. + /// The request carried credentials, which are bound for the origin, so its + /// response is one reader's and never enters a shared cache. #[display("request carried Authorization")] AuthorizedRequest, - /// Not a 200. This is also what covers a `DataDome` block, which replaces the - /// document with a `403` (`integrations/datadome/protection.rs:778`). + /// Not a 200. This is also what covers a bot protection module's block, which + /// replaces the document with a `403`. #[display("status was not 200 OK")] NonOkStatus, /// Not HTML, so there is no template to transform. @@ -6708,6 +7447,61 @@ pub(crate) fn template_ad_slots_script( } } +/// The permission-state `", + escaped + ) +} + +/// The permission state as page JSON, substituting the empty state for an unset +/// value. +/// +/// [`PermissionState::page_json`] never returns an empty string, so this only +/// covers a params value nothing filled in. `JSON.parse("")` throws, and a +/// thrown head script takes the rest of the snippet with it, so an unset value +/// renders as the empty state rather than as broken JavaScript. +fn permissions_json_or_empty(permissions_json: &str) -> Cow<'_, str> { + if permissions_json.is_empty() { + Cow::Owned(PermissionState::default().page_json()) + } else { + Cow::Borrowed(permissions_json) + } +} + /// Build the `tsjs.adSlots` `") - .expect("should substitute the placeholder the transform emitted"); - - assert_eq!( - replaced, br#"

article

"#, - "should splice the payload exactly where the parser marked the body end" - ); - } - - #[test] - fn a_body_close_written_in_script_data_does_not_attract_the_seam() { - // The reverse byte search this replaced picked the string literal, spliced a - // `` terminated the - // publisher's script — in the served page and in the stored template alike. - let document = format!( - r#"

article

{TEMPLATE_SEAM_PLACEHOLDER}"# - ) - .into_bytes(); - - let replaced = replace_seam_placeholder(document, b"") - .expect("should substitute the placeholder the transform emitted"); + assert_eq!( + header_of(&response, header::HeaderName::from_static("x-ts-assembly")), + Some("byte-seam-fallback") + ); + let document = String::from_utf8(body_of(response).await) + .expect("served document should be UTF-8"); - assert_eq!( - replaced, - br#"

article

"#, - "should leave a body-close sequence inside script data untouched" - ); + assert!(document.to_ascii_lowercase().contains(expected_directive)); + assert!(document.contains("scheduleInitialAdInit")); + assert!( + cache + .entries + .lock() + .expect("should lock entries") + .is_empty(), + "publisher-authored ESI must never enter the shared template cache" + ); + assert_eq!( + assembler.calls.load(Ordering::Relaxed), + 0, + "publisher-authored ESI must never reach the platform parser" + ); + } } - #[test] - fn a_publisher_copy_of_the_placeholder_refuses_substitution() { - let document = format!( - "

{TEMPLATE_SEAM_PLACEHOLDER}

article{TEMPLATE_SEAM_PLACEHOLDER}" - ) - .into_bytes(); - - let (returned, error) = - replace_seam_placeholder(document.clone(), b"") - .expect_err("should refuse a document that collides with the placeholder"); - - assert!( - matches!(error, SeamError::Repeated), - "should name the collision rather than guess an occurrence" - ); - assert_eq!( - returned, document, - "should hand back the publisher document byte for byte" + #[tokio::test] + async fn ineligible_requests_and_responses_never_use_the_platform_assembler() { + let inline_stub = Arc::new(StubHttpClient::new()); + let inline_cache = Arc::new(MemoryTemplateCache::default()); + let inline_assembler = Arc::new(RecordingTemplateAssembler::default()); + let inline_settings = Arc::new(settings_with_mode("inline")); + let inline_services = services_with_assembler( + Arc::clone(&inline_stub), + Arc::clone(&inline_cache), + Arc::clone(&inline_assembler), ); - } - - #[test] - fn a_document_without_the_placeholder_refuses_substitution() { - let document = b"

article

".to_vec(); + queue_shareable_html(&inline_stub); - let (returned, error) = - replace_seam_placeholder(document.clone(), b"") - .expect_err("should refuse a document the transform did not mark"); + let _ = + body_of(run(&inline_settings, &inline_services, navigation_request()).await).await; + assert_eq!(inline_assembler.calls.load(Ordering::Relaxed), 0); - assert!( - matches!(error, SeamError::Missing), - "should name the absent placeholder rather than append blindly" - ); - assert_eq!( - returned, document, - "should hand back the document unchanged" + let private_stub = Arc::new(StubHttpClient::new()); + let private_cache = Arc::new(MemoryTemplateCache::default()); + let private_assembler = Arc::new(RecordingTemplateAssembler::default()); + let esi_settings = Arc::new(settings_with_mode("esi")); + let private_services = services_with_assembler( + Arc::clone(&private_stub), + Arc::clone(&private_cache), + Arc::clone(&private_assembler), ); - } - - #[test] - fn parser_validation_does_not_change_the_cached_schema() { - assert_eq!(crate::platform::TEMPLATE_SCHEMA_VERSION, 5); - assert_eq!(AD_ASSEMBLY_SEAM, ""); - assert!(!contains_publisher_esi_directive( - AD_ASSEMBLY_SEAM.as_bytes() - )); - } - - /// Shareable HTML that already contains the seam marker. - /// - /// The marker is reserved, but publisher content can still contain it. The - /// transform adds its own terminal placeholder; repeated markers then make the - /// response bypass the template cache rather than requiring normalization. - fn queue_html_that_collides_with_the_marker(stub: &StubHttpClient) { - stub.push_response_with_headers( + private_stub.push_response_with_headers( 200, - format!( - "

origin

" - ) - .into_bytes(), + b"private origin".to_vec(), vec![ ("content-type", "text/html; charset=utf-8"), ("cache-control", "public, max-age=300"), + ("set-cookie", "reader=personalized; Path=/"), ], ); - } - #[tokio::test] - async fn origin_marker_collision_bypasses_template_cache_without_mutating_publisher_bytes() - { - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let settings = Arc::new(settings_with_mode("esi")); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - queue_html_that_collides_with_the_marker(&stub); - queue_html_that_collides_with_the_marker(&stub); + let _ = + body_of(run(&esi_settings, &private_services, navigation_request()).await).await; + assert_eq!(private_assembler.calls.load(Ordering::Relaxed), 0); - let cold = String::from_utf8( - body_of(run(&settings, &services, navigation_request()).await).await, - ) - .expect("cold document should be UTF-8"); - let warm = String::from_utf8( - body_of(run(&settings, &services, navigation_request()).await).await, + let authenticated_stub = Arc::new(StubHttpClient::new()); + let authenticated_cache = Arc::new(MemoryTemplateCache::default()); + let authenticated_assembler = Arc::new(RecordingTemplateAssembler::default()); + let authenticated_services = services_with_assembler( + Arc::clone(&authenticated_stub), + Arc::clone(&authenticated_cache), + Arc::clone(&authenticated_assembler), + ); + queue_shareable_html(&authenticated_stub); + let mut authenticated_request = navigation_request(); + authenticated_request.headers_mut().insert( + header::AUTHORIZATION, + HeaderValue::from_static("Bearer example-token"), + ); + + let _ = body_of( + run( + &esi_settings, + &authenticated_services, + authenticated_request, + ) + .await, ) - .expect("warm document should be UTF-8"); + .await; + assert_eq!(authenticated_assembler.calls.load(Ordering::Relaxed), 0); - assert!( - cold.contains("origin") && cold.contains("window.tsjs"), - "a reserved-comment collision must not turn a valid origin 200 into a 500" - ); - for document in [&cold, &warm] { - assert!( - document.contains(&format!("window.publisherMarker=\"{AD_ASSEMBLY_SEAM}\"")), - "should preserve a publisher marker inside script data: {document}" - ); - } - assert_eq!(stub.recorded_request_uris().len(), 2); - assert!( - cache - .entries - .lock() - .expect("should lock entries") - .is_empty(), - "should not store a template with a publisher marker collision" + let failed_cache_stub = Arc::new(StubHttpClient::new()); + let failed_cache = Arc::new(MemoryTemplateCache::default()); + failed_cache.fail_lookup.store(true, Ordering::Relaxed); + let failed_cache_assembler = Arc::new(RecordingTemplateAssembler::default()); + let failed_cache_services = services_with_assembler( + Arc::clone(&failed_cache_stub), + Arc::clone(&failed_cache), + Arc::clone(&failed_cache_assembler), ); + queue_shareable_html(&failed_cache_stub); + + let _ = body_of(run(&esi_settings, &failed_cache_services, navigation_request()).await) + .await; + assert_eq!(failed_cache_assembler.calls.load(Ordering::Relaxed), 0); } #[tokio::test] - async fn html_without_an_explicit_body_gets_a_terminal_seam_instead_of_a_500() { + async fn fastly_surrogate_control_still_allows_a_cold_fill_and_warm_hit() { let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); - let settings = Arc::new(settings_with_mode("esi")); + let settings = Arc::new(settings_with_mode_and_template_cache_max_age("esi", 1_200)); let services = services(Arc::clone(&stub), Arc::clone(&cache)); + + // This is the policy observed on the publisher that exposed the bug. Fastly + // gives the surrogate lifetime edge precedence; the configured safety + // ceiling allows the origin's twenty-minute edge freshness. stub.push_response_with_headers( 200, - b"
origin fragment
".to_vec(), + b"origin".to_vec(), vec![ ("content-type", "text/html; charset=utf-8"), - ("cache-control", "public, max-age=300"), + ("cache-control", "public, max-age=60"), + ( + "surrogate-control", + "max-age=1200, stale-while-revalidate=21600, \ + stale-if-error=604800", + ), ], ); - let cold = String::from_utf8( - body_of(run(&settings, &services, navigation_request()).await).await, - ) - .expect("cold fragment should be UTF-8"); - let warm = String::from_utf8( - body_of(run(&settings, &services, navigation_request()).await).await, - ) - .expect("warm fragment should be UTF-8"); - - for document in [&cold, &warm] { - assert!(document.contains("origin fragment")); - assert!(!document.contains(AD_ASSEMBLY_SEAM)); - } - assert_eq!(stub.recorded_request_uris().len(), 1); + let cold = run(&settings, &services, navigation_request()).await; + assert_eq!( + cold.headers() + .get(HEADER_X_TS_TEMPLATE_CACHE) + .and_then(|value| value.to_str().ok()), + Some("miss-stored") + ); + let _ = body_of(cold).await; + let stored_max_age = cache + .stored_max_ages + .lock() + .expect("should lock stored max ages")[0]; assert!( - !cache - .entries - .lock() - .expect("should lock entries") - .is_empty(), - "the recovered template should remain cacheable" + stored_max_age <= Duration::from_secs(1_200) + && stored_max_age > Duration::from_secs(1_195), + "the real store path should receive the configured twenty-minute ceiling, got {stored_max_age:?}" + ); + + let warm = run(&settings, &services, navigation_request()).await; + assert_eq!( + warm.headers() + .get(HEADER_X_TS_TEMPLATE_CACHE) + .and_then(|value| value.to_str().ok()), + Some("hit") + ); + let _ = body_of(warm).await; + assert_eq!( + stub.recorded_request_uris().len(), + 1, + "the warm request must reuse the template authorized by Surrogate-Control" ); } #[tokio::test] - async fn an_unsplittable_cached_template_is_a_miss_before_any_header_commits() { - // The hit path used to check only that a marker existed *somewhere* and - // leave the exactly-one check to the finalizer. By then the 200 and its - // headers were committed and, on the streaming adapter, the document head - // was already on the wire — so the only available failure was a truncated - // response. Falling back to the origin is a slower correct page. + async fn one_identity_template_is_negotiated_per_reader_on_miss_and_hit() { let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); let settings = Arc::new(settings_with_mode("esi")); let services = services(Arc::clone(&stub), Arc::clone(&cache)); - queue_shareable_html(&stub); + stub.push_response_with_headers( + 200, + gzip_encode(b"origin"), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("content-encoding", "gzip"), + ("cache-control", "public, max-age=300"), + ("vary", "accept-encoding"), + ], + ); - // Fill the cache legitimately, then corrupt the stored body in place so the - // entry is found under exactly the key the next request derives. - let _ = body_of(run(&settings, &services, navigation_request()).await).await; - { - let stored_key = cache - .stored_keys - .lock() - .expect("should lock stored keys") - .first() - .cloned() - .expect("the cold request should have stored a template"); - let mut entries = cache.entries.lock().expect("should lock entries"); - let entry = entries - .get_mut(&stored_key.to_cache_key()) - .expect("the stored template should be readable"); - entry.body = format!( - "origin{AD_ASSEMBLY_SEAM}\ - {AD_ASSEMBLY_SEAM}" - ) - .into_bytes(); - } + let mut gzip_request = navigation_request(); + gzip_request + .headers_mut() + .insert(header::ACCEPT_ENCODING, HeaderValue::from_static("gzip")); + let cold = run(&settings, &services, gzip_request).await; + assert_eq!( + cold.headers() + .get(header::CONTENT_ENCODING) + .and_then(|value| value.to_str().ok()), + Some("gzip") + ); + assert!( + cold.headers().get_all(header::VARY).iter().any(|value| { + value.to_str().is_ok_and(|value| { + value + .split(',') + .any(|name| name.trim().eq_ignore_ascii_case("accept-encoding")) + }) + }), + "the encoded cold response must declare reader negotiation" + ); + let cold_decoded = gzip_decode(&body_of(cold).await); - // The fall-back must be able to reach the origin. - queue_shareable_html(&stub); - let response = run(&settings, &services, navigation_request()).await; - let status = response.status(); + let warm_identity = run(&settings, &services, navigation_request()).await; + assert!( + warm_identity + .headers() + .get(header::CONTENT_ENCODING) + .is_none(), + "a reader that did not advertise compression must receive identity" + ); + let warm_identity_body = body_of(warm_identity).await; + + let mut warm_gzip_request = navigation_request(); + warm_gzip_request + .headers_mut() + .insert(header::ACCEPT_ENCODING, HeaderValue::from_static("gzip")); + let warm_gzip = run(&settings, &services, warm_gzip_request).await; assert_eq!( - response + warm_gzip .headers() - .get(HEADER_X_TS_TEMPLATE_CACHE) + .get(header::CONTENT_ENCODING) .and_then(|value| value.to_str().ok()), - Some("invalid"), - "an invalid-object recovery must be observable without exposing its key" + Some("gzip") ); - let served = String::from_utf8(body_of(response).await) - .expect("served document should be UTF-8"); - assert_eq!( - status, - StatusCode::OK, - "the reader should get a complete page, not a committed-then-broken one" + warm_gzip + .headers() + .get(header::VARY) + .and_then(|value| value.to_str().ok()), + Some("Accept-Encoding") + ); + let warm_gzip_decoded = gzip_decode(&body_of(warm_gzip).await); + + let mut warm_brotli_request = navigation_request(); + warm_brotli_request.headers_mut().insert( + header::ACCEPT_ENCODING, + HeaderValue::from_static("br, gzip;q=0.5"), ); + let warm_brotli = run(&settings, &services, warm_brotli_request).await; assert_eq!( - stub.recorded_request_uris().len(), - 2, - "an unusable entry must be treated as a miss and refetched" + warm_brotli + .headers() + .get(header::CONTENT_ENCODING) + .and_then(|value| value.to_str().ok()), + Some("br") + ); + let warm_brotli_decoded = brotli_decode(&body_of(warm_brotli).await); + + assert_eq!(stub.recorded_request_uris().len(), 1); + assert_eq!(cold_decoded, warm_identity_body); + assert_eq!(warm_gzip_decoded, warm_identity_body); + assert_eq!(warm_brotli_decoded, warm_identity_body); + } + + #[tokio::test] + async fn esi_unsupported_reader_encoding_still_injects_tsjs() { + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + queue_shareable_html(&stub); + let mut request = navigation_request(); + request.headers_mut().insert( + header::ACCEPT_ENCODING, + HeaderValue::from_static("zstd, gzip;q=0, deflate;q=0, br;q=0, identity;q=0"), ); + + let body = String::from_utf8(body_of(run(&settings, &services, request).await).await) + .expect("should return UTF-8 HTML"); + + let forwarded = stub.recorded_request_headers(); assert!( - !served.contains(AD_ASSEMBLY_SEAM), - "no marker may survive into the served page: {served}" + forwarded[0].iter().any(|(name, value)| { + name.eq_ignore_ascii_case("accept-encoding") && value == "identity" + }), + "should force an identity origin representation when no reader encoding is \ + available for assembly" ); assert!( - served.contains("window.tsjs"), - "and the fallback must still deliver the seam: {served}" + body.contains("window.tsjs"), + "should preserve TSJS injection while bypassing shared assembly" ); assert!( cache - .entries + .lookups .lock() - .expect("should lock entries") + .expect("should lock lookups") .is_empty(), - "an unusable fresh object must be purged instead of forcing inline fallback \ - until its TTL expires" + "should bypass template cache when the reader accepts no supported representation" ); } - /// The schema version whose templates carried an executable ESI tag at the seam. - /// - /// Pinned as a literal rather than derived from [`TEMPLATE_SCHEMA_VERSION`]: the - /// point is that the current version is *not* this one, and a derived value - /// would move with it and assert nothing. - const ESI_INCLUDE_SCHEMA_VERSION: u32 = 1; - - /// The exact schema-v1 marker retained only as a cache-compatibility fixture. - const LEGACY_ESI_INCLUDE: &str = ""; - #[tokio::test] - async fn a_template_written_under_the_previous_schema_version_is_never_read() { - // v1 put an executable ESI include at the seam. v2 puts an inert comment - // there and hands slots to the scheduler, so a v1 entry has no marker this - // binary can find. A real v1 entry also carries an older build digest, but - // this fixture reuses the current fingerprint, so here `schema_version` is - // the only thing keeping the two apart. Nothing purges template cache on - // deploy. - assert_ne!( - crate::platform::TEMPLATE_SCHEMA_VERSION, - ESI_INCLUDE_SCHEMA_VERSION, - "the seam marker changed shape, so the schema version must have moved \ - off the value under which the old marker was stored" - ); - + async fn an_unsupported_origin_encoding_bypasses_template_cache_without_relabeling_the_body() + { let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); let settings = Arc::new(settings_with_mode("esi")); let services = services(Arc::clone(&stub), Arc::clone(&cache)); - queue_shareable_html(&stub); - - // Fill the cache the ordinary way, then plant a predecessor's entry - // alongside it: same request, previous schema version, previous marker. - // Re-keyed from the key the store actually used, so the fixture cannot - // drift from what the request derives. - let _ = body_of(run(&settings, &services, navigation_request()).await).await; - { - let stored_key = cache - .stored_keys - .lock() - .expect("should lock stored keys") - .first() - .cloned() - .expect("the cold request should have stored a template"); - let mut entries = cache.entries.lock().expect("should lock entries"); - let mut predecessor = entries - .get(&stored_key.to_cache_key()) - .cloned() - .expect("the stored template should be readable"); - predecessor.body = - format!("origin{LEGACY_ESI_INCLUDE}") - .into_bytes(); - predecessor.metadata.schema_version = ESI_INCLUDE_SCHEMA_VERSION; - let mut old_key = stored_key; - old_key.schema_version = ESI_INCLUDE_SCHEMA_VERSION; - entries.insert(old_key.to_cache_key(), predecessor); - } - - let served = String::from_utf8( - body_of(run(&settings, &services, navigation_request()).await).await, - ) - .expect("served document should be UTF-8"); + let wire_body = b"opaque-zstd-wire-bytes".to_vec(); + stub.push_response_with_headers( + 200, + wire_body.clone(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("content-encoding", "zstd"), + ("cache-control", "public, max-age=300"), + ], + ); + let response = run(&settings, &services, navigation_request()).await; + assert_eq!( + response + .headers() + .get(header::CONTENT_ENCODING) + .and_then(|value| value.to_str().ok()), + Some("zstd"), + "the pass-through bytes must keep the origin's representation label" + ); + assert_eq!( + response + .headers() + .get(HEADER_X_TS_TEMPLATE_CACHE) + .and_then(|value| value.to_str().ok()), + Some("bypass-response") + ); + assert_eq!(body_of(response).await, wire_body); assert!( cache - .lookups + .entries .lock() - .expect("should lock lookups") - .iter() - .all(|key| key.schema_version != ESI_INCLUDE_SCHEMA_VERSION), - "no lookup may name a schema version whose templates this binary cannot \ - assemble" - ); - assert!( - !served.contains(", name: header::HeaderName) -> Option<&str> { + response.headers().get(name).and_then(|v| v.to_str().ok()) + } - // Bid first, page second: one FIFO queue, and the auction is dispatched - // before the origin fetch. - queue_bid_response(&stub); - queue_shareable_html(&stub); + /// The bid payload the seam actually embedded, decoded from the served page. + /// + /// Deliberately **not** written as the inverse of [`html_escape_for_script`]: the + /// escaper emits `\"`, `\\` and `\uXXXX`, which are all JSON string escapes, so + /// re-quoting the payload and handing it to `serde_json`'s own string parser + /// decodes it without this test agreeing with the encoder about anything. + /// + /// Panics rather than returning an error, because a served page with no decodable + /// bid payload is the failure under test and its diagnostics belong at the + /// assertion. + fn seam_bids(document: &str) -> serde_json::Map { + const OPEN: &str = "var b=JSON.parse(\""; + // The seam's fixed shape. A bare `")` could occur inside the payload; this + // nine-byte terminator cannot. + const CLOSE: &str = "\");var s="; + let start = document + .find(OPEN) + .unwrap_or_else(|| panic!("the seam must carry a bids payload: {document}")); + let rest = &document[start + OPEN.len()..]; + let end = rest + .find(CLOSE) + .unwrap_or_else(|| panic!("the bids payload must terminate: {document}")); + let unescaped: String = serde_json::from_str(&format!("\"{}\"", &rest[..end])) + .expect("the embedded payload should be a valid JSON string literal"); + serde_json::from_str(&unescaped).expect("the embedded payload should be a JSON object") + } + + #[test] + fn the_seam_payload_replaces_the_transform_owned_placeholder() { + let document = + format!(r#"

article

{TEMPLATE_SEAM_PLACEHOLDER}"#).into_bytes(); + + let replaced = replace_seam_placeholder(document, b"") + .expect("should substitute the placeholder the transform emitted"); - let cold = String::from_utf8( - body_of(run_bidding(&settings, &services, navigation_request()).await).await, - ) - .expect("cold document should be UTF-8"); - let cold_bids = seam_bids(&cold); - assert_eq!( - cold_bids - .get("test-slot") - .and_then(|bid| bid.get("hb_pb")) - .and_then(serde_json::Value::as_str), - Some("3.50"), - "the miss path must deliver the winning bid's bucketed price, not an \ - empty map: {cold}" - ); assert_eq!( - cold_bids - .get("test-slot") - .and_then(|bid| bid.get("hb_bidder")) - .and_then(serde_json::Value::as_str), - Some(STUB_BIDDER), - "the miss path must name the winning bidder: {cold}" + replaced, br#"

article

"#, + "should splice the payload exactly where the parser marked the body end" ); + } - // The warm request needs only the bid response: a hit skips the origin - // entirely, so an unused page fixture here would mask a fetch. - queue_bid_response(&stub); - let warm = String::from_utf8( - body_of(run_bidding(&settings, &services, navigation_request()).await).await, + #[test] + fn a_body_close_written_in_script_data_does_not_attract_the_seam() { + // The reverse byte search this replaced picked the string literal, spliced a + // `` terminated the + // publisher's script — in the served page and in the stored template alike. + let document = format!( + r#"

article

{TEMPLATE_SEAM_PLACEHOLDER}"# ) - .expect("warm document should be UTF-8"); + .into_bytes(); + + let replaced = replace_seam_placeholder(document, b"") + .expect("should substitute the placeholder the transform emitted"); + assert_eq!( - stub.recorded_request_uris() - .iter() - .filter(|uri| uri.contains("/article")) - .count(), - 1, - "the warm request must be served from the template cache" + replaced, + br#"

article

"#, + "should leave a body-close sequence inside script data untouched" ); - let warm_bids = seam_bids(&warm); - assert_eq!( - warm_bids - .get("test-slot") - .and_then(|bid| bid.get("hb_pb")) - .and_then(serde_json::Value::as_str), - Some("3.50"), - "the hit path must deliver this reader's winning bid, not an empty \ - map: {warm}" + } + + #[test] + fn a_publisher_copy_of_the_placeholder_refuses_substitution() { + let document = format!( + "

{TEMPLATE_SEAM_PLACEHOLDER}

article{TEMPLATE_SEAM_PLACEHOLDER}" + ) + .into_bytes(); + + let (returned, error) = + replace_seam_placeholder(document.clone(), b"") + .expect_err("should refuse a document that collides with the placeholder"); + + assert!( + matches!(error, SeamError::Repeated), + "should name the collision rather than guess an occurrence" ); assert_eq!( - warm_bids, cold_bids, - "both paths must splice the same bids for the same auction" + returned, document, + "should hand back the publisher document byte for byte" ); } - #[tokio::test] - async fn max_age_zero_reload_reuses_a_warm_template_and_runs_a_fresh_auction() { - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let settings = Arc::new(settings_with_bidder("esi")); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - - queue_bid_response(&stub); - queue_shareable_html(&stub); - let cold = run_bidding(&settings, &services, navigation_request()).await; - assert_eq!( - cold.headers() - .get(HEADER_X_TS_TEMPLATE_CACHE) - .and_then(|value| value.to_str().ok()), - Some("miss-stored") - ); - let _ = body_of(cold).await; + #[test] + fn a_document_without_the_placeholder_refuses_substitution() { + let document = b"

article

".to_vec(); - // Keep an origin response available so the old bypass behavior completes - // normally and fails on the observable cache state rather than a fixture - // underflow. A correct warm reload leaves this response untouched. - queue_bid_response(&stub); - queue_shareable_html(&stub); - let mut reload = navigation_request(); - reload - .headers_mut() - .insert(header::CACHE_CONTROL, HeaderValue::from_static("max-age=0")); - let warm = run_bidding(&settings, &services, reload).await; - assert_eq!( - warm.headers() - .get(HEADER_X_TS_TEMPLATE_CACHE) - .and_then(|value| value.to_str().ok()), - Some("hit"), - "a browser reload must re-run per-reader assembly without refetching the \ - reader-neutral template" - ); - assert_eq!( - warm.headers() - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("no-store, private"), - "reusing template cache must not make the assembled response browser-cacheable" - ); - let warm = String::from_utf8(body_of(warm).await) - .expect("warm reload document should be UTF-8"); + let (returned, error) = + replace_seam_placeholder(document.clone(), b"") + .expect_err("should refuse a document the transform did not mark"); - let requests = stub.recorded_request_uris(); - assert_eq!( - requests - .iter() - .filter(|uri| uri.contains("/article")) - .count(), - 1, - "the reload must reuse the warm article template" + assert!( + matches!(error, SeamError::Missing), + "should name the absent placeholder rather than append blindly" ); assert_eq!( - stub.recorded_backend_names() - .iter() - .filter(|backend| backend.as_str() == "stub-bidder-backend") - .count(), - 2, - "each navigation must dispatch its own auction even when template cache hits" + returned, document, + "should hand back the document unchanged" ); - assert_eq!( - seam_bids(&warm) - .get("test-slot") - .and_then(|bid| bid.get("hb_pb")) - .and_then(serde_json::Value::as_str), - Some("3.50"), - "the reload's auction result must reach the assembled seam" + } + + #[test] + fn parser_validation_does_not_change_the_cached_schema() { + assert_eq!(crate::platform::TEMPLATE_SCHEMA_VERSION, 5); + assert_eq!(AD_ASSEMBLY_SEAM, ""); + assert!(!contains_publisher_esi_directive( + AD_ASSEMBLY_SEAM.as_bytes() + )); + } + + /// Shareable HTML that already contains the seam marker. + /// + /// The marker is reserved, but publisher content can still contain it. The + /// transform adds its own terminal placeholder; repeated markers then make the + /// response bypass the template cache rather than requiring normalization. + fn queue_html_that_collides_with_the_marker(stub: &StubHttpClient) { + stub.push_response_with_headers( + 200, + format!( + "

origin

" + ) + .into_bytes(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("cache-control", "public, max-age=300"), + ], ); } #[tokio::test] - async fn the_marker_never_reaches_the_browser_on_either_path() { - // The user-visible failure this whole chain exists to avoid: a page that - // returns 200, parses fine, renders no ads, and reports no error, because - // the legacy executable ESI tag was served literally. - // - // Both paths are checked. The miss path assembles after the transform; the - // hit path assembles after reading the cache. They are separate call sites - // and only one of them running would still look like success on the other. + async fn origin_marker_collision_bypasses_template_cache_without_mutating_publisher_bytes() + { let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); let settings = Arc::new(settings_with_mode("esi")); let services = services(Arc::clone(&stub), Arc::clone(&cache)); - queue_shareable_html(&stub); + queue_html_that_collides_with_the_marker(&stub); + queue_html_that_collides_with_the_marker(&stub); let cold = String::from_utf8( body_of(run(&settings, &services, navigation_request()).await).await, ) - .expect("cold response should be utf-8"); + .expect("cold document should be UTF-8"); let warm = String::from_utf8( body_of(run(&settings, &services, navigation_request()).await).await, ) - .expect("warm response should be utf-8"); + .expect("warm document should be UTF-8"); - assert_eq!( - stub.recorded_request_uris().len(), - 1, - "the second request must be a hit, or this only tests one path" + assert!( + cold.contains("origin") && cold.contains("window.tsjs"), + "a reserved-comment collision must not turn a valid origin 200 into a 500" ); - for (label, document) in [("miss", &cold), ("hit", &warm)] { - assert!( - !document.contains(AD_ASSEMBLY_SEAM), - "{label}: an unresolved marker reached the browser: {document}" - ); + for document in [&cold, &warm] { assert!( - document.contains("window.tsjs"), - "{label}: assembly must leave a bids script behind: {document}" + document.contains(&format!("window.publisherMarker=\"{AD_ASSEMBLY_SEAM}\"")), + "should preserve a publisher marker inside script data: {document}" ); } + assert_eq!(stub.recorded_request_uris().len(), 2); + assert!( + cache + .entries + .lock() + .expect("should lock entries") + .is_empty(), + "should not store a template with a publisher marker collision" + ); } #[tokio::test] - async fn the_cached_template_holds_the_marker_and_never_the_bids() { - // Store the reader-neutral template before assembling the final per-user - // response, which must never enter the forbidden assembled-response cache. If - // these were swapped, the cache would hold one visitor's bids and serve them - // to the next — and every test above would still pass, because the served - // page would look correct. + async fn html_without_an_explicit_body_gets_a_terminal_seam_instead_of_a_500() { let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); let settings = Arc::new(settings_with_mode("esi")); let services = services(Arc::clone(&stub), Arc::clone(&cache)); - queue_shareable_html(&stub); - - let _ = run(&settings, &services, navigation_request()).await; + stub.push_response_with_headers( + 200, + b"
origin fragment
".to_vec(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("cache-control", "public, max-age=300"), + ], + ); - let entries = cache.entries.lock().expect("should lock entries"); - let stored = entries - .values() - .next() - .expect("a template should be stored"); - let template = core::str::from_utf8(&stored.body).expect("template should be utf-8"); + let cold = String::from_utf8( + body_of(run(&settings, &services, navigation_request()).await).await, + ) + .expect("cold fragment should be UTF-8"); + let warm = String::from_utf8( + body_of(run(&settings, &services, navigation_request()).await).await, + ) + .expect("warm fragment should be UTF-8"); + for document in [&cold, &warm] { + assert!(document.contains("origin fragment")); + assert!(!document.contains(AD_ASSEMBLY_SEAM)); + } + assert_eq!(stub.recorded_request_uris().len(), 1); assert!( - template.contains(AD_ASSEMBLY_SEAM), - "the cached template must still carry the unresolved marker: {template}" - ); - assert!( - !template.contains("window.tsjs"), - "the cached template must not carry a bids script: {template}" + !cache + .entries + .lock() + .expect("should lock entries") + .is_empty(), + "the recovered template should remain cacheable" ); } #[tokio::test] - async fn nextjs_stored_templates_are_identical_across_request_shapes() { - let mut settings = settings_with_mode("esi"); - settings.publisher.origin_url = "https://origin.example.com".to_owned(); - settings - .integrations - .insert_config( - "nextjs", - &serde_json::json!({ - "enabled": true, - "rewrite_attributes": ["href", "link", "url"], - }), + async fn an_unsplittable_cached_template_is_a_miss_before_any_header_commits() { + // The hit path used to check only that a marker existed *somewhere* and + // leave the exactly-one check to the finalizer. By then the 200 and its + // headers were committed and, on the streaming adapter, the document head + // was already on the wire — so the only available failure was a truncated + // response. Falling back to the origin is a slower correct page. + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + queue_shareable_html(&stub); + + // Fill the cache legitimately, then corrupt the stored body in place so the + // entry is found under exactly the key the next request derives. + let _ = body_of(run(&settings, &services, navigation_request()).await).await; + { + let stored_key = cache + .stored_keys + .lock() + .expect("should lock stored keys") + .first() + .cloned() + .expect("the cold request should have stored a template"); + let mut entries = cache.entries.lock().expect("should lock entries"); + let entry = entries + .get_mut(&stored_key.to_cache_key()) + .expect("the stored template should be readable"); + entry.body = format!( + "origin{AD_ASSEMBLY_SEAM}\ + {AD_ASSEMBLY_SEAM}" ) - .expect("should enable Next.js processing"); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); - assert!( - !registry.html_stream_processor_factories().is_empty(), - "should exercise the Next.js stream processor" + .into_bytes(); + } + + // The fall-back must be able to reach the origin. + queue_shareable_html(&stub); + let response = run(&settings, &services, navigation_request()).await; + let status = response.status(); + assert_eq!( + response + .headers() + .get(HEADER_X_TS_TEMPLATE_CACHE) + .and_then(|value| value.to_str().ok()), + Some("invalid"), + "an invalid-object recovery must be observable without exposing its key" ); - let settings = Arc::new(settings); - let html = br#"
"#; - - for finalizer in [Finalizer::Streaming, Finalizer::Buffered] { - let mut templates = Vec::new(); - for request in [navigation_request(), prefetch_navigation_request()] { - // Independent fills force distinct per-request RSC namespaces. - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - stub.push_response_with_headers( - 200, - html.to_vec(), - vec![ - ("content-type", "text/html"), - ("cache-control", "public, max-age=300"), - ], - ); - - let response = run_via(&settings, &services, request, finalizer).await; - let _ = body_of(response).await; - - let entries = cache.entries.lock().expect("should lock stored templates"); - assert_eq!( - entries.len(), - 1, - "should store a Next.js template on every fill" - ); - let template = entries - .values() - .next() - .expect("should find stored template") - .body - .clone(); - let text = core::str::from_utf8(&template).expect("should store UTF-8 HTML"); - assert!( - text.contains("ts.example.com/page"), - "should rewrite the RSC URL" - ); - assert!( - text.contains(AD_ASSEMBLY_SEAM), - "should store before per-reader assembly" - ); - for marker in [ - "__ts_rsc_", - ".adSlots", - ".bids=", - "gpt-diagnostics", - "gpt_history", - ] { - assert!( - !text.contains(marker), - "should exclude request-scoped marker {marker}" - ); - } - templates.push(template); - } - assert_eq!( - templates[0], templates[1], - "should store identical bytes across request shapes and RSC namespaces" - ); - } - } - - #[tokio::test] - async fn a_transform_that_overruns_its_buffer_stores_nothing() { - // The 16 MB cap in production, shrunk here. A partial template in template cache is the - // worst outcome available: it would be served to every subsequent visitor as - // a truncated document, indefinitely, with no error after the first request. - // - // Safe by construction — the cap error propagates before the store runs — but - // "by construction" is exactly the kind of claim that stops being true after - // an unrelated refactor moves one line. - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let mut raw = settings_with_mode("esi"); - raw.publisher.max_buffered_body_bytes = 8; - let settings = Arc::new(raw); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - queue_shareable_html(&stub); - - let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); - let consent = crate::consent::ConsentContext { - jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, - ..Default::default() - }; - let mut ec_context = EcContext::new_for_test(None, consent); - let publisher_response = handle_publisher_request( - &settings, - &services, - None, - &mut ec_context, - AuctionDispatch { - orchestrator: &orchestrator, - slots: &[article_slot()], - registry: None, - }, - navigation_request(), - EdgeCacheHeader::SMaxageFallback, - ) - .await - .expect("the request itself should succeed; the cap trips during streaming"); - - let result = publisher_response_into_streaming_response( - publisher_response, - &Method::GET, - Arc::clone(&settings), - ®istry, - orchestrator, - services.clone(), - ) - .await; + let served = String::from_utf8(body_of(response).await) + .expect("served document should be UTF-8"); + assert_eq!( + status, + StatusCode::OK, + "the reader should get a complete page, not a committed-then-broken one" + ); + assert_eq!( + stub.recorded_request_uris().len(), + 2, + "an unusable entry must be treated as a miss and refetched" + ); assert!( - result.is_err(), - "overrunning the buffer must fail rather than truncate" + !served.contains(AD_ASSEMBLY_SEAM), + "no marker may survive into the served page: {served}" + ); + assert!( + served.contains("window.tsjs"), + "and the fallback must still deliver the seam: {served}" ); assert!( cache @@ -12737,2949 +13882,3009 @@ mod tests { .lock() .expect("should lock entries") .is_empty(), - "a failed transform must leave nothing in the shared cache" + "an unusable fresh object must be purged instead of forcing inline fallback \ + until its TTL expires" ); } - #[tokio::test] - async fn a_cache_hit_is_never_shared_cacheable() { - // Asserted positively, because the obvious negative check does not work. - // Forbidding `public`, `s-maxage` and `Surrogate-Control` passes trivially - // on a response that carries no `Cache-Control` at all — and *that* is the - // real failure mode here, since a template cache hit returns before the point where the - // publisher path stamps the response private. HTML with no `Cache-Control` - // is heuristically cacheable, so "nothing to forbid" is not safety. - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let settings = Arc::new(settings_with_mode("esi")); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - queue_shareable_html(&stub); + /// The schema version whose templates carried an executable ESI tag at the seam. + /// + /// Pinned as a literal rather than derived from [`TEMPLATE_SCHEMA_VERSION`]: the + /// point is that the current version is *not* this one, and a derived value + /// would move with it and assert nothing. + const ESI_INCLUDE_SCHEMA_VERSION: u32 = 1; - let _cold = run(&settings, &services, navigation_request()).await; - let warm = run(&settings, &services, navigation_request()).await; + /// The exact schema-v1 marker retained only as a cache-compatibility fixture. + const LEGACY_ESI_INCLUDE: &str = ""; - assert_eq!( - stub.recorded_request_uris().len(), - 1, - "the second request must be a hit, or this asserts nothing" - ); - assert_eq!( - header_of(&warm, header::CACHE_CONTROL), - Some("no-store, private"), - "an assembled response is per-user even when its template is not" + #[tokio::test] + async fn a_template_written_under_the_previous_schema_version_is_never_read() { + // v1 put an executable ESI include at the seam. v2 puts an inert comment + // there and hands slots to the scheduler, so a v1 entry has no marker this + // binary can find. A real v1 entry also carries an older build digest, but + // this fixture reuses the current fingerprint, so here `schema_version` is + // the only thing keeping the two apart. Nothing purges template cache on + // deploy. + assert_ne!( + crate::platform::TEMPLATE_SCHEMA_VERSION, + ESI_INCLUDE_SCHEMA_VERSION, + "the seam marker changed shape, so the schema version must have moved \ + off the value under which the old marker was stored" ); - // Validators would let a client revalidate into a shared copy, and the CDN - // directives would instruct an intermediary to store one outright. The - // origin fixture sends a `public, max-age=300` that must not survive. - for stripped in [header::ETAG, header::LAST_MODIFIED, header::EXPIRES] { - assert_eq!( - header_of(&warm, stripped.clone()), - None, - "{stripped} must not survive onto an assembled response" - ); - } - } - - #[tokio::test] - async fn a_returning_visitor_gets_the_same_privacy_headers() { - // The case with no backstop. A first-visit response sets an EC cookie, so - // the adapter's cookie-privacy net force-privatizes it regardless of what - // this path does. A returning visitor sets no cookie, so that net never - // fires and this path is the only thing standing between an assembled - // per-user document and a shared cache. let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); let settings = Arc::new(settings_with_mode("esi")); let services = services(Arc::clone(&stub), Arc::clone(&cache)); queue_shareable_html(&stub); - let _cold = run(&settings, &services, navigation_request()).await; - let warm = run(&settings, &services, navigation_request()).await; + // Fill the cache the ordinary way, then plant a predecessor's entry + // alongside it: same request, previous schema version, previous marker. + // Re-keyed from the key the store actually used, so the fixture cannot + // drift from what the request derives. + let _ = body_of(run(&settings, &services, navigation_request()).await).await; + { + let stored_key = cache + .stored_keys + .lock() + .expect("should lock stored keys") + .first() + .cloned() + .expect("the cold request should have stored a template"); + let mut entries = cache.entries.lock().expect("should lock entries"); + let mut predecessor = entries + .get(&stored_key.to_cache_key()) + .cloned() + .expect("the stored template should be readable"); + predecessor.body = + format!("origin{LEGACY_ESI_INCLUDE}") + .into_bytes(); + predecessor.metadata.schema_version = ESI_INCLUDE_SCHEMA_VERSION; + let mut old_key = stored_key; + old_key.schema_version = ESI_INCLUDE_SCHEMA_VERSION; + entries.insert(old_key.to_cache_key(), predecessor); + } + + let served = String::from_utf8( + body_of(run(&settings, &services, navigation_request()).await).await, + ) + .expect("served document should be UTF-8"); assert!( - header_of(&warm, header::SET_COOKIE).is_none(), - "no cookie here means no privacy net, which is the point of this test" + cache + .lookups + .lock() + .expect("should lock lookups") + .iter() + .all(|key| key.schema_version != ESI_INCLUDE_SCHEMA_VERSION), + "no lookup may name a schema version whose templates this binary cannot \ + assemble" ); - assert_eq!( - header_of(&warm, header::CACHE_CONTROL), - Some("no-store, private") + assert!( + !served.contains(", - ) -> Result, Report> { - Ok(Some(GeoInfo { - city: self.0.to_string(), - country: self.0.to_string(), - continent: self.0.to_string(), - latitude: 1.0, - longitude: 2.0, - metro_code: 3, - region: Some(self.0.to_string()), - asn: Some(4), - })) - } - } - - /// One synthetic user: an identity, a consent posture, and a location. - struct SyntheticUser { - ec_id: &'static str, - jurisdiction: crate::consent::jurisdiction::Jurisdiction, - geo_marker: &'static str, - } - - /// Runs one synthetic user against a fresh cache and returns the stored template. - /// - /// Fresh cache per user deliberately: the point is to compare what each *would* - /// store, so sharing a cache would let the first user's entry answer for the - /// second and the comparison would prove nothing. - async fn stored_template_for(user: &SyntheticUser) -> Vec { - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let settings = Arc::new(settings_with_mode("esi")); - let services = RuntimeServices::builder() - .config_store(Arc::new(NoopConfigStore)) - .secret_store(Arc::new(NoopSecretStore)) - .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) - .backend(Arc::new(StubBackend)) - .http_client(Arc::clone(&stub) as Arc) - .geo(Arc::new(StubGeo(user.geo_marker))) - .client_info(ClientInfo::default()) - .template_cache( - Arc::clone(&cache) as Arc + // A normal navigation fills the cache; only then is a hit available for + // the opted-out request, which is the whole point — the gate held on the + // miss and was ignored on the hit. + let cold = String::from_utf8( + body_of(run_via(&settings, &services, navigation_request(), finalizer).await) + .await, ) - .build(); - queue_shareable_html(&stub); - - let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); - let consent = crate::consent::ConsentContext { - jurisdiction: user.jurisdiction.clone(), - ..Default::default() - }; - let mut ec_context = EcContext::new_for_test(Some(user.ec_id.to_string()), consent); - let publisher_response = handle_publisher_request( - &settings, - &services, - None, - &mut ec_context, - AuctionDispatch { - orchestrator: &orchestrator, - slots: &[article_slot()], - registry: None, - }, - navigation_request(), - EdgeCacheHeader::SMaxageFallback, - ) - .await - .expect("should proxy publisher request"); - let _ = publisher_response_into_streaming_response( - publisher_response, - &Method::GET, - Arc::clone(&settings), - ®istry, - orchestrator, - services.clone(), - ) - .await - .expect("should finalize publisher response"); - - let entries = cache.entries.lock().expect("should lock entries"); - entries - .values() - .next() - .expect("a template should have been stored") - .body - .clone() - } - - #[tokio::test] - async fn two_users_differing_in_identity_consent_and_geo_store_the_same_template() { - // The gate the whole design rests on. The template is shared between - // visitors, so anything request-scoped that reaches it is one visitor's data - // served to the next. Byte-identity is the assertion because it does not - // depend on guessing which field might leak. - let alice = SyntheticUser { - ec_id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.alice1", - jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, - geo_marker: "AliceCity", - }; - let bob = SyntheticUser { - ec_id: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.bobbb1", - jurisdiction: crate::consent::jurisdiction::Jurisdiction::Gdpr, - geo_marker: "BobCity", - }; - - let alice_template = stored_template_for(&alice).await; - let bob_template = stored_template_for(&bob).await; + .expect("cold document should be UTF-8"); + assert!( + cold.contains("scheduleInitialAdInit"), + "the fixture must produce a seam for ordinary traffic, or this test \ + passes for the wrong reason: {cold}" + ); - assert_eq!( - alice_template, bob_template, - "two users differing in identity, consent and geo must produce the same \ - shared template" - ); + let served = String::from_utf8( + body_of( + run_via( + &settings, + &services, + prefetch_navigation_request(), + finalizer, + ) + .await, + ) + .await, + ) + .expect("served document should be UTF-8"); - // Belt and braces: byte-identity would also hold if *both* templates leaked - // the same wrong thing, so name the values that must be absent. - let template = String::from_utf8(alice_template).expect("template should be utf-8"); - for forbidden in [ - alice.ec_id, - bob.ec_id, - alice.geo_marker, - bob.geo_marker, - "adSlots", - "window.tsjs", - ] { + assert_eq!( + stub.recorded_request_uris().len(), + 1, + "the opted-out request must be a cache hit, or the hit finalizer is \ + never reached" + ); assert!( - !template.contains(forbidden), - "`{forbidden}` must not appear in a shared template: {template}" + !served.contains("scheduleInitialAdInit"), + "a reader whose ad stack did not run must not have `adInit` \ + scheduled: {served}" + ); + assert!( + !served.contains("t.adSlots="), + "no slot assignment may reach opted-out traffic: {served}" + ); + assert!( + !served.contains(AD_ASSEMBLY_SEAM), + "the marker must still be consumed even when nothing replaces it: \ + {served}" + ); + assert!( + served.contains("origin"), + "the page itself must still be served: {served}" ); } } #[tokio::test] - async fn inline_mode_never_reads_or_writes_the_cache() { - // The shipped path. If this ever cached, per-user ad state would be shared - // between visitors — the exact failure the whole design exists to avoid. + async fn a_winning_bid_reaches_the_page_on_both_the_miss_and_the_hit_path() { + // The fixture every other test here lacks. With no registered provider the + // auction always resolves to `{}`, so a seam that discarded every non-empty + // bid map looked identical to one that worked — which is precisely how the + // shared modes shipped serving zero bids. let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); - let settings = Arc::new(settings_with_mode("inline")); + let settings = Arc::new(settings_with_bidder("esi")); let services = services(Arc::clone(&stub), Arc::clone(&cache)); - queue_shareable_html(&stub); - queue_shareable_html(&stub); - let _ = run(&settings, &services, navigation_request()).await; - let _ = run(&settings, &services, navigation_request()).await; + // Bid first, page second: one FIFO queue, and the auction is dispatched + // before the origin fetch. + queue_bid_response(&stub); + queue_shareable_html(&stub); + let cold = String::from_utf8( + body_of(run_bidding(&settings, &services, navigation_request()).await).await, + ) + .expect("cold document should be UTF-8"); + let cold_bids = seam_bids(&cold); assert_eq!( - stub.recorded_request_uris().len(), - 2, - "inline must fetch the origin every time" + cold_bids + .get("test-slot") + .and_then(|bid| bid.get("hb_pb")) + .and_then(serde_json::Value::as_str), + Some("3.50"), + "the miss path must deliver the winning bid's bucketed price, not an \ + empty map: {cold}" ); - assert!( - cache - .entries - .lock() - .expect("should lock entries") - .is_empty(), - "inline must never write a shared template" + assert_eq!( + cold_bids + .get("test-slot") + .and_then(|bid| bid.get("hb_bidder")) + .and_then(serde_json::Value::as_str), + Some(STUB_BIDDER), + "the miss path must name the winning bidder: {cold}" ); - } - /// [`settings_with_mode`], with one integration configured a stated way. - /// - /// Edits the parsed `[integrations]` map rather than appending TOML, so two - /// fixtures differ in exactly the field under test. - fn settings_with_prebid_timeout(mode: &str, timeout_ms: u32) -> Settings { - let mut settings = settings_with_mode(mode); - settings.integrations.insert( - "prebid".to_string(), - serde_json::json!({ - "enabled": true, - "external_bundle_url": "https://assets.example.com/prebid/bundle.js", - "timeout_ms": timeout_ms, - }), + // The warm request needs only the bid response: a hit skips the origin + // entirely, so an unused page fixture here would mask a fetch. + queue_bid_response(&stub); + let warm = String::from_utf8( + body_of(run_bidding(&settings, &services, navigation_request()).await).await, + ) + .expect("warm document should be UTF-8"); + assert_eq!( + stub.recorded_request_uris() + .iter() + .filter(|uri| uri.contains("/article")) + .count(), + 1, + "the warm request must be served from the template cache" + ); + let warm_bids = seam_bids(&warm); + assert_eq!( + warm_bids + .get("test-slot") + .and_then(|bid| bid.get("hb_pb")) + .and_then(serde_json::Value::as_str), + Some("3.50"), + "the hit path must deliver this reader's winning bid, not an empty \ + map: {warm}" + ); + assert_eq!( + warm_bids, cold_bids, + "both paths must splice the same bids for the same auction" ); - settings - } - - /// Every key the cache was asked to store, rendered. - fn stored_cache_keys(cache: &MemoryTemplateCache) -> Vec { - cache - .stored_keys - .lock() - .expect("should lock stored keys") - .iter() - .map(crate::platform::TemplateCacheKey::to_cache_key) - .collect() - } - - /// Every key the cache was asked to read, rendered. - fn looked_up_cache_keys(cache: &MemoryTemplateCache) -> Vec { - cache - .lookups - .lock() - .expect("should lock lookups") - .iter() - .map(crate::platform::TemplateCacheKey::to_cache_key) - .collect() } #[tokio::test] - async fn two_integration_configurations_never_share_a_template() { - // `template_fingerprint` folds the complete typed config, and its own - // tests call it directly — so reverting the *call site* back to the - // bundle-only hash, a constant for a given binary, left the entire suite - // green while every configuration silently shared one template. - // - // This drives the real request path and asserts on the keys the cache was - // actually handed, which is the only place that mutation is visible. + async fn max_age_zero_reload_reuses_a_warm_template_and_runs_a_fresh_auction() { let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_bidder("esi")); let services = services(Arc::clone(&stub), Arc::clone(&cache)); - let first = Arc::new(settings_with_prebid_timeout("esi", 1000)); - let second = Arc::new(settings_with_prebid_timeout("esi", 2500)); - queue_shareable_html(&stub); - queue_shareable_html(&stub); - let _ = run(&first, &services, navigation_request()).await; - let _ = run(&second, &services, navigation_request()).await; + queue_bid_response(&stub); + queue_shareable_html(&stub); + let cold = run_bidding(&settings, &services, navigation_request()).await; + assert_eq!( + cold.headers() + .get(HEADER_X_TS_TEMPLATE_CACHE) + .and_then(|value| value.to_str().ok()), + Some("miss-stored") + ); + let _ = body_of(cold).await; - let stored = stored_cache_keys(&cache); + // Keep an origin response available so the old bypass behavior completes + // normally and fails on the observable cache state rather than a fixture + // underflow. A correct warm reload leaves this response untouched. + queue_bid_response(&stub); + queue_shareable_html(&stub); + let mut reload = navigation_request(); + reload + .headers_mut() + .insert(header::CACHE_CONTROL, HeaderValue::from_static("max-age=0")); + let warm = run_bidding(&settings, &services, reload).await; assert_eq!( - stored.len(), - 2, - "each configuration must store its own template, got {stored:?}" + warm.headers() + .get(HEADER_X_TS_TEMPLATE_CACHE) + .and_then(|value| value.to_str().ok()), + Some("hit"), + "a browser reload must re-run per-reader assembly without refetching the \ + reader-neutral template" ); - assert_ne!( - stored[0], stored[1], - "two `[integrations]` configurations must key different templates; one key \ - serves the first configuration's injected markup to the second" + assert_eq!( + warm.headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("no-store, private"), + "reusing template cache must not make the assembled response browser-cacheable" + ); + let warm = String::from_utf8(body_of(warm).await) + .expect("warm reload document should be UTF-8"); + + let requests = stub.recorded_request_uris(); + assert_eq!( + requests + .iter() + .filter(|uri| uri.contains("/article")) + .count(), + 1, + "the reload must reuse the warm article template" ); assert_eq!( - stub.recorded_request_uris().len(), + stub.recorded_backend_names() + .iter() + .filter(|backend| backend.as_str() == "stub-bidder-backend") + .count(), 2, - "the second configuration must have missed and fetched the origin rather \ - than reading the first's template" + "each navigation must dispatch its own auction even when template cache hits" + ); + assert_eq!( + seam_bids(&warm) + .get("test-slot") + .and_then(|bid| bid.get("hb_pb")) + .and_then(serde_json::Value::as_str), + Some("3.50"), + "the reload's auction result must reach the assembled seam" ); } #[tokio::test] - async fn one_integration_configuration_keys_one_template() { - // The converse, and the failure mode a fingerprint fix can introduce: - // over-invalidating is as total as under-invalidating. A fingerprint that - // moves between two equal configurations is a cache that never hits, which - // this would read as "no measurable benefit" rather than as a bug. + async fn the_marker_never_reaches_the_browser_on_either_path() { + // The user-visible failure this whole chain exists to avoid: a page that + // returns 200, parses fine, renders no ads, and reports no error, because + // the legacy executable ESI tag was served literally. // - // The two `Settings` are parsed independently, so their `[integrations]` - // maps iterate in different orders — which is what exercises the sort. + // Both paths are checked. The miss path assembles after the transform; the + // hit path assembles after reading the cache. They are separate call sites + // and only one of them running would still look like success on the other. let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); let services = services(Arc::clone(&stub), Arc::clone(&cache)); - let first = Arc::new(settings_with_prebid_timeout("esi", 1000)); - let second = Arc::new(settings_with_prebid_timeout("esi", 1000)); queue_shareable_html(&stub); - let _ = run(&first, &services, navigation_request()).await; - let _ = run(&second, &services, navigation_request()).await; + let cold = String::from_utf8( + body_of(run(&settings, &services, navigation_request()).await).await, + ) + .expect("cold response should be utf-8"); + let warm = String::from_utf8( + body_of(run(&settings, &services, navigation_request()).await).await, + ) + .expect("warm response should be utf-8"); - let looked_up = looked_up_cache_keys(&cache); - assert_eq!( - looked_up.len(), - 2, - "both requests must consult the cache, got {looked_up:?}" - ); - assert_eq!( - looked_up[0], looked_up[1], - "equal configurations must name one key, or the cache never hits" - ); assert_eq!( stub.recorded_request_uris().len(), 1, - "the second request must be served from the first's template" + "the second request must be a hit, or this only tests one path" ); + for (label, document) in [("miss", &cold), ("hit", &warm)] { + assert!( + !document.contains(AD_ASSEMBLY_SEAM), + "{label}: an unresolved marker reached the browser: {document}" + ); + assert!( + document.contains("window.tsjs"), + "{label}: assembly must leave a bids script behind: {document}" + ); + } } #[tokio::test] - async fn one_context_key_allowlist_keys_one_template() { - let mut source = serde_json::to_value(settings_with_mode("esi")) - .expect("should serialize cache settings"); - source["auction"]["allowed_context_keys"] = - serde_json::json!(["zeta", "alpha", "gamma", "beta", "epsilon", "delta"]); + async fn the_cached_template_holds_the_marker_and_never_the_bids() { + // Store the reader-neutral template before assembling the final per-user + // response, which must never enter the forbidden assembled-response cache. If + // these were swapped, the cache would hold one visitor's bids and serve them + // to the next — and every test above would still pass, because the served + // page would look correct. let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); let services = services(Arc::clone(&stub), Arc::clone(&cache)); + queue_shareable_html(&stub); - for _ in 0..16 { - let settings = Arc::new( - serde_json::from_value(source.clone()).expect("should parse cache settings"), - ); - queue_shareable_html(&stub); - let _ = run(&settings, &services, navigation_request()).await; - } + let _ = run(&settings, &services, navigation_request()).await; + + let entries = cache.entries.lock().expect("should lock entries"); + let stored = entries + .values() + .next() + .expect("a template should be stored"); + let template = core::str::from_utf8(&stored.body).expect("template should be utf-8"); - let keys = looked_up_cache_keys(&cache); - assert_eq!(keys.len(), 16, "should consult the cache for every request"); assert!( - keys.iter().all(|key| key == &keys[0]), - "should reuse one key across independently parsed allowlists" - ); - assert_eq!( - stored_cache_keys(&cache).len(), - 1, - "should store one template" + template.contains(AD_ASSEMBLY_SEAM), + "the cached template must still carry the unresolved marker: {template}" ); - assert_eq!( - stub.recorded_request_uris().len(), - 1, - "should fetch the origin only for the first request" + assert!( + !template.contains("window.tsjs"), + "the cached template must not carry a bids script: {template}" ); } - fn cookie_navigation_request() -> Request { - HttpRequest::builder() - .method(Method::GET) - .uri("https://ts.example.com/article") - .header(header::HOST, "ts.example.com") - .header("sec-fetch-dest", "document") - .header("sec-fetch-mode", "navigate") - .header(header::COOKIE, "ts-ec=abc123") - .body(EdgeBody::empty()) - .expect("should build cookie-bearing request") - } - #[tokio::test] - async fn readthrough_never_collapses_template_cookie_variants() { - let mut settings = cookie_policy_settings(Some(&["ab_bucket"]), None, true); - Arc::make_mut(&mut settings) - .creative_opportunities - .as_mut() - .expect("should configure opportunities") - .origin_readthrough_enabled = Some(true); - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - for arm in ["absent", "A", "B", "empty"] { - stub.push_response_with_headers( - 200, - format!("arm-{arm}").into_bytes(), - vec![ - ("content-type", "text/html"), - ("cache-control", "public, max-age=300"), - ], - ); - } - for (index, (cookie, arm)) in [ - (None, "absent"), - (Some("ab_bucket=A"), "A"), - (Some("ab_bucket=B"), "B"), - (Some("ab_bucket="), "empty"), - (Some("ab_bucket=A"), "A"), - (Some("ab_bucket=B"), "B"), - ] - .iter() - .enumerate() - { - let fields: Vec<&[u8]> = cookie.iter().map(|value| value.as_bytes()).collect(); - let response = run(&settings, &services, cookie_policy_request(&fields)).await; - assert_eq!( - response.headers()[HEADER_X_TS_TEMPLATE_CACHE], - if index < 4 { "miss-stored" } else { "hit" }, - "should still cache separate templates" - ); - let body = String::from_utf8(body_of(response).await).expect("should decode HTML"); - assert!( - body.contains(&format!("arm-{arm}")), - "should preserve each cookie variant" - ); - } - assert_eq!( - stub.recorded_cache_intents(), - vec![PlatformCacheIntent::Bypass; 4], - "should never cache origin bytes without the template cookie dimensions, even when the key cookie is absent" + async fn two_pass_integration_templates_are_identical_across_request_shapes() { + let mut settings = settings_with_mode("esi"); + settings.publisher.origin_url = "https://origin.example.com".to_owned(); + crate::integrations::registry_test_support::payload_fixture::select_and_place( + &mut settings, ); - assert_eq!( - stored_cache_keys(&cache).len(), - 4, - "should store each template variant" + let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + assert!( + !registry + .middleware_chain( + &settings.fetch, + crate::middleware::MiddlewarePhase::Fetch, + crate::middleware::HTML_MEDIA_TYPE, + "/", + ) + .is_empty(), + "should exercise the stand-in's middleware, which supplies a stream processor" ); - } + let settings = Arc::new(settings); + let html = br#"
"#; - #[tokio::test] - async fn readthrough_respects_named_session_bypass_without_disabling_anonymous_sharing() { - let mut settings = cookie_policy_settings(None, Some(&["session"]), true); - Arc::make_mut(&mut settings) - .creative_opportunities - .as_mut() - .expect("should configure opportunities") - .origin_readthrough_enabled = Some(true); + for finalizer in [Finalizer::Streaming, Finalizer::Buffered] { + let mut templates = Vec::new(); + for request in [navigation_request(), prefetch_navigation_request()] { + // Independent fills force distinct per-document namespaces. + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + stub.push_response_with_headers( + 200, + html.to_vec(), + vec![ + ("content-type", "text/html"), + ("cache-control", "public, max-age=300"), + ], + ); + + let response = run_via(&settings, &services, request, finalizer).await; + let _ = body_of(response).await; + + let entries = cache.entries.lock().expect("should lock stored templates"); + assert_eq!(entries.len(), 1, "should store a template on every fill"); + let template = entries + .values() + .next() + .expect("should find stored template") + .body + .clone(); + let text = core::str::from_utf8(&template).expect("should store UTF-8 HTML"); + assert!( + text.contains("ts.example.com/page"), + "should rewrite the URL in the payload" + ); + assert!( + text.contains(AD_ASSEMBLY_SEAM), + "should store before per-reader assembly" + ); + for marker in [ + crate::integrations::registry_test_support::payload_fixture::PLACEHOLDER_PREFIX, + ".adSlots", + ".bids=", + "gpt-diagnostics", + "gpt_history", + ] { + assert!( + !text.contains(marker), + "should exclude request-scoped marker {marker}" + ); + } + templates.push(template); + } + assert_eq!( + templates[0], templates[1], + "should store identical bytes across request shapes and per-document namespaces" + ); + } + } + + #[tokio::test] + async fn a_transform_that_overruns_its_buffer_stores_nothing() { + // The 16 MB cap in production, shrunk here. A partial template in template cache is the + // worst outcome available: it would be served to every subsequent visitor as + // a truncated document, indefinitely, with no error after the first request. + // + // Safe by construction — the cap error propagates before the store runs — but + // "by construction" is exactly the kind of claim that stops being true after + // an unrelated refactor moves one line. let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); + let mut raw = settings_with_mode("esi"); + raw.publisher.max_buffered_body_bytes = 8; + let settings = Arc::new(raw); let services = services(Arc::clone(&stub), Arc::clone(&cache)); queue_shareable_html(&stub); + + let orchestrator = Arc::new(AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + )); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let consent = crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + }; + let mut ec_context = EcContext::new_for_test(None, consent); + let publisher_response = handle_publisher_request( + AppContext { + settings: &settings, + integration_registry: ®istry, + }, + &services, + None, + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &[article_slot()], + registry: None, + }, + navigation_request(), + EdgeCacheHeader::SMaxageFallback, + ) + .await + .expect("the request itself should succeed; the cap trips during streaming"); + + let result = publisher_response_into_streaming_response( + publisher_response, + &Method::GET, + Arc::clone(&settings), + ®istry, + orchestrator, + services.clone(), + ) + .await; + + assert!( + result.is_err(), + "overrunning the buffer must fail rather than truncate" + ); + assert!( + cache + .entries + .lock() + .expect("should lock entries") + .is_empty(), + "a failed transform must leave nothing in the shared cache" + ); + } + + #[tokio::test] + async fn a_cache_hit_is_never_shared_cacheable() { + // Asserted positively, because the obvious negative check does not work. + // Forbidding `public`, `s-maxage` and `Surrogate-Control` passes trivially + // on a response that carries no `Cache-Control` at all — and *that* is the + // real failure mode here, since a template cache hit returns before the point where the + // publisher path stamps the response private. HTML with no `Cache-Control` + // is heuristically cacheable, so "nothing to forbid" is not safety. + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); queue_shareable_html(&stub); - let _ = run(&settings, &services, cookie_policy_request(&[])).await; - let response = run(&settings, &services, cookie_policy_request(&[b"session="])).await; + + let _cold = run(&settings, &services, navigation_request()).await; + let warm = run(&settings, &services, navigation_request()).await; + assert_eq!( - response.headers()[HEADER_X_TS_TEMPLATE_CACHE], - "bypass-request", - "should not read an anonymous template for a session" + stub.recorded_request_uris().len(), + 1, + "the second request must be a hit, or this asserts nothing" ); assert_eq!( - stub.recorded_cache_intents(), - vec![ - PlatformCacheIntent::Shared { - surrogate_key: format!( - "ts-template {}", - reader_url_surrogate_key("http://ts.example.com/article") - ) - }, - PlatformCacheIntent::Bypass, - ], - "should share anonymous bytes while refusing even an empty named session" + header_of(&warm, header::CACHE_CONTROL), + Some("no-store, private"), + "an assembled response is per-user even when its template is not" ); + + // Validators would let a client revalidate into a shared copy, and the CDN + // directives would instruct an intermediary to store one outright. The + // origin fixture sends a `public, max-age=300` that must not survive. + for stripped in [header::ETAG, header::LAST_MODIFIED, header::EXPIRES] { + assert_eq!( + header_of(&warm, stripped.clone()), + None, + "{stripped} must not survive onto an assembled response" + ); + } } #[tokio::test] - async fn readthrough_preload_bypasses_configured_cookie_dimensions() { - let mut settings = cookie_policy_settings(Some(&["ab_bucket"]), None, true); - let config = Arc::make_mut(&mut settings) - .creative_opportunities - .as_mut() - .expect("should configure opportunities"); - config.origin_readthrough_enabled = Some(true); - config.assembly_mode = Some(AssemblyMode::Inline); + async fn a_returning_visitor_gets_the_same_privacy_headers() { + // The case with no backstop. A first-visit response sets an EC cookie, so + // the adapter's cookie-privacy net force-privatizes it regardless of what + // this path does. A returning visitor sets no cookie, so that net never + // fires and this path is the only thing standing between an assembled + // per-user document and a shared cache. let stub = Arc::new(StubHttpClient::new()); - stub.set_pending_streaming_responses_supported(true); - let services = services(Arc::clone(&stub), Arc::new(MemoryTemplateCache::default())); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); queue_shareable_html(&stub); - run_through_ec_preload( - &settings, - &services, - cookie_policy_request(&[b"ab_bucket=A"]), - ) - .await; + + let _cold = run(&settings, &services, navigation_request()).await; + let warm = run(&settings, &services, navigation_request()).await; + + assert!( + header_of(&warm, header::SET_COOKIE).is_none(), + "no cookie here means no privacy net, which is the point of this test" + ); assert_eq!( - stub.recorded_cache_intents(), - vec![PlatformCacheIntent::Bypass], - "should apply the cookie-dimension exclusion on the pending origin path too" + header_of(&warm, header::CACHE_CONTROL), + Some("no-store, private") ); } - fn cookie_policy_settings( - key_names: Option<&[&str]>, - bypass_names: Option<&[&str]>, - independent: bool, - ) -> Arc { - let mut settings = settings_with_mode("esi"); - let config = settings - .creative_opportunities - .as_mut() - .expect("should configure opportunities"); - config.template_cache_key_cookies = - key_names.map(|names| names.iter().map(|name| (*name).to_string()).collect()); - config.template_cache_bypass_cookies = - bypass_names.map(|names| names.iter().map(|name| (*name).to_string()).collect()); - config.origin_is_cookie_independent = Some(independent); - config - .validate_runtime() - .expect("should validate cookie policy fixture"); - Arc::new(settings) - } + /// Geo that reports a fixed, recognizable location. + /// + /// A distinct value per synthetic user, so a geo leak into the template shows up + /// as a substring rather than requiring inference. + struct StubGeo(&'static str); - fn cookie_policy_request(fields: &[&[u8]]) -> Request { - let mut request = navigation_request(); - for field in fields { - request.headers_mut().append( - header::COOKIE, - HeaderValue::from_bytes(field).expect("should build cookie field"), - ); + #[async_trait::async_trait(?Send)] + impl crate::platform::PlatformGeo for StubGeo { + async fn lookup( + &self, + _client_ip: Option, + _services: &crate::platform::RuntimeServices, + ) -> Result, Report> { + Ok(Some(GeoInfo { + city: self.0.to_string(), + country: self.0.to_string(), + continent: self.0.to_string(), + latitude: 1.0, + longitude: 2.0, + metro_code: 3, + region: Some(self.0.to_string()), + asn: Some(4), + })) } - request } - // Exercise raw fields at the prepared-request boundary. Ordinary diagnostics - // preparation removes invalid fields/empty pairs before generic cookie handling. - fn prepared_cookie_policy_request(fields: &[&[u8]]) -> Request { - let mut request = cookie_policy_request(fields); - request.extensions_mut().insert( - crate::integrations::gpt_diagnostics::GptDiagnosticsRequestDecision::default(), - ); - request + /// One synthetic user: an identity, a consent posture, and a location. + struct SyntheticUser { + ec_id: &'static str, + jurisdiction: crate::consent::jurisdiction::Jurisdiction, + geo_marker: &'static str, } - #[tokio::test] - async fn template_cookie_publisher_key_only_admits_listed_cookies() { - for unused in [None, Some([].as_slice())] { - let settings = cookie_policy_settings(Some(&["ab_bucket"]), unused, false); - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - queue_shareable_html(&stub); - queue_shareable_html(&stub); - for _ in 0..2 { - let _ = body_of( - run( - &settings, - &services, - cookie_policy_request(&[b"ab_bucket=A"]), - ) - .await, - ) - .await; - } - assert_eq!( - stub.recorded_request_uris().len(), - 1, - "should share a listed-only variant even with independence false" - ); - assert_eq!( - stored_cache_keys(&cache).len(), - 1, - "should store one variant" - ); - let lookups = looked_up_cache_keys(&cache).len(); - let _ = body_of( - run( - &settings, - &services, - cookie_policy_request(&[b"ab_bucket=A; unknown=1"]), - ) - .await, + /// Runs one synthetic user against a fresh cache and returns the stored template. + /// + /// Fresh cache per user deliberately: the point is to compare what each *would* + /// store, so sharing a cache would let the first user's entry answer for the + /// second and the comparison would prove nothing. + async fn stored_template_for(user: &SyntheticUser) -> Vec { + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = RuntimeServices::builder() + .config_store(Arc::new(NoopConfigStore)) + .secret_store(Arc::new(NoopSecretStore)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::new(StubBackend)) + .http_client(Arc::clone(&stub) as Arc) + .geo(Arc::new(StubGeo(user.geo_marker))) + .client_info(ClientInfo::default()) + .template_cache( + Arc::clone(&cache) as Arc ) - .await; - assert_eq!( - looked_up_cache_keys(&cache).len(), - lookups, - "should bypass for any unlisted cookie" - ); - assert_eq!( - stored_cache_keys(&cache).len(), - 1, - "should not store an unlisted-cookie response" - ); - assert_eq!( - stub.recorded_request_uris().len(), - 2, - "should fetch unlisted-cookie origin HTML" - ); - } + .build(); + queue_shareable_html(&stub); + + let orchestrator = Arc::new(AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + )); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let consent = crate::consent::ConsentContext { + jurisdiction: user.jurisdiction.clone(), + ..Default::default() + }; + let mut ec_context = EcContext::new_for_test(Some(user.ec_id.to_string()), consent); + let publisher_response = handle_publisher_request( + AppContext { + settings: &settings, + integration_registry: ®istry, + }, + &services, + None, + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &[article_slot()], + registry: None, + }, + navigation_request(), + EdgeCacheHeader::SMaxageFallback, + ) + .await + .expect("should proxy publisher request"); + let _ = publisher_response_into_streaming_response( + publisher_response, + &Method::GET, + Arc::clone(&settings), + ®istry, + orchestrator, + services.clone(), + ) + .await + .expect("should finalize publisher response"); + + let entries = cache.entries.lock().expect("should lock entries"); + entries + .values() + .next() + .expect("a template should have been stored") + .body + .clone() } #[tokio::test] - async fn template_cookie_publisher_session_bypasses_warm_and_cold_cache() { - for finalizer in [Finalizer::Streaming, Finalizer::Buffered] { - for unused in [None, Some([].as_slice()), Some(["ab_bucket"].as_slice())] { - for warm in [false, true] { - let settings = cookie_policy_settings(unused, Some(&["session"]), true); - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - if warm { - queue_shareable_html(&stub); - let _ = body_of( - run_via( - &settings, - &services, - cookie_policy_request(&[b"ab_bucket=A; ts-ec=reader"]), - finalizer, - ) - .await, - ) - .await; - assert_eq!( - stored_cache_keys(&cache).len(), - 1, - "should warm an anonymous template first" - ); - } - let lookups = looked_up_cache_keys(&cache).len(); - let stores = stored_cache_keys(&cache).len(); - for session in [b"session=".as_slice(), b"session=token".as_slice()] { - stub.push_response_with_headers( - 200, - b"personal-account".to_vec(), - vec![ - ("content-type", "text/html"), - ("cache-control", "public, max-age=300"), - ], - ); - let response = run_via( - &settings, - &services, - cookie_policy_request(&[b"ab_bucket=A; ts-ec=reader", session]), - finalizer, - ) - .await; - assert_eq!( - response.headers()[HEADER_X_TS_TEMPLATE_CACHE], - "bypass-request", - "should report request bypass" - ); - assert!( - !response.headers().contains_key(HEADER_X_TS_ASSEMBLY), - "should omit shared-assembly diagnostics on inline responses" - ); - assert!( - String::from_utf8(body_of(response).await) - .expect("should decode HTML") - .contains("personal-account"), - "should render this request's origin HTML" - ); - } - assert_eq!( - looked_up_cache_keys(&cache).len(), - lookups, - "should never look up or reserve for session requests" - ); - assert_eq!( - stored_cache_keys(&cache).len(), - stores, - "should never store session HTML" - ); - assert_eq!( - stub.recorded_request_uris().len(), - usize::from(warm) + 2, - "should fetch every session request" - ); - assert!( - cache - .entries - .lock() - .expect("should lock templates") - .values() - .all(|entry| !String::from_utf8_lossy(&entry.body) - .contains("personal-account")), - "should keep personal bytes out of templates" - ); - } - } + async fn two_users_differing_in_identity_consent_and_geo_store_the_same_template() { + // The gate the whole design rests on. The template is shared between + // visitors, so anything request-scoped that reaches it is one visitor's data + // served to the next. Byte-identity is the assertion because it does not + // depend on guessing which field might leak. + let alice = SyntheticUser { + ec_id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.alice1", + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + geo_marker: "AliceCity", + }; + let bob = SyntheticUser { + ec_id: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.bobbb1", + jurisdiction: crate::consent::jurisdiction::Jurisdiction::Gdpr, + geo_marker: "BobCity", + }; + + let alice_template = stored_template_for(&alice).await; + let bob_template = stored_template_for(&bob).await; + + assert_eq!( + alice_template, bob_template, + "two users differing in identity, consent and geo must produce the same \ + shared template" + ); + + // Belt and braces: byte-identity would also hold if *both* templates leaked + // the same wrong thing, so name the values that must be absent. + let template = String::from_utf8(alice_template).expect("template should be utf-8"); + for forbidden in [ + alice.ec_id, + bob.ec_id, + alice.geo_marker, + bob.geo_marker, + "adSlots", + "window.tsjs", + ] { + assert!( + !template.contains(forbidden), + "`{forbidden}` must not appear in a shared template: {template}" + ); } } #[tokio::test] - async fn template_cookie_publisher_downstream_variants_remain_separate() { - for finalizer in [Finalizer::Streaming, Finalizer::Buffered] { - let mut settings = - cookie_policy_settings(Some(&["ab_bucket"]), Some(&["session"]), true); - Arc::make_mut(&mut settings) - .creative_opportunities - .as_mut() - .expect("should configure opportunities") - .template_cache_vary = Some(vec!["x-exp-variant".to_string()]); - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - for arm in ["A", "B"] { - stub.push_response_with_headers( - 200, - format!("arm-{arm}").into_bytes(), - vec![ - ("content-type", "text/html"), - ("cache-control", "public, max-age=300"), - ("vary", "X-Exp-Variant"), - ], - ); - } - for (index, arm) in ["A", "B", "A", "B"].iter().enumerate() { - let cookies = format!( - "ab_bucket={arm}; ts-ec=reader{index}; ignored=scope{index}; ignored=other{index}" - ); - let request = cookie_policy_request(&[cookies.as_bytes()]); - assert!( - !request.headers().contains_key("x-exp-variant"), - "should reproduce downstream-only header topology" - ); - let response = run_via(&settings, &services, request, finalizer).await; - assert_eq!( - response.headers()[HEADER_X_TS_TEMPLATE_CACHE], - if index < 2 { "miss-stored" } else { "hit" }, - "should share within each arm" - ); - assert!( - response.headers()[header::CACHE_CONTROL] - .to_str() - .expect("should read cache control") - .contains("private"), - "should keep assembled output private" - ); - let html = - String::from_utf8(body_of(response).await).expect("should decode HTML"); - assert!( - html.contains(&format!("arm-{arm}")), - "should render the correct experiment arm" - ); - assert!( - !html.contains(if *arm == "A" { "arm-B" } else { "arm-A" }), - "should never cross-serve arms" - ); - } - assert_eq!( - stub.recorded_request_uris().len(), - 2, - "should skip origin on both warm arms" - ); - let stored = stored_cache_keys(&cache); - assert_eq!(stored.len(), 2, "should store two variants"); - assert_ne!(stored[0], stored[1], "should distinguish arms in the key"); - } + async fn inline_mode_never_reads_or_writes_the_cache() { + // The shipped path. If this ever cached, per-user ad state would be shared + // between visitors — the exact failure the whole design exists to avoid. + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("inline")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + queue_shareable_html(&stub); + queue_shareable_html(&stub); + + let _ = run(&settings, &services, navigation_request()).await; + let _ = run(&settings, &services, navigation_request()).await; + + assert_eq!( + stub.recorded_request_uris().len(), + 2, + "inline must fetch the origin every time" + ); + assert!( + cache + .entries + .lock() + .expect("should lock entries") + .is_empty(), + "inline must never write a shared template" + ); + } + + /// [`settings_with_mode`], with one integration, core's stand-in, + /// configured a stated way. + /// + /// Edits the parsed settings rather than appending TOML, so two + /// fixtures differ in exactly the field under test. + fn settings_with_integration_timeout(mode: &str, timeout_ms: u32) -> Settings { + let mut settings = settings_with_mode(mode); + settings + .insert_module_config( + "testing", + crate::integrations::registry_test_support::deferred_fixture::MODULE, + &serde_json::json!({ + "label": "example", + "timeout_ms": timeout_ms, + }), + ) + .expect("should insert the stand-in's table"); + crate::integrations::registry_test_support::deferred_fixture::place(&mut settings); + settings + } + + /// Every key the cache was asked to store, rendered. + fn stored_cache_keys(cache: &MemoryTemplateCache) -> Vec { + cache + .stored_keys + .lock() + .expect("should lock stored keys") + .iter() + .map(crate::platform::TemplateCacheKey::to_cache_key) + .collect() + } + + /// Every key the cache was asked to read, rendered. + fn looked_up_cache_keys(cache: &MemoryTemplateCache) -> Vec { + cache + .lookups + .lock() + .expect("should lock lookups") + .iter() + .map(crate::platform::TemplateCacheKey::to_cache_key) + .collect() } #[tokio::test] - async fn template_cookie_publisher_absent_and_empty_are_separate() { - let settings = cookie_policy_settings(Some(&["ab_bucket"]), None, true); + async fn two_integration_configurations_never_share_a_template() { + // `template_fingerprint` folds the complete typed config, and its own + // tests call it directly — so reverting the *call site* back to the + // bundle-only hash, a constant for a given binary, left the entire suite + // green while every configuration silently shared one template. + // + // This drives the real request path and asserts on the keys the cache was + // actually handed, which is the only place that mutation is visible. let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); let services = services(Arc::clone(&stub), Arc::clone(&cache)); - for variant in ["absent", "empty"] { - stub.push_response_with_headers( - 200, - format!("variant-{variant}") - .into_bytes(), - vec![ - ("content-type", "text/html"), - ("cache-control", "public, max-age=300"), - ], - ); - } - for (field, expected) in [ - (b"ts-ec=reader1".as_slice(), "absent"), - (b"ab_bucket=".as_slice(), "empty"), - (b"ts-ec=reader2".as_slice(), "absent"), - (b"ab_bucket=; ts-ec=reader3".as_slice(), "empty"), - ] { - let html = String::from_utf8( - body_of(run(&settings, &services, cookie_policy_request(&[field])).await).await, - ) - .expect("should decode HTML"); - assert!( - html.contains(&format!("variant-{expected}")), - "should preserve absent versus empty variants" - ); - } + let first = Arc::new(settings_with_integration_timeout("esi", 1000)); + let second = Arc::new(settings_with_integration_timeout("esi", 2500)); + queue_shareable_html(&stub); + queue_shareable_html(&stub); + + let first_page = String::from_utf8( + body_of(run(&first, &services, navigation_request()).await).await, + ) + .expect("should serve the first configuration's page as UTF-8"); + let second_page = String::from_utf8( + body_of(run(&second, &services, navigation_request()).await).await, + ) + .expect("should serve the second configuration's page as UTF-8"); + + assert!( + first_page.contains("\"timeoutMs\":1000") + && second_page.contains("\"timeoutMs\":2500"), + "each page should carry the markup its own configuration writes" + ); + let stored = stored_cache_keys(&cache); assert_eq!( - stored_cache_keys(&cache).len(), + stored.len(), 2, - "should store both presence variants" + "each configuration must store its own template, got {stored:?}" + ); + assert_ne!( + stored[0], stored[1], + "two configurations of one integration must key different templates, because \ + one key serves the first configuration's injected markup to the second" ); assert_eq!( stub.recorded_request_uris().len(), 2, - "should hit both variants without origin" + "the second configuration must have missed and fetched the origin rather \ + than reading the first's template" ); } #[tokio::test] - async fn template_cookie_publisher_response_guards_remain_effective() { - for guard in [ - vec![("vary", "Cookie")], - vec![("vary", "X-Exp-Variant"), ("vary", "cOoKiE")], - vec![("vary", "X-Exp-Variant, Cookie")], - vec![("vary", "*")], - vec![("vary", "uncovered-header")], - vec![("set-cookie", "origin=value")], - ] { - let mut settings = - cookie_policy_settings(Some(&["ab_bucket"]), Some(&["session"]), true); - Arc::make_mut(&mut settings) - .creative_opportunities - .as_mut() - .expect("should configure opportunities") - .template_cache_vary = Some(vec!["x-exp-variant".to_string()]); - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - let mut response_headers = vec![ - ("content-type", "text/html"), - ("cache-control", "public, max-age=300"), - ]; - response_headers.extend(guard); - for _ in 0..2 { - stub.push_response_with_headers( - 200, - b"origin".to_vec(), - response_headers.clone(), - ); - let response = run( - &settings, - &services, - cookie_policy_request(&[b"ab_bucket=A"]), - ) - .await; - assert_eq!( - response.headers()[HEADER_X_TS_TEMPLATE_CACHE], - "bypass-response", - "should preserve origin response restrictions" - ); - let _ = body_of(response).await; - } - assert!( - stored_cache_keys(&cache).is_empty(), - "should never store a disqualified response" - ); - assert_eq!( - stub.recorded_request_uris().len(), - 2, - "should fetch each disqualified response" - ); - } - } + async fn one_integration_configuration_keys_one_template() { + // The converse, and the failure mode a fingerprint fix can introduce: + // over-invalidating is as total as under-invalidating. A fingerprint that + // moves between two equal configurations is a cache that never hits, which + // this would read as "no measurable benefit" rather than as a bug. + // + // The two `Settings` are parsed independently, so the integration's + // table iterates in different orders, which is what exercises the sort. + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + let first = Arc::new(settings_with_integration_timeout("esi", 1000)); + let second = Arc::new(settings_with_integration_timeout("esi", 1000)); + queue_shareable_html(&stub); - #[tokio::test] - async fn template_cookie_publisher_malformed_later_fields_and_duplicates_bypass() { - for fields in [ - vec![b"ab_bucket=A".as_slice(), b"unknown=\xff".as_slice()], - vec![b"ab_bucket=A".as_slice(), b"ab_bucket=B".as_slice()], - vec![b"ab_bucket=A; ab_bucket=A".as_slice()], - vec![b"ab_bucket=A; broken".as_slice()], - vec![b"ab_bucket=A;".as_slice()], - vec![b"ab_bucket=A".as_slice(), b"".as_slice()], - ] { - for warm in [false, true] { - let settings = cookie_policy_settings(Some(&["ab_bucket"]), None, true); - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - if warm { - queue_shareable_html(&stub); - let _ = body_of( - run( - &settings, - &services, - cookie_policy_request(&[b"ab_bucket=A"]), - ) - .await, - ) - .await; - assert_eq!( - stored_cache_keys(&cache).len(), - 1, - "should warm cache before malformed input" - ); - } - let lookups = looked_up_cache_keys(&cache).len(); - let stores = stored_cache_keys(&cache).len(); - queue_shareable_html(&stub); - let response = run( - &settings, - &services, - prepared_cookie_policy_request(&fields), - ) - .await; - assert_eq!( - response.headers()[HEADER_X_TS_TEMPLATE_CACHE], - "bypass-request", - "should bypass malformed fields reaching the evaluator" - ); - let _ = body_of(response).await; - assert_eq!( - looked_up_cache_keys(&cache).len(), - lookups, - "should never look up ambiguous input" - ); - assert_eq!( - stored_cache_keys(&cache).len(), - stores, - "should never store ambiguous input" - ); - assert_eq!( - stub.recorded_request_uris().len(), - usize::from(warm) + 1, - "should forward requests accepted by earlier validation" - ); - } - } + let _ = run(&first, &services, navigation_request()).await; + let _ = run(&second, &services, navigation_request()).await; + + let looked_up = looked_up_cache_keys(&cache); + assert_eq!( + looked_up.len(), + 2, + "both requests must consult the cache, got {looked_up:?}" + ); + assert_eq!( + looked_up[0], looked_up[1], + "equal configurations must name one key, or the cache never hits" + ); + assert_eq!( + stub.recorded_request_uris().len(), + 1, + "the second request must be served from the first's template" + ); } #[tokio::test] - async fn template_cookie_publisher_selected_invalid_header_keeps_existing_error() { - let settings = cookie_policy_settings(Some(&["ab_bucket"]), None, true); + async fn one_context_key_allowlist_keys_one_template() { + let mut source = serde_json::to_value(settings_with_mode("esi")) + .expect("should serialize cache settings"); + source["auction"]["allowed_context_keys"] = + serde_json::json!(["zeta", "alpha", "gamma", "beta", "epsilon", "delta"]); let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); let services = services(Arc::clone(&stub), Arc::clone(&cache)); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - let mut ec_context = - EcContext::new_for_test(None, crate::consent::ConsentContext::default()); - let error = handle_publisher_request( - &settings, - &services, - None, - &mut ec_context, - AuctionDispatch { - orchestrator: &orchestrator, - slots: &[article_slot()], - registry: None, - }, - prepared_cookie_policy_request(&[b"ab_bucket=\xff"]), - EdgeCacheHeader::SMaxageFallback, - ) - .await - .err() - .expect("should retain the existing selected-header error"); - assert!( - matches!( - error.current_context(), - TrustedServerError::InvalidHeaderValue { .. } - ), - "should preserve the existing error type" - ); + + for _ in 0..16 { + let settings = Arc::new( + serde_json::from_value(source.clone()).expect("should parse cache settings"), + ); + queue_shareable_html(&stub); + let _ = run(&settings, &services, navigation_request()).await; + } + + let keys = looked_up_cache_keys(&cache); + assert_eq!(keys.len(), 16, "should consult the cache for every request"); assert!( - looked_up_cache_keys(&cache).is_empty(), - "should not reach shared lookup" + keys.iter().all(|key| key == &keys[0]), + "should reuse one key across independently parsed allowlists" ); - assert!( - stored_cache_keys(&cache).is_empty(), - "should not store invalid requests" + assert_eq!( + stored_cache_keys(&cache).len(), + 1, + "should store one template" ); - assert!( - stub.recorded_request_uris().is_empty(), - "should retain earlier rejection before origin" + assert_eq!( + stub.recorded_request_uris().len(), + 1, + "should fetch the origin only for the first request" ); } + fn cookie_navigation_request() -> Request { + HttpRequest::builder() + .method(Method::GET) + .uri("https://ts.example.com/article") + .header(header::HOST, "ts.example.com") + .header("sec-fetch-dest", "document") + .header("sec-fetch-mode", "navigate") + .header(header::COOKIE, "ts-ec=abc123") + .body(EdgeBody::empty()) + .expect("should build cookie-bearing request") + } + #[tokio::test] - async fn template_cookie_publisher_ignores_json_without_changing_origin_cookies() { - for finalizer in [Finalizer::Streaming, Finalizer::Buffered] { - let settings = - cookie_policy_settings(Some(&["ab_bucket"]), Some(&["session"]), true); - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - let cold_cookie = r#"ab_bucket=A; g_state={"enabled":true,"count":1}"#; - queue_shareable_html(&stub); - for (cookie, state) in [ - (cold_cookie, "miss-stored"), - (r#"ab_bucket=A; g_state={"enabled":false,"count":2}"#, "hit"), - ] { - let response = run_via( - &settings, - &services, - cookie_policy_request(&[cookie.as_bytes()]), - finalizer, - ) - .await; - assert_eq!( - response.headers()[HEADER_X_TS_TEMPLATE_CACHE], - state, - "should share the template despite unrelated JSON cookie changes" - ); - let _ = body_of(response).await; - } - let forwarded = stub.recorded_request_headers(); - assert_eq!( - forwarded.len(), - 1, - "should fetch origin only for the cold request" - ); - assert_eq!( - forwarded[0] - .iter() - .filter(|(name, _)| name.eq_ignore_ascii_case(header::COOKIE.as_str())) - .map(|(_, value)| value.as_str()) - .collect::>(), - [cold_cookie], - "should preserve the ignored cookie when forwarding to origin" + async fn readthrough_never_collapses_template_cookie_variants() { + let mut settings = cookie_policy_settings(Some(&["ab_bucket"]), None, true); + Arc::make_mut(&mut settings) + .creative_opportunities + .as_mut() + .expect("should configure opportunities") + .origin_readthrough_enabled = Some(true); + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + for arm in ["absent", "A", "B", "empty"] { + stub.push_response_with_headers( + 200, + format!("arm-{arm}").into_bytes(), + vec![ + ("content-type", "text/html"), + ("cache-control", "public, max-age=300"), + ], ); - let lookups = looked_up_cache_keys(&cache).len(); - queue_shareable_html(&stub); - let response = run_via( - &settings, - &services, - cookie_policy_request(&[br#"ab_bucket=A; g_state={"enabled":true}; session="#]), - finalizer, - ) - .await; + } + for (index, (cookie, arm)) in [ + (None, "absent"), + (Some("ab_bucket=A"), "A"), + (Some("ab_bucket=B"), "B"), + (Some("ab_bucket="), "empty"), + (Some("ab_bucket=A"), "A"), + (Some("ab_bucket=B"), "B"), + ] + .iter() + .enumerate() + { + let fields: Vec<&[u8]> = cookie.iter().map(|value| value.as_bytes()).collect(); + let response = run(&settings, &services, cookie_policy_request(&fields)).await; assert_eq!( response.headers()[HEADER_X_TS_TEMPLATE_CACHE], - "bypass-request", - "should honor bypass cookie presence alongside ignored JSON" - ); - let _ = body_of(response).await; - assert_eq!( - looked_up_cache_keys(&cache).len(), - lookups, - "should skip shared lookup for session requests" + if index < 4 { "miss-stored" } else { "hit" }, + "should still cache separate templates" ); - assert_eq!( - stored_cache_keys(&cache).len(), - 1, - "should not store session responses" + let body = String::from_utf8(body_of(response).await).expect("should decode HTML"); + assert!( + body.contains(&format!("arm-{arm}")), + "should preserve each cookie variant" ); } + assert_eq!( + stub.recorded_cache_intents(), + vec![PlatformCacheIntent::Bypass; 4], + "should never cache origin bytes without the template cookie dimensions, even when the key cookie is absent" + ); + assert_eq!( + stored_cache_keys(&cache).len(), + 4, + "should store each template variant" + ); } #[tokio::test] - async fn template_cookie_publisher_uses_cookies_after_existing_preparation() { - let settings = cookie_policy_settings(Some(&["ab_bucket"]), None, false); - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - queue_shareable_html(&stub); - // Normal preparation strips invalid fields and empty pairs before origin - // forwarding. Preserve it; the cache policy sees the prepared request. - let cold = run( + async fn readthrough_respects_named_session_bypass_without_disabling_anonymous_sharing() { + let mut settings = cookie_policy_settings(None, Some(&["session"]), true); + Arc::make_mut(&mut settings) + .creative_opportunities + .as_mut() + .expect("should configure opportunities") + .origin_readthrough_enabled = Some(true); + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + queue_shareable_html(&stub); + queue_shareable_html(&stub); + let _ = run(&settings, &services, cookie_policy_request(&[])).await; + let response = run(&settings, &services, cookie_policy_request(&[b"session="])).await; + assert_eq!( + response.headers()[HEADER_X_TS_TEMPLATE_CACHE], + "bypass-request", + "should not read an anonymous template for a session" + ); + assert_eq!( + stub.recorded_cache_intents(), + vec![ + PlatformCacheIntent::Shared { + surrogate_key: format!( + "ts-template {}", + reader_url_surrogate_key("http://ts.example.com/article") + ) + }, + PlatformCacheIntent::Bypass, + ], + "should share anonymous bytes while refusing even an empty named session" + ); + } + + #[tokio::test] + async fn readthrough_preload_bypasses_configured_cookie_dimensions() { + let mut settings = cookie_policy_settings(Some(&["ab_bucket"]), None, true); + let config = Arc::make_mut(&mut settings) + .creative_opportunities + .as_mut() + .expect("should configure opportunities"); + config.origin_readthrough_enabled = Some(true); + config.assembly_mode = Some(AssemblyMode::Inline); + let stub = Arc::new(StubHttpClient::new()); + stub.set_pending_streaming_responses_supported(true); + let services = services(Arc::clone(&stub), Arc::new(MemoryTemplateCache::default())); + queue_shareable_html(&stub); + run_through_ec_preload( &settings, &services, - cookie_policy_request(&[b"ab_bucket=A;", b"unknown=\xff"]), + cookie_policy_request(&[b"ab_bucket=A"]), ) .await; assert_eq!( - cold.headers()[HEADER_X_TS_TEMPLATE_CACHE], - "miss-stored", - "should classify the cookies actually forwarded" + stub.recorded_cache_intents(), + vec![PlatformCacheIntent::Bypass], + "should apply the cookie-dimension exclusion on the pending origin path too" ); - let _ = body_of(cold).await; - let warm = run( + } + + fn cookie_policy_settings( + key_names: Option<&[&str]>, + bypass_names: Option<&[&str]>, + independent: bool, + ) -> Arc { + let mut settings = settings_with_mode("esi"); + let config = settings + .creative_opportunities + .as_mut() + .expect("should configure opportunities"); + config.template_cache_key_cookies = + key_names.map(|names| names.iter().map(|name| (*name).to_string()).collect()); + config.template_cache_bypass_cookies = + bypass_names.map(|names| names.iter().map(|name| (*name).to_string()).collect()); + config.origin_is_cookie_independent = Some(independent); + config + .validate_runtime() + .expect("should validate cookie policy fixture"); + Arc::new(settings) + } + + /// The settings check refuses a key cookie named as core's own identity + /// cookies are. A deployment's own prefix is known only on a request, + /// so a key cookie naming this request's Edge Cookie shares no + /// template. + #[tokio::test] + async fn a_key_cookie_naming_the_requests_edge_cookie_shares_no_template() { + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = cookie_policy_settings(Some(&["_ex-ec"]), None, true); + let services = services(Arc::clone(&stub), Arc::clone(&cache)).with_prefixes( + crate::prefix::Prefixes::new( + crate::prefix::PublicPrefix::new(["_ex"]).expect("should accept the prefix"), + crate::platform::ServerKeyPrefix::new("ex", "test-publisher.com"), + ), + ); + queue_shareable_html(&stub); + + let response = run( &settings, &services, - cookie_policy_request(&[b"ab_bucket=A"]), + cookie_policy_request(&[b"_ex-ec=reader"]), ) .await; + assert_eq!( - warm.headers()[HEADER_X_TS_TEMPLATE_CACHE], - "hit", - "should share identical prepared origin inputs" - ); - let _ = body_of(warm).await; - assert_eq!( - stub.recorded_request_uris().len(), - 1, - "should preserve existing request preparation" + response.headers()[HEADER_X_TS_TEMPLATE_CACHE], + "bypass-request" ); - let forwarded = stub.recorded_request_headers(); - assert_eq!(forwarded.len(), 1, "should record one origin request"); - let forwarded_cookies = forwarded[0] - .iter() - .filter(|(name, _)| name.eq_ignore_ascii_case(header::COOKIE.as_str())) - .map(|(_, value)| value.as_str()) - .collect::>(); - assert_eq!( - forwarded_cookies, - ["ab_bucket=A"], - "should forward exactly the prepared cookie represented by the cache key" + let _ = body_of(response).await; + assert!( + stored_cache_keys(&cache).is_empty(), + "a reader's identifier must never key a stored template" ); } + fn cookie_policy_request(fields: &[&[u8]]) -> Request { + let mut request = navigation_request(); + for field in fields { + request.headers_mut().append( + header::COOKIE, + HeaderValue::from_bytes(field).expect("should build cookie field"), + ); + } + request + } + + // Exercise raw fields at the prepared-request boundary. A module's preparer + // may remove invalid fields and empty pairs before generic cookie handling. + fn prepared_cookie_policy_request(fields: &[&[u8]]) -> Request { + let mut request = cookie_policy_request(fields); + IntegrationRegistry::mark_prepared_for_tests(&mut request); + request + } + #[tokio::test] - async fn template_cookie_publisher_empty_lists_keep_legacy_behavior() { - for independent in [false, true] { - let settings = cookie_policy_settings(Some(&[]), Some(&[]), independent); + async fn template_cookie_publisher_key_only_admits_listed_cookies() { + for unused in [None, Some([].as_slice())] { + let settings = cookie_policy_settings(Some(&["ab_bucket"]), unused, false); let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); let services = services(Arc::clone(&stub), Arc::clone(&cache)); + queue_shareable_html(&stub); + queue_shareable_html(&stub); for _ in 0..2 { - queue_shareable_html(&stub); - let _ = - body_of(run(&settings, &services, cookie_navigation_request()).await).await; + let _ = body_of( + run( + &settings, + &services, + cookie_policy_request(&[b"ab_bucket=A"]), + ) + .await, + ) + .await; } assert_eq!( stub.recorded_request_uris().len(), - if independent { 1 } else { 2 }, - "should retain legacy boolean eligibility" + 1, + "should share a listed-only variant even with independence false" ); assert_eq!( stored_cache_keys(&cache).len(), - usize::from(independent), - "should preserve legacy storage" + 1, + "should store one variant" ); - } - } - - #[tokio::test] - async fn template_cookie_publisher_policy_changes_invalidate_templates() { - let first = cookie_policy_settings(None, None, true); - let second = cookie_policy_settings(Some(&["ab_bucket"]), None, true); - let third = cookie_policy_settings(Some(&["ab_bucket"]), Some(&["session"]), true); - assert_ne!( - template_fingerprint(&first), - template_fingerprint(&second), - "should fingerprint the key policy" - ); - assert_ne!( - template_fingerprint(&second), - template_fingerprint(&third), - "should fingerprint the bypass policy" - ); - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - for settings in [&first, &second, &third] { - queue_shareable_html(&stub); + let lookups = looked_up_cache_keys(&cache).len(); let _ = body_of( run( - settings, + &settings, &services, - cookie_policy_request(&[b"ab_bucket=A"]), + cookie_policy_request(&[b"ab_bucket=A; unknown=1"]), ) .await, ) .await; - } - let keys = stored_cache_keys(&cache); - assert_eq!( - keys.len(), - 3, - "should store a fresh entry under each policy" - ); - assert_ne!(keys[0], keys[1], "should not reuse pre-policy templates"); - assert_ne!( - keys[1], keys[2], - "should not reuse entries admitted under another bypass policy" - ); - assert_eq!( - stub.recorded_request_uris().len(), - 3, - "should fetch after each policy change" - ); - } - - #[tokio::test] - async fn template_cookie_publisher_warm_variant_runs_fresh_reader_assembly() { - let mut raw = settings_with_bidder("esi"); - let config = raw - .creative_opportunities - .as_mut() - .expect("should configure opportunities"); - config.template_cache_key_cookies = Some(vec!["ab_bucket".to_string()]); - config.origin_is_cookie_independent = Some(true); - let settings = Arc::new(raw); - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - for (index, field) in [ - b"ab_bucket=A; ts-ec=reader1".as_slice(), - b"ab_bucket=A; ts-ec=reader2".as_slice(), - ] - .iter() - .enumerate() - { - queue_bid_response(&stub); - if index == 0 { - queue_shareable_html(&stub); - } - let response = run_bidding_at_price( - &settings, - &services, - cookie_policy_request(&[field]), - if index == 0 { 3.5 } else { 7.5 }, - ) - .await; assert_eq!( - response.headers()[HEADER_X_TS_TEMPLATE_CACHE], - if index == 0 { "miss-stored" } else { "hit" }, - "should serve the second reader from the shared template" - ); - assert!( - response.headers()[header::CACHE_CONTROL] - .to_str() - .expect("should read cache policy") - .contains("private"), - "should keep reader output private" + looked_up_cache_keys(&cache).len(), + lookups, + "should bypass for any unlisted cookie" ); - let document = - String::from_utf8(body_of(response).await).expect("should decode document"); assert_eq!( - seam_bids(&document) - .get("test-slot") - .and_then(|bid| bid.get("hb_pb")) - .and_then(serde_json::Value::as_str), - Some(if index == 0 { "3.50" } else { "7.50" }), - "should assemble this request's winning bid" + stored_cache_keys(&cache).len(), + 1, + "should not store an unlisted-cookie response" ); - } - let requests = stub.recorded_request_uris(); - assert_eq!( - requests - .iter() - .filter(|uri| uri.contains("/article")) - .count(), - 1, - "should fetch one shared origin template" - ); - assert_eq!( - requests.len(), - 3, - "should run a fresh auction on each reader request" - ); - let entries = cache.entries.lock().expect("should lock templates"); - assert_eq!(entries.len(), 1, "should share within a variant"); - let stored = String::from_utf8_lossy( - &entries - .values() - .next() - .expect("should store a template") - .body, - ); - assert!( - stored.contains(AD_ASSEMBLY_SEAM), - "should keep the unresolved reader assembly marker" - ); - for reader_bytes in ["reader1", "reader2", "window.tsjs", "hb_pb"] { - assert!( - !stored.contains(reader_bytes), - "should exclude per-reader state from stored bytes" + assert_eq!( + stub.recorded_request_uris().len(), + 2, + "should fetch unlisted-cookie origin HTML" ); } } #[tokio::test] - async fn template_cookie_publisher_warm_variant_finalizes_ec_withdrawal() { + async fn template_cookie_publisher_session_bypasses_warm_and_cold_cache() { for finalizer in [Finalizer::Streaming, Finalizer::Buffered] { - let mut settings = cookie_policy_settings(Some(&["ab_bucket"]), None, true); - Arc::make_mut(&mut settings).auction.providers = - crate::auction_config_types::AuctionConfig::legacy_provider_map(&[ - SCHEDULING_PROVIDER, - ]); + for unused in [None, Some([].as_slice()), Some(["ab_bucket"].as_slice())] { + for warm in [false, true] { + let settings = cookie_policy_settings(unused, Some(&["session"]), true); + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + if warm { + queue_shareable_html(&stub); + let _ = body_of( + run_via( + &settings, + &services, + cookie_policy_request(&[b"ab_bucket=A; ts-ec=reader"]), + finalizer, + ) + .await, + ) + .await; + assert_eq!( + stored_cache_keys(&cache).len(), + 1, + "should warm an anonymous template first" + ); + } + let lookups = looked_up_cache_keys(&cache).len(); + let stores = stored_cache_keys(&cache).len(); + for session in [b"session=".as_slice(), b"session=token".as_slice()] { + stub.push_response_with_headers( + 200, + b"personal-account".to_vec(), + vec![ + ("content-type", "text/html"), + ("cache-control", "public, max-age=300"), + ], + ); + let response = run_via( + &settings, + &services, + cookie_policy_request(&[b"ab_bucket=A; ts-ec=reader", session]), + finalizer, + ) + .await; + assert_eq!( + response.headers()[HEADER_X_TS_TEMPLATE_CACHE], + "bypass-request", + "should report request bypass" + ); + assert!( + !response.headers().contains_key(HEADER_X_TS_ASSEMBLY), + "should omit shared-assembly diagnostics on inline responses" + ); + assert!( + String::from_utf8(body_of(response).await) + .expect("should decode HTML") + .contains("personal-account"), + "should render this request's origin HTML" + ); + } + assert_eq!( + looked_up_cache_keys(&cache).len(), + lookups, + "should never look up or reserve for session requests" + ); + assert_eq!( + stored_cache_keys(&cache).len(), + stores, + "should never store session HTML" + ); + assert_eq!( + stub.recorded_request_uris().len(), + usize::from(warm) + 2, + "should fetch every session request" + ); + assert!( + cache + .entries + .lock() + .expect("should lock templates") + .values() + .all(|entry| !String::from_utf8_lossy(&entry.body) + .contains("personal-account")), + "should keep personal bytes out of templates" + ); + } + } + } + } + + #[tokio::test] + async fn template_cookie_publisher_downstream_variants_remain_separate() { + for finalizer in [Finalizer::Streaming, Finalizer::Buffered] { + let mut settings = + cookie_policy_settings(Some(&["ab_bucket"]), Some(&["session"]), true); + Arc::make_mut(&mut settings) + .creative_opportunities + .as_mut() + .expect("should configure opportunities") + .template_cache_vary = Some(vec!["x-exp-variant".to_string()]); let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); let services = services(Arc::clone(&stub), Arc::clone(&cache)); - let graph = KvIdentityGraph::in_memory("cookie-withdrawal-store"); - let identities = [ - format!("{}.Read01", "a".repeat(64)), - format!("{}.Read02", "b".repeat(64)), - ]; - for (index, identity) in identities.iter().enumerate() { - assert!( - crate::ec::generation::is_valid_ec_id(identity), - "should use valid EC identities in the fixture" + for arm in ["A", "B"] { + stub.push_response_with_headers( + 200, + format!("arm-{arm}").into_bytes(), + vec![ + ("content-type", "text/html"), + ("cache-control", "public, max-age=300"), + ("vary", "X-Exp-Variant"), + ], ); - graph - .create( - identity, - &crate::ec::kv_types::KvEntry::minimal( - "example.com", - &format!("partner-reader-{index}"), - crate::ec::current_timestamp(), - ), - ) - .expect("should seed a live reader identity"); } - let registry = IntegrationRegistry::new(&settings) - .expect("should create integration registry"); - let partner = serde_json::from_value(serde_json::json!({ - "name": "Example partner", - "source_domain": "example.com", - "bidstream_enabled": true - })) - .expect("should deserialize fixture partner"); - let partners = PartnerRegistry::from_config(&[partner]) - .expect("should create partner registry"); - // Only the cold request has an origin response available. - queue_shareable_html(&stub); - - for (index, (identity, withdrawn)) in [ - (&identities[0], false), - (&identities[1], false), - (&identities[1], true), - ] - .into_iter() - .enumerate() - { - let consent = if withdrawn { - ConsentContext { - jurisdiction: crate::consent::jurisdiction::Jurisdiction::UsState( - "CA".to_owned(), - ), - gpc: true, - ..Default::default() - } - } else { - scheduling_consent() - }; - let mut ec_context = EcContext::new_for_test(Some(identity.clone()), consent); - assert_eq!( - ec_context.ec_allowed(), - !withdrawn, - "should apply reader consent" + for (index, arm) in ["A", "B", "A", "B"].iter().enumerate() { + let cookies = format!( + "ab_bucket={arm}; ts-ec=reader{index}; ignored=scope{index}; ignored=other{index}" ); - let captured = Arc::new(Mutex::new(None)); - let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - orchestrator.register_provider(Arc::new(SchedulingCaptureProvider { - captured: Arc::clone(&captured), - http: Arc::clone(&stub), - lookups: Arc::new(AtomicUsize::new(0)), - })); - let orchestrator = Arc::new(orchestrator); - let cookies = format!("ab_bucket=A; ts-ec={identity}"); - let response = handle_publisher_request( - &settings, - &services, - Some(&graph), - &mut ec_context, - AuctionDispatch { - orchestrator: &orchestrator, - slots: &[article_slot()], - registry: Some(&partners), - }, - cookie_policy_request(&[cookies.as_bytes()]), - EdgeCacheHeader::SMaxageFallback, - ) - .await - .expect("should serve a reader with an existing EC"); + let request = cookie_policy_request(&[cookies.as_bytes()]); assert!( - ec_context.kv_snapshot().entry_for(identity).is_some(), - "should preload this reader's identity even during withdrawal on a hit" - ); - let mut response = finalize_test_publisher_response( - response, - &settings, - &services, - ®istry, - orchestrator, - finalizer, - ) - .await; - crate::ec::finalize::ec_finalize_response( - &settings, - &mut ec_context, - Some(&graph), - &partners, - None, - None, - &mut response, + !request.headers().contains_key("x-exp-variant"), + "should reproduce downstream-only header topology" ); + let response = run_via(&settings, &services, request, finalizer).await; assert_eq!( response.headers()[HEADER_X_TS_TEMPLATE_CACHE], - if index == 0 { "miss-stored" } else { "hit" }, - "should share one variant across identities and withdrawal" + if index < 2 { "miss-stored" } else { "hit" }, + "should share within each arm" ); - let cache_control = response.headers()[header::CACHE_CONTROL] - .to_str() - .expect("should decode cache policy"); - for directive in ["private", "no-store"] { - assert!( - cache_control - .split(',') - .any(|value| value.trim() == directive), - "should keep finalized reader responses private and uncacheable" - ); - } - assert_eq!( - response - .headers() - .get_all(header::SET_COOKIE) - .iter() - .any(|value| { - let value = value.to_str().expect("should decode response cookie"); - value.starts_with("ts-ec=") && value.contains("Max-Age=0") - }), - withdrawn, - "should expire the EC cookie only for the withdrawing reader" + assert!( + response.headers()[header::CACHE_CONTROL] + .to_str() + .expect("should read cache control") + .contains("private"), + "should keep assembled output private" ); - let body = body_of(response).await; - assert!(!body.is_empty(), "should render a complete reader response"); - let captured = captured.lock().expect("should lock captured auction"); - let auction = captured.as_ref().expect("should dispatch a reader auction"); - assert_eq!( - auction.request.user.id.as_deref(), - if withdrawn { - None - } else { - Some(identity.as_str()) - }, - "should use this reader's identity and suppress it after withdrawal" + let html = + String::from_utf8(body_of(response).await).expect("should decode HTML"); + assert!( + html.contains(&format!("arm-{arm}")), + "should render the correct experiment arm" + ); + assert!( + !html.contains(if *arm == "A" { "arm-B" } else { "arm-A" }), + "should never cross-serve arms" ); - if withdrawn { - assert!( - auction.request.user.eids.is_none(), - "should suppress withdrawn EIDs" - ); - } else { - let eids = auction - .request - .user - .eids - .as_ref() - .expect("should include consenting reader EIDs"); - assert_eq!(eids.len(), 1, "should expose only the configured partner"); - assert_eq!( - eids[0].source, "example.com", - "should use the registered source" - ); - assert_eq!( - eids[0].uids[0].id, - format!("partner-reader-{index}"), - "should use this reader's partner identity on cold and warm requests" - ); - } } assert_eq!( stub.recorded_request_uris().len(), - 1, - "should fetch origin only once" - ); - assert_eq!( - looked_up_cache_keys(&cache).len(), - 3, - "should look up every reader" - ); - assert_eq!( - stored_cache_keys(&cache).len(), - 1, - "should store only the cold template" + 2, + "should skip origin on both warm arms" ); - for (index, identity) in identities.iter().enumerate() { - let (entry, _) = graph - .get(identity) - .expect("should read reader identity") - .expect("should retain the live row or its tombstone"); - assert_eq!( - entry.consent.ok, - index == 0, - "should revoke only the second reader" - ); - assert_eq!( - entry.ids.is_empty(), - index == 1, - "should clear only revoked partner IDs" - ); - } - let entries = cache.entries.lock().expect("should lock cached templates"); - let stored = &entries - .values() - .next() - .expect("should retain shared template") - .body; - let stored = String::from_utf8_lossy(stored); - for identity in &identities { - assert!( - !stored.contains(identity), - "should keep reader identities out of cached bytes" - ); - } + let stored = stored_cache_keys(&cache); + assert_eq!(stored.len(), 2, "should store two variants"); + assert_ne!(stored[0], stored[1], "should distinguish arms in the key"); } } #[tokio::test] - async fn by_default_a_cookie_bearing_request_uses_no_shared_cache() { - // The shipped default, and the reason the cache is nearly inert on real - // traffic: TS sets its own identity cookie, so essentially every repeat - // visitor arrives carrying one and is excluded in both directions. + async fn template_cookie_publisher_absent_and_empty_are_separate() { + let settings = cookie_policy_settings(Some(&["ab_bucket"]), None, true); let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); - let settings = Arc::new(settings_with_mode("esi")); let services = services(Arc::clone(&stub), Arc::clone(&cache)); - queue_shareable_html(&stub); - queue_shareable_html(&stub); - - let _ = run(&settings, &services, cookie_navigation_request()).await; - let _ = run(&settings, &services, cookie_navigation_request()).await; - + for variant in ["absent", "empty"] { + stub.push_response_with_headers( + 200, + format!("variant-{variant}") + .into_bytes(), + vec![ + ("content-type", "text/html"), + ("cache-control", "public, max-age=300"), + ], + ); + } + for (field, expected) in [ + (b"ts-ec=reader1".as_slice(), "absent"), + (b"ab_bucket=".as_slice(), "empty"), + (b"ts-ec=reader2".as_slice(), "absent"), + (b"ab_bucket=; ts-ec=reader3".as_slice(), "empty"), + ] { + let html = String::from_utf8( + body_of(run(&settings, &services, cookie_policy_request(&[field])).await).await, + ) + .expect("should decode HTML"); + assert!( + html.contains(&format!("variant-{expected}")), + "should preserve absent versus empty variants" + ); + } assert_eq!( - stub.recorded_request_uris().len(), + stored_cache_keys(&cache).len(), 2, - "both requests must reach the origin" - ); - assert!( - cache - .entries - .lock() - .expect("should lock entries") - .is_empty(), - "and neither may store a template" + "should store both presence variants" ); - } - - #[tokio::test] - async fn a_declared_cookie_independent_origin_lets_repeat_visitors_share() { - // The opt-in. Without it the cache can only ever serve first-ever page - // views, which is not the population the issue cares about. - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let mut raw = settings_with_mode("esi"); - raw.creative_opportunities - .as_mut() - .expect("fixture configures creative opportunities") - .origin_is_cookie_independent = Some(true); - let settings = Arc::new(raw); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - queue_shareable_html(&stub); - queue_shareable_html(&stub); - - let _ = run(&settings, &services, cookie_navigation_request()).await; - let _ = run(&settings, &services, cookie_navigation_request()).await; - assert_eq!( stub.recorded_request_uris().len(), - 1, - "the second cookie-bearing request should be served from the cache" + 2, + "should hit both variants without origin" ); } #[tokio::test] - async fn an_active_diagnostics_request_never_stores_a_template() { - // An independent review reintroduced a diagnostics leak scoped to - // A request-private diagnostics mutation could otherwise leak through a - // shared template. `requires_private_no_store()` is a - // strict superset of the condition under which diagnostics markup is - // emitted, and that stamp lands *before* the template cache gate reads response headers, - // so such a request never stores a template at all. - // - // That is a coincidence between two independent conditions, and the whole - // protection rests on it. This pins the consequence directly, so the - // relationship is checked rather than merely reasoned about. - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let settings = Arc::new(settings_with_mode("esi")); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - queue_shareable_html(&stub); - - let _ = run(&settings, &services, diagnostics_navigation_request()).await; - - assert!( - cache - .entries - .lock() - .expect("should lock entries") - .is_empty(), - "a reader running diagnostics must not contribute to a shared cache" - ); + async fn template_cookie_publisher_response_guards_remain_effective() { + for guard in [ + vec![("vary", "Cookie")], + vec![("vary", "X-Exp-Variant"), ("vary", "cOoKiE")], + vec![("vary", "X-Exp-Variant, Cookie")], + vec![("vary", "*")], + vec![("vary", "uncovered-header")], + vec![("set-cookie", "origin=value")], + ] { + let mut settings = + cookie_policy_settings(Some(&["ab_bucket"]), Some(&["session"]), true); + Arc::make_mut(&mut settings) + .creative_opportunities + .as_mut() + .expect("should configure opportunities") + .template_cache_vary = Some(vec!["x-exp-variant".to_string()]); + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + let mut response_headers = vec![ + ("content-type", "text/html"), + ("cache-control", "public, max-age=300"), + ]; + response_headers.extend(guard); + for _ in 0..2 { + stub.push_response_with_headers( + 200, + b"origin".to_vec(), + response_headers.clone(), + ); + let response = run( + &settings, + &services, + cookie_policy_request(&[b"ab_bucket=A"]), + ) + .await; + assert_eq!( + response.headers()[HEADER_X_TS_TEMPLATE_CACHE], + "bypass-response", + "should preserve origin response restrictions" + ); + let _ = body_of(response).await; + } + assert!( + stored_cache_keys(&cache).is_empty(), + "should never store a disqualified response" + ); + assert_eq!( + stub.recorded_request_uris().len(), + 2, + "should fetch each disqualified response" + ); + } } #[tokio::test] - async fn active_diagnostics_bypass_an_already_warm_template() { - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let settings = Arc::new(settings_with_mode("esi")); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - queue_shareable_html(&stub); - queue_shareable_html(&stub); - - let _ = body_of(run(&settings, &services, navigation_request()).await).await; - let response = run(&settings, &services, diagnostics_navigation_request()).await; - let document = String::from_utf8(body_of(response).await) - .expect("diagnostics document should be UTF-8"); + async fn template_cookie_publisher_malformed_later_fields_and_duplicates_bypass() { + for fields in [ + vec![b"ab_bucket=A".as_slice(), b"unknown=\xff".as_slice()], + vec![b"ab_bucket=A".as_slice(), b"ab_bucket=B".as_slice()], + vec![b"ab_bucket=A; ab_bucket=A".as_slice()], + vec![b"ab_bucket=A; broken".as_slice()], + vec![b"ab_bucket=A;".as_slice()], + vec![b"ab_bucket=A".as_slice(), b"".as_slice()], + ] { + for warm in [false, true] { + let settings = cookie_policy_settings(Some(&["ab_bucket"]), None, true); + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + if warm { + queue_shareable_html(&stub); + let _ = body_of( + run( + &settings, + &services, + cookie_policy_request(&[b"ab_bucket=A"]), + ) + .await, + ) + .await; + assert_eq!( + stored_cache_keys(&cache).len(), + 1, + "should warm cache before malformed input" + ); + } + let lookups = looked_up_cache_keys(&cache).len(); + let stores = stored_cache_keys(&cache).len(); + queue_shareable_html(&stub); + let response = run( + &settings, + &services, + prepared_cookie_policy_request(&fields), + ) + .await; + assert_eq!( + response.headers()[HEADER_X_TS_TEMPLATE_CACHE], + "bypass-request", + "should bypass malformed fields reaching the evaluator" + ); + let _ = body_of(response).await; + assert_eq!( + looked_up_cache_keys(&cache).len(), + lookups, + "should never look up ambiguous input" + ); + assert_eq!( + stored_cache_keys(&cache).len(), + stores, + "should never store ambiguous input" + ); + assert_eq!( + stub.recorded_request_uris().len(), + usize::from(warm) + 1, + "should forward requests accepted by earlier validation" + ); + } + } + } - assert_eq!( - stub.recorded_request_uris().len(), - 2, - "request-private diagnostics must reach the origin even when an ordinary \ - shared template is warm" + #[tokio::test] + async fn template_cookie_publisher_selected_invalid_header_keeps_existing_error() { + let settings = cookie_policy_settings(Some(&["ab_bucket"]), None, true); + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let mut ec_context = + EcContext::new_for_test(None, crate::consent::ConsentContext::default()); + let registry = test_registry(&settings); + let error = handle_publisher_request( + AppContext { + settings: &settings, + integration_registry: ®istry, + }, + &services, + None, + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &[article_slot()], + registry: None, + }, + prepared_cookie_policy_request(&[b"ab_bucket=\xff"]), + EdgeCacheHeader::SMaxageFallback, + ) + .await + .err() + .expect("should retain the existing selected-header error"); + assert!( + matches!( + error.current_context(), + TrustedServerError::InvalidHeaderValue { .. } + ), + "should preserve the existing error type" ); - assert_eq!( - cache.lookups.lock().expect("should lock lookups").len(), - 1, - "the diagnostics request must not consult template cache at all" + assert!( + looked_up_cache_keys(&cache).is_empty(), + "should not reach shared lookup" ); assert!( - document.contains("__tsjs_gpt_diagnostics_active"), - "origin fallback must retain the request-private diagnostics bootstrap" + stored_cache_keys(&cache).is_empty(), + "should not store invalid requests" + ); + assert!( + stub.recorded_request_uris().is_empty(), + "should retain earlier rejection before origin" ); } #[tokio::test] - async fn datadome_suppressed_request_bypasses_a_warm_shared_template() { + async fn template_cookie_publisher_ignores_json_without_changing_origin_cookies() { + for finalizer in [Finalizer::Streaming, Finalizer::Buffered] { + let settings = + cookie_policy_settings(Some(&["ab_bucket"]), Some(&["session"]), true); + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + let cold_cookie = r#"ab_bucket=A; g_state={"enabled":true,"count":1}"#; + queue_shareable_html(&stub); + for (cookie, state) in [ + (cold_cookie, "miss-stored"), + (r#"ab_bucket=A; g_state={"enabled":false,"count":2}"#, "hit"), + ] { + let response = run_via( + &settings, + &services, + cookie_policy_request(&[cookie.as_bytes()]), + finalizer, + ) + .await; + assert_eq!( + response.headers()[HEADER_X_TS_TEMPLATE_CACHE], + state, + "should share the template despite unrelated JSON cookie changes" + ); + let _ = body_of(response).await; + } + let forwarded = stub.recorded_request_headers(); + assert_eq!( + forwarded.len(), + 1, + "should fetch origin only for the cold request" + ); + assert_eq!( + forwarded[0] + .iter() + .filter(|(name, _)| name.eq_ignore_ascii_case(header::COOKIE.as_str())) + .map(|(_, value)| value.as_str()) + .collect::>(), + [cold_cookie], + "should preserve the ignored cookie when forwarding to origin" + ); + let lookups = looked_up_cache_keys(&cache).len(); + queue_shareable_html(&stub); + let response = run_via( + &settings, + &services, + cookie_policy_request(&[br#"ab_bucket=A; g_state={"enabled":true}; session="#]), + finalizer, + ) + .await; + assert_eq!( + response.headers()[HEADER_X_TS_TEMPLATE_CACHE], + "bypass-request", + "should honor bypass cookie presence alongside ignored JSON" + ); + let _ = body_of(response).await; + assert_eq!( + looked_up_cache_keys(&cache).len(), + lookups, + "should skip shared lookup for session requests" + ); + assert_eq!( + stored_cache_keys(&cache).len(), + 1, + "should not store session responses" + ); + } + } + + #[tokio::test] + async fn template_cookie_publisher_uses_cookies_after_existing_preparation() { + let settings = cookie_policy_settings(Some(&["ab_bucket"]), None, false); let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); - let mut raw = settings_with_mode("esi"); - raw.integrations - .insert_config( - "datadome", - &serde_json::json!({ - "enabled": true, - "client_side_key": "test-client-key", - }), - ) - .expect("should configure DataDome integration"); - let settings = Arc::new(raw); let services = services(Arc::clone(&stub), Arc::clone(&cache)); queue_shareable_html(&stub); - queue_shareable_html(&stub); - - let ordinary = run(&settings, &services, navigation_request()).await; - let ordinary_document = String::from_utf8(body_of(ordinary).await) - .expect("ordinary document should be UTF-8"); - assert!( - ordinary_document.contains("/integrations/datadome/tags.js"), - "the warm template fixture must contain the ordinary DataDome tag" + // A module's preparer may rewrite the Cookie header before origin + // forwarding, as the stand-in does when it strips the cookie it + // reserves, dropping a field it cannot read and an empty pair as it + // goes. The cache policy sees the prepared request. + let reserved = format!( + "{}=1", + crate::integrations::registry_test_support::request_fixture::COOKIE_NAME ); - - let mut suppressed_request = navigation_request(); - suppressed_request - .headers_mut() - .insert("sec-fetch-dest", HeaderValue::from_static("script")); - suppressed_request - .extensions_mut() - .insert(crate::integrations::datadome::DataDomeClientTagSuppressed); - let suppressed = run(&settings, &services, suppressed_request).await; + let cold = run( + &settings, + &services, + cookie_policy_request(&[b"ab_bucket=A;", b"unknown=\xff", reserved.as_bytes()]), + ) + .await; assert_eq!( - suppressed - .headers() - .get(HEADER_X_TS_TEMPLATE_CACHE) - .and_then(|value| value.to_str().ok()), - Some("bypass-request"), - "request-scoped tag suppression must not read a shared template" + cold.headers()[HEADER_X_TS_TEMPLATE_CACHE], + "miss-stored", + "should classify the cookies actually forwarded" ); - let suppressed_document = String::from_utf8(body_of(suppressed).await) - .expect("suppressed document should be UTF-8"); - + let _ = body_of(cold).await; + let warm = run( + &settings, + &services, + cookie_policy_request(&[b"ab_bucket=A"]), + ) + .await; assert_eq!( - stub.recorded_request_uris().len(), - 2, - "the suppressed navigation must reach the origin even when template cache is warm" + warm.headers()[HEADER_X_TS_TEMPLATE_CACHE], + "hit", + "should share identical prepared origin inputs" ); + let _ = body_of(warm).await; assert_eq!( - cache.lookups.lock().expect("should lock lookups").len(), + stub.recorded_request_uris().len(), 1, - "the suppressed request must bypass template cache before lookup" + "should preserve existing request preparation" ); - assert!( - !suppressed_document.contains("/integrations/datadome/tags.js"), - "the request-scoped suppression decision must survive origin processing" + let forwarded = stub.recorded_request_headers(); + assert_eq!(forwarded.len(), 1, "should record one origin request"); + let forwarded_cookies = forwarded[0] + .iter() + .filter(|(name, _)| name.eq_ignore_ascii_case(header::COOKIE.as_str())) + .map(|(_, value)| value.as_str()) + .collect::>(); + assert_eq!( + forwarded_cookies, + ["ab_bucket=A"], + "should forward exactly the prepared cookie represented by the cache key" ); } #[tokio::test] - async fn real_diagnostics_query_bypasses_template_cache_and_keeps_its_private_bootstrap() { - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let mut raw = settings_with_mode("esi"); - raw.integrations - .insert_config("gpt_diagnostics", &serde_json::json!({ "enabled": true })) - .expect("should configure diagnostics"); - let settings = Arc::new(raw); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - queue_shareable_html(&stub); - - let mut request = navigation_request(); - *request.uri_mut() = "https://ts.example.com/article?ts_console=1" - .parse() - .expect("should parse diagnostics URI"); - let response = run(&settings, &services, request).await; - assert_eq!( - response - .headers() - .get(HEADER_X_TS_TEMPLATE_CACHE) - .and_then(|value| value.to_str().ok()), - Some("bypass-request") - ); - assert_eq!( - response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("no-store, private") - ); - assert!(response.headers().contains_key(header::SET_COOKIE)); - let document = String::from_utf8(body_of(response).await) - .expect("diagnostics document should be UTF-8"); - assert!(document.contains("__tsjs_gpt_diagnostics_active")); - assert!(document.contains("tsjs-gpt_diagnostics.min.js")); - assert!( - cache - .lookups - .lock() - .expect("should lock lookups") - .is_empty(), - "the real query activation must bypass lookup before origin work" - ); + async fn template_cookie_publisher_empty_lists_keep_legacy_behavior() { + for independent in [false, true] { + let settings = cookie_policy_settings(Some(&[]), Some(&[]), independent); + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + for _ in 0..2 { + queue_shareable_html(&stub); + let _ = + body_of(run(&settings, &services, cookie_navigation_request()).await).await; + } + assert_eq!( + stub.recorded_request_uris().len(), + if independent { 1 } else { 2 }, + "should retain legacy boolean eligibility" + ); + assert_eq!( + stored_cache_keys(&cache).len(), + usize::from(independent), + "should preserve legacy storage" + ); + } } #[tokio::test] - async fn real_diagnostics_cookie_bypasses_a_warm_cookie_independent_template() { + async fn template_cookie_publisher_policy_changes_invalidate_templates() { + let first = cookie_policy_settings(None, None, true); + let second = cookie_policy_settings(Some(&["ab_bucket"]), None, true); + let third = cookie_policy_settings(Some(&["ab_bucket"]), Some(&["session"]), true); + // The registry is held constant, so the only thing the + // fingerprints differ on is the cookie policy. + let registry = test_registry(&first); + assert_ne!( + template_fingerprint(&first, ®istry, &crate::prefix::PublicPrefix::default()), + template_fingerprint(&second, ®istry, &crate::prefix::PublicPrefix::default()), + "should fingerprint the key policy" + ); + assert_ne!( + template_fingerprint(&second, ®istry, &crate::prefix::PublicPrefix::default()), + template_fingerprint(&third, ®istry, &crate::prefix::PublicPrefix::default()), + "should fingerprint the bypass policy" + ); let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); - let mut raw = settings_with_mode("esi"); - raw.creative_opportunities - .as_mut() - .expect("fixture configures creative opportunities") - .origin_is_cookie_independent = Some(true); - raw.integrations - .insert_config("gpt_diagnostics", &serde_json::json!({ "enabled": true })) - .expect("should configure diagnostics"); - let settings = Arc::new(raw); let services = services(Arc::clone(&stub), Arc::clone(&cache)); - queue_shareable_html(&stub); - queue_shareable_html(&stub); - - let _ = body_of(run(&settings, &services, navigation_request()).await).await; - let mut diagnostics = navigation_request(); - diagnostics.headers_mut().insert( - header::COOKIE, - HeaderValue::from_static("__Host-ts-console=1"), - ); - let response = run(&settings, &services, diagnostics).await; + for settings in [&first, &second, &third] { + queue_shareable_html(&stub); + let _ = body_of( + run( + settings, + &services, + cookie_policy_request(&[b"ab_bucket=A"]), + ) + .await, + ) + .await; + } + let keys = stored_cache_keys(&cache); assert_eq!( - response - .headers() - .get(HEADER_X_TS_TEMPLATE_CACHE) - .and_then(|value| value.to_str().ok()), - Some("bypass-request") + keys.len(), + 3, + "should store a fresh entry under each policy" ); - let document = String::from_utf8(body_of(response).await) - .expect("diagnostics document should be UTF-8"); - - assert_eq!(stub.recorded_request_uris().len(), 2); - assert_eq!( - cache.lookups.lock().expect("should lock lookups").len(), - 1, - "the diagnostics cookie must bypass the otherwise-eligible warm lookup" + assert_ne!(keys[0], keys[1], "should not reuse pre-policy templates"); + assert_ne!( + keys[1], keys[2], + "should not reuse entries admitted under another bypass policy" ); - assert!(document.contains("__tsjs_gpt_diagnostics_active")); - } - - #[tokio::test] - async fn a_cache_backend_failure_falls_back_to_origin_and_is_observable() { - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - cache.fail_lookup.store(true, Ordering::Relaxed); - let settings = Arc::new(settings_with_mode("esi")); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - queue_shareable_html(&stub); - - let response = run(&settings, &services, navigation_request()).await; - assert_eq!(response.status(), StatusCode::OK); assert_eq!( - response - .headers() - .get(HEADER_X_TS_TEMPLATE_CACHE) - .and_then(|value| value.to_str().ok()), - Some("backend-error") + stub.recorded_request_uris().len(), + 3, + "should fetch after each policy change" ); - let document = String::from_utf8(body_of(response).await) - .expect("fallback document should be UTF-8"); - assert!(document.contains("origin")); - assert_eq!(stub.recorded_request_uris().len(), 1); } #[tokio::test] - async fn a_cache_hit_streams_rather_than_buffering() { - // The property that makes the cache worth having, and the one that regressed - // silently. Buffered assembly held the first byte until the auction resolved - // — measured at ~100x worse TTFB than shipping nothing at all, because the - // inline path already streams and waits only at ``. - // - // Asserted on the body *shape* rather than on timing: a timing test would be - // flaky, and `EdgeBody::Stream` is the structural fact that produces the - // timing. + async fn template_cookie_publisher_warm_variant_runs_fresh_reader_assembly() { + let mut raw = settings_with_bidder("esi"); + let config = raw + .creative_opportunities + .as_mut() + .expect("should configure opportunities"); + config.template_cache_key_cookies = Some(vec!["ab_bucket".to_string()]); + config.origin_is_cookie_independent = Some(true); + let settings = Arc::new(raw); let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); - let settings = Arc::new(settings_with_mode("esi")); let services = services(Arc::clone(&stub), Arc::clone(&cache)); - queue_shareable_html(&stub); - - // Warm the cache, then take the hit. - let _ = run(&settings, &services, navigation_request()).await; - let warm = run(&settings, &services, navigation_request()).await; - + for (index, field) in [ + b"ab_bucket=A; ts-ec=reader1".as_slice(), + b"ab_bucket=A; ts-ec=reader2".as_slice(), + ] + .iter() + .enumerate() + { + queue_bid_response(&stub); + if index == 0 { + queue_shareable_html(&stub); + } + let response = run_bidding_at_price( + &settings, + &services, + cookie_policy_request(&[field]), + if index == 0 { 3.5 } else { 7.5 }, + ) + .await; + assert_eq!( + response.headers()[HEADER_X_TS_TEMPLATE_CACHE], + if index == 0 { "miss-stored" } else { "hit" }, + "should serve the second reader from the shared template" + ); + assert!( + response.headers()[header::CACHE_CONTROL] + .to_str() + .expect("should read cache policy") + .contains("private"), + "should keep reader output private" + ); + let document = + String::from_utf8(body_of(response).await).expect("should decode document"); + assert_eq!( + seam_bids(&document) + .get("test-slot") + .and_then(|bid| bid.get("hb_pb")) + .and_then(serde_json::Value::as_str), + Some(if index == 0 { "3.50" } else { "7.50" }), + "should assemble this request's winning bid" + ); + } + let requests = stub.recorded_request_uris(); assert_eq!( - stub.recorded_request_uris().len(), + requests + .iter() + .filter(|uri| uri.contains("/article")) + .count(), 1, - "the second request must be a hit, or this asserts nothing" + "should fetch one shared origin template" ); - assert!( - warm.headers().get(header::CONTENT_LENGTH).is_none(), - "a streamed assembly has no length until bids resolve, and headers \ - commit before the first byte" + assert_eq!( + requests.len(), + 3, + "should run a fresh auction on each reader request" ); - - // The invariant that actually matters, and the one a body-shape assertion - // misses: a stream that awaited the auction before its first yield would - // still be an `EdgeBody::Stream` and would still hold the first byte. - // - // So pull exactly one chunk and prove the auction has not been collected - // yet — `ad_bids_state` is only written by the collector. No timing - // involved, so nothing to be flaky about. - let EdgeBody::Stream(mut stream) = warm.into_body() else { - panic!("a cache hit must stream, not buffer"); - }; - let first = stream - .next() - .await - .expect("the stream should yield a first chunk") - .expect("the first chunk should read"); - - assert!( - first.starts_with(b"") || first.starts_with(b"` seam emitted - // the marker because the mode was `Esi`, while assembly was skipped because - // the gate had refused a key — a fallback at one seam and not the other. - // - // Bypassing is the *normal* case against a real origin, so this path runs - // far more often than the shared one. It has to produce a working page. - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let settings = Arc::new(settings_with_mode("esi")); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - - // A `Set-Cookie` from the origin disqualifies the response, exactly as a - // real origin does. - stub.push_response_with_headers( - 200, - b"origin".to_vec(), - vec![ - ("content-type", "text/html; charset=utf-8"), - ("cache-control", "public, max-age=300"), - ("set-cookie", "sess=1"), - ], - ); - - let document = String::from_utf8( - body_of(run(&settings, &services, navigation_request()).await).await, - ) - .expect("response should be utf-8"); + async fn template_cookie_publisher_warm_variant_finalizes_ec_withdrawal() { + for finalizer in [Finalizer::Streaming, Finalizer::Buffered] { + let mut settings = cookie_policy_settings(Some(&["ab_bucket"]), None, true); + Arc::make_mut(&mut settings).auction.provider_names = + vec![SCHEDULING_PROVIDER.to_owned()]; + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + let graph = KvIdentityGraph::in_memory("cookie-withdrawal-store"); + let identities = [ + format!("{}.Read01", "a".repeat(64)), + format!("{}.Read02", "b".repeat(64)), + ]; + for (index, identity) in identities.iter().enumerate() { + assert!( + crate::ec::generation::is_valid_ec_id(identity), + "should use valid EC identities in the fixture" + ); + graph + .create( + identity, + &crate::ec::kv_types::KvEntry::minimal( + "example.com", + &format!("partner-reader-{index}"), + crate::ec::current_timestamp(), + ), + ) + .expect("should seed a live reader identity"); + } + let registry = IntegrationRegistry::new(&settings) + .expect("should create integration registry"); + let partner = serde_json::from_value(serde_json::json!({ + "name": "Example partner", + "source_domain": "example.com", + "bidstream_enabled": true + })) + .expect("should deserialize fixture partner"); + let partners = PartnerRegistry::from_config(&[partner]) + .expect("should create partner registry"); + // Only the cold request has an origin response available. + queue_shareable_html(&stub); - assert!( - cache - .entries - .lock() - .expect("should lock entries") - .is_empty(), - "the gate refused, so nothing may be stored" - ); - assert!( - !document.contains(AD_ASSEMBLY_SEAM), - "a marker must never be emitted when nothing will resolve it: {document}" - ); - assert!( - document.contains("window.tsjs"), - "and the reader must still get their bids: {document}" - ); + for (index, (identity, withdrawn)) in [ + (&identities[0], false), + (&identities[1], false), + (&identities[1], true), + ] + .into_iter() + .enumerate() + { + // A US-style opt-out suppresses use without expiring the + // cookie, so the destructive path this test is about needs + // a request that withdrew device storage outright. + let mut ec_context = if withdrawn { + EcContext::new_for_test_withdrawn( + Some(identity.clone()), + ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::UsState( + "CA".to_owned(), + ), + gpc: true, + ..Default::default() + }, + ) + } else { + EcContext::new_for_test(Some(identity.clone()), scheduling_consent()) + }; + assert_eq!( + ec_context.ec_allowed(), + !withdrawn, + "should apply reader consent" + ); + let captured = Arc::new(Mutex::new(None)); + let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + orchestrator.register_provider(Arc::new(SchedulingCaptureProvider { + captured: Arc::clone(&captured), + http: Arc::clone(&stub), + lookups: Arc::new(AtomicUsize::new(0)), + })); + let orchestrator = Arc::new(orchestrator); + let cookies = format!("ab_bucket=A; ts-ec={identity}"); + let response = handle_publisher_request( + AppContext { + settings: &settings, + integration_registry: ®istry, + }, + &services, + Some(&graph), + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &[article_slot()], + registry: Some(&partners), + }, + cookie_policy_request(&[cookies.as_bytes()]), + EdgeCacheHeader::SMaxageFallback, + ) + .await + .expect("should serve a reader with an existing EC"); + assert!( + ec_context.kv_snapshot().entry_for(identity).is_some(), + "should preload this reader's identity even during withdrawal on a hit" + ); + let mut response = finalize_test_publisher_response( + response, + &settings, + &services, + ®istry, + orchestrator, + finalizer, + ) + .await; + crate::ec::finalize::ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &partners, + None, + None, + &mut response, + &services, + ) + .await; + assert_eq!( + response.headers()[HEADER_X_TS_TEMPLATE_CACHE], + if index == 0 { "miss-stored" } else { "hit" }, + "should share one variant across identities and withdrawal" + ); + let cache_control = response.headers()[header::CACHE_CONTROL] + .to_str() + .expect("should decode cache policy"); + for directive in ["private", "no-store"] { + assert!( + cache_control + .split(',') + .any(|value| value.trim() == directive), + "should keep finalized reader responses private and uncacheable" + ); + } + assert_eq!( + response + .headers() + .get_all(header::SET_COOKIE) + .iter() + .any(|value| { + let value = value.to_str().expect("should decode response cookie"); + value.starts_with("ts-ec=") && value.contains("Max-Age=0") + }), + withdrawn, + "should expire the EC cookie only for the withdrawing reader" + ); + let body = body_of(response).await; + assert!(!body.is_empty(), "should render a complete reader response"); + let captured = captured.lock().expect("should lock captured auction"); + let auction = captured.as_ref().expect("should dispatch a reader auction"); + assert_eq!( + auction.request.user.id.as_deref(), + if withdrawn { + None + } else { + Some(identity.as_str()) + }, + "should use this reader's identity and suppress it after withdrawal" + ); + if withdrawn { + assert!( + auction.request.user.eids.is_none(), + "should suppress withdrawn EIDs" + ); + } else { + let eids = auction + .request + .user + .eids + .as_ref() + .expect("should include consenting reader EIDs"); + assert_eq!(eids.len(), 1, "should expose only the configured partner"); + assert_eq!( + eids[0].source, "example.com", + "should use the registered source" + ); + assert_eq!( + eids[0].uids[0].id, + format!("partner-reader-{index}"), + "should use this reader's partner identity on cold and warm requests" + ); + } + } + assert_eq!( + stub.recorded_request_uris().len(), + 1, + "should fetch origin only once" + ); + assert_eq!( + looked_up_cache_keys(&cache).len(), + 3, + "should look up every reader" + ); + assert_eq!( + stored_cache_keys(&cache).len(), + 1, + "should store only the cold template" + ); + for (index, identity) in identities.iter().enumerate() { + let (entry, _) = graph + .get(identity) + .expect("should read reader identity") + .expect("should retain the live row or its tombstone"); + assert_eq!( + entry.consent.ok, + index == 0, + "should revoke only the second reader" + ); + assert_eq!( + entry.ids.is_empty(), + index == 1, + "should clear only revoked partner IDs" + ); + } + let entries = cache.entries.lock().expect("should lock cached templates"); + let stored = &entries + .values() + .next() + .expect("should retain shared template") + .body; + let stored = String::from_utf8_lossy(stored); + for identity in &identities { + assert!( + !stored.contains(identity), + "should keep reader identities out of cached bytes" + ); + } + } } #[tokio::test] - async fn the_seam_carries_slot_definitions_or_the_page_serves_no_ads() { - // Shared modes suppress the head slot script, so the seam is the only place - // `tsjs.adSlots` can come from. Sending only bids left it at its `[]` default, - // `adInit` defined nothing, and the page rendered perfectly with zero TS ads — - // green tests, healthy-looking page, no revenue. + async fn by_default_a_cookie_bearing_request_uses_no_shared_cache() { + // The shipped default, and the reason the cache is nearly inert on real + // traffic: TS sets its own identity cookie, so essentially every repeat + // visitor arrives carrying one and is excluded in both directions. let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); let settings = Arc::new(settings_with_mode("esi")); let services = services(Arc::clone(&stub), Arc::clone(&cache)); queue_shareable_html(&stub); + queue_shareable_html(&stub); - let miss = body_of(run(&settings, &services, navigation_request()).await).await; - let hit = body_of(run(&settings, &services, navigation_request()).await).await; + let _ = run(&settings, &services, cookie_navigation_request()).await; + let _ = run(&settings, &services, cookie_navigation_request()).await; - for (label, body) in [("miss", miss), ("hit", hit)] { - let text = String::from_utf8(body).expect("utf-8"); - assert!( - text.contains("var a=JSON.parse(") && text.contains("s(b,a)"), - "{label}: the seam must carry slot definitions and hand them to the \ - scheduler: {text}" - ); - assert!( - text.contains("test-slot"), - "{label}: the slot definitions must be populated, not `[]`" - ); - } + assert_eq!( + stub.recorded_request_uris().len(), + 2, + "both requests must reach the origin" + ); + assert!( + cache + .entries + .lock() + .expect("should lock entries") + .is_empty(), + "and neither may store a template" + ); } #[tokio::test] - async fn an_assembled_response_is_private_even_when_the_ad_stack_is_off() { - // The private stamp was gated on `should_run_ad_stack`. A bot, prefetch, - // kill-switched or consent-denied request can still assemble an empty-bids - // document, and would have kept the origin's public caching directives — so a - // downstream cache could serve that to a later eligible reader. + async fn a_declared_cookie_independent_origin_lets_repeat_visitors_share() { + // The opt-in. Without it the cache can only ever serve first-ever page + // views, which is not the population the issue cares about. let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); - let settings = Arc::new(settings_with_mode("esi")); + let mut raw = settings_with_mode("esi"); + raw.creative_opportunities + .as_mut() + .expect("fixture configures creative opportunities") + .origin_is_cookie_independent = Some(true); + let settings = Arc::new(raw); let services = services(Arc::clone(&stub), Arc::clone(&cache)); queue_shareable_html(&stub); + queue_shareable_html(&stub); - let bot = HttpRequest::builder() - .method(Method::GET) - .uri("https://ts.example.com/article") - .header(header::HOST, "ts.example.com") - .header("sec-fetch-dest", "document") - .header("sec-fetch-mode", "navigate") - .header( - header::USER_AGENT, - "Googlebot/2.1 (+http://www.google.com/bot.html)", - ) - .body(EdgeBody::empty()) - .expect("should build bot request"); - let response = run(&settings, &services, bot).await; + let _ = run(&settings, &services, cookie_navigation_request()).await; + let _ = run(&settings, &services, cookie_navigation_request()).await; assert_eq!( - response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|v| v.to_str().ok()), - Some("no-store, private"), - "an assembled response must be private whatever the ad stack decided" + stub.recorded_request_uris().len(), + 1, + "the second cookie-bearing request should be served from the cache" ); } #[tokio::test] - async fn a_cache_hit_keeps_the_origin_security_headers() { - // Reconstructing headers keeps origin `Set-Cookie` and caching directives out - // of a shared cache, and also silently dropped Content-Security-Policy — - // a weaker page, served faster. Policy headers are per-URL, so they belong - // with the template. + async fn a_request_a_module_left_state_on_never_stores_a_template() { + // What a module leaves on a request is written into that request's + // document, so a template stored from it would hand one reader's + // document to every later one. The request is kept off the shared + // path before the origin is asked, and this pins the consequence + // directly rather than leaving it to be reasoned about. let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); let settings = Arc::new(settings_with_mode("esi")); let services = services(Arc::clone(&stub), Arc::clone(&cache)); - stub.push_response_with_headers( - 200, - b"origin".to_vec(), - vec![ - ("content-type", "text/html; charset=utf-8"), - ("cache-control", "public, max-age=300"), - ("content-security-policy", "default-src 'self'"), - ("x-frame-options", "SAMEORIGIN"), - ], - ); + queue_shareable_html(&stub); - let _ = run(&settings, &services, navigation_request()).await; - let warm = run(&settings, &services, navigation_request()).await; + let _ = run(&settings, &services, marked_navigation_request()).await; - assert_eq!( - warm.headers() - .get(header::CONTENT_SECURITY_POLICY) - .and_then(|v| v.to_str().ok()), - Some("default-src 'self'"), - "a hit must not drop the origin's CSP" - ); - assert_eq!( - warm.headers() - .get("x-frame-options") - .and_then(|v| v.to_str().ok()), - Some("SAMEORIGIN") + assert!( + cache + .entries + .lock() + .expect("should lock entries") + .is_empty(), + "a request a module left state on must not contribute to a shared cache" ); } #[tokio::test] - async fn a_cache_hit_preserves_every_repeated_policy_header_in_order() { + async fn a_request_a_module_left_state_on_bypasses_an_already_warm_template() { let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); - let settings = Arc::new(settings_with_mode("esi")); + let mut raw = settings_with_mode("esi"); + crate::integrations::registry_test_support::request_fixture::select_and_place(&mut raw); + let settings = Arc::new(raw); let services = services(Arc::clone(&stub), Arc::clone(&cache)); - stub.push_response_with_headers( - 200, - b"origin".to_vec(), - vec![ - ("content-type", "text/html; charset=utf-8"), - ("cache-control", "public, max-age=300"), - ("content-security-policy", "default-src 'self'"), - ("content-security-policy", "script-src 'self'"), - ("link", "; rel=preload; as=script"), - ("link", "; rel=preload; as=style"), - ("cross-origin-opener-policy", "same-origin"), - ("cross-origin-embedder-policy", "require-corp"), - ], - ); + queue_shareable_html(&stub); + queue_shareable_html(&stub); - let _ = run(&settings, &services, navigation_request()).await; - let warm = run(&settings, &services, navigation_request()).await; + let _ = body_of(run(&settings, &services, navigation_request()).await).await; + let response = run(&settings, &services, marked_navigation_request()).await; + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("no-store, private"), + "a document a module's request state is written into is one reader's" + ); + assert!( + response.headers().contains_key(header::SET_COOKIE), + "the module's response finalizer should run on the page path" + ); + let document = String::from_utf8(body_of(response).await) + .expect("the marked request's document should be UTF-8"); - let values = |name: &'static str| { - warm.headers() - .get_all(name) - .iter() - .map(|value| value.to_str().expect("policy header should be text")) - .collect::>() - }; assert_eq!( - values("content-security-policy"), - ["default-src 'self'", "script-src 'self'"] + stub.recorded_request_uris().len(), + 2, + "a request a module left state on must reach the origin even when an \ + ordinary shared template is warm" ); assert_eq!( - values("link"), - [ - "; rel=preload; as=script", - "; rel=preload; as=style" - ] + cache.lookups.lock().expect("should lock lookups").len(), + 1, + "the marked request must not consult template cache at all" + ); + assert!( + document.contains( + crate::integrations::registry_test_support::request_fixture::HEAD_FLAG + ), + "origin fallback must keep what the module writes for the request" ); - assert_eq!(values("cross-origin-opener-policy"), ["same-origin"]); - assert_eq!(values("cross-origin-embedder-policy"), ["require-corp"]); } #[tokio::test] - async fn nonce_bearing_csp_bypasses_the_shared_template_cache() { + async fn a_personalized_request_bypasses_a_warm_shared_template() { let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); let settings = Arc::new(settings_with_mode("esi")); let services = services(Arc::clone(&stub), Arc::clone(&cache)); - for _ in 0..2 { - stub.push_response_with_headers( - 200, - b"origin" - .to_vec(), - vec![ - ("content-type", "text/html; charset=utf-8"), - ("cache-control", "public, max-age=300"), - ( - "content-security-policy", - "default-src 'self'; script-src 'nonce-reader-nonce'", - ), - ], - ); - } + queue_shareable_html(&stub); + queue_shareable_html(&stub); let _ = body_of(run(&settings, &services, navigation_request()).await).await; - let _ = body_of(run(&settings, &services, navigation_request()).await).await; + + let mut personalized_request = navigation_request(); + personalized_request + .headers_mut() + .insert("sec-fetch-dest", HeaderValue::from_static("script")); + personalized_request + .extensions_mut() + .insert(crate::response_privacy::PersonalizedResponse); + let personalized = run(&settings, &services, personalized_request).await; + assert_eq!( + personalized + .headers() + .get(HEADER_X_TS_TEMPLATE_CACHE) + .and_then(|value| value.to_str().ok()), + Some("bypass-request"), + "a personalized request must not read a shared template" + ); + let _ = body_of(personalized).await; assert_eq!( stub.recorded_request_uris().len(), 2, - "a response-bound CSP nonce and its HTML must never be reused from template cache" + "the personalized request must reach the origin even when template cache is warm" ); - assert!( - cache - .entries - .lock() - .expect("should lock entries") - .is_empty() + assert_eq!( + cache.lookups.lock().expect("should lock lookups").len(), + 1, + "the personalized request must bypass template cache before lookup" ); } #[tokio::test] - async fn a_meta_delivered_nonce_policy_bypasses_the_shared_template_cache() { - // The response-header gate cannot see this policy at all. Storing the document - // would replay one response's nonce to every later reader — the exact thing the - // header check exists to prevent. + async fn a_query_a_module_acts_on_bypasses_template_cache_and_keeps_what_the_module_writes() + { let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); - let settings = Arc::new(settings_with_mode("esi")); + let mut raw = settings_with_mode("esi"); + crate::integrations::registry_test_support::request_fixture::select_and_place(&mut raw); + let settings = Arc::new(raw); let services = services(Arc::clone(&stub), Arc::clone(&cache)); - for _ in 0..2 { - stub.push_response_with_headers( - 200, - br#"origin"# - .to_vec(), - vec![ - ("content-type", "text/html; charset=utf-8"), - ("cache-control", "public, max-age=300"), - ], - ); - } + queue_shareable_html(&stub); - let cold = run(&settings, &services, navigation_request()).await; + // The handler prepares a request that reaches it unprepared, so the + // module's preparer reads its query here as it does behind an adapter. + let mut request = navigation_request(); + *request.uri_mut() = format!( + "https://ts.example.com/article?{}", + crate::integrations::registry_test_support::request_fixture::QUERY + ) + .parse() + .expect("should parse the URI"); + let response = run(&settings, &services, request).await; assert_eq!( - cold.headers() + response + .headers() .get(HEADER_X_TS_TEMPLATE_CACHE) .and_then(|value| value.to_str().ok()), - Some("bypass-response"), - "the cold response must refuse to store a nonce-bearing document" + Some("bypass-request") ); - let _ = body_of(cold).await; - let _ = body_of(run(&settings, &services, navigation_request()).await).await; - assert_eq!( - stub.recorded_request_uris().len(), - 2, - "the warm request must reach the origin, not a replayed nonce" + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("no-store, private") ); + assert!( + response.headers().contains_key(header::SET_COOKIE), + "the module's response finalizer should run on the page path" + ); + assert!( + stub.recorded_request_uris().iter().all(|uri| !uri + .contains(crate::integrations::registry_test_support::request_fixture::QUERY)), + "the origin should be asked without the query the module reserved" + ); + let document = + String::from_utf8(body_of(response).await).expect("the document should be UTF-8"); + assert!( + document.contains( + crate::integrations::registry_test_support::request_fixture::HEAD_FLAG + ) + ); + assert!(document.contains( + crate::integrations::registry_test_support::request_fixture::MODULE_FILE + )); assert!( cache - .entries + .lookups .lock() - .expect("should lock entries") + .expect("should lock lookups") .is_empty(), - "nothing nonce-bearing may reach the shared cache" + "a query a module acts on must bypass lookup before origin work" ); } #[tokio::test] - async fn a_nonce_attribute_without_a_policy_bypasses_the_shared_template_cache() { - // Fail closed: the attributes say the document was written for a per-response - // policy, whether or not the policy itself survived to this scan. + async fn a_request_a_module_left_state_on_bypasses_a_warm_cookie_independent_template() { let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); - let settings = Arc::new(settings_with_mode("esi")); + let mut raw = settings_with_mode("esi"); + raw.creative_opportunities + .as_mut() + .expect("fixture configures creative opportunities") + .origin_is_cookie_independent = Some(true); + crate::integrations::registry_test_support::request_fixture::select_and_place(&mut raw); + let settings = Arc::new(raw); let services = services(Arc::clone(&stub), Arc::clone(&cache)); - for _ in 0..2 { - stub.push_response_with_headers( - 200, - br#"origin"# - .to_vec(), - vec![ - ("content-type", "text/html; charset=utf-8"), - ("cache-control", "public, max-age=300"), - ], - ); - } + queue_shareable_html(&stub); + queue_shareable_html(&stub); let _ = body_of(run(&settings, &services, navigation_request()).await).await; - let _ = body_of(run(&settings, &services, navigation_request()).await).await; + let response = run(&settings, &services, marked_navigation_request()).await; + assert_eq!( + response + .headers() + .get(HEADER_X_TS_TEMPLATE_CACHE) + .and_then(|value| value.to_str().ok()), + Some("bypass-request") + ); + let document = + String::from_utf8(body_of(response).await).expect("the document should be UTF-8"); assert_eq!(stub.recorded_request_uris().len(), 2); + assert_eq!( + cache.lookups.lock().expect("should lock lookups").len(), + 1, + "a request a module left state on must bypass the otherwise-eligible warm lookup" + ); assert!( - cache - .entries - .lock() - .expect("should lock entries") - .is_empty() + document.contains( + crate::integrations::registry_test_support::request_fixture::HEAD_FLAG + ) ); } #[tokio::test] - async fn a_body_close_in_script_data_corrupts_neither_the_cold_nor_the_warm_document() { - // No structural `` anywhere: a reverse byte search takes the string - // literal, and the payload's `` then terminates the publisher's script - // — in the response served cold and in the template every warm reader gets. - const PUBLISHER_SCRIPT: &str = r#""#; + async fn a_cache_backend_failure_falls_back_to_origin_and_is_observable() { let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); + cache.fail_lookup.store(true, Ordering::Relaxed); let settings = Arc::new(settings_with_mode("esi")); let services = services(Arc::clone(&stub), Arc::clone(&cache)); - stub.push_response_with_headers( - 200, - format!("{PUBLISHER_SCRIPT}

article

").into_bytes(), - vec![ - ("content-type", "text/html; charset=utf-8"), - ("cache-control", "public, max-age=300"), - ], + queue_shareable_html(&stub); + + let response = run(&settings, &services, navigation_request()).await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response + .headers() + .get(HEADER_X_TS_TEMPLATE_CACHE) + .and_then(|value| value.to_str().ok()), + Some("backend-error") ); + let document = String::from_utf8(body_of(response).await) + .expect("fallback document should be UTF-8"); + assert!(document.contains("origin")); + assert_eq!(stub.recorded_request_uris().len(), 1); + } - let cold = String::from_utf8( - body_of(run(&settings, &services, navigation_request()).await).await, - ) - .expect("the cold document should be UTF-8"); - let warm = String::from_utf8( - body_of(run(&settings, &services, navigation_request()).await).await, - ) - .expect("the warm document should be UTF-8"); + #[tokio::test] + async fn a_cache_hit_streams_rather_than_buffering() { + // The property that makes the cache worth having, and the one that regressed + // silently. Buffered assembly held the first byte until the auction resolved + // — measured at ~100x worse TTFB than shipping nothing at all, because the + // inline path already streams and waits only at ``. + // + // Asserted on the body *shape* rather than on timing: a timing test would be + // flaky, and `EdgeBody::Stream` is the structural fact that produces the + // timing. + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + queue_shareable_html(&stub); + + // Warm the cache, then take the hit. + let _ = run(&settings, &services, navigation_request()).await; + let warm = run(&settings, &services, navigation_request()).await; assert_eq!( stub.recorded_request_uris().len(), 1, - "the document is otherwise shareable and must still be stored and reused" + "the second request must be a hit, or this asserts nothing" + ); + assert!( + warm.headers().get(header::CONTENT_LENGTH).is_none(), + "a streamed assembly has no length until bids resolve, and headers \ + commit before the first byte" + ); + + // The invariant that actually matters, and the one a body-shape assertion + // misses: a stream that awaited the auction before its first yield would + // still be an `EdgeBody::Stream` and would still hold the first byte. + // + // So pull exactly one chunk and prove the auction has not been collected + // yet — `ad_bids_state` is only written by the collector. No timing + // involved, so nothing to be flaky about. + let EdgeBody::Stream(mut stream) = warm.into_body() else { + panic!("a cache hit must stream, not buffer"); + }; + let first = stream + .next() + .await + .expect("the stream should yield a first chunk") + .expect("the first chunk should read"); + + assert!( + first.starts_with(b"") || first.starts_with(b"` sequence, so the seam landed - // inside this comment and the assembled bids never executed. - const TRAILING_COMMENT: &str = ""; + async fn a_bypassed_response_falls_back_to_inline_at_every_seam() { + // Observed against a real origin: a page arrived with a raw + // executable ESI markup in it and no bids at all. The `` seam emitted + // the marker because the mode was `Esi`, while assembly was skipped because + // the gate had refused a key — a fallback at one seam and not the other. + // + // Bypassing is the *normal* case against a real origin, so this path runs + // far more often than the shared one. It has to produce a working page. let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); let settings = Arc::new(settings_with_mode("esi")); let services = services(Arc::clone(&stub), Arc::clone(&cache)); + + // A `Set-Cookie` from the origin disqualifies the response, exactly as a + // real origin does. stub.push_response_with_headers( 200, - format!("

article

{TRAILING_COMMENT}") - .into_bytes(), + b"origin".to_vec(), vec![ ("content-type", "text/html; charset=utf-8"), ("cache-control", "public, max-age=300"), + ("set-cookie", "sess=1"), ], ); - let cold = String::from_utf8( - body_of(run(&settings, &services, navigation_request()).await).await, - ) - .expect("the cold document should be UTF-8"); - let warm = String::from_utf8( + let document = String::from_utf8( body_of(run(&settings, &services, navigation_request()).await).await, ) - .expect("the warm document should be UTF-8"); + .expect("response should be utf-8"); - assert_eq!(stub.recorded_request_uris().len(), 1); - for (label, document) in [("cold", &cold), ("warm", &warm)] { - assert!( - document.contains(TRAILING_COMMENT), - "the {label} document must leave the publisher comment intact: {document}" - ); - assert!( - !document.contains(AD_ASSEMBLY_SEAM), - "the {label} document must not ship an unresolved seam: {document}" - ); - assert!( - !document.contains(TEMPLATE_SEAM_PLACEHOLDER), - "the transform-owned placeholder must never reach a reader: {document}" - ); - } + assert!( + cache + .entries + .lock() + .expect("should lock entries") + .is_empty(), + "the gate refused, so nothing may be stored" + ); + assert!( + !document.contains(AD_ASSEMBLY_SEAM), + "a marker must never be emitted when nothing will resolve it: {document}" + ); + assert!( + document.contains("window.tsjs"), + "and the reader must still get their bids: {document}" + ); } #[tokio::test] - async fn a_post_is_never_answered_from_a_cached_get() { - // `handle_publisher_request` is the `*`-method fallback route, so a publisher - // path that renders a page on GET and accepts a form or webhook on POST reaches - // here for both. Serving the cached GET to the POST swallows the mutating - // request entirely: the origin never sees it, the caller gets 200 and a page, - // and nothing anywhere reports a problem. + async fn the_seam_carries_slot_definitions_or_the_page_serves_no_ads() { + // Shared modes suppress the head slot script, so the seam is the only place + // `tsjs.adSlots` can come from. Sending only bids left it at its `[]` default, + // `adInit` defined nothing, and the page rendered perfectly with zero TS ads — + // green tests, healthy-looking page, no revenue. let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); let settings = Arc::new(settings_with_mode("esi")); let services = services(Arc::clone(&stub), Arc::clone(&cache)); queue_shareable_html(&stub); - queue_shareable_html(&stub); - - // Warm the cache with a GET. - let _ = run(&settings, &services, navigation_request()).await; - assert_eq!(stub.recorded_request_uris().len(), 1); - let post = HttpRequest::builder() - .method(Method::POST) - .uri("https://ts.example.com/article") - .header(header::HOST, "ts.example.com") - .body(EdgeBody::from("field=value")) - .expect("should build post request"); - let _ = run(&settings, &services, post).await; + let miss = body_of(run(&settings, &services, navigation_request()).await).await; + let hit = body_of(run(&settings, &services, navigation_request()).await).await; - assert_eq!( - stub.recorded_request_uris().len(), - 2, - "the POST must reach the origin rather than being answered from the \ - cached GET" - ); - assert_eq!( - cache.entries.lock().expect("should lock entries").len(), - 1, - "and it must not store a template of its own" - ); + for (label, body) in [("miss", miss), ("hit", hit)] { + let text = String::from_utf8(body).expect("utf-8"); + assert!( + text.contains("var a=JSON.parse(") && text.contains("s(b,a)"), + "{label}: the seam must carry slot definitions and hand them to the \ + scheduler: {text}" + ); + assert!( + text.contains("test-slot"), + "{label}: the slot definitions must be populated, not `[]`" + ); + } } #[tokio::test] - async fn an_authenticated_request_is_not_served_a_shared_template() { - // The stored template is perfectly cacheable; this request is not entitled - // to it. The store gate cannot express that, because it is a property of - // the reader rather than of the bytes — which is why the lookup re-checks. + async fn an_assembled_response_is_private_even_when_the_ad_stack_is_off() { + // The private stamp was gated on `should_run_ad_stack`. A bot, prefetch, + // kill-switched or consent-denied request can still assemble an empty-bids + // document, and would have kept the origin's public caching directives — so a + // downstream cache could serve that to a later eligible reader. let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); let settings = Arc::new(settings_with_mode("esi")); let services = services(Arc::clone(&stub), Arc::clone(&cache)); queue_shareable_html(&stub); - queue_shareable_html(&stub); - let _ = run(&settings, &services, navigation_request()).await; - assert_eq!( - cache.entries.lock().expect("should lock entries").len(), - 1, - "the cold request should have populated the cache" - ); - - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - let consent = crate::consent::ConsentContext { - jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, - ..Default::default() - }; - let mut ec_context = EcContext::new_for_test(None, consent); - let authenticated = HttpRequest::builder() + let bot = HttpRequest::builder() .method(Method::GET) .uri("https://ts.example.com/article") .header(header::HOST, "ts.example.com") .header("sec-fetch-dest", "document") - .header(header::AUTHORIZATION, "Basic dXNlcjpwYXNz") + .header("sec-fetch-mode", "navigate") + .header( + header::USER_AGENT, + "Googlebot/2.1 (+http://www.google.com/bot.html)", + ) .body(EdgeBody::empty()) - .expect("should build authenticated request"); - let _ = handle_publisher_request( - &settings, - &services, - None, - &mut ec_context, - AuctionDispatch { - orchestrator: &orchestrator, - slots: &[article_slot()], - registry: None, - }, - authenticated, - EdgeCacheHeader::SMaxageFallback, - ) - .await - .expect("should proxy publisher request"); + .expect("should build bot request"); + let response = run(&settings, &services, bot).await; assert_eq!( - stub.recorded_request_uris().len(), - 2, - "an authenticated request must reach the origin rather than read a \ - shared template" + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|v| v.to_str().ok()), + Some("no-store, private"), + "an assembled response must be private whatever the ad stack decided" ); } - } - - mod template_cache_gate_tests { - //! `cache::core` stores whatever it is handed and rejects nothing, so every - //! one of these conditions is the caller's to enforce. Each is a leak vector - //! or an eligibility rule, not a preference. - use super::*; - use crate::creative_opportunities::AssemblyMode; - use edgezero_core::http::HeaderName; + #[tokio::test] + async fn a_cache_hit_keeps_the_origin_security_headers() { + // Reconstructing headers keeps origin `Set-Cookie` and caching directives out + // of a shared cache, and also silently dropped Content-Security-Policy — + // a weaker page, served faster. Policy headers are per-URL, so they belong + // with the template. + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + stub.push_response_with_headers( + 200, + b"origin".to_vec(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("cache-control", "public, max-age=300"), + ("content-security-policy", "default-src 'self'"), + ("x-frame-options", "SAMEORIGIN"), + ], + ); - fn headers(pairs: &[(HeaderName, &str)]) -> edgezero_core::http::HeaderMap { - let mut map = edgezero_core::http::HeaderMap::new(); - for (name, value) in pairs { - map.insert( - name.clone(), - HeaderValue::from_str(value).expect("should build header value"), - ); - } - map - } + let _ = run(&settings, &services, navigation_request()).await; + let warm = run(&settings, &services, navigation_request()).await; - fn shareable() -> edgezero_core::http::HeaderMap { - headers(&[(header::CACHE_CONTROL, "max-age=60")]) + assert_eq!( + warm.headers() + .get(header::CONTENT_SECURITY_POLICY) + .and_then(|v| v.to_str().ok()), + Some("default-src 'self'"), + "a hit must not drop the origin's CSP" + ); + assert_eq!( + warm.headers() + .get("x-frame-options") + .and_then(|v| v.to_str().ok()), + Some("SAMEORIGIN") + ); } - /// The shipped default: no operator has stated what the origin varies on, so the - /// key covers nothing. Responses without a `Vary` are unaffected; any `Vary` at - /// all disqualifies. - fn nothing_covered() -> VarySpec { - VarySpec::new([]) - } + #[tokio::test] + async fn a_cache_hit_preserves_every_repeated_policy_header_in_order() { + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + stub.push_response_with_headers( + 200, + b"origin".to_vec(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("cache-control", "public, max-age=300"), + ("content-security-policy", "default-src 'self'"), + ("content-security-policy", "script-src 'self'"), + ("link", "; rel=preload; as=script"), + ("link", "; rel=preload; as=style"), + ("cross-origin-opener-policy", "same-origin"), + ("cross-origin-embedder-policy", "require-corp"), + ], + ); - #[test] - fn an_unconfigured_deployment_never_caches_a_varying_response() { - // The fail-closed default. An operator who has not stated the origin's Vary - // must not acquire a shared cache by omission — and a real origin varies on - // something, so this is the common path, not an edge case. - // Deliberately not `Accept-Encoding`: the shared path normalizes supported - // content codings to one identity template, so that header is covered - // whatever the operator configured. Using it here would test the - // structural-coverage carve-out rather than the drift guard. - let mut varying = shareable(); - varying.insert(header::VARY, HeaderValue::from_static("rsc")); + let _ = run(&settings, &services, navigation_request()).await; + let warm = run(&settings, &services, navigation_request()).await; + let values = |name: &'static str| { + warm.headers() + .get_all(name) + .iter() + .map(|value| value.to_str().expect("policy header should be text")) + .collect::>() + }; assert_eq!( - template_cache_bypass_reason( - AssemblyMode::Esi, - false, - false, - StatusCode::OK, - "text/html", - &varying, - ¬hing_covered(), - ), - Some(TemplateCacheBypassReason::VaryNotCovered(VaryGap(vec![ - "rsc".to_string() - ]))), - "an unstated Vary must disqualify rather than silently under-key" + values("content-security-policy"), + ["default-src 'self'", "script-src 'self'"] ); - } - - #[test] - fn an_origin_that_varies_on_cookie_is_refused_even_when_declared_independent() { - // The backstop that makes `origin_is_cookie_independent` safe to offer. The - // operator asserts their origin ignores cookies; if the origin then says - // otherwise, the assertion loses. Without this, a wrong assertion would - // silently cross-serve personalized HTML. - let mut varying = shareable(); - varying.insert(header::VARY, HeaderValue::from_static("Cookie")); - assert_eq!( - template_cache_bypass_reason( - AssemblyMode::Esi, - false, - // The operator's assertion has already been applied here: this is - // `false` precisely because they declared independence. - false, - StatusCode::OK, - "text/html", - &varying, - &VarySpec::new(["cookie".to_string()]), - ), - Some(TemplateCacheBypassReason::VaryCookie), - "the origin's declaration must override both cookie independence and an \ - accidentally configured per-cookie key" + values("link"), + [ + "; rel=preload; as=script", + "; rel=preload; as=style" + ] ); + assert_eq!(values("cross-origin-opener-policy"), ["same-origin"]); + assert_eq!(values("cross-origin-embedder-policy"), ["require-corp"]); } - #[test] - fn a_private_directive_on_a_second_cache_control_line_is_refused() { - // `HeaderMap::get` returns the first value only. An origin that sends - // `Cache-Control: public, max-age=300` and then `Cache-Control: private` on a - // separate line means exactly what one comma-joined line would mean, but the - // second line was invisible — so a response the origin marked private was - // written to a cache shared between readers. The `Vary` reads a few lines up - // already use `get_all` for the same reason. - let mut split = edgezero_core::http::HeaderMap::new(); - split.append( - header::CACHE_CONTROL, - HeaderValue::from_static("public, max-age=300"), - ); - split.append(header::CACHE_CONTROL, HeaderValue::from_static("private")); + #[tokio::test] + async fn nonce_bearing_csp_bypasses_the_shared_template_cache() { + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + for _ in 0..2 { + stub.push_response_with_headers( + 200, + b"origin" + .to_vec(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("cache-control", "public, max-age=300"), + ( + "content-security-policy", + "default-src 'self'; script-src 'nonce-reader-nonce'", + ), + ], + ); + } + + let _ = body_of(run(&settings, &services, navigation_request()).await).await; + let _ = body_of(run(&settings, &services, navigation_request()).await).await; assert_eq!( - template_cache_bypass_reason( - AssemblyMode::Esi, - false, - false, - StatusCode::OK, - "text/html", - &split, - ¬hing_covered(), - ), - Some(TemplateCacheBypassReason::OriginNotShareable), - "a directive on any Cache-Control line must disqualify the response" + stub.recorded_request_uris().len(), + 2, + "a response-bound CSP nonce and its HTML must never be reused from template cache" + ); + assert!( + cache + .entries + .lock() + .expect("should lock entries") + .is_empty() ); } - #[test] - fn a_no_store_directive_on_a_second_cache_control_line_is_refused() { - // Same defect, the other directive that matters — `no-store` is the one an - // origin uses for a response that must not be written down anywhere. - let mut split = edgezero_core::http::HeaderMap::new(); - split.append( - header::CACHE_CONTROL, - HeaderValue::from_static("max-age=60"), + #[tokio::test] + async fn a_meta_delivered_nonce_policy_bypasses_the_shared_template_cache() { + // The response-header gate cannot see this policy at all. Storing the document + // would replay one response's nonce to every later reader — the exact thing the + // header check exists to prevent. + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + for _ in 0..2 { + stub.push_response_with_headers( + 200, + br#"origin"# + .to_vec(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("cache-control", "public, max-age=300"), + ], + ); + } + + let cold = run(&settings, &services, navigation_request()).await; + assert_eq!( + cold.headers() + .get(HEADER_X_TS_TEMPLATE_CACHE) + .and_then(|value| value.to_str().ok()), + Some("bypass-response"), + "the cold response must refuse to store a nonce-bearing document" ); - split.append(header::CACHE_CONTROL, HeaderValue::from_static("no-store")); + let _ = body_of(cold).await; + let _ = body_of(run(&settings, &services, navigation_request()).await).await; assert_eq!( - template_cache_bypass_reason( - AssemblyMode::Esi, - false, - false, - StatusCode::OK, - "text/html", - &split, - ¬hing_covered(), - ), - Some(TemplateCacheBypassReason::OriginNotShareable) + stub.recorded_request_uris().len(), + 2, + "the warm request must reach the origin, not a replayed nonce" + ); + assert!( + cache + .entries + .lock() + .expect("should lock entries") + .is_empty(), + "nothing nonce-bearing may reach the shared cache" ); } - #[test] - fn cdn_specific_cache_policy_cannot_be_overridden_by_public_cache_control() { - for name in crate::cache_policy::EDGE_CACHE_HEADER_NAMES { - let mut split = shareable(); - split.insert( - header::HeaderName::from_static(name), - HeaderValue::from_static("no-store"), - ); - - assert_eq!( - template_cache_bypass_reason( - AssemblyMode::Esi, - false, - false, - StatusCode::OK, - "text/html", - &split, - ¬hing_covered(), - ), - Some(TemplateCacheBypassReason::OriginNotShareable), - "template cache must fail closed on the CDN-specific policy header {name}" + #[tokio::test] + async fn a_nonce_attribute_without_a_policy_bypasses_the_shared_template_cache() { + // Fail closed: the attributes say the document was written for a per-response + // policy, whether or not the policy itself survived to this scan. + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + for _ in 0..2 { + stub.push_response_with_headers( + 200, + br#"origin"# + .to_vec(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("cache-control", "public, max-age=300"), + ], ); } - } - #[test] - fn unsupported_vendor_freshness_does_not_authorize_template_cache() { - for name in crate::cache_policy::EDGE_CACHE_HEADER_NAMES - .iter() - .filter(|name| **name != "surrogate-control") - { - let mut split = shareable(); - split.insert( - header::HeaderName::from_static(name), - HeaderValue::from_static("max-age=60"), - ); + let _ = body_of(run(&settings, &services, navigation_request()).await).await; + let _ = body_of(run(&settings, &services, navigation_request()).await).await; - assert_eq!( - template_cache_bypass_reason( - AssemblyMode::Esi, - false, - false, - StatusCode::OK, - "text/html", - &split, - ¬hing_covered(), - ), - Some(TemplateCacheBypassReason::OriginNotShareable), - "the Fastly exception must not authorize the vendor policy {name}" - ); - } + assert_eq!(stub.recorded_request_uris().len(), 2); + assert!( + cache + .entries + .lock() + .expect("should lock entries") + .is_empty() + ); } - #[test] - fn observed_fastly_surrogate_policy_uses_edge_freshness_capped_by_configuration() { - let publisher_headers = headers(&[ - (header::CACHE_CONTROL, "public, max-age=60"), - ( - header::HeaderName::from_static("surrogate-control"), - "max-age=1200, stale-while-revalidate=21600, stale-if-error=604800", - ), - ]); - - assert_eq!( - template_cache_ttl( - AssemblyMode::Esi, - false, - false, - StatusCode::OK, - "text/html", - &publisher_headers, - &TemplateCachePolicy::for_test(¬hing_covered(), Duration::from_secs(300),), - ), - Ok(Duration::from_secs(300)), - "Fastly's edge freshness should take precedence over the shorter browser \ - lifetime, while the configured safety ceiling remains authoritative" + #[tokio::test] + async fn a_body_close_in_script_data_corrupts_neither_the_cold_nor_the_warm_document() { + // No structural `` anywhere: a reverse byte search takes the string + // literal, and the payload's `` then terminates the publisher's script + // — in the response served cold and in the template every warm reader gets. + const PUBLISHER_SCRIPT: &str = r#""#; + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + stub.push_response_with_headers( + 200, + format!("{PUBLISHER_SCRIPT}

article

").into_bytes(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("cache-control", "public, max-age=300"), + ], ); - } - #[test] - fn fastly_surrogate_freshness_takes_precedence_over_standard_freshness() { - for (cache_control, surrogate_control, expected) in [ - ("public, max-age=300", "max-age=30", 30), - ("public, max-age=30", "max-age=300", 300), - ] { - let publisher_headers = headers(&[ - (header::CACHE_CONTROL, cache_control), - ( - header::HeaderName::from_static("surrogate-control"), - surrogate_control, - ), - ]); + let cold = String::from_utf8( + body_of(run(&settings, &services, navigation_request()).await).await, + ) + .expect("the cold document should be UTF-8"); + let warm = String::from_utf8( + body_of(run(&settings, &services, navigation_request()).await).await, + ) + .expect("the warm document should be UTF-8"); - assert_eq!( - template_cache_ttl( - AssemblyMode::Esi, - false, - false, - StatusCode::OK, - "text/html", - &publisher_headers, - &TemplateCachePolicy::for_test( - ¬hing_covered(), - Duration::from_secs(600), - ), - ), - Ok(Duration::from_secs(expected)) + assert_eq!( + stub.recorded_request_uris().len(), + 1, + "the document is otherwise shareable and must still be stored and reused" + ); + for (label, document) in [("cold", &cold), ("warm", &warm)] { + assert!( + document.contains(PUBLISHER_SCRIPT), + "the {label} document must carry the publisher's script byte for byte: {document}" + ); + assert!( + !document.contains(AD_ASSEMBLY_SEAM), + "the {label} document must not ship an unresolved seam: {document}" ); } } - #[test] - fn surrogate_stale_windows_do_not_extend_fresh_reuse() { - let publisher_headers = headers(&[ - (header::CACHE_CONTROL, "public, max-age=300"), - ( - header::HeaderName::from_static("surrogate-control"), - "max-age=20, stale-while-revalidate=600, stale-if-error=1200", - ), - (header::AGE, "10"), - ]); + #[tokio::test] + async fn a_body_close_in_trailing_comment_data_does_not_attract_the_seam() { + // The reverse search took the *last* `` sequence, so the seam landed + // inside this comment and the assembled bids never executed. + const TRAILING_COMMENT: &str = ""; + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + stub.push_response_with_headers( + 200, + format!("

article

{TRAILING_COMMENT}") + .into_bytes(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("cache-control", "public, max-age=300"), + ], + ); - assert_eq!( - template_cache_ttl( - AssemblyMode::Esi, - false, - false, - StatusCode::OK, - "text/html", - &publisher_headers, - &TemplateCachePolicy::for_test(¬hing_covered(), Duration::from_secs(600),), - ), - Ok(Duration::from_secs(10)), - "stale windows are validated metadata, not fresh template cache lifetime" - ); - } - - #[test] - fn ambiguous_or_unsupported_surrogate_policy_fails_closed() { - for (policy, expected) in [ - ("max-age", TemplateCacheBypassReason::MalformedCachePolicy), - ( - "max-age=30, max-age=60", - TemplateCacheBypassReason::MalformedCachePolicy, - ), - ( - "max-age=tomorrow", - TemplateCacheBypassReason::MalformedCachePolicy, - ), - ( - "max-age=30, public", - TemplateCacheBypassReason::MalformedCachePolicy, - ), - ( - "stale-if-error=60", - TemplateCacheBypassReason::NoPositiveFreshness, - ), - ("max-age=0", TemplateCacheBypassReason::NoPositiveFreshness), - ( - "max-age=30,", - TemplateCacheBypassReason::MalformedCachePolicy, - ), - ] { - let mut publisher_headers = shareable(); - publisher_headers.insert( - header::HeaderName::from_static("surrogate-control"), - HeaderValue::from_str(policy).expect("should build Surrogate-Control"), - ); + let cold = String::from_utf8( + body_of(run(&settings, &services, navigation_request()).await).await, + ) + .expect("the cold document should be UTF-8"); + let warm = String::from_utf8( + body_of(run(&settings, &services, navigation_request()).await).await, + ) + .expect("the warm document should be UTF-8"); - assert_eq!( - template_cache_bypass_reason( - AssemblyMode::Esi, - false, - false, - StatusCode::OK, - "text/html", - &publisher_headers, - ¬hing_covered(), - ), - Some(expected), - "`{policy}` must fail closed" + assert_eq!(stub.recorded_request_uris().len(), 1); + for (label, document) in [("cold", &cold), ("warm", &warm)] { + assert!( + document.contains(TRAILING_COMMENT), + "the {label} document must leave the publisher comment intact: {document}" ); - } - } - - #[test] - fn restrictive_surrogate_policy_is_never_overridden_by_standard_freshness() { - for directive in ["private", "no-store", "no-cache"] { - let mut publisher_headers = shareable(); - publisher_headers.insert( - header::HeaderName::from_static("surrogate-control"), - HeaderValue::from_str(directive).expect("should build Surrogate-Control"), + assert!( + !document.contains(AD_ASSEMBLY_SEAM), + "the {label} document must not ship an unresolved seam: {document}" ); - - assert_eq!( - template_cache_bypass_reason( - AssemblyMode::Esi, - false, - false, - StatusCode::OK, - "text/html", - &publisher_headers, - ¬hing_covered(), - ), - Some(TemplateCacheBypassReason::OriginNotShareable), - "`{directive}` must remain authoritative" + assert!( + !document.contains(TEMPLATE_SEAM_PLACEHOLDER), + "the transform-owned placeholder must never reach a reader: {document}" ); } } - #[test] - fn restrictive_standard_policy_is_never_overridden_by_surrogate_freshness() { - for directive in ["private", "no-store", "no-cache"] { - let publisher_headers = headers(&[ - ( - header::CACHE_CONTROL, - &format!("public, max-age=60, {directive}"), - ), - ( - header::HeaderName::from_static("surrogate-control"), - "max-age=1200", - ), - ]); + #[tokio::test] + async fn a_post_is_never_answered_from_a_cached_get() { + // `handle_publisher_request` is the `*`-method fallback route, so a publisher + // path that renders a page on GET and accepts a form or webhook on POST reaches + // here for both. Serving the cached GET to the POST swallows the mutating + // request entirely: the origin never sees it, the caller gets 200 and a page, + // and nothing anywhere reports a problem. + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + queue_shareable_html(&stub); + queue_shareable_html(&stub); - assert_eq!( - template_cache_bypass_reason( - AssemblyMode::Esi, - false, - false, - StatusCode::OK, - "text/html", - &publisher_headers, - ¬hing_covered(), - ), - Some(TemplateCacheBypassReason::OriginNotShareable), - "standard `{directive}` must refuse template cache even with positive edge freshness" - ); - } - } + // Warm the cache with a GET. + let _ = run(&settings, &services, navigation_request()).await; + assert_eq!(stub.recorded_request_uris().len(), 1); - #[test] - fn surrogate_control_can_authorize_fastly_edge_freshness_without_browser_freshness() { - let publisher_headers = headers(&[( - header::HeaderName::from_static("surrogate-control"), - "max-age=1200", - )]); + let post = HttpRequest::builder() + .method(Method::POST) + .uri("https://ts.example.com/article") + .header(header::HOST, "ts.example.com") + .body(EdgeBody::from("field=value")) + .expect("should build post request"); + let _ = run(&settings, &services, post).await; assert_eq!( - template_cache_ttl( - AssemblyMode::Esi, - false, - false, - StatusCode::OK, - "text/html", - &publisher_headers, - &TemplateCachePolicy::for_test(¬hing_covered(), Duration::from_secs(300),), - ), - Ok(Duration::from_secs(300)), - "Fastly edge freshness should not require browser freshness" - ); - } - - #[test] - fn repeated_cache_control_lines_without_a_disqualifier_still_cache() { - // The other direction: reading every value must not turn an ordinary - // multi-line `Cache-Control` into a bypass, or the fix would disable the - // cache instead of tightening it. - let mut split = edgezero_core::http::HeaderMap::new(); - split.append(header::CACHE_CONTROL, HeaderValue::from_static("public")); - split.append( - header::CACHE_CONTROL, - HeaderValue::from_static("max-age=60"), + stub.recorded_request_uris().len(), + 2, + "the POST must reach the origin rather than being answered from the \ + cached GET" ); - assert_eq!( - template_cache_bypass_reason( - AssemblyMode::Esi, - false, - false, - StatusCode::OK, - "text/html", - &split, - ¬hing_covered(), - ), - None + cache.entries.lock().expect("should lock entries").len(), + 1, + "and it must not store a template of its own" ); } - #[test] - fn origin_freshness_is_positive_age_adjusted_and_capped() { - let fresh_headers = headers(&[(header::CACHE_CONTROL, "public, max-age=300")]); - assert_eq!( - template_cache_ttl( - AssemblyMode::Esi, - false, - false, - StatusCode::OK, - "text/html", - &fresh_headers, - &TemplateCachePolicy::for_test(¬hing_covered(), Duration::from_secs(60),), - ), - Ok(Duration::from_secs(60)) - ); + #[tokio::test] + async fn an_authenticated_request_is_not_served_a_shared_template() { + // The stored template is perfectly cacheable; this request is not entitled + // to it. The store gate cannot express that, because it is a property of + // the reader rather than of the bytes — which is why the lookup re-checks. + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + queue_shareable_html(&stub); + queue_shareable_html(&stub); - let aged = headers(&[ - (header::CACHE_CONTROL, "s-maxage=50, max-age=300"), - (header::AGE, "35"), - ]); + let _ = run(&settings, &services, navigation_request()).await; assert_eq!( - template_cache_ttl( - AssemblyMode::Esi, - false, - false, - StatusCode::OK, - "text/html", - &aged, - &TemplateCachePolicy::for_test(¬hing_covered(), Duration::from_secs(60),), - ), - Ok(Duration::from_secs(15)) + cache.entries.lock().expect("should lock entries").len(), + 1, + "the cold request should have populated the cache" ); - let old_date_without_age = headers(&[ - (header::CACHE_CONTROL, "public, max-age=60"), - (header::DATE, "Wed, 12 Aug 2026 08:00:00 GMT"), - ]); - let one_minute_later = httpdate::parse_http_date("Wed, 12 Aug 2026 08:01:00 GMT") - .expect("should parse fixture time"); - assert_eq!( - origin_shared_ttl_at( - &old_date_without_age, - one_minute_later, - Duration::from_secs(60), - ), - Err(TemplateCacheBypassReason::NoPositiveFreshness), - "an old Date is apparent age even when an upstream omitted Age" - ); - } - - #[test] - fn zero_exhausted_missing_and_malformed_freshness_are_refused() { - for (map, expected) in [ - ( - headers(&[(header::CACHE_CONTROL, "max-age=0")]), - TemplateCacheBypassReason::NoPositiveFreshness, - ), - ( - headers(&[(header::CACHE_CONTROL, "max-age=60"), (header::AGE, "60")]), - TemplateCacheBypassReason::NoPositiveFreshness, - ), - ( - headers(&[(header::CACHE_CONTROL, "public")]), - TemplateCacheBypassReason::NoPositiveFreshness, - ), - ( - headers(&[(header::CACHE_CONTROL, "max-age=tomorrow")]), - TemplateCacheBypassReason::MalformedCachePolicy, - ), - ( - headers(&[(header::CACHE_CONTROL, "max-age=\"60")]), - TemplateCacheBypassReason::MalformedCachePolicy, - ), - ( - headers(&[(header::CACHE_CONTROL, "max-age=+60")]), - TemplateCacheBypassReason::MalformedCachePolicy, - ), - ] { - assert_eq!( - template_cache_bypass_reason( - AssemblyMode::Esi, - false, - false, - StatusCode::OK, - "text/html", - &map, - ¬hing_covered(), - ), - Some(expected) - ); - } - } - - #[test] - fn expires_can_authorize_but_never_extend_an_expired_response() { - let now = httpdate::parse_http_date("Wed, 12 Aug 2026 08:00:00 GMT") - .expect("should parse fixture time"); - let fresh = headers(&[ - (header::DATE, "Wed, 12 Aug 2026 08:00:00 GMT"), - (header::EXPIRES, "Wed, 12 Aug 2026 08:00:30 GMT"), - ]); - assert_eq!( - origin_shared_ttl_at(&fresh, now, Duration::from_secs(60)), - Ok(Duration::from_secs(30)) + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), ); + let consent = crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + }; + let mut ec_context = EcContext::new_for_test(None, consent); + let authenticated = HttpRequest::builder() + .method(Method::GET) + .uri("https://ts.example.com/article") + .header(header::HOST, "ts.example.com") + .header("sec-fetch-dest", "document") + .header(header::AUTHORIZATION, "Basic dXNlcjpwYXNz") + .body(EdgeBody::empty()) + .expect("should build authenticated request"); + let registry = test_registry(&settings); + let _ = handle_publisher_request( + AppContext { + settings: &settings, + integration_registry: ®istry, + }, + &services, + None, + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &[article_slot()], + registry: None, + }, + authenticated, + EdgeCacheHeader::SMaxageFallback, + ) + .await + .expect("should proxy publisher request"); - let expired = headers(&[ - (header::DATE, "Wed, 12 Aug 2026 08:01:00 GMT"), - (header::EXPIRES, "Wed, 12 Aug 2026 08:00:30 GMT"), - ]); assert_eq!( - origin_shared_ttl_at(&expired, now, Duration::from_secs(60)), - Err(TemplateCacheBypassReason::NoPositiveFreshness) + stub.recorded_request_uris().len(), + 2, + "an authenticated request must reach the origin rather than read a \ + shared template" ); } + } - #[test] - fn request_semantics_bypass_template_cache_except_for_a_max_age_zero_reload() { - for (name, value) in [ - (header::CACHE_CONTROL, "no-cache"), - (header::CACHE_CONTROL, "max-age=30"), - (header::CACHE_CONTROL, "max-age=\"0"), - (header::CACHE_CONTROL, "min-fresh=10"), - (header::CACHE_CONTROL, "no-store"), - (header::PRAGMA, "no-cache"), - (header::PRAGMA, "legacy-extension, no-cache"), - (header::RANGE, "bytes=0-99"), - (header::IF_NONE_MATCH, "\"etag\""), - (header::IF_MODIFIED_SINCE, "Wed, 12 Aug 2026 08:00:00 GMT"), - ] { - let map = headers(&[(name.clone(), value)]); - assert!( - request_bypasses_template_cache(&map), - "{name}: {value} must bypass" + mod template_cache_gate_tests { + //! `cache::core` stores whatever it is handed and rejects nothing, so every + //! one of these conditions is the caller's to enforce. Each is a leak vector + //! or an eligibility rule, not a preference. + + use super::*; + use crate::creative_opportunities::AssemblyMode; + use edgezero_core::http::HeaderName; + + fn headers(pairs: &[(HeaderName, &str)]) -> edgezero_core::http::HeaderMap { + let mut map = edgezero_core::http::HeaderMap::new(); + for (name, value) in pairs { + map.insert( + name.clone(), + HeaderValue::from_str(value).expect("should build header value"), ); } - assert!( - !request_bypasses_template_cache(&headers(&[(header::CACHE_CONTROL, "max-age=0")])), - "a browser reload may reuse template cache because the assembled response and auction \ - are still rebuilt for this reader" - ); - assert!(!request_bypasses_template_cache(&headers(&[( - header::CACHE_CONTROL, - "public" - )]))); + map + } + + fn shareable() -> edgezero_core::http::HeaderMap { + headers(&[(header::CACHE_CONTROL, "max-age=60")]) + } + + /// The shipped default: no operator has stated what the origin varies on, so the + /// key covers nothing. Responses without a `Vary` are unaffected; any `Vary` at + /// all disqualifies. + fn nothing_covered() -> VarySpec { + VarySpec::new([]) } #[test] - fn a_wildcard_vary_is_refused() { - // `VarySpec::uncovered_by` filters `*` out, with a comment saying the - // eligibility gate handles it. It did not — nothing rejected the wildcard, so - // a response the origin said no key can select was shareable. + fn an_unconfigured_deployment_never_caches_a_varying_response() { + // The fail-closed default. An operator who has not stated the origin's Vary + // must not acquire a shared cache by omission — and a real origin varies on + // something, so this is the common path, not an edge case. + // Deliberately not `Accept-Encoding`: the shared path normalizes supported + // content codings to one identity template, so that header is covered + // whatever the operator configured. Using it here would test the + // structural-coverage carve-out rather than the drift guard. let mut varying = shareable(); - varying.insert(header::VARY, HeaderValue::from_static("*")); + varying.insert(header::VARY, HeaderValue::from_static("rsc")); assert_eq!( template_cache_bypass_reason( @@ -15691,43 +16896,54 @@ mod tests { &varying, ¬hing_covered(), ), - Some(TemplateCacheBypassReason::VaryWildcard) + Some(TemplateCacheBypassReason::VaryNotCovered(VaryGap(vec![ + "rsc".to_string() + ]))), + "an unstated Vary must disqualify rather than silently under-key" ); } #[test] - fn a_fully_covered_vary_is_cacheable() { + fn an_origin_that_varies_on_cookie_is_refused_even_when_declared_independent() { + // The backstop that makes `origin_is_cookie_independent` safe to offer. The + // operator asserts their origin ignores cookies; if the origin then says + // otherwise, the assertion loses. Without this, a wrong assertion would + // silently cross-serve personalized HTML. let mut varying = shareable(); - varying.insert( - header::VARY, - HeaderValue::from_static("rsc, Accept-Encoding"), - ); + varying.insert(header::VARY, HeaderValue::from_static("Cookie")); assert_eq!( template_cache_bypass_reason( AssemblyMode::Esi, false, + // The operator's assertion has already been applied here: this is + // `false` precisely because they declared independence. false, StatusCode::OK, "text/html", &varying, - &VarySpec::new(["rsc".to_string(), "accept-encoding".to_string()]), + &VarySpec::new(["cookie".to_string()]), ), - None, - "a key covering everything the origin varies on is safe to store" + Some(TemplateCacheBypassReason::VaryCookie), + "the origin's declaration must override both cookie independence and an \ + accidentally configured per-cookie key" ); } #[test] - fn config_drift_names_the_missing_header() { - // The failure this guards: the origin adds a header to its Vary, nobody - // updates config, and requests differing only in that header start sharing a - // template. The reason must name it, or diagnosing means a bisect. - let mut varying = shareable(); - varying.insert( - header::VARY, - HeaderValue::from_static("rsc, next-router-prefetch"), + fn a_private_directive_on_a_second_cache_control_line_is_refused() { + // `HeaderMap::get` returns the first value only. An origin that sends + // `Cache-Control: public, max-age=300` and then `Cache-Control: private` on a + // separate line means exactly what one comma-joined line would mean, but the + // second line was invisible — so a response the origin marked private was + // written to a cache shared between readers. The `Vary` reads a few lines up + // already use `get_all` for the same reason. + let mut split = edgezero_core::http::HeaderMap::new(); + split.append( + header::CACHE_CONTROL, + HeaderValue::from_static("public, max-age=300"), ); + split.append(header::CACHE_CONTROL, HeaderValue::from_static("private")); assert_eq!( template_cache_bypass_reason( @@ -15736,41 +16952,25 @@ mod tests { false, StatusCode::OK, "text/html", - &varying, - &VarySpec::new(["rsc".to_string()]), + &split, + ¬hing_covered(), ), - Some(TemplateCacheBypassReason::VaryNotCovered(VaryGap(vec![ - "next-router-prefetch".to_string() - ]))), - "the uncovered header must be named" + Some(TemplateCacheBypassReason::OriginNotShareable), + "a directive on any Cache-Control line must disqualify the response" ); } #[test] - fn a_vary_split_across_repeated_headers_is_still_checked() { - // Vary is a list header, so an origin may send it once or many times. Reading - // only the first would let the rest through unkeyed. - let mut varying = shareable(); - varying.append(header::VARY, HeaderValue::from_static("rsc")); - varying.append(header::VARY, HeaderValue::from_static("cookie")); - - assert_eq!( - template_cache_bypass_reason( - AssemblyMode::Esi, - false, - false, - StatusCode::OK, - "text/html", - &varying, - &VarySpec::new(["rsc".to_string()]), - ), - Some(TemplateCacheBypassReason::VaryCookie), - "a repeated Vary header must not hide names behind the first value" + fn a_no_store_directive_on_a_second_cache_control_line_is_refused() { + // Same defect, the other directive that matters — `no-store` is the one an + // origin uses for a response that must not be written down anywhere. + let mut split = edgezero_core::http::HeaderMap::new(); + split.append( + header::CACHE_CONTROL, + HeaderValue::from_static("max-age=60"), ); - } + split.append(header::CACHE_CONTROL, HeaderValue::from_static("no-store")); - #[test] - fn a_plain_shareable_html_200_is_cacheable() { assert_eq!( template_cache_bypass_reason( AssemblyMode::Esi, @@ -15778,102 +16978,182 @@ mod tests { false, StatusCode::OK, "text/html", - &shareable(), + &split, ¬hing_covered(), ), - None, - "ESI shareable HTML 200 should be eligible" + Some(TemplateCacheBypassReason::OriginNotShareable) ); } #[test] - fn inline_mode_never_writes_a_template() { - assert_eq!( - template_cache_bypass_reason( - AssemblyMode::Inline, - false, - false, - StatusCode::OK, - "text/html", - &shareable(), - ¬hing_covered(), - ), - Some(TemplateCacheBypassReason::InlineMode), - "inline has no shared template to write" - ); + fn cdn_specific_cache_policy_cannot_be_overridden_by_public_cache_control() { + for name in crate::cache_policy::EDGE_CACHE_HEADER_NAMES { + let mut split = shareable(); + split.insert( + header::HeaderName::from_static(name), + HeaderValue::from_static("no-store"), + ); + + assert_eq!( + template_cache_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &split, + ¬hing_covered(), + ), + Some(TemplateCacheBypassReason::OriginNotShareable), + "template cache must fail closed on the CDN-specific policy header {name}" + ); + } } #[test] - fn an_authorized_request_is_never_cached() { - assert_eq!( - template_cache_bypass_reason( - AssemblyMode::Esi, - true, - false, - StatusCode::OK, - "text/html", - &shareable(), - ¬hing_covered(), - ), - Some(TemplateCacheBypassReason::AuthorizedRequest), - "an authenticated response must not enter a shared cache" - ); + fn unsupported_vendor_freshness_does_not_authorize_template_cache() { + for name in crate::cache_policy::EDGE_CACHE_HEADER_NAMES + .iter() + .filter(|name| **name != "surrogate-control") + { + let mut split = shareable(); + split.insert( + header::HeaderName::from_static(name), + HeaderValue::from_static("max-age=60"), + ); + + assert_eq!( + template_cache_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &split, + ¬hing_covered(), + ), + Some(TemplateCacheBypassReason::OriginNotShareable), + "the Fastly exception must not authorize the vendor policy {name}" + ); + } } #[test] - fn cookie_policy_disqualifies_even_without_set_cookie() { - // The dangerous case: session established on an earlier request, so this - // response carries no Set-Cookie, has no Cache-Control at all, is a 200, - // and is HTML — yet is personalized because TS forwarded the Cookie to - // origin unchanged. Every other condition reports it cacheable. - let no_cache_control = edgezero_core::http::HeaderMap::new(); + fn observed_fastly_surrogate_policy_uses_edge_freshness_capped_by_configuration() { + let publisher_headers = headers(&[ + (header::CACHE_CONTROL, "public, max-age=60"), + ( + header::HeaderName::from_static("surrogate-control"), + "max-age=1200, stale-while-revalidate=21600, stale-if-error=604800", + ), + ]); + assert_eq!( - template_cache_bypass_reason( + template_cache_ttl( AssemblyMode::Esi, false, - true, + false, StatusCode::OK, "text/html", - &no_cache_control, - ¬hing_covered(), + &publisher_headers, + &TemplateCachePolicy::for_test(¬hing_covered(), Duration::from_secs(300),), ), - Some(TemplateCacheBypassReason::CookiePolicy), - "cookie-personalized HTML must not become a shared template" + Ok(Duration::from_secs(300)), + "Fastly's edge freshness should take precedence over the shorter browser \ + lifetime, while the configured safety ceiling remains authoritative" ); } #[test] - fn an_origin_set_cookie_is_never_cached() { - let with_cookie = headers(&[ - (header::CACHE_CONTROL, "max-age=60"), - (header::SET_COOKIE, "sid=abc; Path=/"), + fn fastly_surrogate_freshness_takes_precedence_over_standard_freshness() { + for (cache_control, surrogate_control, expected) in [ + ("public, max-age=300", "max-age=30", 30), + ("public, max-age=30", "max-age=300", 300), + ] { + let publisher_headers = headers(&[ + (header::CACHE_CONTROL, cache_control), + ( + header::HeaderName::from_static("surrogate-control"), + surrogate_control, + ), + ]); + + assert_eq!( + template_cache_ttl( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &publisher_headers, + &TemplateCachePolicy::for_test( + ¬hing_covered(), + Duration::from_secs(600), + ), + ), + Ok(Duration::from_secs(expected)) + ); + } + } + + #[test] + fn surrogate_stale_windows_do_not_extend_fresh_reuse() { + let publisher_headers = headers(&[ + (header::CACHE_CONTROL, "public, max-age=300"), + ( + header::HeaderName::from_static("surrogate-control"), + "max-age=20, stale-while-revalidate=600, stale-if-error=1200", + ), + (header::AGE, "10"), ]); + assert_eq!( - template_cache_bypass_reason( + template_cache_ttl( AssemblyMode::Esi, false, false, StatusCode::OK, "text/html", - &with_cookie, - ¬hing_covered(), + &publisher_headers, + &TemplateCachePolicy::for_test(¬hing_covered(), Duration::from_secs(600),), ), - Some(TemplateCacheBypassReason::OriginSetCookie), - "caching this would replay one visitor's cookie to the next" + Ok(Duration::from_secs(10)), + "stale windows are validated metadata, not fresh template cache lifetime" ); } #[test] - fn non_shareable_cache_control_is_refused_case_insensitively() { - for directive in [ - "private", - "no-store", - "no-cache", - "Private, max-age=60", - "NO-STORE", - "public, No-Cache", + fn ambiguous_or_unsupported_surrogate_policy_fails_closed() { + for (policy, expected) in [ + ("max-age", TemplateCacheBypassReason::MalformedCachePolicy), + ( + "max-age=30, max-age=60", + TemplateCacheBypassReason::MalformedCachePolicy, + ), + ( + "max-age=tomorrow", + TemplateCacheBypassReason::MalformedCachePolicy, + ), + ( + "max-age=30, public", + TemplateCacheBypassReason::MalformedCachePolicy, + ), + ( + "stale-if-error=60", + TemplateCacheBypassReason::NoPositiveFreshness, + ), + ("max-age=0", TemplateCacheBypassReason::NoPositiveFreshness), + ( + "max-age=30,", + TemplateCacheBypassReason::MalformedCachePolicy, + ), ] { - let map = headers(&[(header::CACHE_CONTROL, directive)]); + let mut publisher_headers = shareable(); + publisher_headers.insert( + header::HeaderName::from_static("surrogate-control"), + HeaderValue::from_str(policy).expect("should build Surrogate-Control"), + ); + assert_eq!( template_cache_bypass_reason( AssemblyMode::Esi, @@ -15881,79 +17161,104 @@ mod tests { false, StatusCode::OK, "text/html", - &map, + &publisher_headers, ¬hing_covered(), ), - Some(TemplateCacheBypassReason::OriginNotShareable), - "`{directive}` should disqualify the response" + Some(expected), + "`{policy}` must fail closed" ); } } #[test] - fn a_datadome_block_is_refused_by_the_status_check() { - // DataDome replaces the document with a 403 - // (`integrations/datadome/protection.rs:778`). There is no separate - // marker to detect, and none is needed. - assert_eq!( - template_cache_bypass_reason( - AssemblyMode::Esi, - false, - false, - StatusCode::FORBIDDEN, - "text/html", - &shareable(), - ¬hing_covered(), - ), - Some(TemplateCacheBypassReason::NonOkStatus), - "a blocked document must not become the shared template" - ); + fn restrictive_surrogate_policy_is_never_overridden_by_standard_freshness() { + for directive in ["private", "no-store", "no-cache"] { + let mut publisher_headers = shareable(); + publisher_headers.insert( + header::HeaderName::from_static("surrogate-control"), + HeaderValue::from_str(directive).expect("should build Surrogate-Control"), + ); + + assert_eq!( + template_cache_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &publisher_headers, + ¬hing_covered(), + ), + Some(TemplateCacheBypassReason::OriginNotShareable), + "`{directive}` must remain authoritative" + ); + } } #[test] - fn non_html_is_refused() { - for content_type in ["text/x-component", "application/json", ""] { + fn restrictive_standard_policy_is_never_overridden_by_surrogate_freshness() { + for directive in ["private", "no-store", "no-cache"] { + let publisher_headers = headers(&[ + ( + header::CACHE_CONTROL, + &format!("public, max-age=60, {directive}"), + ), + ( + header::HeaderName::from_static("surrogate-control"), + "max-age=1200", + ), + ]); + assert_eq!( template_cache_bypass_reason( AssemblyMode::Esi, false, false, StatusCode::OK, - content_type, - &shareable(), + "text/html", + &publisher_headers, ¬hing_covered(), ), - Some(TemplateCacheBypassReason::NotHtml), - "`{content_type}` has no HTML template to transform" + Some(TemplateCacheBypassReason::OriginNotShareable), + "standard `{directive}` must refuse template cache even with positive edge freshness" ); } } #[test] - fn unsupported_content_encoding_is_refused_before_representation_headers_change() { - let map = headers(&[ - (header::CACHE_CONTROL, "public, max-age=60"), - (header::CONTENT_ENCODING, "zstd"), - ]); + fn surrogate_control_can_authorize_fastly_edge_freshness_without_browser_freshness() { + let publisher_headers = headers(&[( + header::HeaderName::from_static("surrogate-control"), + "max-age=1200", + )]); + assert_eq!( - template_cache_bypass_reason( + template_cache_ttl( AssemblyMode::Esi, false, false, StatusCode::OK, "text/html", - &map, - ¬hing_covered(), + &publisher_headers, + &TemplateCachePolicy::for_test(¬hing_covered(), Duration::from_secs(300),), ), - Some(TemplateCacheBypassReason::UnsupportedContentEncoding) + Ok(Duration::from_secs(300)), + "Fastly edge freshness should not require browser freshness" ); + } - let mut repeated = headers(&[(header::CACHE_CONTROL, "public, max-age=60")]); - repeated.append(header::CONTENT_TYPE, HeaderValue::from_static("text/html")); - repeated.append( - header::CONTENT_TYPE, - HeaderValue::from_static("application/json"), + #[test] + fn repeated_cache_control_lines_without_a_disqualifier_still_cache() { + // The other direction: reading every value must not turn an ordinary + // multi-line `Cache-Control` into a bypass, or the fix would disable the + // cache instead of tightening it. + let mut split = edgezero_core::http::HeaderMap::new(); + split.append(header::CACHE_CONTROL, HeaderValue::from_static("public")); + split.append( + header::CACHE_CONTROL, + HeaderValue::from_static("max-age=60"), ); + assert_eq!( template_cache_bypass_reason( AssemblyMode::Esi, @@ -15961,550 +17266,498 @@ mod tests { false, StatusCode::OK, "text/html", - &repeated, + &split, ¬hing_covered(), ), - Some(TemplateCacheBypassReason::MalformedRepresentationHeaders) + None ); } #[test] - fn leak_vectors_are_reported_before_mere_ineligibility() { - // A response that fails several conditions should name the most serious - // one, so an operator reading the log sees the security reason rather - // than a content-type quibble. - let map = headers(&[ - (header::CACHE_CONTROL, "private"), - (header::SET_COOKIE, "sid=abc"), - ]); + fn origin_freshness_is_positive_age_adjusted_and_capped() { + let fresh_headers = headers(&[(header::CACHE_CONTROL, "public, max-age=300")]); assert_eq!( - template_cache_bypass_reason( + template_cache_ttl( AssemblyMode::Esi, - true, false, - StatusCode::FORBIDDEN, - "application/json", - &map, - ¬hing_covered(), + false, + StatusCode::OK, + "text/html", + &fresh_headers, + &TemplateCachePolicy::for_test(¬hing_covered(), Duration::from_secs(60),), ), - Some(TemplateCacheBypassReason::AuthorizedRequest), - "authorization is the most serious disqualifier and should win" + Ok(Duration::from_secs(60)) ); - } - } - - mod template_neutrality_tests { - //! The gate for #1009's shared-template design. - //! - //! An "absence of per-user values" scan is not sufficient here: the bug - //! that nearly shipped was a *conditionally present* element whose own - //! content was per-URL. These tests assert byte-identity across requests - //! that differ only in the gating decision. - - use super::*; - use crate::creative_opportunities::{ - AssemblyMode, CreativeOpportunityFormat, CreativeOpportunitySlot, - }; - - pub(super) fn slot() -> CreativeOpportunitySlot { - CreativeOpportunitySlot { - id: "atf".to_string(), - gam_unit_path: Some("/99999/example/home".to_string()), - div_id: Some("ad-atf".to_string()), - page_patterns: vec!["/**".to_string()], - formats: vec![CreativeOpportunityFormat { - width: 300, - height: 250, - media_type: MediaType::Banner, - }], - floor_price: None, - targeting: Default::default(), - providers: Default::default(), - compiled_patterns: Vec::new(), - compiled_unit: None, - } - } - - pub(super) fn settings_with_slots() -> Settings { - let mut settings = crate::test_support::tests::create_test_settings(); - // Construct the section rather than mutating it if present: the shared - // fixture does not carry `[creative_opportunities]`, and an `if let - // Some(..)` here would silently no-op and make the inline assertion - // below vacuous. - settings.creative_opportunities = Some(CreativeOpportunitiesConfig { - enabled: true, - gam_network_id: "99999".to_string(), - auction_timeout_ms: Some(500), - price_granularity: Default::default(), - section_root: None, - assembly_mode: None, - template_cache_vary: None, - template_cache_max_age_seconds: None, - template_cache_key_cookies: None, - template_cache_bypass_cookies: None, - origin_is_cookie_independent: None, - origin_readthrough_enabled: None, - section_segment: None, - slot: vec![slot()], - }); - settings - } - - #[test] - fn shared_modes_emit_no_head_script_regardless_of_the_gating_decision() { - let settings = settings_with_slots(); - let slots = [slot()]; - let mode = AssemblyMode::Esi; - let ran = template_ad_slots_script(mode, true, &settings, &slots, "/"); - let did_not_run = template_ad_slots_script(mode, false, &settings, &slots, "/"); + let aged = headers(&[ + (header::CACHE_CONTROL, "s-maxage=50, max-age=300"), + (header::AGE, "35"), + ]); assert_eq!( - ran, did_not_run, - "{mode:?}: the template must be byte-identical whether or not the ad \ - stack ran; a cached object cannot carry one request's consent, bot, \ - prefetch or kill-switch decision" + template_cache_ttl( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &aged, + &TemplateCachePolicy::for_test(¬hing_covered(), Duration::from_secs(60),), + ), + Ok(Duration::from_secs(15)) ); + + let old_date_without_age = headers(&[ + (header::CACHE_CONTROL, "public, max-age=60"), + (header::DATE, "Wed, 12 Aug 2026 08:00:00 GMT"), + ]); + let one_minute_later = httpdate::parse_http_date("Wed, 12 Aug 2026 08:01:00 GMT") + .expect("should parse fixture time"); assert_eq!( - ran, None, - "{mode:?}: adSlots belongs in the per-request seam, not the template" + origin_shared_ttl_at( + &old_date_without_age, + one_minute_later, + Duration::from_secs(60), + ), + Err(TemplateCacheBypassReason::NoPositiveFreshness), + "an old Date is apparent age even when an upstream omitted Age" ); } #[test] - fn inline_mode_keeps_its_request_dependent_behaviour() { - // Inline responses are per-navigation and never shared, so gating is - // correct there. This guards against "fixing" the shared-mode bug by - // breaking the shipped path. - let settings = settings_with_slots(); - let slots = [slot()]; - - assert!( - template_ad_slots_script(AssemblyMode::Inline, true, &settings, &slots, "/") - .is_some(), - "inline should emit adSlots when the ad stack runs" - ); - assert_eq!( - template_ad_slots_script(AssemblyMode::Inline, false, &settings, &slots, "/"), - None, - "inline should emit nothing when the ad stack does not run" - ); + fn zero_exhausted_missing_and_malformed_freshness_are_refused() { + for (map, expected) in [ + ( + headers(&[(header::CACHE_CONTROL, "max-age=0")]), + TemplateCacheBypassReason::NoPositiveFreshness, + ), + ( + headers(&[(header::CACHE_CONTROL, "max-age=60"), (header::AGE, "60")]), + TemplateCacheBypassReason::NoPositiveFreshness, + ), + ( + headers(&[(header::CACHE_CONTROL, "public")]), + TemplateCacheBypassReason::NoPositiveFreshness, + ), + ( + headers(&[(header::CACHE_CONTROL, "max-age=tomorrow")]), + TemplateCacheBypassReason::MalformedCachePolicy, + ), + ( + headers(&[(header::CACHE_CONTROL, "max-age=\"60")]), + TemplateCacheBypassReason::MalformedCachePolicy, + ), + ( + headers(&[(header::CACHE_CONTROL, "max-age=+60")]), + TemplateCacheBypassReason::MalformedCachePolicy, + ), + ] { + assert_eq!( + template_cache_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &map, + ¬hing_covered(), + ), + Some(expected) + ); + } } #[test] - fn shared_modes_are_neutral_across_differing_slot_matches() { - // Slot matching folds in the request path. Under a shared mode even - // that must not reach the template. - let settings = settings_with_slots(); - - let matched = template_ad_slots_script( - AssemblyMode::Esi, - true, - &settings, - &[slot()], - "/news/article", + fn expires_can_authorize_but_never_extend_an_expired_response() { + let now = httpdate::parse_http_date("Wed, 12 Aug 2026 08:00:00 GMT") + .expect("should parse fixture time"); + let fresh = headers(&[ + (header::DATE, "Wed, 12 Aug 2026 08:00:00 GMT"), + (header::EXPIRES, "Wed, 12 Aug 2026 08:00:30 GMT"), + ]); + assert_eq!( + origin_shared_ttl_at(&fresh, now, Duration::from_secs(60)), + Ok(Duration::from_secs(30)) ); - let unmatched = - template_ad_slots_script(AssemblyMode::Esi, true, &settings, &[], "/other"); + let expired = headers(&[ + (header::DATE, "Wed, 12 Aug 2026 08:01:00 GMT"), + (header::EXPIRES, "Wed, 12 Aug 2026 08:00:30 GMT"), + ]); assert_eq!( - matched, unmatched, - "the template must not vary with slot matching under a shared mode" + origin_shared_ttl_at(&expired, now, Duration::from_secs(60)), + Err(TemplateCacheBypassReason::NoPositiveFreshness) ); } - } - #[tokio::test] - async fn publisher_asset_cache_policy_applies_to_non_html_response() { - let settings = Settings::from_toml(&format!( - r#"{} + #[test] + fn request_semantics_bypass_template_cache_except_for_a_max_age_zero_reload() { + for (name, value) in [ + (header::CACHE_CONTROL, "no-cache"), + (header::CACHE_CONTROL, "max-age=30"), + (header::CACHE_CONTROL, "max-age=\"0"), + (header::CACHE_CONTROL, "min-fresh=10"), + (header::CACHE_CONTROL, "no-store"), + (header::PRAGMA, "no-cache"), + (header::PRAGMA, "legacy-extension, no-cache"), + (header::RANGE, "bytes=0-99"), + (header::IF_NONE_MATCH, "\"etag\""), + (header::IF_MODIFIED_SINCE, "Wed, 12 Aug 2026 08:00:00 GMT"), + ] { + let map = headers(&[(name.clone(), value)]); + assert!( + request_bypasses_template_cache(&map), + "{name}: {value} must bypass" + ); + } + assert!( + !request_bypasses_template_cache(&headers(&[(header::CACHE_CONTROL, "max-age=0")])), + "a browser reload may reuse template cache because the assembled response and auction \ + are still rebuilt for this reader" + ); + assert!(!request_bypasses_template_cache(&headers(&[( + header::CACHE_CONTROL, + "public" + )]))); + } - [[cache.asset_rules]] - id = "publisher-fingerprinted-assets" - enabled = true - path_globs = ["/assets/**/*.png"] - fingerprint_style = "hex" - visibility = "public" - browser_ttl_seconds = 31536000 - edge_ttl_seconds = 31536000 - immutable = true - "#, - crate_test_settings_str() - )) - .expect("should parse settings with cache rule"); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - 200, - b"png".to_vec(), - vec![ - (header::CONTENT_TYPE.as_str(), "image/png"), - (header::CACHE_CONTROL.as_str(), "public, max-age=60"), - ], - ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let request = HttpRequest::builder() - .method(Method::GET) - .uri("https://publisher.example/assets/logo.0123abcd.png") - .header(header::HOST, "publisher.example") - .body(EdgeBody::empty()) - .expect("should build request"); + #[test] + fn a_wildcard_vary_is_refused() { + // `VarySpec::uncovered_by` filters `*` out, with a comment saying the + // eligibility gate handles it. It did not — nothing rejected the wildcard, so + // a response the origin said no key can select was shareable. + let mut varying = shareable(); + varying.insert(header::VARY, HeaderValue::from_static("*")); - let response = run_publisher_proxy(&settings, &services, request).await; - let PublisherResponse::PassThrough { response, .. } = response else { - panic!("should pass through non-HTML asset response"); - }; + assert_eq!( + template_cache_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &varying, + ¬hing_covered(), + ), + Some(TemplateCacheBypassReason::VaryWildcard) + ); + } - assert_eq!( - response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("public, max-age=31536000, immutable"), - "matched publisher asset should receive immutable browser policy" - ); - assert_eq!( - response - .headers() - .get("surrogate-control") - .and_then(|value| value.to_str().ok()), - Some("max-age=31536000"), - "matched publisher asset should receive Fastly edge policy" - ); - } + #[test] + fn a_fully_covered_vary_is_cacheable() { + let mut varying = shareable(); + varying.insert( + header::VARY, + HeaderValue::from_static("rsc, Accept-Encoding"), + ); - #[tokio::test] - async fn publisher_asset_policy_response_with_cookie_is_private_after_finalization() { - let settings = Settings::from_toml(&format!( - r#"{} + assert_eq!( + template_cache_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &varying, + &VarySpec::new(["rsc".to_string(), "accept-encoding".to_string()]), + ), + None, + "a key covering everything the origin varies on is safe to store" + ); + } - [[cache.asset_rules]] - id = "publisher-fingerprinted-assets" - enabled = true - path_globs = ["/assets/**/*.png"] - fingerprint_style = "hex" - visibility = "public" - browser_ttl_seconds = 31536000 - edge_ttl_seconds = 31536000 - immutable = true - "#, - crate_test_settings_str() - )) - .expect("should parse settings with cache rule"); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - 200, - b"png".to_vec(), - vec![ - (header::CONTENT_TYPE.as_str(), "image/png"), - (header::CACHE_CONTROL.as_str(), "public, max-age=60"), - (header::SET_COOKIE.as_str(), "viewer=example; Path=/"), - ], - ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let request = HttpRequest::builder() - .method(Method::GET) - .uri("https://publisher.example/assets/logo.0123abcd.png") - .header(header::HOST, "publisher.example") - .body(EdgeBody::empty()) - .expect("should build request"); + #[test] + fn config_drift_names_the_missing_header() { + // The failure this guards: the origin adds a header to its Vary, nobody + // updates config, and requests differing only in that header start sharing a + // template. The reason must name it, or diagnosing means a bisect. + let mut varying = shareable(); + varying.insert( + header::VARY, + HeaderValue::from_static("rsc, next-router-prefetch"), + ); - let response = run_publisher_proxy(&settings, &services, request).await; - let PublisherResponse::PassThrough { mut response, .. } = response else { - panic!("should pass through non-HTML asset response"); - }; - crate::response_privacy::apply_response_headers_with_cache_privacy( - &settings, - &mut response, - ); + assert_eq!( + template_cache_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &varying, + &VarySpec::new(["rsc".to_string()]), + ), + Some(TemplateCacheBypassReason::VaryNotCovered(VaryGap(vec![ + "next-router-prefetch".to_string() + ]))), + "the uncovered header must be named" + ); + } - assert_eq!( - response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("private, max-age=0"), - "publisher asset with Set-Cookie must become private after finalization" - ); - assert!( - response.headers().get("surrogate-control").is_none(), - "publisher asset with Set-Cookie must not retain a shared-cache header" - ); - } + #[test] + fn a_vary_split_across_repeated_headers_is_still_checked() { + // Vary is a list header, so an origin may send it once or many times. Reading + // only the first would let the rest through unkeyed. + let mut varying = shareable(); + varying.append(header::VARY, HeaderValue::from_static("rsc")); + varying.append(header::VARY, HeaderValue::from_static("cookie")); - #[tokio::test] - async fn publisher_asset_cache_policy_skips_html_response() { - let settings = Settings::from_toml(&format!( - r#"{} + assert_eq!( + template_cache_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &varying, + &VarySpec::new(["rsc".to_string()]), + ), + Some(TemplateCacheBypassReason::VaryCookie), + "a repeated Vary header must not hide names behind the first value" + ); + } - [[cache.asset_rules]] - id = "broad-publisher-path" - enabled = true - path_glob = "/news/*.html" - visibility = "public" - browser_ttl_seconds = 31536000 - edge_ttl_seconds = 31536000 - immutable = true - fingerprint_style = "hex" - "#, - crate_test_settings_str() - )) - .expect("should parse settings with cache rule"); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - 200, - b"news".to_vec(), - vec![ - (header::CONTENT_TYPE.as_str(), "text/html; charset=utf-8"), - (header::CACHE_CONTROL.as_str(), "public, max-age=60"), - ], - ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let request = HttpRequest::builder() - .method(Method::GET) - .uri("https://publisher.example/news/story.0123abcd.html") - .header(header::HOST, "publisher.example") - .body(EdgeBody::empty()) - .expect("should build request"); - - let response = run_publisher_proxy(&settings, &services, request).await; - let response = match response { - PublisherResponse::Stream { response, .. } | PublisherResponse::Buffered(response) => { - response - } - PublisherResponse::PassThrough { .. } | PublisherResponse::AssembleTemplate { .. } => { - panic!("should classify HTML response for processing") - } - }; - - assert_eq!( - response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("public, max-age=60"), - "asset policy must not cache publisher HTML" - ); - assert!( - response.headers().get("surrogate-control").is_none(), - "HTML response must not receive a shared-cache header" - ); - } - - mod ssat_cache_policy_tests { - use super::*; - use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; - use crate::auction::telemetry::{AuctionEventBatch, AuctionTelemetrySink}; - use crate::creative_opportunities::{CreativeOpportunityFormat, CreativeOpportunitySlot}; - use crate::platform::test_support::{ - NoopConfigStore, NoopGeo, NoopSecretStore, StubBackend, - }; - use crate::platform::{ - ClientInfo, PlatformError, PlatformHttpClient, PlatformPendingRequest, - PlatformResponse, PlatformSelectResult, - }; - use crate::test_support::tests::crate_test_settings_str; + #[test] + fn a_plain_shareable_html_200_is_cacheable() { + assert_eq!( + template_cache_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &shareable(), + ¬hing_covered(), + ), + None, + "ESI shareable HTML 200 should be eligible" + ); + } - const ORIGIN_ETAG: &str = "\"origin-tag\""; - const ORIGIN_LAST_MODIFIED: &str = "Wed, 21 Oct 2015 07:28:00 GMT"; - const UNEXPECTED_304_PROVIDER: &str = "example-navigation-bidder"; - const UNEXPECTED_304_BACKEND: &str = "example-navigation-bidder-backend"; + #[test] + fn inline_mode_never_writes_a_template() { + assert_eq!( + template_cache_bypass_reason( + AssemblyMode::Inline, + false, + false, + StatusCode::OK, + "text/html", + &shareable(), + ¬hing_covered(), + ), + Some(TemplateCacheBypassReason::InlineMode), + "inline has no shared template to write" + ); + } - struct DispatchingTestProvider; + #[test] + fn an_authorized_request_is_never_cached() { + assert_eq!( + template_cache_bypass_reason( + AssemblyMode::Esi, + true, + false, + StatusCode::OK, + "text/html", + &shareable(), + ¬hing_covered(), + ), + Some(TemplateCacheBypassReason::AuthorizedRequest), + "an authenticated response must not enter a shared cache" + ); + } - struct RangeAwareHttpClient { - stub: StubHttpClient, + #[test] + fn cookie_policy_disqualifies_even_without_set_cookie() { + // The dangerous case: session established on an earlier request, so this + // response carries no Set-Cookie, has no Cache-Control at all, is a 200, + // and is HTML — yet is personalized because TS forwarded the Cookie to + // origin unchanged. Every other condition reports it cacheable. + let no_cache_control = edgezero_core::http::HeaderMap::new(); + assert_eq!( + template_cache_bypass_reason( + AssemblyMode::Esi, + false, + true, + StatusCode::OK, + "text/html", + &no_cache_control, + ¬hing_covered(), + ), + Some(TemplateCacheBypassReason::CookiePolicy), + "cookie-personalized HTML must not become a shared template" + ); } - impl RangeAwareHttpClient { - fn new() -> Self { - Self { - stub: StubHttpClient::new(), - } - } + #[test] + fn an_origin_set_cookie_is_never_cached() { + let with_cookie = headers(&[ + (header::CACHE_CONTROL, "max-age=60"), + (header::SET_COOKIE, "sid=abc; Path=/"), + ]); + assert_eq!( + template_cache_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &with_cookie, + ¬hing_covered(), + ), + Some(TemplateCacheBypassReason::OriginSetCookie), + "caching this would replay one visitor's cookie to the next" + ); } - #[async_trait::async_trait(?Send)] - impl PlatformHttpClient for RangeAwareHttpClient { - async fn send( - &self, - request: PlatformHttpRequest, - ) -> Result> { - if request.request.headers().contains_key(header::RANGE) { - self.stub.push_response_with_headers( - 206, - b"partial".to_vec(), - vec![ - ("content-type", "text/html; charset=utf-8"), - ("content-range", "bytes 0-18/39"), - ], - ); - } else { - self.stub.push_response_with_headers( - 200, - b"origin".to_vec(), - vec![("content-type", "text/html; charset=utf-8")], - ); - } - self.stub.send(request).await + #[test] + fn non_shareable_cache_control_is_refused_case_insensitively() { + for directive in [ + "private", + "no-store", + "no-cache", + "Private, max-age=60", + "NO-STORE", + "public, No-Cache", + ] { + let map = headers(&[(header::CACHE_CONTROL, directive)]); + assert_eq!( + template_cache_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &map, + ¬hing_covered(), + ), + Some(TemplateCacheBypassReason::OriginNotShareable), + "`{directive}` should disqualify the response" + ); } + } - async fn send_async( - &self, - request: PlatformHttpRequest, - ) -> Result> { - self.stub.send_async(request).await - } + #[test] + fn a_bot_protection_block_is_refused_by_the_status_check() { + // A bot protection module replaces the document with a 403. There + // is no separate marker to detect, and none is needed. + assert_eq!( + template_cache_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::FORBIDDEN, + "text/html", + &shareable(), + ¬hing_covered(), + ), + Some(TemplateCacheBypassReason::NonOkStatus), + "a blocked document must not become the shared template" + ); + } - async fn select( - &self, - pending_requests: Vec, - ) -> Result> { - self.stub.select(pending_requests).await + #[test] + fn non_html_is_refused() { + for content_type in ["text/x-component", "application/json", ""] { + assert_eq!( + template_cache_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + content_type, + &shareable(), + ¬hing_covered(), + ), + Some(TemplateCacheBypassReason::NotHtml), + "`{content_type}` has no HTML template to transform" + ); } } - #[async_trait::async_trait(?Send)] - impl AuctionProvider for DispatchingTestProvider { - fn provider_name(&self) -> &str { - UNEXPECTED_304_PROVIDER - } + #[test] + fn unsupported_content_encoding_is_refused_before_representation_headers_change() { + let map = headers(&[ + (header::CACHE_CONTROL, "public, max-age=60"), + (header::CONTENT_ENCODING, "zstd"), + ]); + assert_eq!( + template_cache_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &map, + ¬hing_covered(), + ), + Some(TemplateCacheBypassReason::UnsupportedContentEncoding) + ); - async fn request_bids( - &self, - _request: &AuctionRequest, - context: &AuctionContext<'_>, - ) -> Result> { - let request = PlatformHttpRequest::new( - HttpRequest::builder() - .method(Method::POST) - .uri("https://bidder.example.com/navigation-bids") - .body(EdgeBody::empty()) - .expect("should build test provider request"), - UNEXPECTED_304_BACKEND, - ); - context - .services - .http_client() - .send_async(request) - .await - .change_context(TrustedServerError::Auction { - message: "test provider launch failed".to_string(), - }) - .map(ProviderRequestOutcome::pending) - } - - async fn parse_response( - &self, - _response: PlatformResponse, - _response_time_ms: u64, - ) -> Result> { - panic!("parse_response must not run for an unexpected origin 304"); - } - - fn timeout_ms(&self) -> u32 { - 100 - } - - fn backend_name( - &self, - _services: &RuntimeServices, - _timeout_ms: u32, - ) -> Option { - Some(UNEXPECTED_304_BACKEND.to_string()) - } - } - - #[derive(Default)] - struct RecordingTelemetrySink { - batches: Mutex>, - } - - #[async_trait::async_trait(?Send)] - impl AuctionTelemetrySink for RecordingTelemetrySink { - async fn emit_auction_events( - &self, - _services: &RuntimeServices, - batch: AuctionEventBatch, - ) -> Result<(), Report> { - self.batches - .lock() - .expect("should lock telemetry batches") - .push(batch); - Ok(()) - } - } - - fn settings_with_enabled_auction_and_creative_opportunities() -> Settings { - let toml = format!( - "{}\n[auction]\nenabled = true\n\n\ - [creative_opportunities]\ngam_network_id = \"12345\"\n", - crate_test_settings_str() + let mut repeated = headers(&[(header::CACHE_CONTROL, "public, max-age=60")]); + repeated.append(header::CONTENT_TYPE, HeaderValue::from_static("text/html")); + repeated.append( + header::CONTENT_TYPE, + HeaderValue::from_static("application/json"), ); - Settings::from_toml(&toml) - .expect("should parse settings with auction and creative opportunities enabled") - } - - fn settings_with_disabled_ad_templates() -> Settings { - let toml = format!( - "{}\n[auction]\nenabled = true\n\n\ - [creative_opportunities]\nenabled = false\ngam_network_id = \"12345\"\n", - crate_test_settings_str() + assert_eq!( + template_cache_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &repeated, + ¬hing_covered(), + ), + Some(TemplateCacheBypassReason::MalformedRepresentationHeaders) ); - Settings::from_toml(&toml).expect("should parse settings with disabled ad templates") } - fn settings_with_disabled_auction() -> Settings { - let toml = format!( - "{}\n[auction]\nenabled = false\n\n\ - [creative_opportunities]\ngam_network_id = \"12345\"\n", - crate_test_settings_str() + #[test] + fn leak_vectors_are_reported_before_mere_ineligibility() { + // A response that fails several conditions should name the most serious + // one, so an operator reading the log sees the security reason rather + // than a content-type quibble. + let map = headers(&[ + (header::CACHE_CONTROL, "private"), + (header::SET_COOKIE, "sid=abc"), + ]); + assert_eq!( + template_cache_bypass_reason( + AssemblyMode::Esi, + true, + false, + StatusCode::FORBIDDEN, + "application/json", + &map, + ¬hing_covered(), + ), + Some(TemplateCacheBypassReason::AuthorizedRequest), + "authorization is the most serious disqualifier and should win" ); - Settings::from_toml(&toml).expect("should parse settings with disabled auction") - } - - fn settings_without_creative_opportunities() -> Settings { - Settings::from_toml(&crate_test_settings_str()) - .expect("should parse settings without creative opportunities") } + } - fn settings_with_dispatching_provider() -> Settings { - let toml = format!( - "{}\n[auction]\nenabled = true\n\n[auction.providers.{UNEXPECTED_304_PROVIDER}]\nprotocol = \"openrtb-2.6\"\nendpoint = \"https://unexpected.example/openrtb2/auction\"\nrouting = \"all_eligible\"\n\n\ - [creative_opportunities]\ngam_network_id = \"12345\"\n", - crate_test_settings_str() - ); - Settings::from_toml(&toml) - .expect("should parse settings with the dispatching test provider") - } + mod template_neutrality_tests { + //! The gate for #1009's shared-template design. + //! + //! An "absence of per-user values" scan is not sufficient here: the bug + //! that nearly shipped was a *conditionally present* element whose own + //! content was per-URL. These tests assert byte-identity across requests + //! that differ only in the gating decision. - fn services_with_telemetry( - http_client: Arc, - telemetry_sink: Arc, - ) -> RuntimeServices { - let telemetry_sink: Arc = telemetry_sink; - RuntimeServices::builder() - .config_store(Arc::new(NoopConfigStore)) - .secret_store(Arc::new(NoopSecretStore)) - .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) - .backend(Arc::new(StubBackend)) - .http_client(http_client) - .geo(Arc::new(NoopGeo)) - .auction_telemetry_sink(telemetry_sink) - .client_info(ClientInfo::default()) - .build() - } + use super::*; + use crate::creative_opportunities::{ + AssemblyMode, CreativeOpportunityFormat, CreativeOpportunitySlot, + }; - fn article_slot() -> CreativeOpportunitySlot { + pub(super) fn slot() -> CreativeOpportunitySlot { CreativeOpportunitySlot { - id: "article-slot".to_string(), - gam_unit_path: None, - div_id: None, - page_patterns: vec!["/article".to_string()], + id: "atf".to_string(), + gam_unit_path: Some("/99999/example/home".to_string()), + div_id: Some("ad-atf".to_string()), + page_patterns: vec!["/**".to_string()], formats: vec![CreativeOpportunityFormat { width: 300, height: 250, @@ -16518,1047 +17771,1252 @@ mod tests { } } - fn conditional_navigation_request() -> Request { - HttpRequest::builder() - .method(Method::GET) - .uri("https://ts.example.com/article") - .header(header::HOST, "ts.example.com") - .header("sec-fetch-dest", "document") - .header(header::IF_NONE_MATCH, ORIGIN_ETAG) - .header(header::IF_MODIFIED_SINCE, ORIGIN_LAST_MODIFIED) - .body(EdgeBody::empty()) - .expect("should build conditional navigation request") + pub(super) fn settings_with_slots() -> Settings { + let mut settings = crate::test_support::tests::create_test_settings(); + // Construct the section rather than mutating it if present: the shared + // fixture does not carry `[creative_opportunities]`, and an `if let + // Some(..)` here would silently no-op and make the inline assertion + // below vacuous. + settings.creative_opportunities = Some(CreativeOpportunitiesConfig { + enabled: true, + gam_network_id: "99999".to_string(), + auction_timeout_ms: Some(500), + price_granularity: Default::default(), + section_root: None, + assembly_mode: None, + template_cache_vary: None, + template_cache_max_age_seconds: None, + template_cache_key_cookies: None, + template_cache_bypass_cookies: None, + origin_is_cookie_independent: None, + origin_readthrough_enabled: None, + section_segment: None, + slot: vec![slot()], + }); + settings } - fn queue_html_response_with_cache_control( - stub: &StubHttpClient, - cache_control: &'static str, - ) { - queue_html_response_with_status_and_cache_control(stub, 200, cache_control); - } + #[test] + fn shared_modes_emit_no_head_script_regardless_of_the_gating_decision() { + let settings = settings_with_slots(); + let slots = [slot()]; + let mode = AssemblyMode::Esi; + let ran = template_ad_slots_script(mode, true, &settings, &slots, "/"); + let did_not_run = template_ad_slots_script(mode, false, &settings, &slots, "/"); - fn queue_html_response_with_status_and_cache_control( - stub: &StubHttpClient, - status: u16, - cache_control: &'static str, - ) { - stub.push_response_with_headers( - status, - b"origin".to_vec(), - vec![ - ("content-type", "text/html; charset=utf-8"), - ("cache-control", cache_control), - ("etag", ORIGIN_ETAG), - ("last-modified", ORIGIN_LAST_MODIFIED), - ("surrogate-control", "max-age=300"), - ("fastly-surrogate-control", "max-age=300"), - ("cdn-cache-control", "max-age=300"), - ("cloudflare-cdn-cache-control", "max-age=300"), - ], + assert_eq!( + ran, did_not_run, + "{mode:?}: the template must be byte-identical whether or not the ad \ + stack ran; a cached object cannot carry one request's consent, bot, \ + prefetch or kill-switch decision" + ); + assert_eq!( + ran, None, + "{mode:?}: adSlots belongs in the per-request seam, not the template" ); } - fn non_regulated_consent() -> crate::consent::ConsentContext { - crate::consent::ConsentContext { - jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, - ..Default::default() - } + #[test] + fn inline_mode_keeps_its_request_dependent_behaviour() { + // Inline responses are per-navigation and never shared, so gating is + // correct there. This guards against "fixing" the shared-mode bug by + // breaking the shipped path. + let settings = settings_with_slots(); + let slots = [slot()]; + + assert!( + template_ad_slots_script(AssemblyMode::Inline, true, &settings, &slots, "/") + .is_some(), + "inline should emit adSlots when the ad stack runs" + ); + assert_eq!( + template_ad_slots_script(AssemblyMode::Inline, false, &settings, &slots, "/"), + None, + "inline should emit nothing when the ad stack does not run" + ); } - async fn run_with_slots( - settings: &Settings, - services: &RuntimeServices, - slots: &[CreativeOpportunitySlot], - req: Request, - ) -> PublisherResponse { - run_with_slots_and_consent(settings, services, slots, req, non_regulated_consent()) - .await - } + #[test] + fn shared_modes_are_neutral_across_differing_slot_matches() { + // Slot matching folds in the request path. Under a shared mode even + // that must not reach the template. + let settings = settings_with_slots(); - async fn run_with_slots_and_consent( - settings: &Settings, - services: &RuntimeServices, - slots: &[CreativeOpportunitySlot], - req: Request, - consent: crate::consent::ConsentContext, - ) -> PublisherResponse { - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - run_with_orchestrator_and_consent( - settings, - services, - &orchestrator, - slots, - req, - consent, - ) - .await - } + let matched = template_ad_slots_script( + AssemblyMode::Esi, + true, + &settings, + &[slot()], + "/news/article", + ); + let unmatched = + template_ad_slots_script(AssemblyMode::Esi, true, &settings, &[], "/other"); - async fn run_with_orchestrator( - settings: &Settings, - services: &RuntimeServices, - orchestrator: &AuctionOrchestrator, - slots: &[CreativeOpportunitySlot], - req: Request, - ) -> PublisherResponse { - run_with_orchestrator_and_consent( - settings, - services, - orchestrator, - slots, - req, - non_regulated_consent(), - ) - .await + assert_eq!( + matched, unmatched, + "the template must not vary with slot matching under a shared mode" + ); } + } - async fn run_with_orchestrator_and_consent( - settings: &Settings, - services: &RuntimeServices, - orchestrator: &AuctionOrchestrator, - slots: &[CreativeOpportunitySlot], - req: Request, - consent: crate::consent::ConsentContext, - ) -> PublisherResponse { - let mut ec_context = EcContext::new_for_test(None, consent); - - handle_publisher_request( - settings, - services, - None, - &mut ec_context, - AuctionDispatch { - orchestrator, - slots, - registry: None, - }, - req, - EdgeCacheHeader::SMaxageFallback, - ) - .await - .expect("should proxy publisher request") - } + #[tokio::test] + async fn publisher_asset_cache_policy_applies_to_non_html_response() { + let settings = settings_from_toml(&format!( + r#"{} - fn response_head(response: PublisherResponse) -> http::response::Parts { - match response { - PublisherResponse::Buffered(response) - | PublisherResponse::Stream { response, .. } - | PublisherResponse::AssembleTemplate { response, .. } - | PublisherResponse::PassThrough { response, .. } => response.into_parts().0, - } - } + [[cache.asset_rules]] + id = "publisher-fingerprinted-assets" + enabled = true + path_globs = ["/assets/**/*.png"] + fingerprint_style = "hex" + visibility = "public" + browser_ttl_seconds = 31536000 + edge_ttl_seconds = 31536000 + immutable = true + "#, + crate_test_settings_str() + )); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + b"png".to_vec(), + vec![ + (header::CONTENT_TYPE.as_str(), "image/png"), + (header::CACHE_CONTROL.as_str(), "public, max-age=60"), + ], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let request = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/assets/logo.0123abcd.png") + .header(header::HOST, "publisher.example") + .body(EdgeBody::empty()) + .expect("should build request"); - fn recorded_header<'a>(headers: &'a [(String, String)], name: &str) -> Option<&'a str> { - headers - .iter() - .find(|(header_name, _)| header_name.eq_ignore_ascii_case(name)) - .map(|(_, value)| value.as_str()) - } + let response = run_publisher_proxy(&settings, &services, request).await; + let PublisherResponse::PassThrough { response, .. } = response else { + panic!("should pass through non-HTML asset response"); + }; - fn active_ec_context() -> EcContext { - EcContext::new_for_test_with_ip( - Some(format!("{}.CkId01", "b".repeat(64))), - scheduling_consent(), - Some("203.0.113.7".to_owned()), - ) - } + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("public, max-age=31536000, immutable"), + "matched publisher asset should receive immutable browser policy" + ); + assert_eq!( + response + .headers() + .get("surrogate-control") + .and_then(|value| value.to_str().ok()), + Some("max-age=31536000"), + "matched publisher asset should receive Fastly edge policy" + ); + } - #[tokio::test] - async fn pending_origin_wait_failure_abandons_dispatched_auction_once() { - let settings = settings_with_dispatching_provider(); - let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - orchestrator.register_provider(Arc::new(DispatchingTestProvider)); - let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); - let stub = Arc::new(StubHttpClient::new()); - stub.set_streaming_responses_supported(true); - stub.set_pending_streaming_responses_supported(true); - queue_html_response_with_cache_control(&stub, "public, max-age=300"); - stub.push_response(200, b"unused provider response".to_vec()); - stub.push_wait_error(); - let services = services_with_telemetry( - Arc::clone(&stub) as Arc, - Arc::clone(&telemetry_sink), - ); - let graph = KvIdentityGraph::new(InMemoryEcKv::new("wait-error-store")); - let mut ec_context = active_ec_context(); - let slots = [article_slot()]; + #[tokio::test] + async fn the_origins_answer_beneath_the_route_namespace_loses_the_deployment_headers() { + let settings = create_test_settings(); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 404, + b"not found".to_vec(), + vec![ + (header::CONTENT_TYPE.as_str(), "text/plain"), + ("x-served-by", "cache-node-1"), + ("x-cache", "MISS"), + ("x-cache-hits", "0"), + ], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let request = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/_ts/anything") + .header(header::HOST, "publisher.example") + .body(EdgeBody::empty()) + .expect("should build request"); - let error = match handle_publisher_request( - &settings, - &services, - Some(&graph), - &mut ec_context, - AuctionDispatch { - orchestrator: &orchestrator, - slots: &slots, - registry: None, - }, - conditional_navigation_request(), - EdgeCacheHeader::SurrogateControl, - ) - .await - { - Ok(_) => panic!("pending origin wait failure should surface as a proxy error"), - Err(error) => error, - }; + let response = match run_publisher_proxy(&settings, &services, request).await { + PublisherResponse::Buffered(response) + | PublisherResponse::Stream { response, .. } + | PublisherResponse::AssembleTemplate { response, .. } + | PublisherResponse::PassThrough { response, .. } => response, + }; + for name in ["x-served-by", "x-cache", "x-cache-hits"] { assert!( - format!("{error:?}").contains("Failed to proxy request to origin"), - "should preserve the existing proxy error context: {error:?}" - ); - let batches = telemetry_sink - .batches - .lock() - .expect("should lock telemetry batches"); - let summaries: Vec<_> = batches - .iter() - .flat_map(AuctionEventBatch::rows) - .filter(|row| row.event_kind == "summary") - .collect(); - assert_eq!( - summaries.len(), - 1, - "should emit exactly one terminal summary" - ); - assert_eq!(summaries[0].terminal_status.as_deref(), Some("abandoned")); - assert_eq!( - summaries[0].terminal_reason.as_deref(), - Some("origin_proxy_error") - ); - assert_eq!( - summaries[0].publisher_domain, settings.publisher.domain, - "reordered path should retain configured publisher attribution" + response.headers().get(name).is_none(), + "the origin's {name} should not reach a reader" ); } + } - #[tokio::test] - async fn pending_origin_start_failure_skips_kv_auction_and_telemetry() { - let settings = settings_with_dispatching_provider(); - let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - orchestrator.register_provider(Arc::new(DispatchingTestProvider)); - let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); - let stub = Arc::new(StubHttpClient::new()); - stub.set_streaming_responses_supported(true); - stub.set_pending_streaming_responses_supported(true); - let services = services_with_telemetry( - Arc::clone(&stub) as Arc, - Arc::clone(&telemetry_sink), - ); - let lookups = Arc::new(AtomicUsize::new(0)); - let graph = KvIdentityGraph::new(OrderRecordingKv { - inner: InMemoryEcKv::new("start-error-store"), - http: Arc::clone(&stub), - http_calls_at_lookup: Arc::new(AtomicUsize::new(0)), - lookups: Arc::clone(&lookups), - }); - let mut ec_context = active_ec_context(); - let slots = [article_slot()]; - - let result = handle_publisher_request( - &settings, - &services, - Some(&graph), - &mut ec_context, - AuctionDispatch { - orchestrator: &orchestrator, - slots: &slots, - registry: None, - }, - conditional_navigation_request(), - EdgeCacheHeader::SurrogateControl, - ) - .await; - - assert!(result.is_err(), "origin start failure should be returned"); - assert_eq!(lookups.load(Ordering::SeqCst), 0, "should skip KV lookup"); - assert_eq!( - stub.recorded_backend_names(), - vec!["stub-backend"], - "should not dispatch the auction provider after origin start fails" - ); - assert!( - telemetry_sink - .batches - .lock() - .expect("should lock telemetry batches") - .is_empty(), - "should not emit abandonment for an auction that never dispatched" - ); - } + #[tokio::test] + async fn publisher_asset_policy_response_with_cookie_is_private_after_finalization() { + let settings = settings_from_toml(&format!( + r#"{} - #[tokio::test] - async fn signer_admission_failure_continues_publisher_origin_without_provider_io() { - let mut settings = settings_with_dispatching_provider(); - settings - .request_signing - .as_mut() - .expect("should configure request signing stores") - .enabled = true; - let plan = Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile signed navigation auction"), - ); - let orchestrator = crate::auction::build_orchestrator_with_plan(plan, &settings) - .expect("should build signed plan-backed orchestrator"); - let stub = Arc::new(StubHttpClient::new()); - queue_html_response_with_cache_control(&stub, "public, max-age=300"); - let services = services_with_telemetry( - Arc::clone(&stub) as Arc, - Arc::new(RecordingTelemetrySink::default()), - ); - let slots = [article_slot()]; + [[cache.asset_rules]] + id = "publisher-fingerprinted-assets" + enabled = true + path_globs = ["/assets/**/*.png"] + fingerprint_style = "hex" + visibility = "public" + browser_ttl_seconds = 31536000 + edge_ttl_seconds = 31536000 + immutable = true + "#, + crate_test_settings_str() + )); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + b"png".to_vec(), + vec![ + (header::CONTENT_TYPE.as_str(), "image/png"), + (header::CACHE_CONTROL.as_str(), "public, max-age=60"), + (header::SET_COOKIE.as_str(), "viewer=example; Path=/"), + ], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let request = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/assets/logo.0123abcd.png") + .header(header::HOST, "publisher.example") + .body(EdgeBody::empty()) + .expect("should build request"); - let response = run_with_orchestrator( - &settings, - &services, - &orchestrator, - &slots, - conditional_navigation_request(), - ) - .await; + let response = run_publisher_proxy(&settings, &services, request).await; + let PublisherResponse::PassThrough { mut response, .. } = response else { + panic!("should pass through non-HTML asset response"); + }; + crate::response_privacy::apply_response_headers_with_cache_privacy( + &settings, + &mut response, + ); - assert_eq!(response_head(response).status, StatusCode::OK); - assert_eq!( - stub.recorded_backend_names(), - vec!["stub-backend".to_string()], - "signer admission failure should skip provider I/O and still fetch the publisher origin" - ); - } + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("private, max-age=0"), + "publisher asset with Set-Cookie must become private after finalization" + ); + assert!( + response.headers().get("surrogate-control").is_none(), + "publisher asset with Set-Cookie must not retain a shared-cache header" + ); + } - #[tokio::test] - async fn eligible_navigation_bypasses_cache_and_returns_non_storable_html() { - // Arrange - let settings = settings_with_enabled_auction_and_creative_opportunities(); - let stub = Arc::new(StubHttpClient::new()); - queue_html_response_with_cache_control(&stub, "public, max-age=300"); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let slots = [article_slot()]; - let req = conditional_navigation_request(); + #[tokio::test] + async fn publisher_asset_cache_policy_skips_html_response() { + let settings = settings_from_toml(&format!( + r#"{} - // Act - let response = run_with_slots(&settings, &services, &slots, req).await; - let response_head = response_head(response); + [[cache.asset_rules]] + id = "broad-publisher-path" + enabled = true + path_glob = "/news/*.html" + visibility = "public" + browser_ttl_seconds = 31536000 + edge_ttl_seconds = 31536000 + immutable = true + fingerprint_style = "hex" + "#, + crate_test_settings_str() + )); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + b"news".to_vec(), + vec![ + (header::CONTENT_TYPE.as_str(), "text/html; charset=utf-8"), + (header::CACHE_CONTROL.as_str(), "public, max-age=60"), + ], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let request = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/news/story.0123abcd.html") + .header(header::HOST, "publisher.example") + .body(EdgeBody::empty()) + .expect("should build request"); - // Assert - assert_eq!( - stub.recorded_cache_intents(), - vec![PlatformCacheIntent::Bypass], - "eligible publisher navigation should bypass the platform cache" - ); - let recorded_requests = stub.recorded_request_headers(); - let outbound_headers = recorded_requests - .first() - .expect("should record the outbound publisher request"); - assert_eq!( - recorded_header(outbound_headers, header::IF_NONE_MATCH.as_str()), - None, - "eligible publisher request should not forward If-None-Match" - ); - assert_eq!( - recorded_header(outbound_headers, header::IF_MODIFIED_SINCE.as_str()), - None, - "eligible publisher request should not forward If-Modified-Since" - ); - assert_eq!( - response_head - .headers - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("no-store, private"), - "eligible HTML response should be private and non-storable" - ); - for header_name in [ - header::ETAG, - header::LAST_MODIFIED, - header::HeaderName::from_static("surrogate-control"), - header::HeaderName::from_static("fastly-surrogate-control"), - header::HeaderName::from_static("cdn-cache-control"), - header::HeaderName::from_static("cloudflare-cdn-cache-control"), - ] { - assert!( - !response_head.headers.contains_key(&header_name), - "eligible HTML response should remove {header_name}" - ); + let response = run_publisher_proxy(&settings, &services, request).await; + let response = match response { + PublisherResponse::Stream { response, .. } | PublisherResponse::Buffered(response) => { + response } - } - - #[tokio::test] - async fn eligible_range_navigation_fetches_complete_html() { - // Arrange - let settings = settings_with_enabled_auction_and_creative_opportunities(); - let http_client = Arc::new(RangeAwareHttpClient::new()); - let services = build_services_with_http_client( - Arc::clone(&http_client) as Arc - ); - let slots = [article_slot()]; - let mut req = conditional_navigation_request(); - req.headers_mut() - .insert(header::RANGE, HeaderValue::from_static("bytes=0-18")); - req.headers_mut() - .insert(header::IF_RANGE, HeaderValue::from_static(ORIGIN_ETAG)); + PublisherResponse::PassThrough { .. } | PublisherResponse::AssembleTemplate { .. } => { + panic!("should classify HTML response for processing") + } + }; - // Act - let response = run_with_slots(&settings, &services, &slots, req).await; - let response_head = response_head(response); + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("public, max-age=60"), + "asset policy must not cache publisher HTML" + ); + assert!( + response.headers().get("surrogate-control").is_none(), + "HTML response must not receive a shared-cache header" + ); + } - // Assert - assert_eq!( - response_head.status, - StatusCode::OK, - "eligible range navigation should fetch the complete origin document" - ); - let recorded_requests = http_client.stub.recorded_request_headers(); - let outbound_headers = recorded_requests - .first() - .expect("should record the outbound publisher request"); - for header_name in [header::RANGE, header::IF_RANGE] { - assert_eq!( - recorded_header(outbound_headers, header_name.as_str()), - None, - "eligible publisher request should not forward {header_name}" + mod ssat_cache_policy_tests { + use super::*; + use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; + use crate::auction::telemetry::{AuctionEventBatch, AuctionTelemetrySink}; + use crate::creative_opportunities::{CreativeOpportunityFormat, CreativeOpportunitySlot}; + use crate::platform::test_support::{ + NoopConfigStore, NoopGeo, NoopSecretStore, StubBackend, + }; + use crate::platform::{ + ClientInfo, PlatformError, PlatformHttpClient, PlatformPendingRequest, + PlatformResponse, PlatformSelectResult, + }; + use crate::test_support::tests::crate_test_settings_str; + + const ORIGIN_ETAG: &str = "\"origin-tag\""; + const ORIGIN_LAST_MODIFIED: &str = "Wed, 21 Oct 2015 07:28:00 GMT"; + const UNEXPECTED_304_PROVIDER: &str = "example_navigation_bidder"; + const UNEXPECTED_304_BACKEND: &str = "example-navigation-bidder-backend"; + + struct DispatchingTestProvider; + + struct RangeAwareHttpClient { + stub: StubHttpClient, + } + + impl RangeAwareHttpClient { + fn new() -> Self { + Self { + stub: StubHttpClient::new(), + } + } + } + + #[async_trait::async_trait(?Send)] + impl PlatformHttpClient for RangeAwareHttpClient { + async fn send( + &self, + request: PlatformHttpRequest, + ) -> Result> { + if request.request.headers().contains_key(header::RANGE) { + self.stub.push_response_with_headers( + 206, + b"partial".to_vec(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("content-range", "bytes 0-18/39"), + ], + ); + } else { + self.stub.push_response_with_headers( + 200, + b"origin".to_vec(), + vec![("content-type", "text/html; charset=utf-8")], + ); + } + self.stub.send(request).await + } + + async fn send_async( + &self, + request: PlatformHttpRequest, + ) -> Result> { + self.stub.send_async(request).await + } + + async fn select( + &self, + pending_requests: Vec, + ) -> Result> { + self.stub.select(pending_requests).await + } + } + + #[async_trait::async_trait(?Send)] + impl AuctionProvider for DispatchingTestProvider { + fn provider_name(&self) -> &str { + UNEXPECTED_304_PROVIDER + } + + async fn request_bids( + &self, + _request: &AuctionRequest, + context: &AuctionContext<'_>, + ) -> Result> { + let request = PlatformHttpRequest::new( + HttpRequest::builder() + .method(Method::POST) + .uri("https://bidder.example.com/navigation-bids") + .body(EdgeBody::empty()) + .expect("should build test provider request"), + UNEXPECTED_304_BACKEND, ); + context + .services + .http_client() + .send_async(request) + .await + .change_context(TrustedServerError::Auction { + message: "test provider launch failed".to_string(), + }) + .map(ProviderRequestOutcome::pending) + } + + async fn parse_response( + &self, + _response: PlatformResponse, + _response_time_ms: u64, + ) -> Result> { + panic!("parse_response must not run for an unexpected origin 304"); + } + + fn timeout_ms(&self) -> u32 { + 100 + } + + fn backend_name( + &self, + _services: &RuntimeServices, + _timeout_ms: u32, + ) -> Option { + Some(UNEXPECTED_304_BACKEND.to_string()) } } - #[tokio::test] - async fn navigation_without_matched_slots_uses_short_browser_cache_policy() { - // Arrange - let settings = settings_with_enabled_auction_and_creative_opportunities(); - let stub = Arc::new(StubHttpClient::new()); - queue_html_response_with_cache_control(&stub, "public, max-age=300"); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let mut req = conditional_navigation_request(); - req.headers_mut() - .insert(header::RANGE, HeaderValue::from_static("bytes=0-18")); - req.headers_mut() - .insert(header::IF_RANGE, HeaderValue::from_static(ORIGIN_ETAG)); + #[derive(Default)] + struct RecordingTelemetrySink { + batches: Mutex>, + } - // Act - let response = run_with_slots(&settings, &services, &[], req).await; - let response_head = response_head(response); + #[async_trait::async_trait(?Send)] + impl AuctionTelemetrySink for RecordingTelemetrySink { + async fn emit_auction_events( + &self, + _services: &RuntimeServices, + batch: AuctionEventBatch, + ) -> Result<(), Report> { + self.batches + .lock() + .expect("should lock telemetry batches") + .push(batch); + Ok(()) + } + } - // Assert - assert_eq!( - stub.recorded_cache_intents(), - vec![PlatformCacheIntent::Default], - "should preserve platform caching for non-ad requests while readthrough is disabled", - ); - let recorded_requests = stub.recorded_request_headers(); - let outbound_headers = recorded_requests - .first() - .expect("should record the outbound publisher request"); - assert_eq!( - recorded_header(outbound_headers, header::IF_NONE_MATCH.as_str()), - Some(ORIGIN_ETAG), - "publisher request without matched slots should preserve If-None-Match" - ); - assert_eq!( - recorded_header(outbound_headers, header::IF_MODIFIED_SINCE.as_str()), - Some(ORIGIN_LAST_MODIFIED), - "publisher request without matched slots should preserve If-Modified-Since" + fn settings_with_enabled_auction_and_creative_opportunities() -> Settings { + let toml = format!( + "{}\n\ + [creative_opportunities]\ngam_network_id = \"12345\"\n", + crate_test_settings_str().replace("[auction]\n", "[auction]\nenabled = true\n") ); - assert_eq!( - recorded_header(outbound_headers, header::RANGE.as_str()), - Some("bytes=0-18"), - "publisher request without matched slots should preserve Range" + settings_from_toml(&toml) + } + + fn settings_with_disabled_ad_templates() -> Settings { + let toml = format!( + "{}\n\ + [creative_opportunities]\nenabled = false\ngam_network_id = \"12345\"\n", + crate_test_settings_str().replace("[auction]\n", "[auction]\nenabled = true\n") ); - assert_eq!( - recorded_header(outbound_headers, header::IF_RANGE.as_str()), - Some(ORIGIN_ETAG), - "publisher request without matched slots should preserve If-Range" + settings_from_toml(&toml) + } + + fn settings_with_disabled_auction() -> Settings { + let toml = format!( + "{}\n\ + [creative_opportunities]\ngam_network_id = \"12345\"\n", + crate_test_settings_str().replace("[auction]\n", "[auction]\nenabled = false\n") ); + settings_from_toml(&toml) + } - for (header_name, expected) in [ - (header::CACHE_CONTROL, "private, max-age=60"), - (header::ETAG, ORIGIN_ETAG), - (header::LAST_MODIFIED, ORIGIN_LAST_MODIFIED), - ( - header::HeaderName::from_static("surrogate-control"), - "max-age=300", - ), - ( - header::HeaderName::from_static("fastly-surrogate-control"), - "max-age=300", - ), - ( - header::HeaderName::from_static("cdn-cache-control"), - "max-age=300", - ), - ( - header::HeaderName::from_static("cloudflare-cdn-cache-control"), - "max-age=300", - ), - ] { - assert_eq!( - response_head - .headers - .get(&header_name) - .and_then(|value| value.to_str().ok()), - Some(expected), - "publisher response without matched slots should preserve {header_name}" - ); - } + fn settings_without_creative_opportunities() -> Settings { + settings_from_toml(&crate_test_settings_str()) } - #[tokio::test] - async fn disabled_ad_templates_use_short_browser_cache_policy() { - // Arrange - let mut settings = settings_with_disabled_ad_templates(); - settings.proxy.allowed_domains = - vec!["*.example".to_string(), "*.example.com".to_string()]; - let stub = Arc::new(StubHttpClient::new()); - queue_html_response_with_cache_control(&stub, "public, max-age=300"); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc + fn settings_with_dispatching_provider() -> Settings { + let toml = format!( + "{}\n[demand]\nmodules = [\"{UNEXPECTED_304_PROVIDER}\"]\n\n[demand.{UNEXPECTED_304_PROVIDER}]\nimplementation = \"auction.plain-fixture\"\nendpoint = \"https://unexpected.example/openrtb2/auction\"\nrouting = \"all_eligible\"\n\n\ + [creative_opportunities]\ngam_network_id = \"12345\"\n", + crate_test_settings_str().replace("[auction]\n", "[auction]\nenabled = true\n") ); - let slots = [article_slot()]; + settings_from_toml(&toml) + } - // Act - let response = run_with_slots( - &settings, - &services, - &slots, - conditional_navigation_request(), - ) - .await; - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - let response = buffer_publisher_response_async( - response, - &Method::GET, - &settings, - ®istry, - &orchestrator, - &services, - ) - .await - .expect("should buffer disabled-template response"); - let (response_head, body) = response.into_parts(); - let body = String::from_utf8( - body.into_bytes() - .expect("should return an in-memory publisher body") - .to_vec(), - ) - .expect("should return UTF-8 publisher HTML"); + fn services_with_telemetry( + http_client: Arc, + telemetry_sink: Arc, + ) -> RuntimeServices { + let telemetry_sink: Arc = telemetry_sink; + RuntimeServices::builder() + .config_store(Arc::new(NoopConfigStore)) + .secret_store(Arc::new(NoopSecretStore)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::new(StubBackend)) + .http_client(http_client) + .geo(Arc::new(NoopGeo)) + .auction_telemetry_sink(telemetry_sink) + .client_info(ClientInfo::default()) + .build() + } - // Assert - assert_eq!( - stub.recorded_cache_intents(), - vec![PlatformCacheIntent::Default], - "should preserve platform caching with disabled ad templates and readthrough" - ); - assert_eq!( - response_head - .headers - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("private, max-age=60"), - "disabled server-side ad templates should use the private browser cache policy" - ); - assert!( - !body.contains(".adSlots=JSON.parse"), - "disabled server-side ad templates should not inject ad-slot state" - ); - for (header_name, expected) in [ - (header::ETAG, ORIGIN_ETAG), - (header::LAST_MODIFIED, ORIGIN_LAST_MODIFIED), - ( - header::HeaderName::from_static("surrogate-control"), - "max-age=300", - ), - ( - header::HeaderName::from_static("fastly-surrogate-control"), - "max-age=300", - ), - ( - header::HeaderName::from_static("cdn-cache-control"), - "max-age=300", - ), - ( - header::HeaderName::from_static("cloudflare-cdn-cache-control"), - "max-age=300", - ), - ] { - assert_eq!( - response_head - .headers - .get(&header_name) - .and_then(|value| value.to_str().ok()), - Some(expected), - "disabled server-side ad templates should preserve {header_name}" - ); + fn article_slot() -> CreativeOpportunitySlot { + CreativeOpportunitySlot { + id: "article-slot".to_string(), + gam_unit_path: None, + div_id: None, + page_patterns: vec!["/article".to_string()], + formats: vec![CreativeOpportunityFormat { + width: 300, + height: 250, + media_type: MediaType::Banner, + }], + floor_price: None, + targeting: Default::default(), + providers: Default::default(), + compiled_patterns: Vec::new(), + compiled_unit: None, } } - #[tokio::test] - async fn disabled_auction_uses_private_browser_cache_policy() { - // Arrange - let settings = settings_with_disabled_auction(); - let stub = Arc::new(StubHttpClient::new()); - queue_html_response_with_cache_control(&stub, "no-cache"); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + fn conditional_navigation_request() -> Request { + HttpRequest::builder() + .method(Method::GET) + .uri("https://ts.example.com/article") + .header(header::HOST, "ts.example.com") + .header("sec-fetch-dest", "document") + .header(header::IF_NONE_MATCH, ORIGIN_ETAG) + .header(header::IF_MODIFIED_SINCE, ORIGIN_LAST_MODIFIED) + .body(EdgeBody::empty()) + .expect("should build conditional navigation request") + } - // Act - let response = run_with_slots( - &settings, - &services, - &[article_slot()], - conditional_navigation_request(), - ) - .await; - let response_head = response_head(response); + fn queue_html_response_with_cache_control( + stub: &StubHttpClient, + cache_control: &'static str, + ) { + queue_html_response_with_status_and_cache_control(stub, 200, cache_control); + } - // Assert - assert_eq!( - response_head - .headers - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("private, max-age=60"), - "disabled auction should use the private browser cache policy" + fn queue_html_response_with_status_and_cache_control( + stub: &StubHttpClient, + status: u16, + cache_control: &'static str, + ) { + stub.push_response_with_headers( + status, + b"origin".to_vec(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("cache-control", cache_control), + ("etag", ORIGIN_ETAG), + ("last-modified", ORIGIN_LAST_MODIFIED), + ("surrogate-control", "max-age=300"), + ("fastly-surrogate-control", "max-age=300"), + ("cdn-cache-control", "max-age=300"), + ("cloudflare-cdn-cache-control", "max-age=300"), + ], ); } - #[tokio::test] - async fn navigation_without_matched_slots_replaces_origin_cache_policy() { - let settings = settings_with_enabled_auction_and_creative_opportunities(); + fn non_regulated_consent() -> crate::consent::ConsentContext { + crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + } + } - for cache_control in ["no-cache", "max-age=0", "must-revalidate", "s-maxage=0"] { - // Arrange - let stub = Arc::new(StubHttpClient::new()); - queue_html_response_with_cache_control(&stub, cache_control); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + async fn run_with_slots( + settings: &Settings, + services: &RuntimeServices, + slots: &[CreativeOpportunitySlot], + req: Request, + ) -> PublisherResponse { + run_with_slots_and_consent(settings, services, slots, req, non_regulated_consent()) + .await + } - // Act - let response = - run_with_slots(&settings, &services, &[], conditional_navigation_request()) - .await; - let response_head = response_head(response); + async fn run_with_slots_and_consent( + settings: &Settings, + services: &RuntimeServices, + slots: &[CreativeOpportunitySlot], + req: Request, + consent: crate::consent::ConsentContext, + ) -> PublisherResponse { + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(settings), + ); + run_with_orchestrator_and_consent( + settings, + services, + &orchestrator, + slots, + req, + consent, + ) + .await + } - // Assert - assert_eq!( - response_head - .headers - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("private, max-age=60"), - "inactive server-side ad templates should replace origin {cache_control} policy" - ); - } + async fn run_with_orchestrator( + settings: &Settings, + services: &RuntimeServices, + orchestrator: &AuctionOrchestrator, + slots: &[CreativeOpportunitySlot], + req: Request, + ) -> PublisherResponse { + run_with_orchestrator_and_consent( + settings, + services, + orchestrator, + slots, + req, + non_regulated_consent(), + ) + .await } - #[tokio::test] - async fn navigation_without_matched_slots_preserves_private_origin_cache_policy() { - let settings = settings_with_enabled_auction_and_creative_opportunities(); + async fn run_with_orchestrator_and_consent( + settings: &Settings, + services: &RuntimeServices, + orchestrator: &AuctionOrchestrator, + slots: &[CreativeOpportunitySlot], + req: Request, + consent: crate::consent::ConsentContext, + ) -> PublisherResponse { + let mut ec_context = EcContext::new_for_test(None, consent); - for cache_control in ["private, max-age=0", "No-Store"] { - // Arrange - let stub = Arc::new(StubHttpClient::new()); - queue_html_response_with_cache_control(&stub, cache_control); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - - // Act - let response = - run_with_slots(&settings, &services, &[], conditional_navigation_request()) - .await; - let response_head = response_head(response); + let registry = test_registry(settings); + handle_publisher_request( + AppContext { + settings, + integration_registry: ®istry, + }, + services, + None, + &mut ec_context, + AuctionDispatch { + orchestrator, + slots, + registry: None, + }, + req, + EdgeCacheHeader::SMaxageFallback, + ) + .await + .expect("should proxy publisher request") + } - // Assert - assert_eq!( - response_head - .headers - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some(cache_control), - "inactive server-side ad templates should preserve private origin {cache_control} policy" - ); - for (header_name, expected) in [ - (header::ETAG, ORIGIN_ETAG), - (header::LAST_MODIFIED, ORIGIN_LAST_MODIFIED), - ( - header::HeaderName::from_static("surrogate-control"), - "max-age=300", - ), - ( - header::HeaderName::from_static("fastly-surrogate-control"), - "max-age=300", - ), - ( - header::HeaderName::from_static("cdn-cache-control"), - "max-age=300", - ), - ( - header::HeaderName::from_static("cloudflare-cdn-cache-control"), - "max-age=300", - ), - ] { - assert_eq!( - response_head - .headers - .get(&header_name) - .and_then(|value| value.to_str().ok()), - Some(expected), - "inactive server-side ad templates should preserve {header_name}" - ); - } + fn response_head(response: PublisherResponse) -> http::response::Parts { + match response { + PublisherResponse::Buffered(response) + | PublisherResponse::Stream { response, .. } + | PublisherResponse::AssembleTemplate { response, .. } + | PublisherResponse::PassThrough { response, .. } => response.into_parts().0, } } - #[tokio::test] - async fn request_scoped_ad_stack_suppression_preserves_origin_cache_policy() { - let settings = settings_with_enabled_auction_and_creative_opportunities(); - let slots = [article_slot()]; - let mut bot_request = conditional_navigation_request(); - bot_request.headers_mut().insert( - "user-agent", - HeaderValue::from_static("Mozilla/5.0 (compatible; Googlebot/2.1)"), - ); - let mut prefetch_request = conditional_navigation_request(); - prefetch_request - .headers_mut() - .insert("sec-purpose", HeaderValue::from_static("prefetch")); - - for (skip_reason, request, consent) in [ - ("bot", bot_request, non_regulated_consent()), - ("prefetch", prefetch_request, non_regulated_consent()), - ( - "consent denied", - conditional_navigation_request(), - crate::consent::ConsentContext { - jurisdiction: crate::consent::jurisdiction::Jurisdiction::Gdpr, - ..Default::default() - }, - ), - ] { - // Arrange - let stub = Arc::new(StubHttpClient::new()); - queue_html_response_with_cache_control(&stub, "no-cache"); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - - // Act - let response = - run_with_slots_and_consent(&settings, &services, &slots, request, consent) - .await; - let response_head = response_head(response); + fn recorded_header<'a>(headers: &'a [(String, String)], name: &str) -> Option<&'a str> { + headers + .iter() + .find(|(header_name, _)| header_name.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.as_str()) + } - // Assert - assert_eq!( - response_head - .headers - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("no-cache"), - "{skip_reason} should retain the origin cache policy" - ); - } + fn active_ec_context() -> EcContext { + EcContext::new_for_test_with_ip( + Some(format!("{}.CkId01", "b".repeat(64))), + scheduling_consent(), + Some("203.0.113.7".to_owned()), + ) } #[tokio::test] - async fn absent_creative_opportunities_use_short_browser_cache_policy() { - // Arrange - let settings = settings_without_creative_opportunities(); + async fn pending_origin_wait_failure_abandons_dispatched_auction_once() { + let settings = settings_with_dispatching_provider(); + let mut orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); + orchestrator.register_provider(Arc::new(DispatchingTestProvider)); + let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); let stub = Arc::new(StubHttpClient::new()); - queue_html_response_with_cache_control(&stub, "no-cache"); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc + stub.set_streaming_responses_supported(true); + stub.set_pending_streaming_responses_supported(true); + queue_html_response_with_cache_control(&stub, "public, max-age=300"); + stub.push_response(200, b"unused provider response".to_vec()); + stub.push_wait_error(); + let services = services_with_telemetry( + Arc::clone(&stub) as Arc, + Arc::clone(&telemetry_sink), ); + let graph = KvIdentityGraph::new(InMemoryEcKv::new("wait-error-store")); + let mut ec_context = active_ec_context(); + let slots = [article_slot()]; - // Act - let response = - run_with_slots(&settings, &services, &[], conditional_navigation_request()).await; - let response_head = response_head(response); + let error = match handle_publisher_request( + AppContext { + settings: &settings, + integration_registry: &test_registry(&settings), + }, + &services, + Some(&graph), + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &slots, + registry: None, + }, + conditional_navigation_request(), + EdgeCacheHeader::SurrogateControl, + ) + .await + { + Ok(_) => panic!("pending origin wait failure should surface as a proxy error"), + Err(error) => error, + }; - // Assert + assert!( + format!("{error:?}").contains("Failed to proxy request to origin"), + "should preserve the existing proxy error context: {error:?}" + ); + let batches = telemetry_sink + .batches + .lock() + .expect("should lock telemetry batches"); + let summaries: Vec<_> = batches + .iter() + .flat_map(AuctionEventBatch::rows) + .filter(|row| row.event_kind == "summary") + .collect(); assert_eq!( - response_head - .headers - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("private, max-age=60"), - "absent creative opportunities should use the private inactive-stack cache policy" + summaries.len(), + 1, + "should emit exactly one terminal summary" + ); + assert_eq!(summaries[0].terminal_status.as_deref(), Some("abandoned")); + assert_eq!( + summaries[0].terminal_reason.as_deref(), + Some("origin_proxy_error") + ); + assert_eq!( + summaries[0].publisher_domain, settings.publisher.domain, + "reordered path should retain configured publisher attribution" ); } #[tokio::test] - async fn inactive_ad_stack_preserves_non_ok_response_cache_policy() { - let settings = settings_with_disabled_ad_templates(); + async fn pending_origin_start_failure_skips_kv_auction_and_telemetry() { + let settings = settings_with_dispatching_provider(); + let mut orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); + orchestrator.register_provider(Arc::new(DispatchingTestProvider)); + let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); + let stub = Arc::new(StubHttpClient::new()); + stub.set_streaming_responses_supported(true); + stub.set_pending_streaming_responses_supported(true); + let services = services_with_telemetry( + Arc::clone(&stub) as Arc, + Arc::clone(&telemetry_sink), + ); + let lookups = Arc::new(AtomicUsize::new(0)); + let graph = KvIdentityGraph::new(OrderRecordingKv { + inner: InMemoryEcKv::new("start-error-store"), + http: Arc::clone(&stub), + http_calls_at_lookup: Arc::new(AtomicUsize::new(0)), + lookups: Arc::clone(&lookups), + }); + let mut ec_context = active_ec_context(); + let slots = [article_slot()]; - for status in [206, 404, 500, 503] { - // Arrange - let stub = Arc::new(StubHttpClient::new()); - queue_html_response_with_status_and_cache_control(&stub, status, "no-cache"); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + let result = handle_publisher_request( + AppContext { + settings: &settings, + integration_registry: &test_registry(&settings), + }, + &services, + Some(&graph), + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &slots, + registry: None, + }, + conditional_navigation_request(), + EdgeCacheHeader::SurrogateControl, + ) + .await; - // Act - let response = run_with_slots( - &settings, - &services, - &[article_slot()], - conditional_navigation_request(), - ) - .await; - let response_head = response_head(response); + assert!(result.is_err(), "origin start failure should be returned"); + assert_eq!(lookups.load(Ordering::SeqCst), 0, "should skip KV lookup"); + assert_eq!( + stub.recorded_backend_names(), + vec!["stub-backend"], + "should not dispatch the auction provider after origin start fails" + ); + assert!( + telemetry_sink + .batches + .lock() + .expect("should lock telemetry batches") + .is_empty(), + "should not emit abandonment for an auction that never dispatched" + ); + } - // Assert - assert_eq!( - response_head - .headers - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("no-cache"), - "inactive server-side ad templates should preserve origin policy on {status}" - ); - } + #[tokio::test] + async fn signer_admission_failure_continues_publisher_origin_without_provider_io() { + let mut settings = settings_with_dispatching_provider(); + settings + .request_signing + .as_mut() + .expect("should configure request signing stores") + .enabled = true; + let plan = Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile signed navigation auction"), + ); + let orchestrator = crate::auction::build_orchestrator_with_plan(plan) + .expect("should build signed plan-backed orchestrator"); + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, "public, max-age=300"); + let services = services_with_telemetry( + Arc::clone(&stub) as Arc, + Arc::new(RecordingTelemetrySink::default()), + ); + let slots = [article_slot()]; + + let response = run_with_orchestrator( + &settings, + &services, + &orchestrator, + &slots, + conditional_navigation_request(), + ) + .await; + + assert_eq!(response_head(response).status, StatusCode::OK); + assert_eq!( + stub.recorded_backend_names(), + vec!["stub-backend".to_string()], + "signer admission failure should skip provider I/O and still fetch the publisher origin" + ); } #[tokio::test] - async fn inactive_ad_stack_preserves_gpt_diagnostics_cache_privacy() { + async fn eligible_navigation_bypasses_cache_and_returns_non_storable_html() { // Arrange - let mut settings = settings_with_disabled_ad_templates(); - settings - .integrations - .insert_config("gpt_diagnostics", &serde_json::json!({ "enabled": true })) - .expect("should enable GPT diagnostics"); + let settings = settings_with_enabled_auction_and_creative_opportunities(); let stub = Arc::new(StubHttpClient::new()); - queue_html_response_with_cache_control(&stub, "no-cache"); + queue_html_response_with_cache_control(&stub, "public, max-age=300"); let services = build_services_with_http_client( Arc::clone(&stub) as Arc ); - let request = HttpRequest::builder() - .method(Method::GET) - .uri("https://ts.example.com/article?ts_console=1") - .header(header::HOST, "ts.example.com") - .header("sec-fetch-dest", "document") - .body(EdgeBody::empty()) - .expect("should build GPT diagnostics request"); + let slots = [article_slot()]; + let req = conditional_navigation_request(); // Act - let response = run_with_slots(&settings, &services, &[article_slot()], request).await; + let response = run_with_slots(&settings, &services, &slots, req).await; let response_head = response_head(response); // Assert + assert_eq!( + stub.recorded_cache_intents(), + vec![PlatformCacheIntent::Bypass], + "eligible publisher navigation should bypass the platform cache" + ); + let recorded_requests = stub.recorded_request_headers(); + let outbound_headers = recorded_requests + .first() + .expect("should record the outbound publisher request"); + assert_eq!( + recorded_header(outbound_headers, header::IF_NONE_MATCH.as_str()), + None, + "eligible publisher request should not forward If-None-Match" + ); + assert_eq!( + recorded_header(outbound_headers, header::IF_MODIFIED_SINCE.as_str()), + None, + "eligible publisher request should not forward If-Modified-Since" + ); assert_eq!( response_head .headers .get(header::CACHE_CONTROL) .and_then(|value| value.to_str().ok()), Some("no-store, private"), - "active GPT diagnostics should retain cache privacy when server-side ad templates are inactive" + "eligible HTML response should be private and non-storable" ); + for header_name in [ + header::ETAG, + header::LAST_MODIFIED, + header::HeaderName::from_static("surrogate-control"), + header::HeaderName::from_static("fastly-surrogate-control"), + header::HeaderName::from_static("cdn-cache-control"), + header::HeaderName::from_static("cloudflare-cdn-cache-control"), + ] { + assert!( + !response_head.headers.contains_key(&header_name), + "eligible HTML response should remove {header_name}" + ); + } } #[tokio::test] - async fn inactive_ad_stack_preserves_non_get_and_non_document_cache_policy() { - let settings = settings_with_disabled_ad_templates(); + async fn eligible_range_navigation_fetches_complete_html() { + // Arrange + let settings = settings_with_enabled_auction_and_creative_opportunities(); + let http_client = Arc::new(RangeAwareHttpClient::new()); + let services = build_services_with_http_client( + Arc::clone(&http_client) as Arc + ); + let slots = [article_slot()]; + let mut req = conditional_navigation_request(); + req.headers_mut() + .insert(header::RANGE, HeaderValue::from_static("bytes=0-18")); + req.headers_mut() + .insert(header::IF_RANGE, HeaderValue::from_static(ORIGIN_ETAG)); - for request in [ - HttpRequest::builder() - .method(Method::POST) - .uri("https://ts.example.com/article") - .header(header::HOST, "ts.example.com") - .header("sec-fetch-dest", "document") - .body(EdgeBody::empty()) - .expect("should build non-GET document request"), - HttpRequest::builder() - .method(Method::GET) - .uri("https://ts.example.com/article") - .header(header::HOST, "ts.example.com") - .header("sec-fetch-dest", "empty") - .body(EdgeBody::empty()) - .expect("should build non-document request"), - ] { - // Arrange - let stub = Arc::new(StubHttpClient::new()); - queue_html_response_with_cache_control(&stub, "no-cache"); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc + // Act + let response = run_with_slots(&settings, &services, &slots, req).await; + let response_head = response_head(response); + + // Assert + assert_eq!( + response_head.status, + StatusCode::OK, + "eligible range navigation should fetch the complete origin document" + ); + let recorded_requests = http_client.stub.recorded_request_headers(); + let outbound_headers = recorded_requests + .first() + .expect("should record the outbound publisher request"); + for header_name in [header::RANGE, header::IF_RANGE] { + assert_eq!( + recorded_header(outbound_headers, header_name.as_str()), + None, + "eligible publisher request should not forward {header_name}" ); + } + } - // Act - let response = - run_with_slots(&settings, &services, &[article_slot()], request).await; - let response_head = response_head(response); + #[tokio::test] + async fn navigation_without_matched_slots_uses_short_browser_cache_policy() { + // Arrange + let settings = settings_with_enabled_auction_and_creative_opportunities(); + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, "public, max-age=300"); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let mut req = conditional_navigation_request(); + req.headers_mut() + .insert(header::RANGE, HeaderValue::from_static("bytes=0-18")); + req.headers_mut() + .insert(header::IF_RANGE, HeaderValue::from_static(ORIGIN_ETAG)); - // Assert + // Act + let response = run_with_slots(&settings, &services, &[], req).await; + let response_head = response_head(response); + + // Assert + assert_eq!( + stub.recorded_cache_intents(), + vec![PlatformCacheIntent::Default], + "should preserve platform caching for non-ad requests while readthrough is disabled", + ); + let recorded_requests = stub.recorded_request_headers(); + let outbound_headers = recorded_requests + .first() + .expect("should record the outbound publisher request"); + assert_eq!( + recorded_header(outbound_headers, header::IF_NONE_MATCH.as_str()), + Some(ORIGIN_ETAG), + "publisher request without matched slots should preserve If-None-Match" + ); + assert_eq!( + recorded_header(outbound_headers, header::IF_MODIFIED_SINCE.as_str()), + Some(ORIGIN_LAST_MODIFIED), + "publisher request without matched slots should preserve If-Modified-Since" + ); + assert_eq!( + recorded_header(outbound_headers, header::RANGE.as_str()), + Some("bytes=0-18"), + "publisher request without matched slots should preserve Range" + ); + assert_eq!( + recorded_header(outbound_headers, header::IF_RANGE.as_str()), + Some(ORIGIN_ETAG), + "publisher request without matched slots should preserve If-Range" + ); + + for (header_name, expected) in [ + (header::CACHE_CONTROL, "private, max-age=60"), + (header::ETAG, ORIGIN_ETAG), + (header::LAST_MODIFIED, ORIGIN_LAST_MODIFIED), + ( + header::HeaderName::from_static("surrogate-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("fastly-surrogate-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("cdn-cache-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("cloudflare-cdn-cache-control"), + "max-age=300", + ), + ] { assert_eq!( response_head .headers - .get(header::CACHE_CONTROL) + .get(&header_name) .and_then(|value| value.to_str().ok()), - Some("no-cache"), - "inactive server-side ad templates should preserve non-document request policy" + Some(expected), + "publisher response without matched slots should preserve {header_name}" ); } } - #[tokio::test] - async fn eligible_navigation_rejects_unexpected_origin_304() { - for content_type in [None, Some("text/html; charset=utf-8")] { - // Arrange - let settings = settings_with_dispatching_provider(); - let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - orchestrator.register_provider(Arc::new(DispatchingTestProvider)); - let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); - let stub = Arc::new(StubHttpClient::new()); + fn settings_naming_the_service_version() -> Settings { + let mut settings = settings_without_creative_opportunities(); + settings.serve = + crate::middleware::PhaseEntries::new(vec![crate::middleware::PhaseEntry { + media_type: crate::middleware::HTML_MEDIA_TYPE.to_owned(), + path: None, + middleware: vec![crate::service_version::MODULE.to_owned()], + }]); + settings + } - // `send_async` consumes the first response before the publisher - // origin request consumes the second response. - stub.push_response(200, b"unused provider response".to_vec()); - let mut origin_headers = vec![ - ("cache-control", "public, max-age=300"), - ("etag", ORIGIN_ETAG), - ("last-modified", ORIGIN_LAST_MODIFIED), - ("surrogate-control", "max-age=300"), - ("fastly-surrogate-control", "max-age=300"), - ]; - if let Some(content_type) = content_type { - origin_headers.push(("content-type", content_type)); - } - stub.push_response_with_headers(304, Vec::new(), origin_headers); - let services = services_with_telemetry( - Arc::clone(&stub) as Arc, - Arc::clone(&telemetry_sink), - ); - let slots = [article_slot()]; + /// The registry of `settings` on a platform reporting `version` for + /// its service. + fn registry_on_a_platform_reporting( + settings: &Settings, + version: &str, + ) -> IntegrationRegistry { + IntegrationRegistry::new(settings) + .expect("should create integration registry") + .with_platform(crate::attestation::PlatformIdentity { + service_version: version.to_owned(), + ..crate::attestation::PlatformIdentity::named("example-platform") + }) + } - // Act - let response = run_with_orchestrator( - &settings, - &services, - &orchestrator, - &slots, - conditional_navigation_request(), - ) - .await; + fn service_version_header(response: &Response) -> Option { + response + .headers() + .get("x-ts-service-version") + .map(|value| String::from_utf8_lossy(value.as_bytes()).into_owned()) + } - // Assert - let response = match response { - PublisherResponse::Buffered(response) => response, - PublisherResponse::PassThrough { .. } - | PublisherResponse::Stream { .. } - | PublisherResponse::AssembleTemplate { .. } => { - panic!("unexpected origin 304 should return a buffered response") - } - }; - assert_eq!( - response.status(), - StatusCode::BAD_GATEWAY, - "eligible origin 304 should fail closed with or without Content-Type" - ); - assert_eq!( - response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("no-store, private"), - "eligible origin 304 should return an explicitly non-storable response" - ); - assert!( - response - .extensions() - .get::() - .is_some(), - "invalid origin 304 response should remain private after late response effects" - ); - for header_name in [ - header::ETAG, - header::LAST_MODIFIED, - header::HeaderName::from_static("surrogate-control"), - header::HeaderName::from_static("fastly-surrogate-control"), - ] { - assert!( - !response.headers().contains_key(&header_name), - "eligible origin 304 should not forward {header_name}" - ); - } + fn git_version_header(response: &Response) -> Option<&str> { + response + .headers() + .get("x-ts-version") + .and_then(|value| value.to_str().ok()) + } - let batches = telemetry_sink - .batches - .lock() - .expect("should lock telemetry batches"); - let summary_rows: Vec<_> = batches - .iter() - .flat_map(AuctionEventBatch::rows) - .filter(|row| row.event_kind == "summary") - .collect(); - assert_eq!( - summary_rows.len(), - 1, - "unexpected origin 304 should emit exactly one summary row" - ); - assert_eq!( - summary_rows[0].terminal_status.as_deref(), - Some("abandoned"), - "unexpected origin 304 should abandon the dispatched auction" - ); - assert_eq!( - summary_rows[0].terminal_reason.as_deref(), - Some("unexpected_origin_304"), - "unexpected origin 304 should use the bounded telemetry reason" - ); - } + /// A page from the origin, as the publisher path hands it to a + /// finalizer, with the services it was fetched through. + async fn origin_page(settings: &Settings) -> (PublisherResponse, RuntimeServices) { + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, "public, max-age=300"); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let response = run_with_slots( + settings, + &services, + &[article_slot()], + conditional_navigation_request(), + ) + .await; + (response, services) + } + + async fn buffered( + settings: &Settings, + registry: &IntegrationRegistry, + ) -> Response { + let (response, services) = origin_page(settings).await; + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(settings), + ); + buffer_publisher_response_async( + response, + &Method::GET, + settings, + registry, + &orchestrator, + &services, + ) + .await + .expect("should buffer the page") } #[tokio::test] - async fn inactive_ad_stack_304_uses_short_browser_cache_policy() { + async fn the_service_version_middleware_writes_the_header_on_a_buffered_page() { + let settings = settings_naming_the_service_version(); + let registry = registry_on_a_platform_reporting(&settings, "52"); + + let response = buffered(&settings, ®istry).await; + + assert_eq!( + service_version_header(&response).as_deref(), + Some("52"), + "should write the version the platform reports on the buffered page" + ); + assert_eq!( + git_version_header(&response), + crate::constants::TS_GIT_VERSION, + "should write the git version the build compiled in on the buffered page" + ); + } + + #[tokio::test] + async fn the_service_version_middleware_writes_the_header_on_a_streamed_page() { + let settings = settings_naming_the_service_version(); + let registry = registry_on_a_platform_reporting(&settings, "52"); + let (response, services) = origin_page(&settings).await; + let orchestrator = Arc::new(AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + )); + + let response = publisher_response_into_streaming_response( + response, + &Method::GET, + Arc::new(settings), + ®istry, + orchestrator, + services, + ) + .await + .expect("should stream the page"); + + assert_eq!( + service_version_header(&response).as_deref(), + Some("52"), + "should write the version the platform reports before the body streams" + ); + } + + #[tokio::test] + async fn a_page_no_serve_entry_names_the_middleware_on_carries_no_version() { + let settings = settings_without_creative_opportunities(); + let registry = registry_on_a_platform_reporting(&settings, "52"); + + let response = buffered(&settings, ®istry).await; + + assert_eq!( + service_version_header(&response), + None, + "should write nothing where no entry names the middleware" + ); + assert_eq!( + git_version_header(&response), + None, + "should write no git version where no entry names the middleware" + ); + } + + #[tokio::test] + async fn a_platform_reporting_no_version_writes_no_header() { + let settings = settings_naming_the_service_version(); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + + let response = buffered(&settings, ®istry).await; + + assert_eq!( + service_version_header(&response), + None, + "should write nothing for a version the platform did not report" + ); + } + + #[tokio::test] + async fn disabled_ad_templates_use_short_browser_cache_policy() { // Arrange - let settings = settings_with_enabled_auction_and_creative_opportunities(); + let mut settings = settings_with_disabled_ad_templates(); + settings.proxy.allowed_domains = + vec!["*.example".to_string(), "*.example.com".to_string()]; let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - 304, - Vec::new(), - vec![ - ("cache-control", "public, max-age=300"), - ("etag", ORIGIN_ETAG), - ("last-modified", ORIGIN_LAST_MODIFIED), - ("surrogate-control", "max-age=300"), - ("fastly-surrogate-control", "max-age=300"), - ], - ); + queue_html_response_with_cache_control(&stub, "public, max-age=300"); let services = build_services_with_http_client( Arc::clone(&stub) as Arc ); + let slots = [article_slot()]; // Act - let response = - run_with_slots(&settings, &services, &[], conditional_navigation_request()).await; + let response = run_with_slots( + &settings, + &services, + &slots, + conditional_navigation_request(), + ) + .await; + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); + let response = buffer_publisher_response_async( + response, + &Method::GET, + &settings, + ®istry, + &orchestrator, + &services, + ) + .await + .expect("should buffer disabled-template response"); + let (response_head, body) = response.into_parts(); + let body = String::from_utf8( + body.into_bytes() + .expect("should return an in-memory publisher body") + .to_vec(), + ) + .expect("should return UTF-8 publisher HTML"); // Assert - let response = match response { - PublisherResponse::Buffered(response) => response, - PublisherResponse::PassThrough { .. } - | PublisherResponse::Stream { .. } - | PublisherResponse::AssembleTemplate { .. } => { - panic!("noneligible origin 304 should remain buffered") - } - }; assert_eq!( - response.status(), - StatusCode::NOT_MODIFIED, - "inactive server-side ad templates should preserve the 304 status" + stub.recorded_cache_intents(), + vec![PlatformCacheIntent::Default], + "should preserve platform caching with disabled ad templates and readthrough" ); assert_eq!( - response - .headers() + response_head + .headers .get(header::CACHE_CONTROL) .and_then(|value| value.to_str().ok()), Some("private, max-age=60"), - "inactive server-side ad templates should apply the browser policy on revalidation" + "disabled server-side ad templates should use the private browser cache policy" + ); + assert!( + !body.contains(".adSlots=JSON.parse"), + "disabled server-side ad templates should not inject ad-slot state" ); for (header_name, expected) in [ (header::ETAG, ORIGIN_ETAG), @@ -17571,1833 +19029,3544 @@ mod tests { header::HeaderName::from_static("fastly-surrogate-control"), "max-age=300", ), + ( + header::HeaderName::from_static("cdn-cache-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("cloudflare-cdn-cache-control"), + "max-age=300", + ), ] { assert_eq!( - response - .headers() + response_head + .headers .get(&header_name) .and_then(|value| value.to_str().ok()), Some(expected), - "inactive server-side ad templates should preserve {header_name} on revalidation" + "disabled server-side ad templates should preserve {header_name}" ); } - assert_eq!( - stub.recorded_cache_intents(), - vec![PlatformCacheIntent::Default], - "should preserve platform revalidation for non-ad requests with readthrough disabled" + } + + #[tokio::test] + async fn disabled_auction_uses_private_browser_cache_policy() { + // Arrange + let settings = settings_with_disabled_auction(); + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, "no-cache"); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc ); - let recorded_requests = stub.recorded_request_headers(); - let outbound_headers = recorded_requests - .first() - .expect("should record the outbound publisher request"); + + // Act + let response = run_with_slots( + &settings, + &services, + &[article_slot()], + conditional_navigation_request(), + ) + .await; + let response_head = response_head(response); + + // Assert assert_eq!( - recorded_header(outbound_headers, header::IF_NONE_MATCH.as_str()), - Some(ORIGIN_ETAG), - "noneligible publisher request should preserve If-None-Match" + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("private, max-age=60"), + "disabled auction should use the private browser cache policy" + ); + } + + #[tokio::test] + async fn navigation_without_matched_slots_replaces_origin_cache_policy() { + let settings = settings_with_enabled_auction_and_creative_opportunities(); + + for cache_control in ["no-cache", "max-age=0", "must-revalidate", "s-maxage=0"] { + // Arrange + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, cache_control); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + + // Act + let response = + run_with_slots(&settings, &services, &[], conditional_navigation_request()) + .await; + let response_head = response_head(response); + + // Assert + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("private, max-age=60"), + "inactive server-side ad templates should replace origin {cache_control} policy" + ); + } + } + + #[tokio::test] + async fn navigation_without_matched_slots_preserves_private_origin_cache_policy() { + let settings = settings_with_enabled_auction_and_creative_opportunities(); + + for cache_control in ["private, max-age=0", "No-Store"] { + // Arrange + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, cache_control); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + + // Act + let response = + run_with_slots(&settings, &services, &[], conditional_navigation_request()) + .await; + let response_head = response_head(response); + + // Assert + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some(cache_control), + "inactive server-side ad templates should preserve private origin {cache_control} policy" + ); + for (header_name, expected) in [ + (header::ETAG, ORIGIN_ETAG), + (header::LAST_MODIFIED, ORIGIN_LAST_MODIFIED), + ( + header::HeaderName::from_static("surrogate-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("fastly-surrogate-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("cdn-cache-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("cloudflare-cdn-cache-control"), + "max-age=300", + ), + ] { + assert_eq!( + response_head + .headers + .get(&header_name) + .and_then(|value| value.to_str().ok()), + Some(expected), + "inactive server-side ad templates should preserve {header_name}" + ); + } + } + } + + #[tokio::test] + async fn request_scoped_ad_stack_suppression_preserves_origin_cache_policy() { + let settings = settings_with_enabled_auction_and_creative_opportunities(); + let slots = [article_slot()]; + let mut bot_request = conditional_navigation_request(); + bot_request.headers_mut().insert( + "user-agent", + HeaderValue::from_static("Mozilla/5.0 (compatible; Googlebot/2.1)"), + ); + let mut prefetch_request = conditional_navigation_request(); + prefetch_request + .headers_mut() + .insert("sec-purpose", HeaderValue::from_static("prefetch")); + + for (skip_reason, request, consent) in [ + ("bot", bot_request, non_regulated_consent()), + ("prefetch", prefetch_request, non_regulated_consent()), + ( + "consent denied", + conditional_navigation_request(), + crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::Gdpr, + ..Default::default() + }, + ), + ] { + // Arrange + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, "no-cache"); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + + // Act + let response = + run_with_slots_and_consent(&settings, &services, &slots, request, consent) + .await; + let response_head = response_head(response); + + // Assert + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("no-cache"), + "{skip_reason} should retain the origin cache policy" + ); + } + } + + #[tokio::test] + async fn absent_creative_opportunities_use_short_browser_cache_policy() { + // Arrange + let settings = settings_without_creative_opportunities(); + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, "no-cache"); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc ); + + // Act + let response = + run_with_slots(&settings, &services, &[], conditional_navigation_request()).await; + let response_head = response_head(response); + + // Assert assert_eq!( - recorded_header(outbound_headers, header::IF_MODIFIED_SINCE.as_str()), - Some(ORIGIN_LAST_MODIFIED), - "noneligible publisher request should preserve If-Modified-Since" + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("private, max-age=60"), + "absent creative opportunities should use the private inactive-stack cache policy" ); } - #[tokio::test] - async fn inactive_ad_stack_304_preserves_private_origin_cache_policy() { - let settings = settings_with_enabled_auction_and_creative_opportunities(); + #[tokio::test] + async fn inactive_ad_stack_preserves_non_ok_response_cache_policy() { + let settings = settings_with_disabled_ad_templates(); + + for status in [206, 404, 500, 503] { + // Arrange + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_status_and_cache_control(&stub, status, "no-cache"); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + + // Act + let response = run_with_slots( + &settings, + &services, + &[article_slot()], + conditional_navigation_request(), + ) + .await; + let response_head = response_head(response); + + // Assert + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("no-cache"), + "inactive server-side ad templates should preserve origin policy on {status}" + ); + } + } + + #[tokio::test] + async fn inactive_ad_stack_preserves_the_cache_privacy_of_a_request_a_module_acts_on() { + // Arrange + let mut settings = settings_with_disabled_ad_templates(); + crate::integrations::registry_test_support::request_fixture::select_and_place( + &mut settings, + ); + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, "no-cache"); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let request = HttpRequest::builder() + .method(Method::GET) + .uri(format!( + "https://ts.example.com/article?{}", + crate::integrations::registry_test_support::request_fixture::QUERY + )) + .header(header::HOST, "ts.example.com") + .header("sec-fetch-dest", "document") + .body(EdgeBody::empty()) + .expect("should build the request the module acts on"); + + // Act + let response = run_with_slots(&settings, &services, &[article_slot()], request).await; + let response_head = response_head(response); + + // Assert + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("no-store, private"), + "a document a module wrote into should stay private when server-side ad templates are inactive" + ); + } + + #[tokio::test] + async fn inactive_ad_stack_preserves_non_get_and_non_document_cache_policy() { + let settings = settings_with_disabled_ad_templates(); + + for request in [ + HttpRequest::builder() + .method(Method::POST) + .uri("https://ts.example.com/article") + .header(header::HOST, "ts.example.com") + .header("sec-fetch-dest", "document") + .body(EdgeBody::empty()) + .expect("should build non-GET document request"), + HttpRequest::builder() + .method(Method::GET) + .uri("https://ts.example.com/article") + .header(header::HOST, "ts.example.com") + .header("sec-fetch-dest", "empty") + .body(EdgeBody::empty()) + .expect("should build non-document request"), + ] { + // Arrange + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, "no-cache"); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + + // Act + let response = + run_with_slots(&settings, &services, &[article_slot()], request).await; + let response_head = response_head(response); + + // Assert + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("no-cache"), + "inactive server-side ad templates should preserve non-document request policy" + ); + } + } + + #[tokio::test] + async fn eligible_navigation_rejects_unexpected_origin_304() { + for content_type in [None, Some("text/html; charset=utf-8")] { + // Arrange + let settings = settings_with_dispatching_provider(); + let mut orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); + orchestrator.register_provider(Arc::new(DispatchingTestProvider)); + let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); + let stub = Arc::new(StubHttpClient::new()); + + // `send_async` consumes the first response before the publisher + // origin request consumes the second response. + stub.push_response(200, b"unused provider response".to_vec()); + let mut origin_headers = vec![ + ("cache-control", "public, max-age=300"), + ("etag", ORIGIN_ETAG), + ("last-modified", ORIGIN_LAST_MODIFIED), + ("surrogate-control", "max-age=300"), + ("fastly-surrogate-control", "max-age=300"), + ]; + if let Some(content_type) = content_type { + origin_headers.push(("content-type", content_type)); + } + stub.push_response_with_headers(304, Vec::new(), origin_headers); + let services = services_with_telemetry( + Arc::clone(&stub) as Arc, + Arc::clone(&telemetry_sink), + ); + let slots = [article_slot()]; + + // Act + let response = run_with_orchestrator( + &settings, + &services, + &orchestrator, + &slots, + conditional_navigation_request(), + ) + .await; + + // Assert + let response = match response { + PublisherResponse::Buffered(response) => response, + PublisherResponse::PassThrough { .. } + | PublisherResponse::Stream { .. } + | PublisherResponse::AssembleTemplate { .. } => { + panic!("unexpected origin 304 should return a buffered response") + } + }; + assert_eq!( + response.status(), + StatusCode::BAD_GATEWAY, + "eligible origin 304 should fail closed with or without Content-Type" + ); + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("no-store, private"), + "eligible origin 304 should return an explicitly non-storable response" + ); + assert!( + response + .extensions() + .get::() + .is_some(), + "invalid origin 304 response should remain private after late response effects" + ); + for header_name in [ + header::ETAG, + header::LAST_MODIFIED, + header::HeaderName::from_static("surrogate-control"), + header::HeaderName::from_static("fastly-surrogate-control"), + ] { + assert!( + !response.headers().contains_key(&header_name), + "eligible origin 304 should not forward {header_name}" + ); + } + + let batches = telemetry_sink + .batches + .lock() + .expect("should lock telemetry batches"); + let summary_rows: Vec<_> = batches + .iter() + .flat_map(AuctionEventBatch::rows) + .filter(|row| row.event_kind == "summary") + .collect(); + assert_eq!( + summary_rows.len(), + 1, + "unexpected origin 304 should emit exactly one summary row" + ); + assert_eq!( + summary_rows[0].terminal_status.as_deref(), + Some("abandoned"), + "unexpected origin 304 should abandon the dispatched auction" + ); + assert_eq!( + summary_rows[0].terminal_reason.as_deref(), + Some("unexpected_origin_304"), + "unexpected origin 304 should use the bounded telemetry reason" + ); + } + } + + #[tokio::test] + async fn inactive_ad_stack_304_uses_short_browser_cache_policy() { + // Arrange + let settings = settings_with_enabled_auction_and_creative_opportunities(); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 304, + Vec::new(), + vec![ + ("cache-control", "public, max-age=300"), + ("etag", ORIGIN_ETAG), + ("last-modified", ORIGIN_LAST_MODIFIED), + ("surrogate-control", "max-age=300"), + ("fastly-surrogate-control", "max-age=300"), + ], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + + // Act + let response = + run_with_slots(&settings, &services, &[], conditional_navigation_request()).await; + + // Assert + let response = match response { + PublisherResponse::Buffered(response) => response, + PublisherResponse::PassThrough { .. } + | PublisherResponse::Stream { .. } + | PublisherResponse::AssembleTemplate { .. } => { + panic!("noneligible origin 304 should remain buffered") + } + }; + assert_eq!( + response.status(), + StatusCode::NOT_MODIFIED, + "inactive server-side ad templates should preserve the 304 status" + ); + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("private, max-age=60"), + "inactive server-side ad templates should apply the browser policy on revalidation" + ); + for (header_name, expected) in [ + (header::ETAG, ORIGIN_ETAG), + (header::LAST_MODIFIED, ORIGIN_LAST_MODIFIED), + ( + header::HeaderName::from_static("surrogate-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("fastly-surrogate-control"), + "max-age=300", + ), + ] { + assert_eq!( + response + .headers() + .get(&header_name) + .and_then(|value| value.to_str().ok()), + Some(expected), + "inactive server-side ad templates should preserve {header_name} on revalidation" + ); + } + assert_eq!( + stub.recorded_cache_intents(), + vec![PlatformCacheIntent::Default], + "should preserve platform revalidation for non-ad requests with readthrough disabled" + ); + let recorded_requests = stub.recorded_request_headers(); + let outbound_headers = recorded_requests + .first() + .expect("should record the outbound publisher request"); + assert_eq!( + recorded_header(outbound_headers, header::IF_NONE_MATCH.as_str()), + Some(ORIGIN_ETAG), + "noneligible publisher request should preserve If-None-Match" + ); + assert_eq!( + recorded_header(outbound_headers, header::IF_MODIFIED_SINCE.as_str()), + Some(ORIGIN_LAST_MODIFIED), + "noneligible publisher request should preserve If-Modified-Since" + ); + } + + #[tokio::test] + async fn inactive_ad_stack_304_preserves_private_origin_cache_policy() { + let settings = settings_with_enabled_auction_and_creative_opportunities(); + + for cache_control in ["private, max-age=0", "No-Store"] { + // Arrange + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 304, + Vec::new(), + vec![("cache-control", cache_control), ("etag", ORIGIN_ETAG)], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + + // Act + let response = + run_with_slots(&settings, &services, &[], conditional_navigation_request()) + .await; + let response_head = response_head(response); + + // Assert + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some(cache_control), + "inactive server-side ad templates should preserve origin {cache_control} policy on revalidation" + ); + } + } + } + + #[tokio::test] + async fn publisher_request_uses_platform_http_client_with_http_types() { + let settings = create_test_settings(); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"origin response".to_vec()); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/page") + .header(header::HOST, "publisher.example") + .body(EdgeBody::empty()) + .expect("should build request"); + + let response = match run_publisher_proxy(&settings, &services, req).await { + PublisherResponse::Buffered(r) => r, + PublisherResponse::PassThrough { mut response, body } => { + *response.body_mut() = body; + response + } + PublisherResponse::Stream { response, .. } + | PublisherResponse::AssembleTemplate { response, .. } => response, + }; + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response_body_string(response), "origin response"); + assert_eq!( + stub.recorded_backend_names(), + vec!["stub-backend".to_string()], + "should proxy through the platform http client" + ); + } + + #[tokio::test] + async fn personalized_navigation_removes_conditional_and_range_headers() { + let settings = create_test_settings(); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + b"origin".to_vec(), + vec![("content-type", "text/html; charset=utf-8")], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let mut req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/page") + .header(header::HOST, "publisher.example") + .header("sec-fetch-dest", "document") + .header(header::IF_NONE_MATCH, "\"cached-page\"") + .header(header::IF_MODIFIED_SINCE, "Wed, 21 Oct 2015 07:28:00 GMT") + .header(header::RANGE, "bytes=0-18") + .header(header::IF_RANGE, "\"cached-page\"") + .body(EdgeBody::empty()) + .expect("should build conditional request"); + req.extensions_mut() + .insert(crate::response_privacy::PersonalizedResponse); + + let _response = run_publisher_proxy(&settings, &services, req).await; + + let headers = stub + .recorded_request_headers() + .into_iter() + .next() + .expect("should record one outbound request"); + for header_name in [ + header::IF_NONE_MATCH, + header::IF_MODIFIED_SINCE, + header::RANGE, + header::IF_RANGE, + ] { + assert!( + headers + .iter() + .all(|(name, _)| !name.eq_ignore_ascii_case(header_name.as_str())), + "personalized navigations must not forward {header_name}" + ); + } + } + + #[tokio::test] + async fn personalized_iframe_removes_conditional_and_range_headers() { + let settings = create_test_settings(); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + b"frame".to_vec(), + vec![("content-type", "text/html; charset=utf-8")], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let mut req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/frame") + .header(header::HOST, "publisher.example") + .header("sec-fetch-dest", "iframe") + .header(header::IF_NONE_MATCH, "\"cached-frame\"") + .header(header::IF_MODIFIED_SINCE, "Wed, 21 Oct 2015 07:28:00 GMT") + .header(header::RANGE, "bytes=0-18") + .header(header::IF_RANGE, "\"cached-frame\"") + .body(EdgeBody::empty()) + .expect("should build conditional iframe request"); + req.extensions_mut() + .insert(crate::response_privacy::PersonalizedResponse); + + let _response = run_publisher_proxy(&settings, &services, req).await; + + let headers = stub + .recorded_request_headers() + .into_iter() + .next() + .expect("should record one outbound request"); + for header_name in [ + header::IF_NONE_MATCH, + header::IF_MODIFIED_SINCE, + header::RANGE, + header::IF_RANGE, + ] { + assert!( + headers + .iter() + .all(|(name, _)| !name.eq_ignore_ascii_case(header_name.as_str())), + "personalized iframe documents must not forward {header_name}" + ); + } + } + + #[tokio::test] + async fn personalized_subresource_preserves_conditional_and_range_headers() { + let settings = create_test_settings(); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + b"video".to_vec(), + vec![("content-type", "video/mp4")], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let mut req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/video.mp4") + .header(header::HOST, "publisher.example") + .header("sec-fetch-dest", "video") + .header(header::IF_NONE_MATCH, "\"cached-video\"") + .header(header::IF_MODIFIED_SINCE, "Wed, 21 Oct 2015 07:28:00 GMT") + .header(header::RANGE, "bytes=0-18") + .header(header::IF_RANGE, "\"cached-video\"") + .body(EdgeBody::empty()) + .expect("should build conditional subresource request"); + req.extensions_mut() + .insert(crate::response_privacy::PersonalizedResponse); + + let _response = run_publisher_proxy(&settings, &services, req).await; + + let headers = stub + .recorded_request_headers() + .into_iter() + .next() + .expect("should record one outbound request"); + for (header_name, expected) in [ + (header::IF_NONE_MATCH, "\"cached-video\""), + (header::IF_MODIFIED_SINCE, "Wed, 21 Oct 2015 07:28:00 GMT"), + (header::RANGE, "bytes=0-18"), + (header::IF_RANGE, "\"cached-video\""), + ] { + assert_eq!( + headers + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case(header_name.as_str())) + .map(|(_, value)| value.as_str()), + Some(expected), + "personalized subresources should preserve {header_name}" + ); + } + } + + #[tokio::test] + async fn publisher_origin_fetch_leaves_stream_response_disabled_when_unsupported() { + let settings = create_test_settings(); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + b"origin".to_vec(), + vec![("content-type", "text/html; charset=utf-8")], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/page") + .header(header::HOST, "publisher.example") + .body(EdgeBody::empty()) + .expect("should build request"); + + let _ = run_publisher_proxy(&settings, &services, req).await; + + assert_eq!( + stub.recorded_stream_response_flags(), + vec![false], + "publisher origin fetch must not request streams when the platform does not support them" + ); + } + + #[tokio::test] + async fn publisher_origin_fetch_sets_stream_response_when_supported() { + let settings = create_test_settings(); + let stub = Arc::new(StubHttpClient::new()); + stub.set_streaming_responses_supported(true); + stub.push_response_with_headers( + 200, + b"origin".to_vec(), + vec![("content-type", "text/html; charset=utf-8")], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/page") + .header(header::HOST, "publisher.example") + .body(EdgeBody::empty()) + .expect("should build request"); + + let _ = run_publisher_proxy(&settings, &services, req).await; + + assert_eq!( + stub.recorded_stream_response_flags(), + vec![true], + "publisher origin fetch should request streams when the platform supports them" + ); + } + + #[tokio::test] + async fn handle_publisher_request_does_not_self_generate_ec() { + // EC generation is the adapter's real-browser-gated responsibility. This + // handler must never mint an EC ID on its own: for a navigation from a + // client the adapter did not pre-generate for (e.g. a non-real browser), + // `ec_value` must stay `None` so no IP-derived identifier reaches the + // auction. Consent allows EC creation and a client IP is present here — + // exactly the conditions under which the old inline call would have + // generated one. + let settings = create_test_settings(); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"ok".to_vec()); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + + let consent = crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + }; + let mut ec_context = + EcContext::new_for_test_with_ip(None, consent, Some("203.0.113.7".to_string())); + assert!( + ec_context.ec_allowed(), + "test precondition: consent must allow EC creation" + ); + + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); + let req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/article") + .header(header::HOST, "publisher.example") + .header("sec-fetch-dest", "document") + .body(EdgeBody::empty()) + .expect("should build request"); + + let registry = test_registry(&settings); + let _ = handle_publisher_request( + AppContext { + settings: &settings, + integration_registry: ®istry, + }, + &services, + None, + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &[], + registry: None, + }, + req, + EdgeCacheHeader::SMaxageFallback, + ) + .await + .expect("should proxy publisher request"); + + assert_eq!( + ec_context.ec_value(), + None, + "handler must not self-generate an EC ID; generation is the adapter's real-browser-gated responsibility", + ); + } + + #[tokio::test] + async fn what_a_request_filter_leaves_survives_into_the_publisher_html_pipeline() { + let mut settings = create_test_settings(); + crate::integrations::registry_test_support::request_fixture::select_and_place( + &mut settings, + ); + let registry = IntegrationRegistry::new(&settings) + .expect("should create integration registry with the stand-in"); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + b"content".to_vec(), + vec![("content-type", "text/html; charset=utf-8")], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let mut req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/page") + .header(header::HOST, "publisher.example") + .header("sec-fetch-dest", "document") + .header( + crate::integrations::registry_test_support::request_fixture::FILTER_HEADER, + "1", + ) + .body(EdgeBody::empty()) + .expect("should build request"); + + let filter_outcome = registry + .filter_request(crate::integrations::RequestFilterRegistryInput { + settings: &settings, + services: &services, + req: &mut req, + geo_info: None, + permissions: None, + }) + .await + .expect("should run the stand-in's filter"); + assert!(matches!( + filter_outcome, + crate::integrations::RequestFilterRegistryOutcome::Continue(_) + )); + let publisher_response = run_publisher_proxy(&settings, &services, req).await; + let response = buffer_publisher_response_async( + publisher_response, + &Method::GET, + &settings, + ®istry, + &AuctionOrchestrator::new(crate::auction::test_support::legacy_auction_config( + &settings, + )), + &services, + ) + .await + .expect("should buffer publisher response"); + let html = response_body_string(response); + + assert!( + html.contains(crate::integrations::registry_test_support::request_fixture::HEAD_FLAG), + "what the filter left on the request should reach the document" + ); + assert_eq!( + stub.recorded_backend_names().len(), + 1, + "only the publisher origin should be called" + ); + } + + #[test] + fn personalized_html_is_private_and_not_shared_cached() { + let mut response = Response::builder() + .status(StatusCode::OK) + .header(header::CACHE_CONTROL, "public, max-age=600") + .header("surrogate-control", "max-age=600") + .header("fastly-surrogate-control", "max-age=600") + .header("cloudflare-cdn-cache-control", "max-age=600") + .header("cdn-cache-control", "max-age=600") + .header(header::ETAG, "\"origin-tag\"") + .header(header::LAST_MODIFIED, "Wed, 21 Oct 2015 07:28:00 GMT") + .body(EdgeBody::empty()) + .expect("should build cacheable HTML response"); + + super::apply_personalized_response_cache_privacy( + &mut response, + &Method::GET, + true, + "text/html; charset=utf-8", + ); + + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("no-store, private"), + "personalized HTML should be private and non-storable" + ); + assert!( + response.headers().get("surrogate-control").is_none(), + "personalized HTML should not retain Surrogate-Control" + ); + assert!( + response.headers().get("fastly-surrogate-control").is_none(), + "personalized HTML should not retain Fastly-Surrogate-Control" + ); + assert!( + response + .headers() + .get("cloudflare-cdn-cache-control") + .is_none(), + "personalized HTML should not retain Cloudflare-CDN-Cache-Control" + ); + assert!( + response.headers().get("cdn-cache-control").is_none(), + "personalized HTML should not retain CDN-Cache-Control" + ); + for header_name in [header::ETAG, header::LAST_MODIFIED] { + assert!( + !response.headers().contains_key(&header_name), + "personalized HTML should not retain {header_name}" + ); + } + + let mut no_store_response = Response::builder() + .status(StatusCode::OK) + .header(header::CACHE_CONTROL, "no-store") + .body(EdgeBody::empty()) + .expect("should build no-store HTML response"); + super::apply_personalized_response_cache_privacy( + &mut no_store_response, + &Method::GET, + true, + "text/html; charset=utf-8", + ); + assert_eq!( + no_store_response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("no-store, private"), + "personalized HTML should use the exact synthesized-HTML policy" + ); + } + + #[test] + fn personalized_cache_privacy_does_not_change_non_html_or_ordinary_responses() { + let mut response = Response::builder() + .status(StatusCode::OK) + .header(header::CACHE_CONTROL, "public, max-age=600") + .header("surrogate-control", "max-age=600") + .body(EdgeBody::empty()) + .expect("should build cacheable response"); - for cache_control in ["private, max-age=0", "No-Store"] { - // Arrange - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - 304, - Vec::new(), - vec![("cache-control", cache_control), ("etag", ORIGIN_ETAG)], - ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + super::apply_personalized_response_cache_privacy( + &mut response, + &Method::GET, + false, + "text/html; charset=utf-8", + ); + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("public, max-age=600"), + "ordinary HTML should retain its existing cache policy" + ); - // Act - let response = - run_with_slots(&settings, &services, &[], conditional_navigation_request()) - .await; - let response_head = response_head(response); + super::apply_personalized_response_cache_privacy( + &mut response, + &Method::GET, + true, + "text/css", + ); + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("public, max-age=600"), + "non-HTML should retain its existing cache policy" + ); + } - // Assert - assert_eq!( - response_head - .headers - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some(cache_control), - "inactive server-side ad templates should preserve origin {cache_control} policy on revalidation" - ); - } + #[test] + fn a_personalized_response_marker_from_any_integration_forces_private_caching() { + let mut response = Response::builder() + .status(StatusCode::OK) + .header(header::CACHE_CONTROL, "public, max-age=600") + .header("surrogate-control", "max-age=600") + .header("fastly-surrogate-control", "max-age=600") + .header("cloudflare-cdn-cache-control", "max-age=600") + .header("cdn-cache-control", "max-age=600") + .header(header::ETAG, "\"origin-tag\"") + .header(header::LAST_MODIFIED, "Wed, 21 Oct 2015 07:28:00 GMT") + .body(EdgeBody::empty()) + .expect("should build cacheable HTML response"); + + super::apply_personalized_response_cache_privacy( + &mut response, + &Method::GET, + true, + "text/html; charset=utf-8", + ); + + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("no-store, private"), + "a personalized response should be private and non-storable" + ); + for header_name in [ + "surrogate-control", + "fastly-surrogate-control", + "cloudflare-cdn-cache-control", + "cdn-cache-control", + ] { + assert!( + response.headers().get(header_name).is_none(), + "a personalized response should not retain {header_name}" + ); + } + for header_name in [header::ETAG, header::LAST_MODIFIED] { + assert!( + !response.headers().contains_key(&header_name), + "a personalized response should not retain {header_name}" + ); } } - #[tokio::test] - async fn publisher_request_uses_platform_http_client_with_http_types() { - let settings = create_test_settings(); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"origin response".to_vec()); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let req = HttpRequest::builder() + #[test] + fn the_neutral_marker_alone_marks_a_response_personalized() { + let mut personalized = HttpRequest::builder() .method(Method::GET) .uri("https://publisher.example/page") .header(header::HOST, "publisher.example") + .header("sec-fetch-dest", "document") .body(EdgeBody::empty()) - .expect("should build request"); - - let response = match run_publisher_proxy(&settings, &services, req).await { - PublisherResponse::Buffered(r) => r, - PublisherResponse::PassThrough { mut response, body } => { - *response.body_mut() = body; - response - } - PublisherResponse::Stream { response, .. } - | PublisherResponse::AssembleTemplate { response, .. } => response, - }; + .expect("should build personalized request"); + personalized + .extensions_mut() + .insert(crate::response_privacy::PersonalizedResponse); - assert_eq!(response.status(), StatusCode::OK); - assert_eq!(response_body_string(response), "origin response"); - assert_eq!( - stub.recorded_backend_names(), - vec!["stub-backend".to_string()], - "should proxy through the platform http client" + assert!( + super::request_requires_personalized_delivery(&personalized), + "the neutral marker alone should mark a response personalized" ); - } - #[tokio::test] - async fn suppressed_navigation_removes_conditional_and_range_headers() { - let settings = create_test_settings(); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - 200, - b"origin".to_vec(), - vec![("content-type", "text/html; charset=utf-8")], - ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let mut req = HttpRequest::builder() + let ordinary = HttpRequest::builder() .method(Method::GET) .uri("https://publisher.example/page") .header(header::HOST, "publisher.example") .header("sec-fetch-dest", "document") - .header(header::IF_NONE_MATCH, "\"cached-page\"") - .header(header::IF_MODIFIED_SINCE, "Wed, 21 Oct 2015 07:28:00 GMT") - .header(header::RANGE, "bytes=0-18") - .header(header::IF_RANGE, "\"cached-page\"") .body(EdgeBody::empty()) - .expect("should build conditional request"); - req.extensions_mut() - .insert(crate::integrations::datadome::DataDomeClientTagSuppressed); + .expect("should build ordinary request"); - let _response = run_publisher_proxy(&settings, &services, req).await; + assert!( + !super::request_requires_personalized_delivery(&ordinary), + "a request carrying no marker should not be personalized" + ); + } - let headers = stub - .recorded_request_headers() - .into_iter() - .next() - .expect("should record one outbound request"); - for header_name in [ - header::IF_NONE_MATCH, - header::IF_MODIFIED_SINCE, - header::RANGE, - header::IF_RANGE, - ] { - assert!( - headers - .iter() - .all(|(name, _)| !name.eq_ignore_ascii_case(header_name.as_str())), - "suppressed navigations must not forward {header_name}" + #[test] + fn response_carries_body_preserves_bodiless_metadata() { + // A processable GET 200 buffers a body and recomputes Content-Length. + assert!( + super::response_carries_body(&Method::GET, StatusCode::OK), + "a GET 200 publisher response should carry a buffered body" + ); + // HEAD carries no body; recomputing Content-Length to 0 would mislead + // clients/caches about the GET representation length. + assert!( + !super::response_carries_body(&Method::HEAD, StatusCode::OK), + "HEAD publisher responses must not get a recomputed Content-Length" + ); + // Bodiless statuses keep their metadata regardless of method. + assert!( + !super::response_carries_body(&Method::GET, StatusCode::NO_CONTENT), + "204 responses must not get a recomputed Content-Length" + ); + assert!( + !super::response_carries_body(&Method::GET, StatusCode::RESET_CONTENT), + "205 responses must not get a recomputed Content-Length" + ); + assert!( + !super::response_carries_body(&Method::GET, StatusCode::NOT_MODIFIED), + "304 responses must not get a recomputed Content-Length" + ); + } + + #[test] + fn dispatched_auction_guard_stays_armed_until_collection_completes() { + // `take()` hands the dispatched auction to the async collector, but the + // guard must stay armed across the collection await so a drop while + // collection is still pending (a client disconnect at the await point) + // still logs the loss. Only `disarm()` — called once collection reaches + // a terminal result — clears the warning. + let mut guard = DispatchedAuctionGuard::new(DispatchedAuction::empty_for_test( + test_auction_request(), + 10, + )); + assert!(guard.armed, "a freshly dispatched guard should be armed"); + + let _dispatched = guard + .take() + .expect("guard should yield the dispatched auction for collection"); + assert!( + guard.armed, + "guard must stay armed across the collection await so a drop mid-collection is reported" + ); + + guard.disarm(); + assert!( + !guard.armed, + "guard must disarm once collection reaches a terminal result" + ); + } + + fn response_body_string(response: http::Response) -> String { + String::from_utf8( + response + .into_body() + .into_bytes() + .unwrap_or_default() + .to_vec(), + ) + .expect("response body should be valid UTF-8") + } + + #[test] + fn test_content_type_detection() { + let test_cases = vec![ + ("text/html", true), + ("text/html; charset=utf-8", true), + ("Text/HTML; Charset=utf-8", true), + ("text/css", true), + ("Text/CSS", true), + ("text/javascript", true), + ("application/javascript", true), + ("Application/JavaScript", true), + ("application/json", true), + ("application/json; charset=utf-8", true), + ("Application/JSON; Charset=UTF-8", true), + ("image/jpeg", false), + ("image/png", false), + ("application/pdf", false), + ("video/mp4", false), + ("application/octet-stream", false), + ]; + + for (content_type, expected) in test_cases { + assert_eq!( + is_processable_content_type(content_type), + expected, + "Content-Type '{content_type}' should_process: expected {expected}", ); } } - #[tokio::test] - async fn suppressed_iframe_removes_conditional_and_range_headers() { - let settings = create_test_settings(); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - 200, - b"frame".to_vec(), - vec![("content-type", "text/html; charset=utf-8")], + #[test] + fn supported_content_encoding_accepts_known_values() { + assert!(is_supported_content_encoding(""), "should accept empty"); + assert!( + is_supported_content_encoding("identity"), + "should accept identity" ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc + assert!(is_supported_content_encoding("gzip"), "should accept gzip"); + assert!( + is_supported_content_encoding("deflate"), + "should accept deflate" ); - let mut req = HttpRequest::builder() - .method(Method::GET) - .uri("https://publisher.example/frame") - .header(header::HOST, "publisher.example") - .header("sec-fetch-dest", "iframe") - .header(header::IF_NONE_MATCH, "\"cached-frame\"") - .header(header::IF_MODIFIED_SINCE, "Wed, 21 Oct 2015 07:28:00 GMT") - .header(header::RANGE, "bytes=0-18") - .header(header::IF_RANGE, "\"cached-frame\"") - .body(EdgeBody::empty()) - .expect("should build conditional iframe request"); - req.extensions_mut() - .insert(crate::integrations::datadome::DataDomeClientTagSuppressed); - - let _response = run_publisher_proxy(&settings, &services, req).await; - - let headers = stub - .recorded_request_headers() - .into_iter() - .next() - .expect("should record one outbound request"); - for header_name in [ - header::IF_NONE_MATCH, - header::IF_MODIFIED_SINCE, - header::RANGE, - header::IF_RANGE, - ] { - assert!( - headers - .iter() - .all(|(name, _)| !name.eq_ignore_ascii_case(header_name.as_str())), - "suppressed iframe documents must not forward {header_name}" - ); - } + assert!(is_supported_content_encoding("br"), "should accept br"); } - #[tokio::test] - async fn suppressed_subresource_preserves_conditional_and_range_headers() { - let settings = create_test_settings(); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - 200, - b"video".to_vec(), - vec![("content-type", "video/mp4")], + #[test] + fn supported_content_encoding_rejects_unknown_values() { + assert!(!is_supported_content_encoding("zstd"), "should reject zstd"); + assert!( + !is_supported_content_encoding("compress"), + "should reject compress" ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc + assert!( + !is_supported_content_encoding("snappy"), + "should reject snappy" ); - let mut req = HttpRequest::builder() - .method(Method::GET) - .uri("https://publisher.example/video.mp4") - .header(header::HOST, "publisher.example") - .header("sec-fetch-dest", "video") - .header(header::IF_NONE_MATCH, "\"cached-video\"") - .header(header::IF_MODIFIED_SINCE, "Wed, 21 Oct 2015 07:28:00 GMT") - .header(header::RANGE, "bytes=0-18") - .header(header::IF_RANGE, "\"cached-video\"") - .body(EdgeBody::empty()) - .expect("should build conditional subresource request"); - req.extensions_mut() - .insert(crate::integrations::datadome::DataDomeClientTagSuppressed); - - let _response = run_publisher_proxy(&settings, &services, req).await; - - let headers = stub - .recorded_request_headers() - .into_iter() - .next() - .expect("should record one outbound request"); - for (header_name, expected) in [ - (header::IF_NONE_MATCH, "\"cached-video\""), - (header::IF_MODIFIED_SINCE, "Wed, 21 Oct 2015 07:28:00 GMT"), - (header::RANGE, "bytes=0-18"), - (header::IF_RANGE, "\"cached-video\""), - ] { - assert_eq!( - headers - .iter() - .find(|(name, _)| name.eq_ignore_ascii_case(header_name.as_str())) - .map(|(_, value)| value.as_str()), - Some(expected), - "suppressed subresources should preserve {header_name}" - ); - } } #[tokio::test] - async fn publisher_origin_fetch_leaves_stream_response_disabled_when_unsupported() { + async fn parser_seam_loop_collects_before_writing_post_body_chunks() { let settings = create_test_settings(); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - 200, - b"origin".to_vec(), - vec![("content-type", "text/html; charset=utf-8")], + let services = noop_services(); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc + let dispatched = DispatchedAuction::empty_for_test(test_auction_request(), 500); + let read_count = Arc::new(AtomicUsize::new(0)); + let body_close_processed_at = Arc::new(AtomicUsize::new(0)); + let token = b""; + let reader = ChunkedReader::new( + &[ + b"painted", + b"", + b"", + ], + Arc::clone(&read_count), ); - let req = HttpRequest::builder() - .method(Method::GET) - .uri("https://publisher.example/page") - .header(header::HOST, "publisher.example") - .body(EdgeBody::empty()) - .expect("should build request"); + let mut processor = RecordingProcessor { + read_count: Arc::clone(&read_count), + body_close_processed_at: Arc::clone(&body_close_processed_at), + }; + let ad_bids_state = AdBidsState::default(); + let ctx = AuctionCollectCtx { + dispatched, + telemetry: AuctionTelemetryCarry { + observation: None, + auction_request: None, + }, + deps: AuctionCollectDeps { + price_granularity: PriceGranularity::default(), + ad_bids_state: &ad_bids_state, + orchestrator: &orchestrator, + services: &services, + settings: &settings, + request_origin: String::new(), + }, + }; + let mut output = Vec::new(); - let _ = run_publisher_proxy(&settings, &services, req).await; + body_close_hold_loop( + reader, + &mut output, + &mut processor, + ctx, + Some(token.to_vec()), + ) + .await + .expect("should stream body with auction hold"); assert_eq!( - stub.recorded_stream_response_flags(), - vec![false], - "publisher origin fetch must not request streams when the platform does not support them" + body_close_processed_at.load(Ordering::SeqCst), + 1, + "close-body tail should be processed as soon as it is found, before later chunks are read" + ); + let output = std::str::from_utf8(&output).expect("should be utf8"); + let painted = output + .find("painted") + .expect("should preserve body content"); + let bids = output + .find("var b=JSON.parse(") + .expect("should inject collected bids"); + let close = output.find("").expect("should preserve body close"); + let late = output + .find("late()") + .expect("should preserve trailing script"); + assert!( + painted < bids && bids < close && close < late, + "output order: {output}" ); } #[tokio::test] - async fn publisher_origin_fetch_sets_stream_response_when_supported() { + async fn parser_seam_write_failure_abandons_auction_once() { + struct FailingWriter; + + impl Write for FailingWriter { + fn write(&mut self, _buf: &[u8]) -> io::Result { + Err(io::Error::other("injected write failure")) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + let settings = create_test_settings(); - let stub = Arc::new(StubHttpClient::new()); - stub.set_streaming_responses_supported(true); - stub.push_response_with_headers( - 200, - b"origin".to_vec(), - vec![("content-type", "text/html; charset=utf-8")], - ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let req = HttpRequest::builder() - .method(Method::GET) - .uri("https://publisher.example/page") - .header(header::HOST, "publisher.example") - .body(EdgeBody::empty()) - .expect("should build request"); + let sink = Arc::new(RecordingTelemetrySink::default()); + let services = noop_services_with_telemetry_sink(Arc::clone(&sink) as _); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let ad_bids_state = AdBidsState::default(); + let ec_context = EcContext::new_for_test(None, ConsentContext::default()); + let ctx = AuctionCollectCtx { + dispatched: DispatchedAuction::empty_for_test(test_auction_request(), 500), + telemetry: AuctionTelemetryCarry { + observation: Some(AuctionObservationContext::from_parts( + AuctionSource::InitialNavigation, + "proxy.example.com", + "/article", + 1, + None, + &ec_context, + )), + auction_request: None, + }, + deps: AuctionCollectDeps { + price_granularity: PriceGranularity::default(), + ad_bids_state: &ad_bids_state, + orchestrator: &orchestrator, + services: &services, + settings: &settings, + request_origin: String::new(), + }, + }; + let mut processor = RecordingProcessor { + read_count: Arc::new(AtomicUsize::new(0)), + body_close_processed_at: Arc::new(AtomicUsize::new(0)), + }; - let _ = run_publisher_proxy(&settings, &services, req).await; + let error = body_close_hold_loop( + std::io::Cursor::new(b"ready"), + &mut FailingWriter, + &mut processor, + ctx, + Some(b"".to_vec()), + ) + .await + .expect_err("injected writer failure should surface"); + assert!(format!("{error:?}").contains("Failed to write processed chunk")); + let batches = sink.batches.lock().expect("should lock telemetry batches"); + let summaries: Vec<_> = batches + .iter() + .flat_map(crate::auction::telemetry::AuctionEventBatch::rows) + .filter(|row| row.event_kind == "summary") + .collect(); + assert_eq!(summaries.len(), 1, "should emit one terminal summary"); + assert_eq!(summaries[0].terminal_status.as_deref(), Some("abandoned")); assert_eq!( - stub.recorded_stream_response_flags(), - vec![true], - "publisher origin fetch should request streams when the platform supports them" + summaries[0].terminal_reason.as_deref(), + Some("stream_write_error") ); } #[tokio::test] - async fn handle_publisher_request_does_not_self_generate_ec() { - // EC generation is the adapter's real-browser-gated responsibility. This - // handler must never mint an EC ID on its own: for a navigation from a - // client the adapter did not pre-generate for (e.g. a non-real browser), - // `ec_value` must stay `None` so no IP-derived identifier reaches the - // auction. Consent allows EC creation and a client IP is present here — - // exactly the conditions under which the old inline call would have - // generated one. - let settings = create_test_settings(); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"ok".to_vec()); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + async fn parser_seam_async_sink_failures_abandon_auction_once() { + struct FailingWriter { + fail_flush: bool, + } - let consent = crate::consent::ConsentContext { - jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, - ..Default::default() - }; - let mut ec_context = - EcContext::new_for_test_with_ip(None, consent, Some("203.0.113.7".to_string())); - assert!( - ec_context.ec_allowed(), - "test precondition: consent must allow EC creation" - ); + impl Write for FailingWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + if self.fail_flush { + Ok(buf.len()) + } else { + Err(io::Error::other("injected write failure")) + } + } - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - let req = HttpRequest::builder() - .method(Method::GET) - .uri("https://publisher.example/article") - .header(header::HOST, "publisher.example") - .header("sec-fetch-dest", "document") - .body(EdgeBody::empty()) - .expect("should build request"); + fn flush(&mut self) -> io::Result<()> { + Err(io::Error::other("injected flush failure")) + } + } - let _ = handle_publisher_request( - &settings, - &services, - None, - &mut ec_context, - AuctionDispatch { - orchestrator: &orchestrator, - slots: &[], - registry: None, - }, - req, - EdgeCacheHeader::SMaxageFallback, - ) - .await - .expect("should proxy publisher request"); + for fail_flush in [false, true] { + for with_close in [false, true] { + let settings = create_test_settings(); + let sink = Arc::new(RecordingTelemetrySink::default()); + let services = noop_services_with_telemetry_sink(Arc::clone(&sink) as _); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let ad_bids_state = AdBidsState::default(); + let ec_context = EcContext::new_for_test(None, ConsentContext::default()); + let ctx = AuctionCollectCtx { + dispatched: DispatchedAuction::empty_for_test(test_auction_request(), 500), + telemetry: AuctionTelemetryCarry { + observation: Some(AuctionObservationContext::from_parts( + AuctionSource::InitialNavigation, + "proxy.example.com", + "/article", + 1, + None, + &ec_context, + )), + auction_request: None, + }, + deps: AuctionCollectDeps { + price_granularity: PriceGranularity::default(), + ad_bids_state: &ad_bids_state, + orchestrator: &orchestrator, + services: &services, + settings: &settings, + request_origin: String::new(), + }, + }; + let mut processor = RecordingProcessor { + read_count: Arc::new(AtomicUsize::new(0)), + body_close_processed_at: Arc::new(AtomicUsize::new(0)), + }; + let html = if with_close { + "ready" + } else { + "ready" + }; + let body = + EdgeBody::stream(futures::stream::iter(vec![bytes::Bytes::from_static( + html.as_bytes(), + )])); - assert_eq!( - ec_context.ec_value(), - None, - "handler must not self-generate an EC ID; generation is the adapter's real-browser-gated responsibility", - ); + let error = body_close_hold_loop_stream( + body, + &mut FailingWriter { fail_flush }, + &mut processor, + Compression::None, + Compression::None, + ctx, + settings.publisher.max_buffered_body_bytes, + Some(b"".to_vec()), + ) + .await + .expect_err("should propagate the sink failure before collecting"); + let expected_error = if fail_flush { + "Failed to flush output before auction collection" + } else { + "Failed to write encoded chunk" + }; + assert!( + format!("{error:?}").contains(expected_error), + "should preserve the sink error: {error:?}" + ); + + let batches = sink.batches.lock().expect("should lock telemetry batches"); + let summaries: Vec<_> = batches + .iter() + .flat_map(crate::auction::telemetry::AuctionEventBatch::rows) + .filter(|row| row.event_kind == "summary") + .collect(); + assert_eq!( + summaries.len(), + 1, + "should emit one terminal summary for fail_flush={fail_flush}, with_close={with_close}" + ); + assert_eq!( + summaries[0].terminal_status.as_deref(), + Some("abandoned"), + "should abandon the uncollected auction" + ); + assert_eq!( + summaries[0].terminal_reason.as_deref(), + Some("stream_write_error"), + "should classify write and flush failures consistently" + ); + } + } } #[tokio::test] - async fn datadome_filter_marker_survives_into_publisher_html_pipeline() { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config( - "datadome", - &serde_json::json!({ - "enabled": true, - "enable_protection": true, - "server_side_key_secret_name": "server-side-key", - "protection_excluded_ip_cidrs": ["192.0.2.0/24"], - "client_side_key": "test-client-key", - }), - ) - .expect("should configure DataDome integration"); - let registry = IntegrationRegistry::new(&settings) - .expect("should create integration registry with DataDome"); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - 200, - b"content".to_vec(), - vec![("content-type", "text/html; charset=utf-8")], + async fn hold_step_yields_ready_prefix_before_collecting_auction() { + // A small page whose `` lands in the first source chunk must + // still stream its document prefix immediately. `hold_step_decoded_chunk` + // reports the ready prefix and `close_found` without collecting; only + // `hold_collect_close_tail` awaits collection. Regression guard for the + // #849 FCP objective: the prefix must become ready while collection + // remains pending. + let settings = create_test_settings(); + let services = noop_services(); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), ); - let services = build_services_with_secret_http_client_and_client_ip( - NoopSecretStore, - Arc::clone(&stub) as Arc, - Some("192.0.2.10".parse().expect("should parse client IP")), + let ad_bids_state = AdBidsState::default(); + let token = b""; + let mut state = AuctionHoldState::new( + DispatchedAuctionGuard::new(DispatchedAuction::empty_for_test( + test_auction_request(), + 500, + )), + AuctionTelemetryCarry { + observation: None, + auction_request: None, + }, + Some(token.to_vec()), ); - let mut req = HttpRequest::builder() - .method(Method::GET) - .uri("https://publisher.example/page") - .header(header::HOST, "publisher.example") - .header("sec-fetch-dest", "document") - .body(EdgeBody::empty()) - .expect("should build request"); + let collect_refs = AuctionCollectDeps { + price_granularity: PriceGranularity::default(), + ad_bids_state: &ad_bids_state, + orchestrator: &orchestrator, + services: &services, + settings: &settings, + request_origin: String::new(), + }; + // Passthrough processor: the ordering contract is about collection, not + // HTML rewriting, so keep the emitted bytes verbatim. + let mut processor = RecordingProcessor { + read_count: Arc::new(AtomicUsize::new(0)), + body_close_processed_at: Arc::new(AtomicUsize::new(0)), + }; + let mut encoder = BodyStreamEncoder::new(Compression::None); - let filter_outcome = registry - .filter_request(crate::integrations::RequestFilterRegistryInput { - settings: &settings, - services: &services, - req: &mut req, - geo_info: None, - }) - .await - .expect("should run DataDome filter"); - assert!(matches!( - filter_outcome, - crate::integrations::RequestFilterRegistryOutcome::Continue(_) - )); - let publisher_response = run_publisher_proxy(&settings, &services, req).await; - let response = buffer_publisher_response_async( - publisher_response, - &Method::GET, - &settings, - ®istry, - &AuctionOrchestrator::new(settings.auction.clone()), - &services, + let step = hold_step_decoded_chunk( + &mut processor, + &mut encoder, + b"painted", + &mut state, + &collect_refs, ) .await - .expect("should buffer publisher response"); - let html = response_body_string(response); + .expect("hold step should succeed"); - assert!(!html.contains("window.ddjskey")); - assert!(!html.contains("/integrations/datadome/tags.js")); + assert!( + step.close_found, + "parser marker in the first chunk must be detected" + ); + let ready: Vec = step.ready.iter().flat_map(|b| b.to_vec()).collect(); assert_eq!( - stub.recorded_backend_names().len(), - 1, - "only the publisher origin should be called" + std::str::from_utf8(&ready).expect("ready prefix should be utf8"), + "painted", + "the prefix up to must be ready before collection" + ); + assert!( + ad_bids_state + .script_cell() + .lock() + .expect("should lock bid state") + .is_none(), + "auction must not be collected while the ready prefix is emitted" ); - } - - #[test] - fn suppressed_datadome_tag_reaches_publisher_html_pipeline() { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config( - "datadome", - &serde_json::json!({ - "enabled": true, - "client_side_key": "test-client-key", - }), - ) - .expect("should configure DataDome integration"); - let registry = IntegrationRegistry::new(&settings) - .expect("should create integration registry with DataDome"); - let mut params = make_stream_params(&settings, "identity"); - params.content_type = "text/html; charset=utf-8".to_string(); - params.suppress_datadome_client_side_tag = true; - let mut output = Vec::new(); - - stream_publisher_body( - EdgeBody::from(b"content".to_vec()), - &mut output, - ¶ms, - &settings, - ®istry, - ) - .expect("should process suppressed HTML"); - let html = String::from_utf8(output).expect("should produce UTF-8 HTML"); + let tail = hold_collect_close_tail(&mut encoder, &mut state, &collect_refs) + .await + .expect("collect should succeed"); + let tail_bytes: Vec = tail.iter().flat_map(|b| b.to_vec()).collect(); + let tail = std::str::from_utf8(&tail_bytes).expect("held tail should be utf8"); assert!( - !html.contains("window.ddjskey"), - "publisher processing should omit the DataDome client configuration" + tail.contains("var b=JSON.parse(") && tail.ends_with(""), + "collected bids and the held close tail must be emitted together: {tail}" ); assert!( - !html.contains("/integrations/datadome/tags.js"), - "publisher processing should omit the DataDome client tag URL" + ad_bids_state + .script_cell() + .lock() + .expect("should lock bid state") + .is_some(), + "collection must run when the held tail is emitted" ); } #[test] - fn suppressed_datadome_html_is_private_and_not_shared_cached() { - let mut response = Response::builder() - .status(StatusCode::OK) - .header(header::CACHE_CONTROL, "public, max-age=600") - .header("surrogate-control", "max-age=600") - .header("fastly-surrogate-control", "max-age=600") - .header("cloudflare-cdn-cache-control", "max-age=600") - .header("cdn-cache-control", "max-age=600") - .header(header::ETAG, "\"origin-tag\"") - .header(header::LAST_MODIFIED, "Wed, 21 Oct 2015 07:28:00 GMT") - .body(EdgeBody::empty()) - .expect("should build cacheable HTML response"); + fn inline_body_close_seam_holds_tail_in_single_chunk() { + let token = b""; + let mut hold = InlineBodyCloseSeam::new(token.to_vec()); - super::apply_datadome_client_tag_cache_privacy( - &mut response, - &Method::GET, - true, - "text/html; charset=utf-8", - ); + let ready = hold.push(b"painted"); + let held = hold.finish(); assert_eq!( - response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("no-store, private"), - "suppressed HTML should be private and non-storable" - ); - assert!( - response.headers().get("surrogate-control").is_none(), - "suppressed HTML should not retain Surrogate-Control" + std::str::from_utf8(&ready).expect("should be utf8"), + "painted", + "content before should stream before auction collection" ); - assert!( - response.headers().get("fastly-surrogate-control").is_none(), - "suppressed HTML should not retain Fastly-Surrogate-Control" + assert_eq!( + std::str::from_utf8(&held).expect("should be utf8"), + "", + "the close-body tag and trailing bytes should be held" ); - assert!( - response - .headers() - .get("cloudflare-cdn-cache-control") - .is_none(), - "suppressed HTML should not retain Cloudflare-CDN-Cache-Control" + } + + #[test] + fn inline_body_close_seam_holds_tail_across_chunks() { + let token = b""; + let mut hold = InlineBodyCloseSeam::new(token.to_vec()); + + let first = hold.push(b"painted"); + let held = hold.finish(); + + let streamed = [first, second].concat(); + assert_eq!( + std::str::from_utf8(&streamed).expect("should be utf8"), + "painted", + "split bytes must not leak before auction collection" ); - assert!( - response.headers().get("cdn-cache-control").is_none(), - "suppressed HTML should not retain CDN-Cache-Control" + assert_eq!( + std::str::from_utf8(&held).expect("should be utf8"), + "", + "split close-body tag should be held intact" ); - for header_name in [header::ETAG, header::LAST_MODIFIED] { - assert!( - !response.headers().contains_key(&header_name), - "suppressed HTML should not retain {header_name}" + } + + #[test] + fn inline_body_close_seam_matches_every_token_split_and_ignores_other_tokens() { + let token = b""; + for split in 0..=token.len() { + let mut seam = InlineBodyCloseSeam::new(token.to_vec()); + let mut ready = seam.push(b""); + ready.extend(seam.push(&token[..split])); + ready.extend(seam.push(&token[split..])); + assert!(seam.found(), "should match token split at {split}"); + assert_eq!( + ready, b"", + "should release all bytes before split {split}" ); + assert!(seam.finish().is_empty()); } - let mut no_store_response = Response::builder() - .status(StatusCode::OK) - .header(header::CACHE_CONTROL, "no-store") - .body(EdgeBody::empty()) - .expect("should build no-store HTML response"); - super::apply_datadome_client_tag_cache_privacy( - &mut no_store_response, - &Method::GET, - true, - "text/html; charset=utf-8", + let other = b""; + let mut seam = InlineBodyCloseSeam::new(token.to_vec()); + let ready = seam.push(other); + assert!(!seam.found()); + assert_eq!([ready, seam.finish()].concat(), other); + } + + #[test] + fn unsupported_encoding_response_is_returned_unmodified() { + assert_eq!( + classify_response_route( + StatusCode::OK, + "text/html; charset=utf-8", + "zstd", + "example.com" + ), + ResponseRoute::BufferedUnmodified, ); + } + + #[test] + fn test_publisher_origin_host_extraction() { + let settings = create_test_settings(); + let origin_host = settings.publisher.origin_host(); + assert_eq!(origin_host, "origin.test-publisher.com"); + + let mut settings_with_port = create_test_settings(); + settings_with_port.publisher.origin_url = "origin.test-publisher.com:8080".to_string(); assert_eq!( - no_store_response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("no-store, private"), - "suppressed HTML should use the exact synthesized-HTML policy" + settings_with_port.publisher.origin_host(), + "origin.test-publisher.com:8080" ); } #[test] - fn datadome_cache_privacy_does_not_change_non_html_or_unsuppressed_responses() { - let mut response = Response::builder() - .status(StatusCode::OK) - .header(header::CACHE_CONTROL, "public, max-age=600") - .header("surrogate-control", "max-age=600") - .body(EdgeBody::empty()) - .expect("should build cacheable response"); + fn test_invalid_utf8_handling() { + let invalid_utf8_bytes = vec![0xFF, 0xFE, 0xFD]; + assert!(String::from_utf8(invalid_utf8_bytes.clone()).is_err()); + } - super::apply_datadome_client_tag_cache_privacy( - &mut response, - &Method::GET, - false, - "text/html; charset=utf-8", + #[test] + fn test_utf8_conversion_edge_cases() { + let test_cases = vec![ + (vec![0xE2, 0x98, 0x83], true), + (vec![0xF0, 0x9F, 0x98, 0x80], true), + (vec![0xFF, 0xFE], false), + (vec![0xC0, 0x80], false), + (vec![0xED, 0xA0, 0x80], false), + ]; + + for (bytes, should_be_valid) in test_cases { + let result = String::from_utf8(bytes.clone()); + assert_eq!( + result.is_ok(), + should_be_valid, + "UTF-8 validation failed for bytes: {:?}", + bytes + ); + } + } + + #[test] + fn route_streams_2xx_html_without_post_processors() { + assert_eq!( + classify_response_route( + StatusCode::OK, + "text/html; charset=utf-8", + "gzip", + "example.com" + ), + ResponseRoute::Stream, ); + } + + #[test] + fn route_streams_mixed_case_html_content_type() { assert_eq!( - response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("public, max-age=600"), - "unsuppressed HTML should retain its existing cache policy" + classify_response_route( + StatusCode::OK, + "Text/HTML; Charset=utf-8", + "gzip", + "example.com" + ), + ResponseRoute::Stream, + "HTML MIME type matching must be case-insensitive", + ); + } + + #[test] + fn route_streams_html_with_post_processors() { + assert_eq!( + classify_response_route( + StatusCode::OK, + "text/html; charset=utf-8", + "gzip", + "example.com" + ), + ResponseRoute::Stream, ); + } - super::apply_datadome_client_tag_cache_privacy( - &mut response, - &Method::GET, - true, - "text/css", - ); + #[test] + fn route_streams_non_html_even_with_post_processors_registered() { assert_eq!( - response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("public, max-age=600"), - "non-HTML should retain its existing cache policy" + classify_response_route(StatusCode::OK, "application/json", "gzip", "example.com"), + ResponseRoute::Stream, ); } #[test] - fn response_carries_body_preserves_bodiless_metadata() { - // A processable GET 200 buffers a body and recomputes Content-Length. - assert!( - super::response_carries_body(&Method::GET, StatusCode::OK), - "a GET 200 publisher response should carry a buffered body" - ); - // HEAD carries no body; recomputing Content-Length to 0 would mislead - // clients/caches about the GET representation length. - assert!( - !super::response_carries_body(&Method::HEAD, StatusCode::OK), - "HEAD publisher responses must not get a recomputed Content-Length" - ); - // Bodiless statuses keep their metadata regardless of method. - assert!( - !super::response_carries_body(&Method::GET, StatusCode::NO_CONTENT), - "204 responses must not get a recomputed Content-Length" - ); - assert!( - !super::response_carries_body(&Method::GET, StatusCode::RESET_CONTENT), - "205 responses must not get a recomputed Content-Length" - ); - assert!( - !super::response_carries_body(&Method::GET, StatusCode::NOT_MODIFIED), - "304 responses must not get a recomputed Content-Length" + fn route_buffers_unmodified_on_unsupported_encoding() { + assert_eq!( + classify_response_route(StatusCode::OK, "text/html", "zstd", "example.com"), + ResponseRoute::BufferedUnmodified, ); } #[test] - fn dispatched_auction_guard_stays_armed_until_collection_completes() { - // `take()` hands the dispatched auction to the async collector, but the - // guard must stay armed across the collection await so a drop while - // collection is still pending (a client disconnect at the await point) - // still logs the loss. Only `disarm()` — called once collection reaches - // a terminal result — clears the warning. - let mut guard = DispatchedAuctionGuard::new(DispatchedAuction::empty_for_test( - test_auction_request(), - 10, - )); - assert!(guard.armed, "a freshly dispatched guard should be armed"); + fn route_passes_through_non_processable_2xx() { + assert_eq!( + classify_response_route(StatusCode::OK, "image/png", "", "example.com"), + ResponseRoute::PassThrough, + ); + } - let _dispatched = guard - .take() - .expect("guard should yield the dispatched auction for collection"); - assert!( - guard.armed, - "guard must stay armed across the collection await so a drop mid-collection is reported" + #[test] + fn route_buffers_non_processable_error_responses() { + assert_eq!( + classify_response_route(StatusCode::NOT_FOUND, "image/png", "", "example.com"), + ResponseRoute::BufferedUnmodified, ); + } - guard.disarm(); - assert!( - !guard.armed, - "guard must disarm once collection reaches a terminal result" + #[test] + fn route_excludes_204_from_pass_through() { + assert_eq!( + classify_response_route(StatusCode::NO_CONTENT, "image/png", "", "example.com"), + ResponseRoute::BufferedUnmodified, ); } - fn response_body_string(response: http::Response) -> String { - String::from_utf8( - response - .into_body() - .into_bytes() - .unwrap_or_default() - .to_vec(), - ) - .expect("response body should be valid UTF-8") + #[test] + fn route_excludes_205_from_pass_through() { + assert_eq!( + classify_response_route(StatusCode::RESET_CONTENT, "image/png", "", "example.com"), + ResponseRoute::BufferedUnmodified, + ); } #[test] - fn test_content_type_detection() { - let test_cases = vec![ - ("text/html", true), - ("text/html; charset=utf-8", true), - ("Text/HTML; Charset=utf-8", true), - ("text/css", true), - ("Text/CSS", true), - ("text/javascript", true), - ("application/javascript", true), - ("Application/JavaScript", true), - ("application/json", true), - ("application/json; charset=utf-8", true), - ("Application/JSON; Charset=UTF-8", true), - ("image/jpeg", false), - ("image/png", false), - ("application/pdf", false), - ("video/mp4", false), - ("application/octet-stream", false), - ]; + fn route_excludes_204_for_processable_content_types() { + assert_eq!( + classify_response_route( + StatusCode::NO_CONTENT, + "text/html; charset=utf-8", + "gzip", + "example.com" + ), + ResponseRoute::BufferedUnmodified, + "204 + HTML must not route to Stream", + ); + assert_eq!( + classify_response_route( + StatusCode::NO_CONTENT, + "text/html; charset=utf-8", + "gzip", + "example.com" + ), + ResponseRoute::BufferedUnmodified, + "204 + HTML + post-processors must not route to Stream", + ); + } - for (content_type, expected) in test_cases { - assert_eq!( - is_processable_content_type(content_type), - expected, - "Content-Type '{content_type}' should_process: expected {expected}", - ); - } + #[test] + fn route_excludes_205_for_processable_content_types() { + assert_eq!( + classify_response_route( + StatusCode::RESET_CONTENT, + "application/json", + "", + "example.com" + ), + ResponseRoute::BufferedUnmodified, + "205 + JSON must not route to Stream", + ); } #[test] - fn supported_content_encoding_accepts_known_values() { - assert!(is_supported_content_encoding(""), "should accept empty"); - assert!( - is_supported_content_encoding("identity"), - "should accept identity" + fn route_streams_non_2xx_processable_content() { + assert_eq!( + classify_response_route( + StatusCode::NOT_FOUND, + "text/html; charset=utf-8", + "gzip", + "example.com" + ), + ResponseRoute::Stream, ); - assert!(is_supported_content_encoding("gzip"), "should accept gzip"); - assert!( - is_supported_content_encoding("deflate"), - "should accept deflate" + assert_eq!( + classify_response_route( + StatusCode::INTERNAL_SERVER_ERROR, + "application/json", + "gzip", + "example.com" + ), + ResponseRoute::Stream, ); - assert!(is_supported_content_encoding("br"), "should accept br"); } #[test] - fn supported_content_encoding_rejects_unknown_values() { - assert!(!is_supported_content_encoding("zstd"), "should reject zstd"); - assert!( - !is_supported_content_encoding("compress"), - "should reject compress" + fn route_streams_non_2xx_html_with_post_processors() { + assert_eq!( + classify_response_route( + StatusCode::NOT_FOUND, + "text/html; charset=utf-8", + "gzip", + "example.com" + ), + ResponseRoute::Stream, ); - assert!( - !is_supported_content_encoding("snappy"), - "should reject snappy" + } + + #[test] + fn route_passes_through_non_processable_even_with_empty_request_host() { + assert_eq!( + classify_response_route(StatusCode::OK, "image/png", "", ""), + ResponseRoute::PassThrough, ); } - #[tokio::test] - async fn parser_seam_loop_collects_before_writing_post_body_chunks() { - let settings = create_test_settings(); - let services = noop_services(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - let dispatched = DispatchedAuction::empty_for_test(test_auction_request(), 500); - let read_count = Arc::new(AtomicUsize::new(0)); - let body_close_processed_at = Arc::new(AtomicUsize::new(0)); - let token = b""; - let reader = ChunkedReader::new( - &[ - b"painted", - b"", - b"", - ], - Arc::clone(&read_count), + #[test] + fn route_buffers_processable_content_with_empty_request_host() { + assert_eq!( + classify_response_route(StatusCode::OK, "text/html", "gzip", ""), + ResponseRoute::BufferedUnmodified, ); - let mut processor = RecordingProcessor { - read_count: Arc::clone(&read_count), - body_close_processed_at: Arc::clone(&body_close_processed_at), - }; - let ad_bids_state = AdBidsState::default(); - let ctx = AuctionCollectCtx { - dispatched, - telemetry: AuctionTelemetryCarry { - observation: None, - auction_request: None, - }, - deps: AuctionCollectDeps { - price_granularity: PriceGranularity::default(), - ad_bids_state: &ad_bids_state, - orchestrator: &orchestrator, - services: &services, - settings: &settings, - request_origin: String::new(), - }, - }; - let mut output = Vec::new(); + } + + #[test] + fn pass_through_preserves_body_and_content_length() { + let image_bytes: Vec = (0..=255).cycle().take(4096).collect(); - body_close_hold_loop( - reader, - &mut output, - &mut processor, - ctx, - Some(token.to_vec()), - ) - .await - .expect("should stream body with auction hold"); + let mut response = Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "image/png") + .header(header::CONTENT_LENGTH, image_bytes.len() as u64) + .body(EdgeBody::from(image_bytes.clone())) + .expect("should build test response"); + // Simulate PassThrough: take body then reattach + let body = std::mem::replace(response.body_mut(), EdgeBody::empty()); + // Body is unmodified — Content-Length stays correct assert_eq!( - body_close_processed_at.load(Ordering::SeqCst), - 1, - "close-body tail should be processed as soon as it is found, before later chunks are read" + response + .headers() + .get(header::CONTENT_LENGTH) + .and_then(|v| v.to_str().ok()) + .expect("should have content-length"), + "4096", + "Content-Length should be preserved for pass-through" ); - let output = std::str::from_utf8(&output).expect("should be utf8"); - let painted = output - .find("painted") - .expect("should preserve body content"); - let bids = output - .find("var b=JSON.parse(") - .expect("should inject collected bids"); - let close = output.find("").expect("should preserve body close"); - let late = output - .find("late()") - .expect("should preserve trailing script"); - assert!( - painted < bids && bids < close && close < late, - "output order: {output}" + + // Reattach and verify body content + *response.body_mut() = body; + let (_, final_body) = response.into_parts(); + let output = final_body.into_bytes().unwrap_or_default(); + assert_eq!( + output, image_bytes, + "pass-through should preserve body byte-for-byte" ); } - #[tokio::test] - async fn parser_seam_write_failure_abandons_auction_once() { - struct FailingWriter; + #[test] + fn test_content_encoding_detection() { + let test_encodings = vec!["gzip", "deflate", "br", "identity", ""]; - impl Write for FailingWriter { - fn write(&mut self, _buf: &[u8]) -> io::Result { - Err(io::Error::other("injected write failure")) - } + for encoding in test_encodings { + let mut req = build_request(Method::GET, "https://test.example.com/page"); + req.headers_mut().insert( + header::ACCEPT_ENCODING, + http::HeaderValue::from_static("gzip, deflate, br"), + ); - fn flush(&mut self) -> io::Result<()> { - Ok(()) + if !encoding.is_empty() { + req.headers_mut().insert( + header::CONTENT_ENCODING, + http::HeaderValue::from_str(encoding) + .expect("content encoding should be valid"), + ); } + + let content_encoding = req + .headers() + .get(header::CONTENT_ENCODING) + .map(|h| h.to_str().unwrap_or_default()) + .unwrap_or_default(); + + assert_eq!(content_encoding, encoding); } + } - let settings = create_test_settings(); - let sink = Arc::new(RecordingTelemetrySink::default()); - let services = noop_services_with_telemetry_sink(Arc::clone(&sink) as _); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - let ad_bids_state = AdBidsState::default(); - let ec_context = EcContext::new_for_test(None, ConsentContext::default()); - let ctx = AuctionCollectCtx { - dispatched: DispatchedAuction::empty_for_test(test_auction_request(), 500), - telemetry: AuctionTelemetryCarry { - observation: Some(AuctionObservationContext::from_parts( - AuctionSource::InitialNavigation, - "proxy.example.com", - "/article", - 1, - None, - &ec_context, - )), - auction_request: None, - }, - deps: AuctionCollectDeps { - price_granularity: PriceGranularity::default(), - ad_bids_state: &ad_bids_state, - orchestrator: &orchestrator, - services: &services, - settings: &settings, - request_origin: String::new(), - }, - }; - let mut processor = RecordingProcessor { - read_count: Arc::new(AtomicUsize::new(0)), - body_close_processed_at: Arc::new(AtomicUsize::new(0)), - }; + #[test] + fn publisher_proxy_does_not_add_accept_encoding_when_absent() { + let mut req = build_request(Method::GET, "https://test.example.com/page"); + // No Accept-Encoding header set by the client. - let error = body_close_hold_loop( - std::io::Cursor::new(b"ready"), - &mut FailingWriter, - &mut processor, - ctx, - Some(b"".to_vec()), - ) - .await - .expect_err("injected writer failure should surface"); - assert!(format!("{error:?}").contains("Failed to write processed chunk")); + restrict_accept_encoding(&mut req); - let batches = sink.batches.lock().expect("should lock telemetry batches"); - let summaries: Vec<_> = batches - .iter() - .flat_map(crate::auction::telemetry::AuctionEventBatch::rows) - .filter(|row| row.event_kind == "summary") - .collect(); - assert_eq!(summaries.len(), 1, "should emit one terminal summary"); - assert_eq!(summaries[0].terminal_status.as_deref(), Some("abandoned")); assert_eq!( - summaries[0].terminal_reason.as_deref(), - Some("stream_write_error") + req.headers().get(header::ACCEPT_ENCODING), + None, + "publisher proxy should not inject Accept-Encoding when the client sent none" ); } - #[tokio::test] - async fn parser_seam_async_sink_failures_abandon_auction_once() { - struct FailingWriter { - fail_flush: bool, - } + #[test] + fn publisher_proxy_limits_accept_encoding_to_supported_values() { + let mut req = build_request(Method::GET, "https://test.example.com/page"); + req.headers_mut().insert( + header::ACCEPT_ENCODING, + http::HeaderValue::from_static("gzip, deflate, br, zstd"), + ); - impl Write for FailingWriter { - fn write(&mut self, buf: &[u8]) -> io::Result { - if self.fail_flush { - Ok(buf.len()) - } else { - Err(io::Error::other("injected write failure")) - } - } + restrict_accept_encoding(&mut req); - fn flush(&mut self) -> io::Result<()> { - Err(io::Error::other("injected flush failure")) - } - } + assert_eq!( + req.headers() + .get(header::ACCEPT_ENCODING) + .and_then(|value| value.to_str().ok()), + Some("gzip, deflate, br"), + "publisher fallback should only advertise encodings the rewrite pipeline supports" + ); + } - for fail_flush in [false, true] { - for with_close in [false, true] { - let settings = create_test_settings(); - let sink = Arc::new(RecordingTelemetrySink::default()); - let services = noop_services_with_telemetry_sink(Arc::clone(&sink) as _); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - let ad_bids_state = AdBidsState::default(); - let ec_context = EcContext::new_for_test(None, ConsentContext::default()); - let ctx = AuctionCollectCtx { - dispatched: DispatchedAuction::empty_for_test(test_auction_request(), 500), - telemetry: AuctionTelemetryCarry { - observation: Some(AuctionObservationContext::from_parts( - AuctionSource::InitialNavigation, - "proxy.example.com", - "/article", - 1, - None, - &ec_context, - )), - auction_request: None, - }, - deps: AuctionCollectDeps { - price_granularity: PriceGranularity::default(), - ad_bids_state: &ad_bids_state, - orchestrator: &orchestrator, - services: &services, - settings: &settings, - request_origin: String::new(), - }, - }; - let mut processor = RecordingProcessor { - read_count: Arc::new(AtomicUsize::new(0)), - body_close_processed_at: Arc::new(AtomicUsize::new(0)), - }; - let html = if with_close { - "ready" - } else { - "ready" - }; - let body = - EdgeBody::stream(futures::stream::iter(vec![bytes::Bytes::from_static( - html.as_bytes(), - )])); + #[test] + fn publisher_proxy_preserves_identity_only_accept_encoding() { + let mut req = build_request(Method::GET, "https://test.example.com/page"); + req.headers_mut().insert( + header::ACCEPT_ENCODING, + http::HeaderValue::from_static("identity"), + ); + + restrict_accept_encoding(&mut req); + + assert_eq!( + req.headers() + .get(header::ACCEPT_ENCODING) + .and_then(|value| value.to_str().ok()), + Some("identity"), + "publisher fallback should preserve identity-only clients" + ); + } + + #[test] + fn publisher_proxy_respects_supported_client_subset() { + let mut req = build_request(Method::GET, "https://test.example.com/page"); + req.headers_mut().insert( + header::ACCEPT_ENCODING, + http::HeaderValue::from_static("br, gzip;q=0, zstd"), + ); + + restrict_accept_encoding(&mut req); + + assert_eq!( + req.headers() + .get(header::ACCEPT_ENCODING) + .and_then(|value| value.to_str().ok()), + Some("br"), + "publisher fallback should only advertise the supported encodings the client accepts" + ); + } - let error = body_close_hold_loop_stream( - body, - &mut FailingWriter { fail_flush }, - &mut processor, - Compression::None, - Compression::None, - ctx, - settings.publisher.max_buffered_body_bytes, - Some(b"".to_vec()), - ) - .await - .expect_err("should propagate the sink failure before collecting"); - let expected_error = if fail_flush { - "Failed to flush output before auction collection" - } else { - "Failed to write encoded chunk" - }; - assert!( - format!("{error:?}").contains(expected_error), - "should preserve the sink error: {error:?}" - ); + #[test] + fn publisher_proxy_falls_back_to_identity_for_unsupported_client_encodings() { + let mut req = build_request(Method::GET, "https://test.example.com/page"); + req.headers_mut().insert( + header::ACCEPT_ENCODING, + http::HeaderValue::from_static("zstd"), + ); - let batches = sink.batches.lock().expect("should lock telemetry batches"); - let summaries: Vec<_> = batches - .iter() - .flat_map(crate::auction::telemetry::AuctionEventBatch::rows) - .filter(|row| row.event_kind == "summary") - .collect(); - assert_eq!( - summaries.len(), - 1, - "should emit one terminal summary for fail_flush={fail_flush}, with_close={with_close}" - ); - assert_eq!( - summaries[0].terminal_status.as_deref(), - Some("abandoned"), - "should abandon the uncollected auction" - ); - assert_eq!( - summaries[0].terminal_reason.as_deref(), - Some("stream_write_error"), - "should classify write and flush failures consistently" + restrict_accept_encoding(&mut req); + + assert_eq!( + req.headers() + .get(header::ACCEPT_ENCODING) + .and_then(|value| value.to_str().ok()), + Some("identity"), + "publisher fallback should request identity when the client only accepts unsupported encodings" + ); + } + + #[test] + fn esi_reader_encoding_negotiation_honours_quality_identity_and_repeated_fields() { + let headers = |values: &[&str]| { + let mut headers = edgezero_core::http::HeaderMap::new(); + for value in values { + headers.append( + header::ACCEPT_ENCODING, + HeaderValue::from_str(value).expect("should build accept-encoding"), ); } + headers + }; + + assert_eq!( + negotiate_reader_compression(&headers(&[])), + Ok(Compression::None) + ); + assert_eq!( + negotiate_reader_compression(&headers(&["gzip;q=0.8", "br;q=0.4, identity;q=0.1"])), + Ok(Compression::Gzip) + ); + assert_eq!( + negotiate_reader_compression(&headers(&["gzip, br"])), + Ok(Compression::Brotli), + "server preference breaks an equal-quality tie" + ); + assert_eq!( + negotiate_reader_compression(&headers(&["gzip;q=0.5"])), + Ok(Compression::None), + "implicit identity has q=1" + ); + assert_eq!( + negotiate_reader_compression(&headers(&["zstd, identity;q=0"])), + Err(ReaderEncodingError::NoAcceptableEncoding) + ); + assert_eq!( + negotiate_reader_compression(&headers(&["gzip;q=invalid"])), + Err(ReaderEncodingError::Malformed) + ); + for malformed in [ + "gzip;q=1e-1", + "gzip;q=0.1234", + "gzip;q=1.001", + "not a coding;q=1", + ] { + assert_eq!( + negotiate_reader_compression(&headers(&[malformed])), + Err(ReaderEncodingError::Malformed), + "{malformed} is not valid Accept-Encoding syntax" + ); } } - #[tokio::test] - async fn hold_step_yields_ready_prefix_before_collecting_auction() { - // A small page whose `` lands in the first source chunk must - // still stream its document prefix immediately. `hold_step_decoded_chunk` - // reports the ready prefix and `close_found` without collecting; only - // `hold_collect_close_tail` awaits collection. Regression guard for the - // #849 FCP objective: the prefix must become ready while collection - // remains pending. + #[test] + fn tsjs_dynamic_returns_not_found_for_unknown_filename() { let settings = create_test_settings(); - let services = noop_services(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - let ad_bids_state = AdBidsState::default(); - let token = b""; - let mut state = AuctionHoldState::new( - DispatchedAuctionGuard::new(DispatchedAuction::empty_for_test( - test_auction_request(), - 500, - )), - AuctionTelemetryCarry { - observation: None, - auction_request: None, - }, - Some(token.to_vec()), + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"); + let req = build_request( + Method::GET, + "https://publisher.example/static/tsjs=unknown.js", ); - let collect_refs = AuctionCollectDeps { - price_granularity: PriceGranularity::default(), - ad_bids_state: &ad_bids_state, - orchestrator: &orchestrator, - services: &services, - settings: &settings, - request_origin: String::new(), - }; - // Passthrough processor: the ordering contract is about collection, not - // HTML rewriting, so keep the emitted bytes verbatim. - let mut processor = RecordingProcessor { - read_count: Arc::new(AtomicUsize::new(0)), - body_close_processed_at: Arc::new(AtomicUsize::new(0)), - }; - let mut encoder = BodyStreamEncoder::new(Compression::None); - let step = hold_step_decoded_chunk( - &mut processor, - &mut encoder, - b"painted", - &mut state, - &collect_refs, + let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SMaxageFallback) + .expect("should handle tsjs request"); + assert_eq!(response.status(), StatusCode::NOT_FOUND); + } + + #[test] + fn tsjs_dynamic_serves_unified_bundle_for_known_filename() { + let settings = create_test_settings(); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), ) - .await - .expect("hold step should succeed"); + .expect("should create integration registry"); + let req = build_request( + Method::GET, + "https://publisher.example/static/tsjs=tsjs-unified.min.js", + ); - assert!( - step.close_found, - "parser marker in the first chunk must be detected" + let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SMaxageFallback) + .expect("should handle tsjs request"); + assert_eq!(response.status(), StatusCode::OK); + } + + #[test] + fn tsjs_dynamic_serves_a_standalone_module_without_cookie_variance() { + let mut settings = create_test_settings(); + settings.select_module( + "testing", + crate::integrations::registry_test_support::request_fixture::MODULE, ); - let ready: Vec = step.ready.iter().flat_map(|b| b.to_vec()).collect(); - assert_eq!( - std::str::from_utf8(&ready).expect("ready prefix should be utf8"), - "painted", - "the prefix up to must be ready before collection" + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let mut req = build_request( + Method::GET, + &format!( + "https://publisher.example/static/tsjs={}", + crate::integrations::registry_test_support::request_fixture::MODULE_FILE + ), ); - assert!( - ad_bids_state - .script_cell() - .lock() - .expect("should lock bid state") - .is_none(), - "auction must not be collected while the ready prefix is emitted" + req.headers_mut().insert( + header::COOKIE, + HeaderValue::from_static("ts-request-fixture=1"), ); - let tail = hold_collect_close_tail(&mut encoder, &mut state, &collect_refs) - .await - .expect("collect should succeed"); - let tail_bytes: Vec = tail.iter().flat_map(|b| b.to_vec()).collect(); - let tail = std::str::from_utf8(&tail_bytes).expect("held tail should be utf8"); - assert!( - tail.contains("var b=JSON.parse(") && tail.ends_with(""), - "collected bids and the held close tail must be emitted together: {tail}" - ); + let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SMaxageFallback) + .expect("should handle tsjs request"); + + assert_eq!(response.status(), StatusCode::OK); + assert!(!response.headers().contains_key(header::SET_COOKIE)); assert!( - ad_bids_state - .script_cell() - .lock() - .expect("should lock bid state") - .is_some(), - "collection must run when the held tail is emitted" + !response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.contains("private") || value.contains("no-store")), + "standalone module should remain cookie-independent and publicly cacheable" ); } #[test] - fn inline_body_close_seam_holds_tail_in_single_chunk() { - let token = b""; - let mut hold = InlineBodyCloseSeam::new(token.to_vec()); - - let ready = hold.push(b"painted"); - let held = hold.finish(); + fn parse_single_module_filename_extracts_known_id() { + let settings = create_test_settings(); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); assert_eq!( - std::str::from_utf8(&ready).expect("should be utf8"), - "painted", - "content before should stream before auction collection" + parse_single_module_filename("tsjs-creative.min.js", ®istry), + Some("creative"), + "should extract the module id from a minified filename" ); assert_eq!( - std::str::from_utf8(&held).expect("should be utf8"), - "", - "the close-body tag and trailing bytes should be held" + parse_single_module_filename("tsjs-creative.js", ®istry), + Some("creative"), + "should extract the module id from an unminified filename" ); } #[test] - fn inline_body_close_seam_holds_tail_across_chunks() { - let token = b""; - let mut hold = InlineBodyCloseSeam::new(token.to_vec()); + fn parse_single_module_filename_resolves_a_carried_module_id() { + let mut settings = create_test_settings(); + settings.select_module("testing", "testing.probe"); + let extra = [IntegrationBuilder::new( + "probe", + "seam-probe", + carried_probe_registration, + validate_nothing, + ) + .with_module_name("testing.probe")]; + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should build a registry with a carried module"); - let first = hold.push(b"painted"); - let held = hold.finish(); + assert_eq!( + parse_single_module_filename("tsjs-probe.min.js", ®istry), + Some("probe"), + "should resolve a module id trusted-server-js has never heard of" + ); + } + + #[test] + fn parse_single_module_filename_rejects_unknown_ids() { + let settings = create_test_settings(); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); - let streamed = [first, second].concat(); assert_eq!( - std::str::from_utf8(&streamed).expect("should be utf8"), - "painted", - "split bytes must not leak before auction collection" + parse_single_module_filename("tsjs-evil.min.js", ®istry), + None, + "should reject unknown module names" ); assert_eq!( - std::str::from_utf8(&held).expect("should be utf8"), - "", - "split close-body tag should be held intact" + parse_single_module_filename("tsjs-core.min.js", ®istry), + None, + "should not resolve core, which is only ever served inside the bundle" + ); + assert_eq!( + parse_single_module_filename("creative.min.js", ®istry), + None, + "should reject without tsjs- prefix" + ); + assert_eq!( + parse_single_module_filename("tsjs-creative.txt", ®istry), + None, + "should reject non-js extension" ); } #[test] - fn inline_body_close_seam_matches_every_token_split_and_ignores_other_tokens() { - let token = b""; - for split in 0..=token.len() { - let mut seam = InlineBodyCloseSeam::new(token.to_vec()); - let mut ready = seam.push(b""); - ready.extend(seam.push(&token[..split])); - ready.extend(seam.push(&token[split..])); - assert!(seam.found(), "should match token split at {split}"); - assert_eq!( - ready, b"", - "should release all bytes before split {split}" - ); - assert!(seam.finish().is_empty()); - } + fn tsjs_dynamic_serves_a_deferred_module_when_it_is_selected() { + use crate::integrations::registry_test_support::deferred_fixture; - let other = b""; - let mut seam = InlineBodyCloseSeam::new(token.to_vec()); - let ready = seam.push(other); - assert!(!seam.found()); - assert_eq!([ready, seam.finish()].concat(), other); - } + let mut settings = create_test_settings(); + settings.select_module("testing", deferred_fixture::MODULE); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"); + let req = build_request( + Method::GET, + &format!( + "https://publisher.example/static/tsjs={}", + deferred_fixture::MODULE_FILE + ), + ); - #[test] - fn unsupported_encoding_response_is_returned_unmodified() { + let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SMaxageFallback) + .expect("should handle tsjs request"); assert_eq!( - classify_response_route( - StatusCode::OK, - "text/html; charset=utf-8", - "zstd", - "example.com" - ), - ResponseRoute::BufferedUnmodified, + response.status(), + StatusCode::OK, + "should serve the deferred module when its integration is selected" ); } #[test] - fn test_publisher_origin_host_extraction() { + fn tsjs_dynamic_returns_not_found_for_a_module_that_is_not_named() { let settings = create_test_settings(); - let origin_host = settings.publisher.origin_host(); - assert_eq!(origin_host, "origin.test-publisher.com"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"); + let req = build_request( + Method::GET, + &format!( + "https://publisher.example/static/tsjs={}", + crate::integrations::registry_test_support::deferred_fixture::MODULE_FILE + ), + ); - let mut settings_with_port = create_test_settings(); - settings_with_port.publisher.origin_url = "origin.test-publisher.com:8080".to_string(); + let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SMaxageFallback) + .expect("should handle tsjs request"); assert_eq!( - settings_with_port.publisher.origin_host(), - "origin.test-publisher.com:8080" + response.status(), + StatusCode::NOT_FOUND, + "should return 404 for a deferred module that is not named" ); } #[test] - fn test_invalid_utf8_handling() { - let invalid_utf8_bytes = vec![0xFF, 0xFE, 0xFD]; - assert!(String::from_utf8(invalid_utf8_bytes.clone()).is_err()); + fn tsjs_dynamic_returns_not_found_for_arbitrary_module_name() { + let settings = create_test_settings(); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"); + let req = build_request( + Method::GET, + "https://publisher.example/static/tsjs=tsjs-evil.min.js", + ); + + let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SMaxageFallback) + .expect("should handle tsjs request"); + assert_eq!( + response.status(), + StatusCode::NOT_FOUND, + "should reject unknown module names" + ); } #[test] - fn test_utf8_conversion_edge_cases() { - let test_cases = vec![ - (vec![0xE2, 0x98, 0x83], true), - (vec![0xF0, 0x9F, 0x98, 0x80], true), - (vec![0xFF, 0xFE], false), - (vec![0xC0, 0x80], false), - (vec![0xED, 0xA0, 0x80], false), - ]; + fn tsjs_dynamic_uses_immutable_cache_for_matching_hash() { + let settings = create_test_settings(); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let module_ids = registry.js_module_ids_immediate(); + let hash = trusted_server_js::concatenated_hash(&module_ids); + let request = build_request( + Method::GET, + &format!("https://publisher.example/static/tsjs=tsjs-unified.min.js?v={hash}"), + ); - for (bytes, should_be_valid) in test_cases { - let result = String::from_utf8(bytes.clone()); - assert_eq!( - result.is_ok(), - should_be_valid, - "UTF-8 validation failed for bytes: {:?}", - bytes - ); - } - } + let response = handle_tsjs_dynamic(&request, ®istry, EdgeCacheHeader::SurrogateControl) + .expect("should handle tsjs request"); - #[test] - fn route_streams_2xx_html_without_post_processors() { + assert_eq!(response.status(), StatusCode::OK); assert_eq!( - classify_response_route( - StatusCode::OK, - "text/html; charset=utf-8", - "gzip", - "example.com" - ), - ResponseRoute::Stream, + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("public, max-age=31536000, immutable"), + "matching content-versioned bundle should be immutable" ); - } - - #[test] - fn route_streams_mixed_case_html_content_type() { assert_eq!( - classify_response_route( - StatusCode::OK, - "Text/HTML; Charset=utf-8", - "gzip", - "example.com" - ), - ResponseRoute::Stream, - "HTML MIME type matching must be case-insensitive", + response + .headers() + .get("surrogate-control") + .and_then(|value| value.to_str().ok()), + Some("max-age=31536000"), + "matching content-versioned bundle should set the Fastly edge TTL" ); } #[test] - fn route_streams_html_with_post_processors() { + fn tsjs_dynamic_uses_cloudflare_edge_header_when_selected() { + let settings = create_test_settings(); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let module_ids = registry.js_module_ids_immediate(); + let hash = trusted_server_js::concatenated_hash(&module_ids); + let request = build_request( + Method::GET, + &format!("https://publisher.example/static/tsjs=tsjs-unified.min.js?v={hash}"), + ); + + let response = handle_tsjs_dynamic( + &request, + ®istry, + EdgeCacheHeader::CloudflareCdnCacheControl, + ) + .expect("should handle tsjs request"); + assert_eq!( - classify_response_route( - StatusCode::OK, - "text/html; charset=utf-8", - "gzip", - "example.com" - ), - ResponseRoute::Stream, + response + .headers() + .get("cloudflare-cdn-cache-control") + .and_then(|value| value.to_str().ok()), + Some("max-age=31536000"), + "Cloudflare requests should use its edge cache header" + ); + assert!( + response.headers().get("surrogate-control").is_none(), + "Cloudflare requests should not emit Fastly's edge cache header" ); } + fn body_text(response: http::Response) -> String { + let bytes = response + .into_body() + .into_bytes() + .expect("should read the tsjs response body") + .to_vec(); + String::from_utf8(bytes).expect("should serve UTF-8 JavaScript") + } + + fn cache_control_text(response: &http::Response) -> String { + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()) + .unwrap_or_default() + .to_string() + } + + fn carried_deferred_probe_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("probe") + .with_js_module(CarriedJsModule { + source: PROBE_JS, + sha256: PROBE_JS_SHA256, + }) + .with_deferred_js() + .build(), + )) + } + #[test] - fn route_streams_non_html_even_with_post_processors_registered() { - assert_eq!( - classify_response_route(StatusCode::OK, "application/json", "gzip", "example.com"), - ResponseRoute::Stream, + fn tsjs_dynamic_serves_a_carried_module_in_the_unified_bundle_under_the_composed_hash() { + let mut settings = create_test_settings(); + settings.select_module("testing", "testing.probe"); + let extra = [IntegrationBuilder::new( + "probe", + "seam-probe", + carried_probe_registration, + validate_nothing, + ) + .with_module_name("testing.probe")]; + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should build a registry with a carried module"); + let parts = registry.js_parts_immediate(); + let expected_hash = crate::tsjs_bundle::compose_hash(&parts); + let request = build_request( + Method::GET, + &format!("https://publisher.example/static/tsjs=tsjs-unified.min.js?v={expected_hash}"), + ); + + let response = handle_tsjs_dynamic(&request, ®istry, EdgeCacheHeader::SMaxageFallback) + .expect("should handle tsjs request"); + + assert_eq!(response.status(), StatusCode::OK, "should serve the bundle"); + assert!( + cache_control_text(&response).contains("immutable"), + "should treat the composed hash as the matching version" + ); + let body = body_text(response); + let core = trusted_server_js::module_bundle("core").expect("should have compiled core in"); + assert!( + body.starts_with(core), + "should put the compile-time core first in the bundle" + ); + assert!( + body.contains("window.__probe=1"), + "should serve the carried module inside the unified bundle" ); } #[test] - fn route_buffers_unmodified_on_unsupported_encoding() { - assert_eq!( - classify_response_route(StatusCode::OK, "text/html", "zstd", "example.com"), - ResponseRoute::BufferedUnmodified, + fn tsjs_dynamic_serves_a_carried_deferred_module_standalone() { + let mut settings = create_test_settings(); + settings.select_module("testing", "testing.probe"); + let extra = [IntegrationBuilder::new( + "probe", + "seam-probe", + carried_deferred_probe_registration, + validate_nothing, + ) + .with_module_name("testing.probe")]; + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should build a registry with a carried deferred module"); + let request = build_request( + Method::GET, + &format!("https://publisher.example/static/tsjs=tsjs-probe.min.js?v={PROBE_JS_SHA256}"), ); - } - #[test] - fn route_passes_through_non_processable_2xx() { + let response = handle_tsjs_dynamic(&request, ®istry, EdgeCacheHeader::SMaxageFallback) + .expect("should handle tsjs request"); + assert_eq!( - classify_response_route(StatusCode::OK, "image/png", "", "example.com"), - ResponseRoute::PassThrough, + response.status(), + StatusCode::OK, + "should serve a carried deferred module as its own file" + ); + assert!( + cache_control_text(&response).contains("immutable"), + "should treat the carried module's own hash as the matching version" ); - } - - #[test] - fn route_buffers_non_processable_error_responses() { assert_eq!( - classify_response_route(StatusCode::NOT_FOUND, "image/png", "", "example.com"), - ResponseRoute::BufferedUnmodified, + body_text(response), + PROBE_JS, + "should serve the carried source verbatim" ); } - #[test] - fn route_excludes_204_from_pass_through() { - assert_eq!( - classify_response_route(StatusCode::NO_CONTENT, "image/png", "", "example.com"), - ResponseRoute::BufferedUnmodified, - ); + fn carried_standalone_probe_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("probe") + .with_js_module(CarriedJsModule { + source: PROBE_JS, + sha256: PROBE_JS_SHA256, + }) + .with_standalone_js() + .build(), + )) } - #[test] - fn route_excludes_205_from_pass_through() { - assert_eq!( - classify_response_route(StatusCode::RESET_CONTENT, "image/png", "", "example.com"), - ResponseRoute::BufferedUnmodified, - ); + /// The settings of a deployment running the middleware fixture, with the + /// `[[fetch]]` entry given. + fn settings_running_the_middleware_fixture(fetch: Vec) -> Settings { + use crate::integrations::registry_test_support::middleware_fixture as fixture; + + let mut settings = create_test_settings(); + settings.select_module("testing", fixture::MODULE); + settings.fetch = PhaseEntries::new(fetch); + settings } - #[test] - fn route_excludes_204_for_processable_content_types() { - assert_eq!( - classify_response_route( - StatusCode::NO_CONTENT, - "text/html; charset=utf-8", - "gzip", - "example.com" - ), - ResponseRoute::BufferedUnmodified, - "204 + HTML must not route to Stream", - ); - assert_eq!( - classify_response_route( - StatusCode::NO_CONTENT, - "text/html; charset=utf-8", - "gzip", - "example.com" - ), - ResponseRoute::BufferedUnmodified, - "204 + HTML + post-processors must not route to Stream", - ); + /// A text body of `content_type` streamed through the publisher path, as + /// a reader gets it. + fn text_body_through_the_publisher( + settings: &Settings, + content_type: &str, + body: &str, + ) -> String { + let registry = IntegrationRegistry::new(settings).expect("should build a registry"); + let params = OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), + csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), + content_encoding: "identity".to_string(), + origin_host: "origin.example.com".to_string(), + origin_url: "https://origin.example.com".to_string(), + request_host: "proxy.example.com".to_string(), + request_scheme: "https".to_string(), + request_path: "/a.txt".to_owned(), + content_type: content_type.to_string(), + permissions_json: String::new(), + ad_slots_script: None, + ad_bids_state: AdBidsState::default(), + auction_observation: None, + auction_request: None, + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), + request_state: IntegrationRequestState::default(), + }; + let mut output = Vec::new(); + stream_publisher_body( + EdgeBody::from(body.as_bytes().to_vec()), + &mut output, + ¶ms, + settings, + ®istry, + ) + .expect("should stream a text body"); + String::from_utf8(output).expect("text") } #[test] - fn route_excludes_205_for_processable_content_types() { + fn a_text_body_passes_through_the_middleware_of_the_entry_covering_it() { + use crate::integrations::registry_test_support::middleware_fixture as fixture; + + let settings = settings_running_the_middleware_fixture(vec![PhaseEntry { + media_type: fixture::TEXT_MEDIA_TYPE.to_owned(), + path: None, + middleware: vec![fixture::TEXT.to_owned()], + }]); + let body = "User-agent: *\nSee https://origin.example.com/more\n"; + + let output = text_body_through_the_publisher(&settings, "text/plain; charset=utf-8", body); + assert_eq!( - classify_response_route( - StatusCode::RESET_CONTENT, - "application/json", - "", - "example.com" + output, + format!( + "User-agent: *\nSee https://proxy.example.com/more\n{}", + fixture::TEXT_LINE ), - ResponseRoute::BufferedUnmodified, - "205 + JSON must not route to Stream", + "should move the origin's address and then run the entry's middleware" ); } - #[test] - fn route_streams_non_2xx_processable_content() { - assert_eq!( - classify_response_route( - StatusCode::NOT_FOUND, - "text/html; charset=utf-8", - "gzip", - "example.com" - ), - ResponseRoute::Stream, + /// The same through the response path a streamed body takes, which + /// builds its processor apart from the buffered path's. + #[tokio::test] + async fn a_streamed_text_body_passes_through_the_middleware_of_the_entry_covering_it() { + use crate::integrations::registry_test_support::middleware_fixture as fixture; + use crate::platform::test_support::noop_services; + + let settings = settings_running_the_middleware_fixture(vec![PhaseEntry { + media_type: fixture::TEXT_MEDIA_TYPE.to_owned(), + path: None, + middleware: vec![fixture::TEXT.to_owned()], + }]); + let registry = IntegrationRegistry::new(&settings).expect("should build a registry"); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), ); + let services = noop_services(); + let mut params = OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), + csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), + content_encoding: String::new(), + origin_host: "origin.example.com".to_string(), + origin_url: "https://origin.example.com".to_string(), + request_host: "proxy.example.com".to_string(), + request_scheme: "https".to_string(), + request_path: "/a.txt".to_owned(), + content_type: "text/plain; charset=utf-8".to_string(), + permissions_json: String::new(), + ad_slots_script: None, + ad_bids_state: AdBidsState::default(), + auction_observation: None, + auction_request: None, + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), + request_state: IntegrationRequestState::default(), + }; + let body = EdgeBody::stream(futures::stream::iter(vec![ + bytes::Bytes::from_static(b"User-agent: *\nSee https://origin.example.com/"), + bytes::Bytes::from_static(b"more\n"), + ])); + let mut output = Vec::new(); + + stream_publisher_body_async( + body, + &mut output, + &mut params, + &settings, + ®istry, + &orchestrator, + &services, + ) + .await + .expect("should stream a text body"); + assert_eq!( - classify_response_route( - StatusCode::INTERNAL_SERVER_ERROR, - "application/json", - "gzip", - "example.com" + String::from_utf8(output).expect("text"), + format!( + "User-agent: *\nSee https://proxy.example.com/more\n{}", + fixture::TEXT_LINE ), - ResponseRoute::Stream, + "should move the origin's address and then run the entry's middleware" ); } - #[test] - fn route_streams_non_2xx_html_with_post_processors() { - assert_eq!( - classify_response_route( - StatusCode::NOT_FOUND, - "text/html; charset=utf-8", - "gzip", - "example.com" - ), - ResponseRoute::Stream, - ); + /// The parameters of a text response at `/a.txt`, from the origin as + /// `content_type` says, for a reader of `proxy.example.com`. + fn text_params(content_type: &str) -> OwnedProcessResponseParams { + OwnedProcessResponseParams { + csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), + public_prefix: crate::prefix::PublicPrefix::default(), + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), + content_encoding: "identity".to_string(), + origin_host: "origin.example.com".to_string(), + origin_url: "https://origin.example.com".to_string(), + request_host: "proxy.example.com".to_string(), + request_scheme: "https".to_string(), + request_path: "/a.txt".to_owned(), + content_type: content_type.to_string(), + permissions_json: String::new(), + ad_slots_script: None, + ad_bids_state: AdBidsState::default(), + auction_observation: None, + auction_request: None, + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), + request_state: IntegrationRequestState::default(), + } } - #[test] - fn route_passes_through_non_processable_even_with_empty_request_host() { - assert_eq!( - classify_response_route(StatusCode::OK, "image/png", "", ""), - ResponseRoute::PassThrough, - ); + /// Settings running the fixture with a serve entry on `text/plain` + /// naming its header-writing middleware. + fn settings_serving_the_text_header() -> Settings { + use crate::integrations::registry_test_support::middleware_fixture as fixture; + + let mut settings = settings_running_the_middleware_fixture(Vec::new()); + settings.serve = PhaseEntries::new(vec![PhaseEntry { + media_type: fixture::TEXT_MEDIA_TYPE.to_owned(), + path: None, + middleware: vec![fixture::TEXT_HEADER.to_owned()], + }]); + settings } - #[test] - fn route_buffers_processable_content_with_empty_request_host() { - assert_eq!( - classify_response_route(StatusCode::OK, "text/html", "gzip", ""), - ResponseRoute::BufferedUnmodified, - ); + fn named_headers(headers: &[ResponseHeader]) -> Vec<(&str, &[u8])> { + headers + .iter() + .map(|(name, value)| (name.as_str(), value.as_bytes())) + .collect() } #[test] - fn pass_through_preserves_body_and_content_length() { - let image_bytes: Vec = (0..=255).cycle().take(4096).collect(); + fn a_text_body_s_serve_middleware_write_their_headers_on_the_response() { + use crate::integrations::registry_test_support::middleware_fixture as fixture; - let mut response = Response::builder() - .status(StatusCode::OK) - .header(header::CONTENT_TYPE, "image/png") - .header(header::CONTENT_LENGTH, image_bytes.len() as u64) - .body(EdgeBody::from(image_bytes.clone())) - .expect("should build test response"); + let settings = settings_serving_the_text_header(); + let registry = IntegrationRegistry::new(&settings).expect("should build a registry"); + let params = text_params("text/plain; charset=utf-8"); + let mut output = Vec::new(); - // Simulate PassThrough: take body then reattach - let body = std::mem::replace(response.body_mut(), EdgeBody::empty()); - // Body is unmodified — Content-Length stays correct - assert_eq!( - response - .headers() - .get(header::CONTENT_LENGTH) - .and_then(|v| v.to_str().ok()) - .expect("should have content-length"), - "4096", - "Content-Length should be preserved for pass-through" - ); + let headers = stream_publisher_body( + EdgeBody::from(b"plain\n".to_vec()), + &mut output, + ¶ms, + &settings, + ®istry, + ) + .expect("should stream a text body"); - // Reattach and verify body content - *response.body_mut() = body; - let (_, final_body) = response.into_parts(); - let output = final_body.into_bytes().unwrap_or_default(); assert_eq!( - output, image_bytes, - "pass-through should preserve body byte-for-byte" + named_headers(&headers), + [( + fixture::TEXT_HEADER_NAME, + fixture::TEXT_HEADER_VALUE.as_bytes() + )], + "should hand back the header the serve middleware writes on a text response" ); + assert_eq!(String::from_utf8(output).expect("text"), "plain\n"); } - #[test] - fn test_content_encoding_detection() { - let test_encodings = vec!["gzip", "deflate", "br", "identity", ""]; - - for encoding in test_encodings { - let mut req = build_request(Method::GET, "https://test.example.com/page"); - req.headers_mut().insert( - header::ACCEPT_ENCODING, - http::HeaderValue::from_static("gzip, deflate, br"), - ); + #[tokio::test] + async fn a_streamed_text_body_s_serve_middleware_write_their_headers_on_the_response() { + use crate::integrations::registry_test_support::middleware_fixture as fixture; + use crate::platform::test_support::noop_services; - if !encoding.is_empty() { - req.headers_mut().insert( - header::CONTENT_ENCODING, - http::HeaderValue::from_str(encoding) - .expect("content encoding should be valid"), - ); - } + let settings = settings_serving_the_text_header(); + let registry = IntegrationRegistry::new(&settings).expect("should build a registry"); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); + let services = noop_services(); + let mut params = text_params("text/plain; charset=utf-8"); + let body = EdgeBody::stream(futures::stream::iter(vec![ + bytes::Bytes::from_static(b"pla"), + bytes::Bytes::from_static(b"in\n"), + ])); + let mut output = Vec::new(); - let content_encoding = req - .headers() - .get(header::CONTENT_ENCODING) - .map(|h| h.to_str().unwrap_or_default()) - .unwrap_or_default(); + let headers = stream_publisher_body_async( + body, + &mut output, + &mut params, + &settings, + ®istry, + &orchestrator, + &services, + ) + .await + .expect("should stream a text body"); - assert_eq!(content_encoding, encoding); - } + assert_eq!( + named_headers(&headers), + [( + fixture::TEXT_HEADER_NAME, + fixture::TEXT_HEADER_VALUE.as_bytes() + )], + "should hand back the header the serve middleware writes on a streamed text response" + ); + assert_eq!(String::from_utf8(output).expect("text"), "plain\n"); } #[test] - fn publisher_proxy_does_not_add_accept_encoding_when_absent() { - let mut req = build_request(Method::GET, "https://test.example.com/page"); - // No Accept-Encoding header set by the client. + fn a_text_body_no_entry_covers_is_left_as_it_was() { + use crate::integrations::registry_test_support::middleware_fixture as fixture; - restrict_accept_encoding(&mut req); + let settings = settings_running_the_middleware_fixture(vec![PhaseEntry { + media_type: fixture::TEXT_MEDIA_TYPE.to_owned(), + path: Some("/elsewhere/".to_owned()), + middleware: vec![fixture::TEXT.to_owned()], + }]); + + let output = text_body_through_the_publisher(&settings, "text/plain", "plain\n"); assert_eq!( - req.headers().get(header::ACCEPT_ENCODING), - None, - "publisher proxy should not inject Accept-Encoding when the client sent none" + output, "plain\n", + "should run nothing on a path the entry does not cover" ); } - #[test] - fn publisher_proxy_limits_accept_encoding_to_supported_values() { - let mut req = build_request(Method::GET, "https://test.example.com/page"); - req.headers_mut().insert( - header::ACCEPT_ENCODING, - http::HeaderValue::from_static("gzip, deflate, br, zstd"), - ); + /// What the publisher path answered one request with. + struct Proxied { + settings: Settings, + registry: IntegrationRegistry, + services: crate::platform::RuntimeServices, + orchestrator: AuctionOrchestrator, + response: PublisherResponse, + } - restrict_accept_encoding(&mut req); + /// A request through the publisher path, the origin answering `status`. + async fn proxy(fetch: Vec, method: Method, path: &str, status: u16) -> Proxied { + proxy_with( + settings_running_the_middleware_fixture(fetch), + method, + path, + status, + ) + .await + } - assert_eq!( - req.headers() - .get(header::ACCEPT_ENCODING) - .and_then(|value| value.to_str().ok()), - Some("gzip, deflate, br"), - "publisher fallback should only advertise encodings the rewrite pipeline supports" - ); + /// As [`proxy`], with the settings given. + async fn proxy_with(settings: Settings, method: Method, path: &str, status: u16) -> Proxied { + proxy_origin( + settings, + method, + path, + status, + b"the origin's page", + "text/html; charset=utf-8", + ) + .await } - #[test] - fn publisher_proxy_preserves_identity_only_accept_encoding() { - let mut req = build_request(Method::GET, "https://test.example.com/page"); - req.headers_mut().insert( - header::ACCEPT_ENCODING, - http::HeaderValue::from_static("identity"), + /// As [`proxy_with`], with the origin answering `status` with `body` as + /// `content_type`. + async fn proxy_origin( + settings: Settings, + method: Method, + path: &str, + status: u16, + body: &[u8], + content_type: &'static str, + ) -> Proxied { + let registry = IntegrationRegistry::new(&settings).expect("should build a registry"); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + status, + body.to_vec(), + vec![("content-type", content_type)], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let consent = crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + }; + let mut ec_context = EcContext::new_for_test_with_ip(None, consent, None); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), ); + let req = HttpRequest::builder() + .method(method) + .uri(format!("https://publisher.example{path}")) + .header(header::HOST, "publisher.example") + .body(EdgeBody::empty()) + .expect("should build request"); + let response = handle_publisher_request( + AppContext { + settings: &settings, + integration_registry: ®istry, + }, + &services, + None, + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &[], + registry: None, + }, + req, + EdgeCacheHeader::SMaxageFallback, + ) + .await + .expect("should proxy the request"); + Proxied { + settings, + registry, + services, + orchestrator, + response, + } + } - restrict_accept_encoding(&mut req); + /// One entry naming the fixture's text middleware, for `path` or for + /// every path. + fn text_entry_for(path: Option<&str>) -> Vec { + use crate::integrations::registry_test_support::middleware_fixture as fixture; - assert_eq!( - req.headers() - .get(header::ACCEPT_ENCODING) - .and_then(|value| value.to_str().ok()), - Some("identity"), - "publisher fallback should preserve identity-only clients" - ); + vec![PhaseEntry { + media_type: fixture::TEXT_MEDIA_TYPE.to_owned(), + path: path.map(str::to_owned), + middleware: vec![fixture::TEXT.to_owned()], + }] } - #[test] - fn publisher_proxy_respects_supported_client_subset() { - let mut req = build_request(Method::GET, "https://test.example.com/page"); - req.headers_mut().insert( - header::ACCEPT_ENCODING, - http::HeaderValue::from_static("br, gzip;q=0, zstd"), + /// The response of a proxied request, however it is to be sent. + fn response_of(response: PublisherResponse) -> Response { + match response { + PublisherResponse::Buffered(response) + | PublisherResponse::Stream { response, .. } + | PublisherResponse::PassThrough { response, .. } => response, + PublisherResponse::AssembleTemplate { .. } => { + panic!("a template is not an answer to these requests") + } + } + } + + fn content_type_of(response: &Response) -> Option<&str> { + response + .headers() + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + } + + /// Settings refusing every crawler, with the entry that places core's + /// robots.txt middleware on the file. + fn settings_refusing_every_crawler() -> Settings { + use crate::robots_txt::middleware as robots_txt; + + let mut settings = settings_running_the_middleware_fixture(vec![PhaseEntry { + media_type: robots_txt::MEDIA_TYPE.to_owned(), + path: Some(crate::robots_txt::ROBOTS_TXT_PATH.to_owned()), + middleware: vec![robots_txt::MODULE.to_owned()], + }]); + let mut refusing = crate::robots_txt::RobotsTxtConfig::refuse_all(); + refusing.always_allow = vec!["/ads.txt".to_owned()]; + settings.robots_txt = Some(refusing); + settings + } + + /// The file a proxied robots.txt request streams, with its status and + /// content type checked. + async fn streamed_robots_txt(proxied: Proxied) -> String { + let PublisherResponse::Stream { + response, + body, + params, + } = proxied.response + else { + panic!("the file should stream through the entry's middleware"); + }; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + content_type_of(&response), + Some("text/plain; charset=utf-8"), + "should be a document of the entry's media type" ); + let mut params = *params; + let mut output = Vec::new(); + stream_publisher_body_async( + body, + &mut output, + &mut params, + &proxied.settings, + &proxied.registry, + &proxied.orchestrator, + &proxied.services, + ) + .await + .expect("should stream the file"); + String::from_utf8(output).expect("the file is text") + } - restrict_accept_encoding(&mut req); + /// The file the middleware makes replaces the origin's own. + #[tokio::test] + async fn robots_txt_replaces_the_origins_file() { + let proxied = proxy_origin( + settings_refusing_every_crawler(), + Method::GET, + crate::robots_txt::ROBOTS_TXT_PATH, + 200, + b"User-agent: *\nAllow: /\n", + "text/plain; charset=utf-8", + ) + .await; assert_eq!( - req.headers() - .get(header::ACCEPT_ENCODING) - .and_then(|value| value.to_str().ok()), - Some("br"), - "publisher fallback should only advertise the supported encodings the client accepts" + streamed_robots_txt(proxied).await, + "User-agent: *\nAllow: /ads.txt\nDisallow: /\n", + "the refusal replaces the origin's file" ); } - #[test] - fn publisher_proxy_falls_back_to_identity_for_unsupported_client_encodings() { - let mut req = build_request(Method::GET, "https://test.example.com/page"); - req.headers_mut().insert( - header::ACCEPT_ENCODING, - http::HeaderValue::from_static("zstd"), + /// An origin that answers every path with its page, as a single-page + /// application's catch-all does, has no file, so the middleware makes it + /// rather than the crawler being handed the page. + #[tokio::test] + async fn robots_txt_replaces_a_page_the_origin_answers_at_its_path() { + let proxied = proxy_with( + settings_refusing_every_crawler(), + Method::GET, + crate::robots_txt::ROBOTS_TXT_PATH, + 200, + ) + .await; + + assert_eq!( + streamed_robots_txt(proxied).await, + "User-agent: *\nAllow: /ads.txt\nDisallow: /\n", + "the file replaces the page" ); + } - restrict_accept_encoding(&mut req); + /// Where the origin has no file, the creating rule makes it. + #[tokio::test] + async fn robots_txt_is_made_where_the_origin_has_none() { + let proxied = proxy_with( + settings_refusing_every_crawler(), + Method::GET, + crate::robots_txt::ROBOTS_TXT_PATH, + 404, + ) + .await; assert_eq!( - req.headers() - .get(header::ACCEPT_ENCODING) - .and_then(|value| value.to_str().ok()), - Some("identity"), - "publisher fallback should request identity when the client only accepts unsupported encodings" + streamed_robots_txt(proxied).await, + "User-agent: *\nAllow: /ads.txt\nDisallow: /\n", + "the file is made" ); } - #[test] - fn esi_reader_encoding_negotiation_honours_quality_identity_and_repeated_fields() { - let headers = |values: &[&str]| { - let mut headers = edgezero_core::http::HeaderMap::new(); - for value in values { - headers.append( - header::ACCEPT_ENCODING, - HeaderValue::from_str(value).expect("should build accept-encoding"), - ); - } - headers - }; + #[tokio::test] + async fn a_file_the_origin_does_not_have_is_made_by_the_middleware_named_for_its_path() { + use crate::integrations::registry_test_support::middleware_fixture as fixture; + let proxied = proxy( + text_entry_for(Some("/ads.txt")), + Method::GET, + "/ads.txt", + 404, + ) + .await; + let PublisherResponse::Stream { + response, + body, + params, + } = proxied.response + else { + panic!("a made file should stream through the entry's middleware"); + }; + assert_eq!(response.status(), StatusCode::OK); assert_eq!( - negotiate_reader_compression(&headers(&[])), - Ok(Compression::None) + content_type_of(&response), + Some("text/plain; charset=utf-8"), + "should be a document of the entry's media type" ); + let mut params = *params; + let mut output = Vec::new(); + stream_publisher_body_async( + body, + &mut output, + &mut params, + &proxied.settings, + &proxied.registry, + &proxied.orchestrator, + &proxied.services, + ) + .await + .expect("should stream the made file"); assert_eq!( - negotiate_reader_compression(&headers(&["gzip;q=0.8", "br;q=0.4, identity;q=0.1"])), - Ok(Compression::Gzip) + String::from_utf8(output).expect("text"), + fixture::TEXT_LINE, + "should be what the middleware writes on an empty document" ); + } + + #[tokio::test] + async fn a_head_request_for_a_made_file_gets_the_same_answer() { + let proxied = proxy( + text_entry_for(Some("/ads.txt")), + Method::HEAD, + "/ads.txt", + 404, + ) + .await; + let response = buffer_publisher_response_async( + proxied.response, + &Method::HEAD, + &proxied.settings, + &proxied.registry, + &proxied.orchestrator, + &proxied.services, + ) + .await + .expect("should buffer the HEAD answer"); + + assert_eq!(response.status(), StatusCode::OK); assert_eq!( - negotiate_reader_compression(&headers(&["gzip, br"])), - Ok(Compression::Brotli), - "server preference breaks an equal-quality tie" + content_type_of(&response), + Some("text/plain; charset=utf-8"), + "should answer a HEAD as the GET is answered" ); + assert!( + response + .into_body() + .into_bytes() + .expect("should hold the HEAD answer in memory") + .is_empty(), + "should answer a HEAD with no body" + ); + } + + #[tokio::test] + async fn an_html_entry_with_a_path_makes_no_missing_page() { + use crate::integrations::registry_test_support::middleware_fixture as fixture; + use crate::middleware::HTML_MEDIA_TYPE; + + let proxied = proxy( + vec![PhaseEntry { + media_type: HTML_MEDIA_TYPE.to_owned(), + path: Some("/news/".to_owned()), + middleware: vec![fixture::HEAD.to_owned()], + }], + Method::GET, + "/news/missing", + 404, + ) + .await; + assert_eq!( - negotiate_reader_compression(&headers(&["gzip;q=0.5"])), - Ok(Compression::None), - "implicit identity has q=1" + response_of(proxied.response).status(), + StatusCode::NOT_FOUND, + "should keep the origin's answer for a page it does not have" ); + } + + #[tokio::test] + async fn an_html_entry_ahead_does_not_shadow_the_entry_for_a_missing_file() { + use crate::integrations::registry_test_support::middleware_fixture as fixture; + use crate::middleware::HTML_MEDIA_TYPE; + + let mut fetch = vec![PhaseEntry { + media_type: HTML_MEDIA_TYPE.to_owned(), + path: Some("/".to_owned()), + middleware: vec![fixture::HEAD.to_owned()], + }]; + fetch.extend(text_entry_for(Some("/ads.txt"))); + let proxied = proxy(fetch, Method::GET, "/ads.txt", 404).await; + let response = response_of(proxied.response); + + assert_eq!(response.status(), StatusCode::OK); assert_eq!( - negotiate_reader_compression(&headers(&["zstd, identity;q=0"])), - Err(ReaderEncodingError::NoAcceptableEncoding) + content_type_of(&response), + Some("text/plain; charset=utf-8"), + "should make the file from the text/plain entry, not as HTML" ); + } + + #[tokio::test] + async fn an_entry_covering_every_path_makes_no_missing_page() { + let proxied = proxy(text_entry_for(None), Method::GET, "/missing.txt", 404).await; + assert_eq!( - negotiate_reader_compression(&headers(&["gzip;q=invalid"])), - Err(ReaderEncodingError::Malformed) + response_of(proxied.response).status(), + StatusCode::NOT_FOUND, + "should keep the origin's answer, because a page the origin does not have is its answer" ); - for malformed in [ - "gzip;q=1e-1", - "gzip;q=0.1234", - "gzip;q=1.001", - "not a coding;q=1", - ] { - assert_eq!( - negotiate_reader_compression(&headers(&[malformed])), - Err(ReaderEncodingError::Malformed), - "{malformed} is not valid Accept-Encoding syntax" - ); - } } - #[test] - fn tsjs_dynamic_returns_not_found_for_unknown_filename() { - let settings = create_test_settings(); - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) - .expect("should create integration registry"); - let req = build_request( + #[tokio::test] + async fn a_missing_file_outside_the_entry_path_keeps_the_origin_answer() { + let proxied = proxy( + text_entry_for(Some("/ads.txt")), Method::GET, - "https://publisher.example/static/tsjs=unknown.js", - ); + "/robots.txt", + 404, + ) + .await; - let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SMaxageFallback) - .expect("should handle tsjs request"); - assert_eq!(response.status(), StatusCode::NOT_FOUND); + assert_eq!( + response_of(proxied.response).status(), + StatusCode::NOT_FOUND + ); } - #[test] - fn tsjs_dynamic_serves_unified_bundle_for_known_filename() { - let settings = create_test_settings(); - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) - .expect("should create integration registry"); - let req = build_request( + #[tokio::test] + async fn an_origin_error_on_the_entry_path_keeps_the_origin_answer() { + let proxied = proxy( + text_entry_for(Some("/ads.txt")), Method::GET, - "https://publisher.example/static/tsjs=tsjs-unified.min.js", - ); + "/ads.txt", + 500, + ) + .await; - let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SMaxageFallback) - .expect("should handle tsjs request"); - assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response_of(proxied.response).status(), + StatusCode::INTERNAL_SERVER_ERROR, + "should make a file only where the origin has none" + ); } - #[test] - fn tsjs_dynamic_serves_diagnostics_standalone_without_cookie_variance() { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config("gpt_diagnostics", &serde_json::json!({ "enabled": true })) - .expect("should enable diagnostics"); - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) - .expect("should create integration registry"); - let mut req = build_request( + #[tokio::test] + async fn what_a_middleware_prepares_reaches_its_create() { + use crate::integrations::registry_test_support::middleware_fixture as fixture; + use crate::middleware::HTML_MEDIA_TYPE; + + let proxied = proxy( + vec![PhaseEntry { + media_type: HTML_MEDIA_TYPE.to_owned(), + path: None, + middleware: vec![fixture::PREPARING.to_owned()], + }], Method::GET, - "https://publisher.example/static/tsjs=tsjs-gpt_diagnostics.min.js", + "/article", + 200, + ) + .await; + let PublisherResponse::Stream { body, params, .. } = proxied.response else { + panic!("a page should stream through the entry's middleware"); + }; + let mut params = *params; + let mut output = Vec::new(); + stream_publisher_body_async( + body, + &mut output, + &mut params, + &proxied.settings, + &proxied.registry, + &proxied.orchestrator, + &proxied.services, + ) + .await + .expect("should stream the page"); + let page = String::from_utf8(output).expect("html"); + + assert!( + page.contains(&fixture::prepared_marker( + fixture::PREPARING, + "publisher.example" + )), + "should write what the preparation answered, called as the middleware for the \ + reader's host: {page}" ); - req.headers_mut().insert( - header::COOKIE, - HeaderValue::from_static("__Host-ts-console=1"), + assert!( + !page.contains(fixture::NOTHING_PREPARED), + "should not ask create before the preparation" ); + } - let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SMaxageFallback) - .expect("should handle tsjs request"); + #[tokio::test] + async fn a_serve_middleware_prepares_for_each_reader() { + use crate::integrations::registry_test_support::middleware_fixture as fixture; + use crate::middleware::HTML_MEDIA_TYPE; + + let mut settings = settings_running_the_middleware_fixture(Vec::new()); + settings.serve = PhaseEntries::new(vec![PhaseEntry { + media_type: HTML_MEDIA_TYPE.to_owned(), + path: None, + middleware: vec![fixture::PREPARING.to_owned()], + }]); + let proxied = proxy_with(settings, Method::GET, "/article", 200).await; + let PublisherResponse::Stream { body, params, .. } = proxied.response else { + panic!("a page should stream through the entry's middleware"); + }; + let mut params = *params; + let mut output = Vec::new(); + stream_publisher_body_async( + body, + &mut output, + &mut params, + &proxied.settings, + &proxied.registry, + &proxied.orchestrator, + &proxied.services, + ) + .await + .expect("should stream the page"); + let page = String::from_utf8(output).expect("html"); - assert_eq!(response.status(), StatusCode::OK); - assert!(!response.headers().contains_key(header::SET_COOKIE)); assert!( - !response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()) - .is_some_and(|value| value.contains("private") || value.contains("no-store")), - "standalone module should remain cookie-independent and publicly cacheable" + page.contains(&fixture::prepared_marker( + fixture::PREPARING, + "publisher.example" + )), + "should prepare in the serve phase for this reader: {page}" ); } - #[test] - fn parse_single_module_filename_extracts_known_id() { + #[tokio::test] + async fn a_refusal_answers_the_request_with_its_status_and_stores_nothing() { + use crate::integrations::registry_test_support::middleware_fixture as fixture; + use crate::middleware::HTML_MEDIA_TYPE; + + let proxied = proxy( + vec![PhaseEntry { + media_type: HTML_MEDIA_TYPE.to_owned(), + path: None, + middleware: vec![fixture::REFUSING.to_owned()], + }], + Method::GET, + "/article", + 200, + ) + .await; + let PublisherResponse::Buffered(response) = proxied.response else { + panic!("a refusal is answered whole, with no document to stream"); + }; + let header_value = |name: header::HeaderName| { + response + .headers() + .get(name) + .and_then(|value| value.to_str().ok()) + .map(str::to_owned) + }; + + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(header_value(header::RETRY_AFTER).as_deref(), Some("30")); assert_eq!( - parse_single_module_filename("tsjs-sourcepoint.min.js"), - Some("sourcepoint"), - "should extract sourcepoint from minified filename" + header_value(header::CONTENT_TYPE).as_deref(), + Some("text/plain; charset=utf-8") ); assert_eq!( - parse_single_module_filename("tsjs-sourcepoint.js"), - Some("sourcepoint"), - "should extract sourcepoint from unminified filename" + header_value(header::CACHE_CONTROL).as_deref(), + Some("no-store, private"), + "should be stored and shared by nothing" + ); + assert!( + response + .extensions() + .get::() + .is_some(), + "should be marked as this request's alone" ); + let body = response.into_body().into_bytes().expect("a whole body"); + assert_eq!(String::from_utf8_lossy(&body), fixture::REFUSAL_MESSAGE); } #[test] - fn parse_single_module_filename_rejects_unknown_ids() { - assert_eq!( - parse_single_module_filename("tsjs-evil.min.js"), - None, - "should reject unknown module names" - ); - assert_eq!( - parse_single_module_filename("tsjs-core.min.js"), - Some("core"), - "should accept any known module ID (deferred check happens in caller)" - ); - assert_eq!( - parse_single_module_filename("prebid.min.js"), - None, - "should reject without tsjs- prefix" - ); + fn a_text_body_in_a_charset_other_than_utf8_is_left_as_it_was() { + use crate::integrations::registry_test_support::middleware_fixture as fixture; + + let settings = settings_running_the_middleware_fixture(vec![PhaseEntry { + media_type: fixture::TEXT_MEDIA_TYPE.to_owned(), + path: None, + middleware: vec![fixture::TEXT.to_owned()], + }]); + + let output = + text_body_through_the_publisher(&settings, "text/plain; charset=utf-16", "plain\n"); + assert_eq!( - parse_single_module_filename("tsjs-sourcepoint.txt"), - None, - "should reject non-js extension" + output, "plain\n", + "should hand a middleware no body it would corrupt by writing UTF-8" ); } #[test] - fn tsjs_dynamic_serves_prebid_shim_when_enabled() { - let settings = create_test_settings(); - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) - .expect("should create integration registry"); - let req = build_request( - Method::GET, - "https://publisher.example/static/tsjs=tsjs-prebid.min.js", - ); + fn a_refusal_rounds_retry_after_up_to_whole_seconds() { + let response = refusal_response(&Refusal::service_unavailable( + std::time::Duration::from_millis(1_500), + "later", + )) + .expect("should build the refusal"); - let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SMaxageFallback) - .expect("should handle tsjs request"); assert_eq!( - response.status(), - StatusCode::OK, - "should serve the deferred prebid shim module when prebid is enabled" + response + .headers() + .get(header::RETRY_AFTER) + .and_then(|value| value.to_str().ok()), + Some("2"), + "should not ask the caller back before the refusal's time" ); } #[test] - fn tsjs_dynamic_returns_not_found_for_disabled_deferred_module() { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config( - "prebid", - &serde_json::json!({ - "enabled": false, - "external_bundle_url": "https://assets.example/prebid/trusted-prebid.js", - }), - ) - .expect("should update prebid config"); - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), + fn a_fetch_preparation_of_a_document_that_may_be_stored_is_not_handed_the_reader() { + use crate::module_context::{EdgeCookie, FromModuleContext}; + + let ec_context = EcContext::new_for_test( + Some(crate::ec::tests::CANONICAL_COOKIE_VALUE.to_owned()), + ConsentContext::default(), ) - .expect("should create integration registry"); - let req = build_request( - Method::GET, - "https://publisher.example/static/tsjs=tsjs-prebid.min.js", - ); + .with_module_for_test(Arc::new(crate::ec::tests::CanonicalizingModule)); + let services = noop_services(); + let settings = settings_running_the_middleware_fixture(Vec::new()); + let method = Method::GET; + let handed = |stored: bool| { + let request = ModuleRequest::new(&method, "publisher.example", "https", "/article"); + let context = preparation_context(request, stored, &settings, &ec_context, &services); + let call = context.call("testing.preparation", PermissionSet::none()); + ( + <&PermissionState>::from_module_context(&call).is_ok(), + EdgeCookie::from_module_context(&call).is_ok(), + ) + }; - let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SMaxageFallback) - .expect("should handle tsjs request"); assert_eq!( - response.status(), - StatusCode::NOT_FOUND, - "should return 404 for disabled deferred module" + handed(false), + (true, true), + "should hand a document that is not stored the reader's permissions and Edge Cookie" + ); + assert_eq!( + handed(true), + (false, false), + "should hand a document that may be stored neither the reader's permissions nor Edge Cookie" ); } #[test] - fn tsjs_dynamic_returns_not_found_for_arbitrary_module_name() { - let settings = create_test_settings(); - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), + fn the_serve_phase_of_a_document_that_may_be_stored_is_prepared_for_the_reader() { + use crate::module_context::{EdgeCookie, FromModuleContext}; + + let ec_context = EcContext::new_for_test( + Some(crate::ec::tests::CANONICAL_COOKIE_VALUE.to_owned()), + ConsentContext::default(), ) - .expect("should create integration registry"); - let req = build_request( - Method::GET, - "https://publisher.example/static/tsjs=tsjs-evil.min.js", - ); + .with_module_for_test(Arc::new(crate::ec::tests::CanonicalizingModule)); + let services = noop_services(); + let settings = settings_running_the_middleware_fixture(Vec::new()); + let method = Method::GET; + let handed = |context: &ModuleContext<'_>| { + let call = context.call("testing.preparation", PermissionSet::none()); + ( + <&PermissionState>::from_module_context(&call).is_ok(), + EdgeCookie::from_module_context(&call).is_ok(), + ) + }; - let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SMaxageFallback) - .expect("should handle tsjs request"); - assert_eq!( - response.status(), - StatusCode::NOT_FOUND, - "should reject unknown module names" - ); + for stored in [true, false] { + let (fetch, serve) = preparation_contexts( + || ModuleRequest::new(&method, "publisher.example", "https", "/article"), + stored, + &settings, + &ec_context, + &services, + ); + + assert_eq!( + handed(&fetch), + (!stored, !stored), + "the fetch phase should see the reader only when the document is not stored" + ); + assert_eq!( + handed(&serve), + (true, true), + "the serve phase should see the reader whether or not the document is stored" + ); + } } #[test] - fn tsjs_dynamic_uses_immutable_cache_for_matching_hash() { - let settings = create_test_settings(); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); - let module_ids = registry.js_module_ids_immediate(); - let hash = trusted_server_js::concatenated_hash(&module_ids); - let request = build_request( - Method::GET, - &format!("https://publisher.example/static/tsjs=tsjs-unified.min.js?v={hash}"), + fn tsjs_dynamic_serves_a_standalone_module_by_registration_flag_not_by_name() { + // The module the registration marks standalone is one a vendor crate + // carries, so nothing in the binary knows its name. A branch that named + // an integration instead of reading the flag could not serve it, and a + // reintroduced constant would fail this test rather than pass it. + let mut settings = create_test_settings(); + settings.select_module("testing", "testing.probe"); + let extra = [IntegrationBuilder::new( + "probe", + "standalone-probe", + carried_standalone_probe_registration, + validate_nothing, + ) + .with_module_name("testing.probe")]; + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should build a registry with a carried standalone module"); + assert!( + registry.js_module_ids_immediate().contains(&"creative"), + "fixture should put core's creative module in the unified bundle" + ); + assert!( + !registry.js_module_ids().contains(&"probe"), + "a standalone module should stay out of the bundled module ids" ); - let response = handle_tsjs_dynamic(&request, ®istry, EdgeCacheHeader::SurrogateControl) - .expect("should handle tsjs request"); - - assert_eq!(response.status(), StatusCode::OK); + let standalone = handle_tsjs_dynamic( + &build_request( + Method::GET, + &format!( + "https://publisher.example/static/tsjs=tsjs-probe.min.js?v={PROBE_JS_SHA256}" + ), + ), + ®istry, + EdgeCacheHeader::SMaxageFallback, + ) + .expect("should handle tsjs request"); assert_eq!( - response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("public, max-age=31536000, immutable"), - "matching content-versioned bundle should be immutable" + standalone.status(), + StatusCode::OK, + "should serve a module its registration marks standalone" + ); + assert!( + cache_control_text(&standalone).contains("immutable"), + "should treat the carried module's own hash as the matching version" ); assert_eq!( - response - .headers() - .get("surrogate-control") - .and_then(|value| value.to_str().ok()), - Some("max-age=31536000"), - "matching content-versioned bundle should set the Fastly edge TTL" + body_text(standalone), + PROBE_JS, + "should serve the standalone module as its own file" ); - } - #[test] - fn tsjs_dynamic_uses_cloudflare_edge_header_when_selected() { - let settings = create_test_settings(); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); - let module_ids = registry.js_module_ids_immediate(); - let hash = trusted_server_js::concatenated_hash(&module_ids); - let request = build_request( - Method::GET, - &format!("https://publisher.example/static/tsjs=tsjs-unified.min.js?v={hash}"), + let unified = handle_tsjs_dynamic( + &build_request( + Method::GET, + "https://publisher.example/static/tsjs=tsjs-unified.min.js", + ), + ®istry, + EdgeCacheHeader::SMaxageFallback, + ) + .expect("should handle tsjs request"); + assert!( + !body_text(unified).contains(PROBE_JS), + "a standalone module should stay out of the unified bundle" ); - - let response = handle_tsjs_dynamic( - &request, + + let bundled_only = handle_tsjs_dynamic( + &build_request( + Method::GET, + "https://publisher.example/static/tsjs=tsjs-lockr.min.js", + ), ®istry, - EdgeCacheHeader::CloudflareCdnCacheControl, + EdgeCacheHeader::SMaxageFallback, ) .expect("should handle tsjs request"); - assert_eq!( - response - .headers() - .get("cloudflare-cdn-cache-control") - .and_then(|value| value.to_str().ok()), - Some("max-age=31536000"), - "Cloudflare requests should use its edge cache header" - ); - assert!( - response.headers().get("surrogate-control").is_none(), - "Cloudflare requests should not emit Fastly's edge cache header" + bundled_only.status(), + StatusCode::NOT_FOUND, + "should not serve a bundle-only module as a single file" ); } @@ -19495,7 +22664,9 @@ mod tests { let body = EdgeBody::from(compressed); let params = OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -19504,15 +22675,16 @@ mod tests { origin_url: "https://origin.example.com".to_string(), request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/".to_owned(), content_type: "text/css".to_string(), + permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let mut output = Vec::new(); @@ -19538,6 +22710,300 @@ mod tests { ); } + /// A selector is chosen for each document, so one that does not parse + /// cannot be refused when the settings load. The response is refused + /// instead of the parser panicking. + #[test] + fn a_middleware_whose_selector_does_not_parse_refuses_the_response() { + use crate::error::IntoHttpResponse as _; + use crate::integrations::registry_test_support::middleware_fixture as fixture; + use crate::middleware::{HTML_MEDIA_TYPE, PhaseEntries, PhaseEntry}; + + let mut settings = create_test_settings(); + settings.select_module("testing", fixture::MODULE); + settings.fetch = PhaseEntries::new(vec![PhaseEntry { + media_type: HTML_MEDIA_TYPE.to_owned(), + path: None, + middleware: vec![fixture::BROKEN.to_owned()], + }]); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let params = OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), + csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), + content_encoding: String::new(), + origin_host: "origin.example.com".to_string(), + origin_url: "https://origin.example.com".to_string(), + request_host: "proxy.example.com".to_string(), + request_scheme: "https".to_string(), + request_path: "/".to_owned(), + content_type: "text/html; charset=utf-8".to_string(), + permissions_json: String::new(), + ad_slots_script: None, + ad_bids_state: AdBidsState::default(), + auction_observation: None, + auction_request: None, + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), + request_state: IntegrationRequestState::default(), + }; + + let mut output = Vec::new(); + let error = stream_publisher_body( + EdgeBody::from(b"x".to_vec()), + &mut output, + ¶ms, + &settings, + ®istry, + ) + .expect_err("should refuse a page whose middleware cannot run"); + + let message = error.to_string(); + assert!( + message.contains(&format!( + "the fetch middleware `{}` cannot run", + fixture::BROKEN + )) && message.contains(&format!( + "`{}` is not a usable selector", + fixture::BROKEN_SELECTOR + )), + "should name the middleware and the selector: {message}" + ); + assert_eq!( + error.current_context().status_code(), + StatusCode::INTERNAL_SERVER_ERROR, + "should answer as a fault of the deployment's own" + ); + assert!(output.is_empty(), "should write nothing of the page"); + } + + /// The same refusal through each finalizer, so an adapter answers with an + /// error where it would otherwise send the headers of a page it cannot + /// deliver. + #[tokio::test] + async fn a_middleware_whose_selector_does_not_parse_fails_the_response_through_each_finalizer() + { + use crate::error::IntoHttpResponse as _; + use crate::integrations::registry_test_support::middleware_fixture as fixture; + use crate::middleware::{HTML_MEDIA_TYPE, PhaseEntries, PhaseEntry}; + + let mut settings = create_test_settings(); + settings.select_module("testing", fixture::MODULE); + settings.fetch = PhaseEntries::new(vec![PhaseEntry { + media_type: HTML_MEDIA_TYPE.to_owned(), + path: None, + middleware: vec![fixture::BROKEN.to_owned()], + }]); + let settings = Arc::new(settings); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let orchestrator = Arc::new(AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + )); + let services = noop_services(); + let page = || PublisherResponse::Stream { + response: Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "text/html; charset=utf-8") + .body(EdgeBody::empty()) + .expect("should build the response"), + body: EdgeBody::from( + b"x".to_vec(), + ), + params: Box::new(OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), + csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), + content_encoding: String::new(), + origin_host: "origin.example.com".to_string(), + origin_url: "https://origin.example.com".to_string(), + request_host: "proxy.example.com".to_string(), + request_scheme: "https".to_string(), + request_path: "/".to_owned(), + content_type: "text/html; charset=utf-8".to_string(), + permissions_json: String::new(), + ad_slots_script: None, + ad_bids_state: AdBidsState::default(), + auction_observation: None, + auction_request: None, + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), + request_state: IntegrationRequestState::default(), + }), + }; + + let streamed = publisher_response_into_streaming_response( + page(), + &Method::GET, + Arc::clone(&settings), + ®istry, + Arc::clone(&orchestrator), + services.clone(), + ) + .await; + let buffered = buffer_publisher_response_async( + page(), + &Method::GET, + &settings, + ®istry, + &orchestrator, + &services, + ) + .await; + + for (finalizer, result) in [("streaming", streamed), ("buffered", buffered)] { + let error = result + .err() + .unwrap_or_else(|| panic!("should fail the response in the {finalizer} finalizer")); + assert!( + error.to_string().contains(&format!( + "the fetch middleware `{}` cannot run", + fixture::BROKEN + )), + "should name the middleware in the {finalizer} finalizer: {error}" + ); + assert_eq!( + error.current_context().status_code(), + StatusCode::INTERNAL_SERVER_ERROR, + "should answer as a fault of the deployment's own in the {finalizer} finalizer" + ); + } + } + + /// A serve entry that cannot run fails the response at each place a page + /// leaves from, being a page that is not stored and a page from the + /// store, through each finalizer. + #[tokio::test] + async fn a_serve_middleware_whose_selector_does_not_parse_fails_the_response_wherever_it_leaves() + { + use crate::error::IntoHttpResponse as _; + use crate::integrations::registry_test_support::middleware_fixture as fixture; + use crate::middleware::{HTML_MEDIA_TYPE, PhaseEntries, PhaseEntry}; + + let mut settings = create_test_settings(); + settings.select_module("testing", fixture::MODULE); + settings.serve = PhaseEntries::new(vec![PhaseEntry { + media_type: HTML_MEDIA_TYPE.to_owned(), + path: None, + middleware: vec![fixture::BROKEN_READER.to_owned()], + }]); + let settings = Arc::new(settings); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let orchestrator = Arc::new(AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + )); + let services = noop_services(); + let response = || { + Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "text/html; charset=utf-8") + .body(EdgeBody::empty()) + .expect("should build the response") + }; + let params = || { + Box::new(OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), + csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), + content_encoding: String::new(), + origin_host: "origin.example.com".to_string(), + origin_url: "https://origin.example.com".to_string(), + request_host: "proxy.example.com".to_string(), + request_scheme: "https".to_string(), + request_path: "/".to_owned(), + content_type: "text/html; charset=utf-8".to_string(), + permissions_json: String::new(), + ad_slots_script: None, + ad_bids_state: AdBidsState::default(), + auction_observation: None, + auction_request: None, + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), + request_state: IntegrationRequestState::default(), + }) + }; + let from_the_origin = || PublisherResponse::Stream { + response: response(), + body: EdgeBody::from( + b"x".to_vec(), + ), + params: params(), + }; + let from_the_store = || PublisherResponse::AssembleTemplate { + response: response(), + template: format!( + "x{AD_ASSEMBLY_SEAM}" + ) + .into_bytes(), + params: params(), + }; + + let mut results = Vec::new(); + let pages: [(&str, &dyn Fn() -> PublisherResponse); 2] = [ + ("from the origin", &from_the_origin), + ("from the store", &from_the_store), + ]; + for (page, publisher_response) in pages { + results.push(( + page, + "streaming", + publisher_response_into_streaming_response( + publisher_response(), + &Method::GET, + Arc::clone(&settings), + ®istry, + Arc::clone(&orchestrator), + services.clone(), + ) + .await, + )); + results.push(( + page, + "buffered", + buffer_publisher_response_async( + publisher_response(), + &Method::GET, + &settings, + ®istry, + &orchestrator, + &services, + ) + .await, + )); + } + + for (page, finalizer, result) in results { + let error = result.err().unwrap_or_else(|| { + panic!("should fail a page {page} in the {finalizer} finalizer") + }); + assert!( + error.to_string().contains(&format!( + "the serve middleware `{}` cannot run", + fixture::BROKEN_READER + )), + "should name the middleware for a page {page} in the {finalizer} finalizer: \ + {error}" + ); + assert_eq!( + error.current_context().status_code(), + StatusCode::INTERNAL_SERVER_ERROR, + "should answer as a fault of the deployment's own for a page {page} in the \ + {finalizer} finalizer" + ); + } + } + /// Empty origin body on the streaming route must produce no output /// without erroring. Exercises the `Ok(0)` branch of `process_chunks` /// plus the processor's `is_last=true, chunk=[]` terminal call. @@ -19554,7 +23020,9 @@ mod tests { .expect("should create integration registry"); let params = OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -19563,15 +23031,16 @@ mod tests { origin_url: "https://origin.example.com".to_string(), request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), + permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let mut output = Vec::new(); @@ -19602,7 +23071,9 @@ mod tests { ) .expect("should create integration registry"); let params = OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -19611,15 +23082,16 @@ mod tests { origin_url: "https://origin.example.com".to_string(), request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), + permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let body = EdgeBody::from_stream(futures::stream::iter(vec![Ok::<_, io::Error>( bytes::Bytes::from_static(b"live"), @@ -19635,8 +23107,8 @@ mod tests { ); } - #[test] - fn body_chunk_source_yields_once_body_in_chunks() { + #[tokio::test] + async fn body_chunk_source_yields_once_body_in_chunks() { futures::executor::block_on(async { let body = EdgeBody::from_bytes(bytes::Bytes::from_static(b"abcdef")); let mut source = BodyChunkSource::new(body, 3).with_max_bytes(16); @@ -19658,8 +23130,8 @@ mod tests { }); } - #[test] - fn body_chunk_source_preserves_stream_chunks() { + #[tokio::test] + async fn body_chunk_source_preserves_stream_chunks() { futures::executor::block_on(async { let body = EdgeBody::stream(futures::stream::iter(vec![ bytes::Bytes::from_static(b"first"), @@ -19684,8 +23156,8 @@ mod tests { }); } - #[test] - fn body_chunk_source_enforces_cumulative_raw_cap() { + #[tokio::test] + async fn body_chunk_source_enforces_cumulative_raw_cap() { futures::executor::block_on(async { let body = EdgeBody::stream(futures::stream::iter(vec![ bytes::Bytes::from_static(b"1234"), @@ -19713,8 +23185,8 @@ mod tests { }); } - #[test] - fn stream_publisher_body_async_processes_stream_without_auction() { + #[tokio::test] + async fn stream_publisher_body_async_processes_stream_without_auction() { futures::executor::block_on(async { let settings = create_test_settings(); let registry = IntegrationRegistry::with_plan( @@ -19725,10 +23197,14 @@ mod tests { ), ) .expect("should create integration registry"); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let services = noop_services(); let mut params = OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -19737,15 +23213,16 @@ mod tests { origin_url: "https://origin.example.com".to_string(), request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/".to_owned(), content_type: "text/css".to_string(), + permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let body = EdgeBody::stream(futures::stream::iter(vec![ bytes::Bytes::from_static(b"body{background:url('https://origin.example.com/"), @@ -19777,8 +23254,8 @@ mod tests { }); } - #[test] - fn stream_publisher_body_async_processes_gzip_stream_without_auction() { + #[tokio::test] + async fn stream_publisher_body_async_processes_gzip_stream_without_auction() { futures::executor::block_on(async { let settings = create_test_settings(); let registry = IntegrationRegistry::with_plan( @@ -19789,10 +23266,14 @@ mod tests { ), ) .expect("should create integration registry"); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let services = noop_services(); let mut params = OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -19801,15 +23282,16 @@ mod tests { origin_url: "https://origin.example.com".to_string(), request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/".to_owned(), content_type: "text/css".to_string(), + permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let compressed = gzip_encode(b"body{background:url('https://origin.example.com/asset.png')}"); @@ -19844,8 +23326,8 @@ mod tests { }); } - #[test] - fn stream_publisher_body_async_processes_deflate_stream_without_auction() { + #[tokio::test] + async fn stream_publisher_body_async_processes_deflate_stream_without_auction() { futures::executor::block_on(async { let settings = create_test_settings(); let registry = IntegrationRegistry::with_plan( @@ -19856,10 +23338,14 @@ mod tests { ), ) .expect("should create integration registry"); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let services = noop_services(); let mut params = OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -19868,15 +23354,16 @@ mod tests { origin_url: "https://origin.example.com".to_string(), request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/".to_owned(), content_type: "text/css".to_string(), + permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let compressed = deflate_encode(b"body{background:url('https://origin.example.com/asset.png')}"); @@ -19911,8 +23398,8 @@ mod tests { }); } - #[test] - fn stream_publisher_body_async_processes_brotli_stream_without_auction() { + #[tokio::test] + async fn stream_publisher_body_async_processes_brotli_stream_without_auction() { futures::executor::block_on(async { let settings = create_test_settings(); let registry = IntegrationRegistry::with_plan( @@ -19923,10 +23410,14 @@ mod tests { ), ) .expect("should create integration registry"); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let services = noop_services(); let mut params = OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -19935,15 +23426,16 @@ mod tests { origin_url: "https://origin.example.com".to_string(), request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/".to_owned(), content_type: "text/css".to_string(), + permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let compressed = brotli_encode(b"body{background:url('https://origin.example.com/asset.png')}"); @@ -19978,8 +23470,8 @@ mod tests { }); } - #[test] - fn stream_publisher_body_async_rejects_truncated_brotli_stream() { + #[tokio::test] + async fn stream_publisher_body_async_rejects_truncated_brotli_stream() { futures::executor::block_on(async { let settings = create_test_settings(); let registry = IntegrationRegistry::with_plan( @@ -19990,10 +23482,14 @@ mod tests { ), ) .expect("should create integration registry"); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let services = noop_services(); let mut params = OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -20002,15 +23498,16 @@ mod tests { origin_url: "https://origin.example.com".to_string(), request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/".to_owned(), content_type: "text/css".to_string(), + permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let compressed = brotli_encode(b"body{background:url('https://origin.example.com/asset.png')}"); @@ -20042,7 +23539,9 @@ mod tests { fn non_html_stream_params(content_encoding: &str) -> OwnedProcessResponseParams { OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -20051,20 +23550,21 @@ mod tests { origin_url: "https://origin.example.com".to_string(), request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/".to_owned(), content_type: "text/css".to_string(), + permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), } } - #[test] - fn stream_publisher_body_async_rejects_truncated_gzip_stream() { + #[tokio::test] + async fn stream_publisher_body_async_rejects_truncated_gzip_stream() { futures::executor::block_on(async { let settings = create_test_settings(); let registry = IntegrationRegistry::with_plan( @@ -20075,7 +23575,9 @@ mod tests { ), ) .expect("should create integration registry"); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let services = noop_services(); let mut params = non_html_stream_params("gzip"); let compressed = @@ -20106,8 +23608,8 @@ mod tests { }); } - #[test] - fn stream_publisher_body_async_rejects_truncated_deflate_stream() { + #[tokio::test] + async fn stream_publisher_body_async_rejects_truncated_deflate_stream() { futures::executor::block_on(async { let settings = create_test_settings(); let registry = IntegrationRegistry::with_plan( @@ -20118,7 +23620,9 @@ mod tests { ), ) .expect("should create integration registry"); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let services = noop_services(); let mut params = non_html_stream_params("deflate"); let compressed = @@ -20150,8 +23654,8 @@ mod tests { }); } - #[test] - fn stream_publisher_body_async_enforces_decoded_byte_cap() { + #[tokio::test] + async fn stream_publisher_body_async_enforces_decoded_byte_cap() { futures::executor::block_on(async { let mut settings = create_test_settings(); // Raw compressed input stays tiny (well under the cap); only the @@ -20166,7 +23670,9 @@ mod tests { ), ) .expect("should create integration registry"); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let services = noop_services(); let mut params = non_html_stream_params("gzip"); let compressed = gzip_encode(&vec![b'a'; 64 * 1024]); @@ -20197,8 +23703,8 @@ mod tests { }); } - #[test] - fn body_chunk_source_resumes_after_cancelled_poll() { + #[tokio::test] + async fn body_chunk_source_resumes_after_cancelled_poll() { futures::executor::block_on(async { let mut pending_once = true; let mut yielded = false; @@ -20243,8 +23749,8 @@ mod tests { }); } - #[test] - fn stream_publisher_body_async_processes_stream_with_auction_hold() { + #[tokio::test] + async fn stream_publisher_body_async_processes_stream_with_auction_hold() { futures::executor::block_on(async { let settings = create_test_settings(); let registry = IntegrationRegistry::with_plan( @@ -20255,11 +23761,15 @@ mod tests { ), ) .expect("should create integration registry"); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let services = noop_services(); let state = AdBidsState::default(); let mut params = OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -20268,7 +23778,9 @@ mod tests { origin_url: "https://origin.example.com".to_string(), request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), + permissions_json: String::new(), ad_slots_script: Some( r#""# .to_string(), @@ -20281,8 +23793,7 @@ mod tests { 10, )), price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let body = EdgeBody::stream(futures::stream::iter(vec![ bytes::Bytes::from_static(b"hello"), @@ -20318,8 +23829,8 @@ mod tests { }); } - #[test] - fn stream_publisher_body_async_auction_hold_decodes_multi_member_gzip_buffered() { + #[tokio::test] + async fn stream_publisher_body_async_auction_hold_decodes_multi_member_gzip_buffered() { futures::executor::block_on(async { let settings = create_test_settings(); let registry = IntegrationRegistry::with_plan( @@ -20330,11 +23841,15 @@ mod tests { ), ) .expect("should create integration registry"); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let services = noop_services(); let state = AdBidsState::default(); let mut params = OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -20343,7 +23858,9 @@ mod tests { origin_url: "https://origin.example.com".to_string(), request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), + permissions_json: String::new(), ad_slots_script: Some( r#""# .to_string(), @@ -20356,8 +23873,7 @@ mod tests { 10, )), price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; // The `` that triggers bid injection lives in the SECOND gzip // member. `flate2::read::GzDecoder` decodes only the first member, so @@ -20396,8 +23912,8 @@ mod tests { }); } - #[test] - fn stream_publisher_body_async_processes_non_html_stream_after_auction_collect() { + #[tokio::test] + async fn stream_publisher_body_async_processes_non_html_stream_after_auction_collect() { futures::executor::block_on(async { let settings = create_test_settings(); let registry = IntegrationRegistry::with_plan( @@ -20408,10 +23924,14 @@ mod tests { ), ) .expect("should create integration registry"); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let services = noop_services(); let mut params = OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -20420,7 +23940,9 @@ mod tests { origin_url: "https://origin.example.com".to_string(), request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/".to_owned(), content_type: "text/css".to_string(), + permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), auction_observation: None, @@ -20430,8 +23952,7 @@ mod tests { 10, )), price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let body = EdgeBody::stream(futures::stream::iter(vec![bytes::Bytes::from_static( b"body{background:url('https://origin.example.com/asset.png')}", @@ -20474,7 +23995,9 @@ mod tests { ) .expect("should create integration registry"), ); - let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let orchestrator = Arc::new(AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + )); let services = noop_services(); let response = Response::builder() .status(StatusCode::OK) @@ -20482,7 +24005,9 @@ mod tests { .body(EdgeBody::empty()) .expect("should build response"); let params = OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -20491,15 +24016,16 @@ mod tests { origin_url: "https://origin.example.com".to_string(), request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/".to_owned(), content_type: "text/css".to_string(), + permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let publisher_response = PublisherResponse::Stream { response, @@ -20606,7 +24132,9 @@ mod tests { ) .expect("should create integration registry"), ); - let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let orchestrator = Arc::new(AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + )); let services = noop_services(); let response = Response::builder() .status(StatusCode::OK) @@ -20635,7 +24163,9 @@ mod tests { dispatched_auction: Option, ) -> OwnedProcessResponseParams { OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -20644,7 +24174,9 @@ mod tests { origin_url: "https://origin.example.com".to_string(), request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), + permissions_json: String::new(), ad_slots_script: Some( r#""# .to_string(), @@ -20654,24 +24186,29 @@ mod tests { auction_request: dispatched_auction.as_ref().map(|_| test_auction_request()), dispatched_auction, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), } } #[test] - fn streaming_finalize_emits_gam_attribution_head_before_origin_eof() { + fn streaming_finalize_emits_an_integration_s_head_before_origin_eof() { + use crate::integrations::registry_test_support::tag_fixture as tag; + let mut settings = create_test_settings(); settings - .integrations - .insert_config( - "gpt", + .insert_module_config( + "testing", + tag::MODULE, &serde_json::json!({ - "enabled": true, - "gam_attribution_enabled": true + "mark_bundle": true }), ) - .expect("should insert GPT config"); + .expect("should insert the stand-in's settings"); + crate::html_processor::test_support::place_on_every_page( + &mut settings, + crate::middleware::MiddlewarePhase::Fetch, + &[tag::MODULE], + ); let body = streaming_finalize_response_with_settings( html_stream_params("", None), @@ -20684,12 +24221,12 @@ mod tests { .expect("should emit UTF-8 HTML"); assert!( - html.contains("__tsjs_gam_attribution_enabled=true"), - "first rewritten head chunk should carry the primary activation flag: {html}" + html.contains(tag::HEAD_FLAG), + "first rewritten head chunk should carry the integration's head insert: {html}" ); assert!( - html.contains("data-ts-gam-attribution=\"true\""), - "first rewritten head chunk should authorize the bundle fallback: {html}" + html.contains(&format!("{}=\"true\"", tag::BUNDLE_ATTRIBUTE)), + "first rewritten head chunk should carry the bundle tag's attribute: {html}" ); } @@ -20737,18 +24274,11 @@ mod tests { fn streaming_finalize_auction_hold_emits_prefix_before_origin_eof() { // A body-close literal in script data must not stop streaming. Only the // request token emitted by lol_html at the structural end is a seam. - let page = br#"
still streaming
"#; + let page = br#"
still streaming
"#; let mut settings = create_test_settings(); - settings - .integrations - .insert_config( - "nextjs", - &serde_json::json!({ - "enabled": true, - "rewrite_attributes": ["href", "link", "url"], - }), - ) - .expect("should enable Next.js"); + crate::integrations::registry_test_support::payload_fixture::select_and_place( + &mut settings, + ); let params = html_stream_params( "", Some(DispatchedAuction::empty_for_test( @@ -20766,11 +24296,11 @@ mod tests { let html = String::from_utf8(first.to_vec()).expect("should be valid UTF-8"); assert!( html.contains("") && html.contains("still streaming"), - "RSC script data and later article bytes must stream before EOF. Got: {html}" + "script data and later article bytes must stream before EOF. Got: {html}" ); assert!( html.contains("proxy.example.com/app") && !html.contains("origin.example.com/app"), - "Next.js rewriting must complete before the parser seam: {html}" + "the stand-in's rewriting must complete before the parser seam: {html}" ); assert!( html.contains(".adSlots=JSON.parse"), @@ -20871,7 +24401,9 @@ mod tests { #[async_trait::async_trait(?Send)] impl AuctionProvider for GatedAuctionProvider { fn provider_name(&self) -> &'static str { - "seam-test" + // A demand name is snake_case, and it has to match the + // `[demand]` entry the test selects for the auction to route to it. + "seam_test" } async fn request_bids( @@ -20914,24 +24446,15 @@ mod tests { } #[test] - fn parser_confirmed_auction_seam_streams_nextjs_for_every_encoding() { + fn parser_confirmed_auction_seam_streams_a_two_pass_integration_for_every_encoding() { for encoding in ["", "gzip", "deflate", "br"] { let mut settings = create_test_settings(); settings.auction.enabled = true; - settings.auction.providers = - crate::auction::AuctionConfig::legacy_provider_map(&["seam-test"]); + settings.auction.provider_names = vec!["seam_test".to_owned()]; settings.auction.timeout_ms = 60_000; - settings.auction.mediator = None; - settings - .integrations - .insert_config( - "nextjs", - &serde_json::json!({ - "enabled": true, - "rewrite_attributes": ["href", "link", "url"], - }), - ) - .expect("should enable Next.js"); + crate::integrations::registry_test_support::payload_fixture::select_and_place( + &mut settings, + ); let client = Arc::new(GatedAuctionHttpClient { inner: StubHttpClient::new(), released: std::sync::atomic::AtomicBool::new(false), @@ -20958,15 +24481,10 @@ mod tests { else { panic!("should dispatch a pending auction"); }; - let payload = r#"{"url":"https://origin.example.com/path","text":""}"#; - let split = payload.find("/path").expect("should find payload split"); - let first_payload = format!("1:T{:x},{}", payload.len(), &payload[..split]); - let first_script = - serde_json::to_string(&first_payload).expect("should encode first payload"); - let second_script = - serde_json::to_string(&payload[split..]).expect("should encode second payload"); - let prefix = format!( - "

before RSC

between scripts
still streaming
" + // One group split across two scripts, so the stand-in holds its + // output from the first script until the second arrives. + let prefix = String::from( + "

before the group

between scripts
still streaming
", ); let page = format!("{prefix}"); let encoded = match encoding { @@ -21029,7 +24547,7 @@ mod tests { ); assert!( prefix.contains("proxy.example.com") && !prefix.contains("origin.example.com"), - "should rewrite the split RSC group for {encoding}: {prefix}" + "should rewrite the split group for {encoding}: {prefix}" ); assert!( !prefix.contains("var b=JSON.parse("), @@ -21077,7 +24595,8 @@ mod tests { "should inject once" ); assert!( - !html.contains("ts-inline-body-close-") && !html.contains("__ts_rsc_"), + !html.contains("ts-inline-body-close-") + && !html.contains(crate::integrations::registry_test_support::payload_fixture::PLACEHOLDER_PREFIX), "should remove internal markers for {encoding}: {html}" ); assert_eq!( @@ -21159,7 +24678,9 @@ mod tests { ) .expect("should create integration registry"), ); - let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let orchestrator = Arc::new(AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + )); for (method, status, expected_length, expected_transfer_encoding) in BODILESS_FRAMING_CASES { @@ -21224,7 +24745,9 @@ mod tests { ), ) .expect("should create integration registry"); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let services = noop_services(); for (method, status, expected_length, expected_transfer_encoding) in BODILESS_FRAMING_CASES @@ -21310,13 +24833,17 @@ mod tests { ), ) .expect("should create integration registry"); - let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let orchestrator = Arc::new(AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + )); let make_params = || { let ec_context = EcContext::new_for_test(None, crate::consent::types::ConsentContext::default()); OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -21325,7 +24852,9 @@ mod tests { origin_url: "https://origin.example.com".to_string(), request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), + permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), auction_observation: Some(AuctionObservationContext::from_parts( @@ -21342,8 +24871,7 @@ mod tests { 10, )), price_granularity: PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), } }; let make_stream_response = || PublisherResponse::Stream { @@ -21491,7 +25019,9 @@ mod tests { ) .expect("should create integration registry"), ); - let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let orchestrator = Arc::new(AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + )); let services = noop_services(); let response = Response::builder() .status(StatusCode::OK) @@ -21511,7 +25041,9 @@ mod tests { .map(bytes::Bytes::copy_from_slice) .collect(); let params = OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -21520,7 +25052,9 @@ mod tests { origin_url: "https://origin.example.com".to_string(), request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), + permissions_json: String::new(), ad_slots_script: Some( r#""# .to_string(), @@ -21533,8 +25067,7 @@ mod tests { 10, )), price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let publisher_response = PublisherResponse::Stream { response, @@ -21592,7 +25125,9 @@ mod tests { r#""#; let state = AdBidsState::with_script(bids_script); let params = OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -21601,7 +25136,9 @@ mod tests { origin_url: "https://origin.example.com".to_string(), request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/".to_owned(), content_type: "Text/HTML; Charset=utf-8".to_string(), + permissions_json: String::new(), ad_slots_script: Some( r#""# .to_string(), @@ -21611,8 +25148,7 @@ mod tests { auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let mut output = Vec::new(); @@ -21656,7 +25192,9 @@ mod tests { // Claim gzip encoding but feed non-gzip bytes. The GzDecoder will // error as soon as it tries to read the gzip header. let params = OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -21665,15 +25203,16 @@ mod tests { origin_url: "https://origin.example.com".to_string(), request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/".to_owned(), content_type: "text/html".to_string(), + permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let bogus_body = EdgeBody::from(b"not gzip".to_vec()); @@ -21731,18 +25270,12 @@ mod tests { /// routes through `Stream`, and the shared processor pipeline applies it. #[test] fn streaming_html_with_stream_processors_rewrites_body() { - // Configure nextjs so a stream processor is registered. + // Select the stand-in and place its middleware, so the page has a + // stream processor. let mut settings = create_test_settings(); - settings - .integrations - .insert_config( - "nextjs", - &serde_json::json!({ - "enabled": true, - "rewrite_attributes": ["href", "link", "url"], - }), - ) - .expect("should update nextjs config"); + crate::integrations::registry_test_support::payload_fixture::select_and_place( + &mut settings, + ); let registry = IntegrationRegistry::with_plan( &settings, @@ -21754,8 +25287,15 @@ mod tests { .expect("should create integration registry"); assert!( - !registry.html_stream_processor_factories().is_empty(), - "nextjs integration must register an HTML stream processor" + !registry + .middleware_chain( + &settings.fetch, + crate::middleware::MiddlewarePhase::Fetch, + crate::middleware::HTML_MEDIA_TYPE, + "/", + ) + .is_empty(), + "the stand-in's middleware, which supplies an HTML stream processor, must run on the page" ); assert_eq!( classify_response_route( @@ -21774,7 +25314,9 @@ mod tests { let body = EdgeBody::from(html.to_vec()); let params = OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -21783,15 +25325,16 @@ mod tests { origin_url: "https://origin.example.com".to_string(), request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), + permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let mut output = Vec::new(); stream_publisher_body(body, &mut output, ¶ms, &settings, ®istry) @@ -21812,23 +25355,15 @@ mod tests { ); } - /// Document-state survives from the parser pass into the stream processor. - /// `NextJsRscPlaceholderRewriter` writes into `IntegrationDocumentState` - /// during parsing; the request-local stream processor reads it and substitutes. - /// Regression test: placeholders must be inserted and removed from final output. - #[test] - fn document_state_placeholders_substitute_through_streaming_path() { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config( - "nextjs", - &serde_json::json!({ - "enabled": true, - "rewrite_attributes": ["href", "link", "url"], - }), - ) - .expect("should update nextjs config"); + /// What a text handler captures during parsing reaches the stream + /// processor of the same document, which substitutes it. + /// Regression test: placeholders must be inserted and removed from final output. + #[test] + fn a_handler_s_placeholders_are_substituted_through_the_streaming_path() { + let mut settings = create_test_settings(); + crate::integrations::registry_test_support::payload_fixture::select_and_place( + &mut settings, + ); let registry = IntegrationRegistry::with_plan( &settings, Arc::new( @@ -21838,10 +25373,12 @@ mod tests { ) .expect("should create integration registry"); - // Small, single-fragment RSC script — placeholder path (not fallback). - let html = br#""#; + // One small script whose payload is swapped for a placeholder. + let html = br#""#; let params = OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -21850,15 +25387,16 @@ mod tests { origin_url: "https://origin.example.com".to_string(), request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/".to_owned(), content_type: "text/html".to_string(), + permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let mut output = Vec::new(); @@ -21869,11 +25407,13 @@ mod tests { &settings, ®istry, ) - .expect("should process RSC push"); + .expect("should process the payload script"); let processed = String::from_utf8(output).expect("valid UTF-8"); assert!( - !processed.contains("__ts_rsc_payload_"), + !processed.contains( + crate::integrations::registry_test_support::payload_fixture::PLACEHOLDER_PREFIX + ), "placeholder must be substituted before reaching output. Got: {processed}" ); assert!( @@ -21889,11 +25429,11 @@ mod tests { #[cfg(test)] mod creative_opportunities_tests { use super::super::{ - AdBidsState, MatchedSlotsContext, build_ad_slots_script, build_auction_request, - build_bid_map, build_bids_script, diagnostics_auction_id, html_escape_for_script, + AdBidsState, MatchedSlotsContext, auction_token, build_ad_slots_script, + build_auction_request, build_bid_map, build_bids_script, html_escape_for_script, write_bids_to_state, }; - use crate::auction::types::{ApsRendererV1, ApsTagType, Bid, BidRenderer, MediaType}; + use crate::auction::types::{Bid, BidRenderer, MediaType}; use crate::consent::ConsentContext; use crate::creative_opportunities::{ CreativeOpportunitiesConfig, CreativeOpportunityFormat, CreativeOpportunitySlot, @@ -22159,28 +25699,28 @@ mod tests { } /// Guards the browser-visible token every auction path shares: it must - /// be fresh per auction and absent unless diagnostics can consume it. + /// be fresh per auction and absent unless a module reads it. #[test] - fn diagnostics_auction_id_is_fresh_and_gated() { - let mut settings = test_settings(); + fn auction_token_is_fresh_and_made_only_where_a_module_reads_it() { + let settings = test_settings(); + let orchestrator = crate::auction::orchestrator::AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); assert_eq!( - diagnostics_auction_id(&settings), + auction_token(&orchestrator), None, - "no token should be minted without the diagnostics integration" + "no token should be minted where no module reads one" ); - settings - .integrations - .insert_config("gpt_diagnostics", &serde_json::json!({ "enabled": true })) - .expect("should enable diagnostics"); + let orchestrator = orchestrator.publishing_auction_token(); let first = - diagnostics_auction_id(&settings).expect("enabled diagnostics should mint a token"); + auction_token(&orchestrator).expect("should mint a token where a module reads one"); let second = - diagnostics_auction_id(&settings).expect("enabled diagnostics should mint a token"); + auction_token(&orchestrator).expect("should mint a token where a module reads one"); assert!( first.starts_with("ts-auc-"), - "token should use the diagnostics prefix, got `{first}`" + "token should use the auction token prefix, got `{first}`" ); assert_ne!(first, second, "each auction should mint its own token"); } @@ -23063,38 +26603,40 @@ mod tests { } #[test] - fn bid_map_exposes_aps_renderer_and_selected_bid_id() { + fn bid_map_exposes_a_typed_renderer_and_the_bid_it_picks() { // Sanitization is opt-in, so enable it: the script-only creative // below is what drives this bid onto the renderer path. Left at the // default it would survive processing as an ordinary creative. let mut settings = test_settings(); settings.auction.sanitize_creatives = true; - let mut bid = make_bid("atf_sidebar_ad", 1.50, "aps", "fallback-ad", "", ""); + let mut bid = make_bid("atf_sidebar_ad", 1.50, "example", "fallback-ad", "", ""); bid.bid_id = Some("selected-bid".to_string()); bid.creative = Some("".to_string()); bid.nurl = None; bid.burl = None; - bid.renderer = Some(BidRenderer::Aps(ApsRendererV1 { - version: 1, - account_id: "example-account".to_string(), - bid_id: "selected-bid".to_string(), - creative_id: None, - tag_type: ApsTagType::Iframe, - creative_url: "https://creative.example/render".to_string(), - aax_response: "fictional-base64".to_string(), - width: 300, - height: 250, - })); + bid.renderer = Some( + BidRenderer::new( + "example", + serde_json::json!({ + "version": 1, + "bidId": "selected-bid", + "creativeUrl": "https://creative.example/render", + "envelope": "fictional-base64", + }), + ) + .expect("should build the example renderer descriptor") + .picking_bid_by("bidId"), + ); let winning_bids = HashMap::from([("atf_sidebar_ad".to_string(), bid)]); let map = build_bid_map(&winning_bids, PriceGranularity::Dense, &settings, "", false); let obj = map["atf_sidebar_ad"] .as_object() - .expect("should include APS bid"); + .expect("should include the renderer bid"); - assert_eq!(obj["hb_bidder"], "aps"); + assert_eq!(obj["hb_bidder"], "example"); assert_eq!(obj["hb_adid"], "selected-bid"); - assert_eq!(obj["renderer"]["type"], "aps"); + assert_eq!(obj["renderer"]["type"], "example"); assert_eq!(obj["renderer"]["bidId"], "selected-bid"); assert!(obj.get("adm").is_none()); @@ -23103,6 +26645,67 @@ mod tests { assert!(script.contains("\\u003C/script\\u003E")); } + #[test] + fn bid_map_prefers_the_renderer_bid_id_over_ad_id_and_the_openrtb_bid_id() { + // Every hb_adid source carries a different value, so the assertion + // below passes only when the renderer field is the one read. The + // envelope is oversized on purpose: reading this field must not + // copy it. + let mut settings = test_settings(); + settings.auction.sanitize_creatives = true; + let mut bid = make_bid("atf_sidebar_ad", 1.50, "example", "ad-id-value", "", ""); + bid.bid_id = Some("openrtb-bid-id".to_string()); + bid.cache_id = None; + bid.creative = Some("".to_string()); + bid.renderer = Some( + BidRenderer::new( + "example", + serde_json::json!({ + "version": 1, + "bidId": "renderer-bid-id", + "creativeUrl": "https://creative.example/render", + "envelope": "A".repeat(200 * 1024), + }), + ) + .expect("should build the example renderer descriptor") + .picking_bid_by("bidId"), + ); + let winning_bids = HashMap::from([("atf_sidebar_ad".to_string(), bid)]); + + let map = build_bid_map(&winning_bids, PriceGranularity::Dense, &settings, "", false); + let obj = map["atf_sidebar_ad"] + .as_object() + .expect("should include the renderer bid"); + + assert_eq!( + obj["hb_adid"], "renderer-bid-id", + "should read hb_adid from the renderer payload rather than ad_id or bid_id" + ); + } + + #[test] + fn bid_map_ignores_a_renderer_bid_id_its_builder_did_not_state() { + let mut settings = test_settings(); + settings.auction.sanitize_creatives = true; + let mut bid = make_bid("atf_sidebar_ad", 1.50, "example", "ad-id-value", "", ""); + bid.cache_id = None; + bid.renderer = Some( + BidRenderer::new("example", serde_json::json!({ "bidId": "renderer-bid-id" })) + .expect("should build renderer descriptor"), + ); + let winning_bids = HashMap::from([("atf_sidebar_ad".to_string(), bid)]); + + let map = build_bid_map(&winning_bids, PriceGranularity::Dense, &settings, "", false); + let obj = map["atf_sidebar_ad"] + .as_object() + .expect("should include the bid"); + + assert_eq!( + obj["hb_adid"], "ad-id-value", + "should ignore a bidId the code that built the renderer did not state it picks its bid by" + ); + } + #[test] fn bid_map_omits_creative_rejected_by_processing_without_renderer() { // Sanitization is opt-in, so enable it: script-only markup is what @@ -23595,8 +27198,8 @@ mod tests { fn settings_with_co() -> Settings { let toml = format!( - "{}\n[auction]\nenabled = true\n\n[creative_opportunities]\ngam_network_id = \"12345\"\n", - crate_test_settings_str() + "{}\n[creative_opportunities]\ngam_network_id = \"12345\"\n", + crate_test_settings_str().replace("[auction]\n", "[auction]\nenabled = true\n") ); Settings::from_toml(&toml).expect("should parse settings with creative_opportunities") } @@ -23604,23 +27207,26 @@ mod tests { /// Settings for a deployment that has no `[creative_opportunities]` /// section, so page-bids answers `404`. fn settings_without_co() -> Settings { - let toml = format!("{}\n[auction]\nenabled = true\n", crate_test_settings_str()); + let toml = format!( + "{}\n", + crate_test_settings_str().replace("[auction]\n", "[auction]\nenabled = true\n") + ); Settings::from_toml(&toml) .expect("should parse settings without creative_opportunities") } fn settings_with_co_auction_disabled() -> Settings { let toml = format!( - "{}\n[auction]\nenabled = false\n\n[creative_opportunities]\ngam_network_id = \"12345\"\n", - crate_test_settings_str() + "{}\n[creative_opportunities]\ngam_network_id = \"12345\"\n", + crate_test_settings_str().replace("[auction]\n", "[auction]\nenabled = false\n") ); Settings::from_toml(&toml).expect("should parse settings with creative_opportunities") } fn settings_with_co_templates_disabled() -> Settings { let toml = format!( - "{}\n[auction]\nenabled = true\n\n[creative_opportunities]\nenabled = false\ngam_network_id = \"12345\"\n", - crate_test_settings_str() + "{}\n[creative_opportunities]\nenabled = false\ngam_network_id = \"12345\"\n", + crate_test_settings_str().replace("[auction]\n", "[auction]\nenabled = true\n") ); Settings::from_toml(&toml).expect("should parse settings with disabled templates") } @@ -23691,7 +27297,9 @@ mod tests { #[tokio::test] async fn page_bids_format_absent_or_json_returns_json() { let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); for path_and_format in ["/2024/article", "/2024/article&format=json"] { let response = run_page_bids_response( &settings, @@ -23723,7 +27331,9 @@ mod tests { #[tokio::test] async fn page_bids_format_rejects_removed_unknown_and_empty_values() { let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); for format in ["fragment", "scrpit", ""] { let response = run_page_bids_response( &settings, @@ -23802,23 +27412,34 @@ mod tests { captured_request: Arc>>, winning_bid: bool, ) -> AuctionOrchestrator { - let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let mut orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(settings), + ); orchestrator.register_provider(Arc::new(AuctionIdTestProvider { captured_request, winning_bid, })); + // The plan compiled for a deployment publishes a token when a + // selected module reads one, and the stand-in is such a module. + if settings + .selects_module(crate::integrations::registry_test_support::request_fixture::MODULE) + { + orchestrator = orchestrator.publishing_auction_token(); + } orchestrator } #[tokio::test] async fn page_bids_response_includes_auction_id_only_for_winning_bids() { let mut settings = settings_with_co(); - settings.auction.providers = - crate::auction::AuctionConfig::legacy_provider_map(&[AUCTION_ID_TEST_PROVIDER]); - settings - .integrations - .insert_config("gpt_diagnostics", &serde_json::json!({ "enabled": true })) - .expect("should enable diagnostics"); + settings.demand = crate::auction::test_support::demand_named( + crate::auction::test_support::plain_fixture::MODULE, + &[AUCTION_ID_TEST_PROVIDER], + ); + settings.select_module( + "testing", + crate::integrations::registry_test_support::request_fixture::MODULE, + ); let slots = article_slot(); let winning_stub = Arc::new(StubHttpClient::new()); winning_stub.push_response(200, b"winner".to_vec()); @@ -23873,7 +27494,7 @@ mod tests { .to_string(); assert!( winning_auction_id.starts_with("ts-auc-"), - "page-bids should expose a freshly minted diagnostics token, got `{winning_auction_id}`" + "page-bids should expose a freshly minted auction token, got `{winning_auction_id}`" ); assert_ne!( winning_auction_id, auction_request.id, @@ -23923,10 +27544,10 @@ mod tests { } /// The browser-visible auction ID is minted per auction and only for - /// deployments that run the diagnostics integration, so it can neither + /// deployments that run a module that reads it, so it can neither /// carry EC identity across auctions nor reach pages that ignore it. #[tokio::test] - async fn page_bids_auction_id_is_per_auction_and_gated_on_diagnostics() { + async fn page_bids_auction_id_is_per_auction_and_made_only_where_a_module_reads_it() { async fn winning_auction_id(settings: &Settings) -> Option { let slots = article_slot(); let stub = Arc::new(StubHttpClient::new()); @@ -23970,32 +27591,34 @@ mod tests { } let mut settings = settings_with_co(); - settings.auction.providers = - crate::auction::AuctionConfig::legacy_provider_map(&[AUCTION_ID_TEST_PROVIDER]); - settings - .integrations - .insert_config("gpt_diagnostics", &serde_json::json!({ "enabled": true })) - .expect("should enable diagnostics"); + settings.demand = crate::auction::test_support::demand_named( + crate::auction::test_support::plain_fixture::MODULE, + &[AUCTION_ID_TEST_PROVIDER], + ); + settings.select_module( + "testing", + crate::integrations::registry_test_support::request_fixture::MODULE, + ); let first = winning_auction_id(&settings) .await - .expect("first auction should expose a diagnostics token"); + .expect("first auction should expose an auction token"); let second = winning_auction_id(&settings) .await - .expect("second auction should expose a diagnostics token"); + .expect("second auction should expose an auction token"); assert_ne!( first, second, "each auction for the same visitor should mint its own token" ); - settings - .integrations - .insert_config("gpt_diagnostics", &serde_json::json!({ "enabled": false })) - .expect("should disable diagnostics"); + settings.remove_module( + "testing", + crate::integrations::registry_test_support::request_fixture::MODULE, + ); assert_eq!( winning_auction_id(&settings).await, None, - "no auction metadata should reach the page without the diagnostics integration" + "no auction metadata should reach the page where no module reads it" ); } @@ -24010,7 +27633,9 @@ mod tests { #[tokio::test] async fn deprecated_alias_response_matches_canonical_path() { let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let canonical = run_page_bids_response( &settings, @@ -24047,7 +27672,9 @@ mod tests { #[tokio::test] async fn deprecated_alias_response_is_marked_deprecated() { let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let canonical = run_page_bids_response( &settings, @@ -24091,7 +27718,9 @@ mod tests { settings.creative_opportunities.is_none(), "test settings should have no creative opportunities configured" ); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let response = run_page_bids_response( &settings, @@ -24118,7 +27747,9 @@ mod tests { #[tokio::test] async fn cross_site_request_is_denied_before_configuration_is_revealed() { let settings = settings_without_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let mut req = Request::builder() .method(Method::GET) .uri(format!("https://test-publisher.com{PAGE_BIDS_PATH}?path=/")) @@ -24138,7 +27769,9 @@ mod tests { #[tokio::test] async fn cross_site_fetch_metadata_is_rejected() { let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let mut req = make_page_bids_request("/2024/01/my-article/"); set_test_header(&mut req, "sec-fetch-site", "cross-site"); @@ -24155,7 +27788,9 @@ mod tests { #[tokio::test] async fn missing_fetch_metadata_without_tsjs_header_is_rejected() { let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let mut req = make_page_bids_request("/2024/01/my-article/"); req.headers_mut().remove("sec-fetch-site"); @@ -24172,7 +27807,9 @@ mod tests { #[tokio::test] async fn missing_fetch_metadata_with_tsjs_header_is_allowed() { let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let mut req = make_page_bids_request("/2024/01/my-article/"); req.headers_mut().remove("sec-fetch-site"); set_test_header(&mut req, "x-tsjs-page-bids", "1"); @@ -24190,7 +27827,9 @@ mod tests { #[tokio::test] async fn same_site_fetch_metadata_is_rejected() { let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let mut req = make_page_bids_request("/2024/01/my-article/"); // `same-site` admits sibling origins under the same registrable // domain — not trusted to spend SSP quota. @@ -24211,7 +27850,9 @@ mod tests { // Spec §8 kill-switch: creative-opportunities.toml with zero slots disables // all server-side auction activity and injection. let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let req = make_page_bids_request("/2024/01/my-article/"); let body = run_page_bids(&settings, &orchestrator, &[], req).await; @@ -24240,7 +27881,9 @@ mod tests { // but the server must not burn SSP request quota running a real auction // for them. Same gate the publisher path applies. let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let slots = article_slot(); let mut req = make_page_bids_request("/2024/01/my-article/"); set_test_header( @@ -24274,7 +27917,9 @@ mod tests { // Navigations triggered by Sec-Purpose=prefetch should not fire real // SSP auctions — the user has not yet visited the page. let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let slots = article_slot(); let mut req = make_page_bids_request("/2024/01/my-article/"); set_test_header(&mut req, "sec-purpose", "prefetch"); @@ -24302,7 +27947,9 @@ mod tests { #[tokio::test] async fn page_bids_omits_only_over_limit_dynamic_slot() { let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let mut over_limit = article_slot() .into_iter() .next() @@ -24343,7 +27990,9 @@ mod tests { async fn url_not_matching_any_pattern_returns_empty_response() { // Slots exist but request path does not match — no auction, no injection. let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let slots = article_slot(); // slot matches /20** only let req = make_page_bids_request("/about"); // does not match @@ -24405,7 +28054,9 @@ mod tests { // the auction is off. Consent is allowed here so the test isolates // the kill switch. let settings = settings_with_co_auction_disabled(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let slots = article_slot(); let req = make_page_bids_request("/2024/01/my-article/"); @@ -24434,7 +28085,9 @@ mod tests { // The dedicated template switch must suppress publisher/page-bids // delivery without using the global auction switch. let settings = settings_with_co_templates_disabled(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let slots = article_slot(); let req = make_page_bids_request("/2024/01/my-article/"); @@ -24465,7 +28118,9 @@ mod tests { // hook does not create GPT slots client-side — matching the publisher // navigation path's shared `evaluate_ad_stack_gate` gate. let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let slots = article_slot(); let req = make_page_bids_request("/2024/01/my-article/"); @@ -24535,6 +28190,11 @@ mod tests { /// tests drive the real handlers with a divergent edge host and assert on /// the auction request the orchestrator dispatched and on the telemetry rows /// the handler emitted. + /// + /// The same capturing setup also covers how both paths key the identity + /// graph. A module whose canonical form differs from the cookie value has + /// its row read under the canonical key, so the partner ID stored there + /// reaches the dispatched auction request. mod navigation_publisher_domain_tests { use super::*; use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; @@ -24542,6 +28202,7 @@ mod tests { use crate::auction::types::AuctionRequest; use crate::auction::{AuctionContext, AuctionOrchestrator}; use crate::creative_opportunities::{CreativeOpportunityFormat, CreativeOpportunitySlot}; + use crate::ec::tests::{CANONICAL_COOKIE_VALUE, CANONICAL_KV_KEY, CanonicalizingModule}; use crate::platform::test_support::{ NoopConfigStore, NoopGeo, NoopSecretStore, StubBackend, }; @@ -24556,7 +28217,7 @@ mod tests { /// `[publisher] domain` from [`crate_test_settings_str`]. const CONFIGURED_DOMAIN: &str = "test-publisher.com"; - const CAPTURING_PROVIDER: &str = "request-capturing-provider"; + const CAPTURING_PROVIDER: &str = "request_capturing_provider"; /// Records the [`AuctionRequest`] the orchestrator dispatched, then /// fails its launch so no real transport handle is needed. @@ -24625,11 +28286,11 @@ mod tests { fn settings_with_capturing_provider() -> Settings { let toml = format!( - "{}\n[auction]\nenabled = true\n\n[auction.providers.{CAPTURING_PROVIDER}]\nprotocol = \"openrtb-2.6\"\nendpoint = \"https://capture.example/openrtb2/auction\"\nrouting = \"all_eligible\"\n\n\ + "{}\n[demand]\nmodules = [\"{CAPTURING_PROVIDER}\"]\n\n[demand.{CAPTURING_PROVIDER}]\nimplementation = \"auction.plain-fixture\"\nendpoint = \"https://capture.example/openrtb2/auction\"\nrouting = \"all_eligible\"\n\n\ [creative_opportunities]\ngam_network_id = \"12345\"\n", - crate_test_settings_str() + crate_test_settings_str().replace("[auction]\n", "[auction]\nenabled = true\n") ); - Settings::from_toml(&toml).expect("should parse settings with a capturing provider") + settings_from_toml(&toml) } fn article_slot() -> Vec { @@ -24713,7 +28374,9 @@ mod tests { settings: &Settings, captured: &Arc>>, ) -> AuctionOrchestrator { - let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let mut orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(settings), + ); orchestrator.register_provider(Arc::new(RequestCapturingProvider { captured: Arc::clone(captured), })); @@ -24798,8 +28461,12 @@ mod tests { .body(EdgeBody::empty()) .expect("should build test request"); + let registry = test_registry(&settings); let _ = handle_publisher_request( - &settings, + AppContext { + settings: &settings, + integration_registry: ®istry, + }, &services, None, &mut ec_context, @@ -24882,8 +28549,12 @@ mod tests { .expect("should build test request"); let slots = slots_with_over_limit_dynamic_sibling(); + let registry = test_registry(&settings); let _ = handle_publisher_request( - &settings, + AppContext { + settings: &settings, + integration_registry: ®istry, + }, &services, None, &mut ec_context, @@ -24944,5 +28615,284 @@ mod tests { assert_only_renderable_slot_was_auctioned(&captured); } + + /// The bidstream partner whose stored ID the canonical row carries. + const CANONICAL_ROW_PARTNER: &str = "ssp.example.com"; + + /// The partner ID the canonical row stores for [`CANONICAL_ROW_PARTNER`]. + const CANONICAL_ROW_UID: &str = "partner-uid-123"; + + /// An identity graph holding one live row under [`CANONICAL_KV_KEY`], + /// and a registry that forwards that row's partner ID as an EID. + fn canonical_row_graph_and_registry() -> (KvIdentityGraph, PartnerRegistry) { + let graph = KvIdentityGraph::in_memory("navigation-canonical-store"); + graph + .create( + CANONICAL_KV_KEY, + &crate::ec::kv_types::KvEntry::minimal( + CANONICAL_ROW_PARTNER, + CANONICAL_ROW_UID, + 1_741_824_000, + ), + ) + .expect("should seed the row under the canonical key"); + let registry = PartnerRegistry::from_config(&[crate::settings::EcPartner { + name: "Canonical row partner".to_owned(), + source_domain: CANONICAL_ROW_PARTNER.to_owned(), + openrtb_atype: crate::settings::EcPartner::default_openrtb_atype(), + bidstream_enabled: true, + api_token: Some(crate::redacted::Redacted::new( + "canonical-row-partner-token-32-bytes".to_owned(), + )), + batch_rate_limit: crate::settings::EcPartner::default_batch_rate_limit(), + pull_sync_enabled: false, + pull_sync_url: None, + pull_sync_allowed_domains: vec![], + pull_sync_ttl_sec: crate::settings::EcPartner::default_pull_sync_ttl_sec(), + pull_sync_rate_limit: crate::settings::EcPartner::default_pull_sync_rate_limit(), + ts_pull_token: None, + }]) + .expect("should build a registry with one bidstream partner"); + (graph, registry) + } + + /// A returning visitor carrying the identifier [`CanonicalizingModule`] + /// creates, with consent that permits the server-side auction. + fn canonicalizing_returning_visitor() -> EcContext { + let consent = crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + }; + EcContext::new_for_test(Some(CANONICAL_COOKIE_VALUE.to_owned()), consent) + .with_module_for_test(Arc::new(CanonicalizingModule)) + } + + /// Asserts the dispatched auction request carried the partner ID the + /// canonical row stores, and the request snapshot is bound to the + /// canonical key. + fn assert_auction_used_the_canonical_row( + captured: &Arc>>, + ec_context: &EcContext, + ) { + let request = captured + .lock() + .expect("should lock captured request") + .clone() + .expect("should dispatch an auction request"); + let eids = request + .user + .eids + .expect("the auction should carry server-side EIDs from the identity graph"); + assert!( + eids.iter().any(|eid| eid.source == CANONICAL_ROW_PARTNER + && eid.uids.iter().any(|uid| uid.id == CANONICAL_ROW_UID)), + "the auction should carry the partner ID stored under the canonical key, got {eids:?}" + ); + assert!( + ec_context + .kv_snapshot() + .entry_for(CANONICAL_KV_KEY) + .is_some(), + "the request snapshot should be bound to the canonical key" + ); + } + + #[tokio::test] + async fn initial_navigation_reads_the_identity_row_under_the_canonical_key() { + // The identity graph stores a row under the owning module's + // canonical form of the identifier. Preloaded and resolved under + // the identifier as issued, a module whose canonical form differs + // from the cookie value found no row, so the auction carried no + // server-side EIDs. + let settings = settings_with_capturing_provider(); + let captured = Arc::new(Mutex::new(None)); + let orchestrator = orchestrator_capturing_request(&settings, &captured); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"ok".to_vec()); + let services = services_with( + Arc::clone(&stub) as Arc, + Arc::new(RecordingTelemetrySink::default()), + ); + let (graph, registry) = canonical_row_graph_and_registry(); + let mut ec_context = canonicalizing_returning_visitor(); + let req = HttpRequest::builder() + .method(Method::GET) + .uri(format!("https://{EDGE_HOST}/2024/01/my-article/")) + .header(header::HOST, EDGE_HOST) + .header("sec-fetch-dest", "document") + .body(EdgeBody::empty()) + .expect("should build test request"); + + let _ = handle_publisher_request( + AppContext { + settings: &settings, + integration_registry: &test_registry(&settings), + }, + &services, + Some(&graph), + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &article_slot(), + registry: Some(®istry), + }, + req, + EdgeCacheHeader::SMaxageFallback, + ) + .await + .expect("should proxy publisher request"); + + assert_auction_used_the_canonical_row(&captured, &ec_context); + } + + #[tokio::test] + async fn initial_navigation_keeps_a_new_identifiers_cookie() { + // Generation binds the request snapshot to the canonical key. The + // navigation preload that follows read under the identifier as + // issued, found nothing there, and replaced that snapshot with a + // miss, so EC finalization skipped the cookie for the identifier + // this request had just created. The second store's first point + // read misses the row generation just wrote, as an eventually + // consistent store can right after a write. The preload keeps + // generation's snapshot in that case only when it compares both + // snapshots under the canonical key. + let stores = [ + ( + "a consistent store", + KvIdentityGraph::in_memory("navigation-new-identifier-store"), + ), + ( + "a store whose first point read misses", + KvIdentityGraph::stale_lookup("navigation-stale-read-store", 1), + ), + ]; + for (store, graph) in stores { + let settings = settings_with_capturing_provider(); + let captured = Arc::new(Mutex::new(None)); + let orchestrator = orchestrator_capturing_request(&settings, &captured); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"ok".to_vec()); + let services = services_with( + Arc::clone(&stub) as Arc, + Arc::new(RecordingTelemetrySink::default()), + ); + let consent = crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + }; + let mut ec_context = EcContext::new_for_test(None, consent) + .with_module_for_test(Arc::new(CanonicalizingModule)); + ec_context + .generate_if_needed( + &settings, + Some(&graph), + &crate::platform::test_support::noop_services(), + ) + .await + .expect("should create the identifier through the module"); + assert_eq!( + ec_context.ec_value(), + Some(CANONICAL_COOKIE_VALUE), + "test precondition: the module should create its identifier" + ); + let req = HttpRequest::builder() + .method(Method::GET) + .uri(format!("https://{EDGE_HOST}/2024/01/my-article/")) + .header(header::HOST, EDGE_HOST) + .header("sec-fetch-dest", "document") + .body(EdgeBody::empty()) + .expect("should build test request"); + + let _ = handle_publisher_request( + AppContext { + settings: &settings, + integration_registry: &test_registry(&settings), + }, + &services, + Some(&graph), + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &[], + registry: None, + }, + req, + EdgeCacheHeader::SMaxageFallback, + ) + .await + .expect("should proxy publisher request"); + + let mut response = Response::new(EdgeBody::empty()); + crate::ec::finalize::ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + &crate::platform::test_support::noop_services(), + ) + .await; + + let cookies: Vec<&str> = response + .headers() + .get_all(header::SET_COOKIE) + .iter() + .filter_map(|value| value.to_str().ok()) + .collect(); + assert!( + cookies + .iter() + .any(|cookie| cookie.starts_with("ts-ec=") && !cookie.contains("Max-Age=0")), + "the identifier this request created should reach the browser with {store}, \ + got {cookies:?}" + ); + } + } + + #[tokio::test] + async fn page_bids_reads_the_identity_row_under_the_canonical_key() { + // The same keying for the SPA re-auction endpoint, which loads the + // row itself only once a live auction will run. + let settings = settings_with_capturing_provider(); + let captured = Arc::new(Mutex::new(None)); + let orchestrator = orchestrator_capturing_request(&settings, &captured); + let services = services_with( + Arc::new(crate::platform::test_support::NoopHttpClient), + Arc::new(RecordingTelemetrySink::default()), + ); + let (graph, registry) = canonical_row_graph_and_registry(); + let mut ec_context = canonicalizing_returning_visitor(); + let mut req = HttpRequest::builder() + .method(Method::GET) + .uri(format!( + "https://{EDGE_HOST}/_ts/page-bids?path=/2024/01/my-article/" + )) + .header(header::HOST, EDGE_HOST) + .body(EdgeBody::empty()) + .expect("should build test request"); + req.headers_mut().insert( + header::HeaderName::from_static("sec-fetch-site"), + HeaderValue::from_static("same-origin"), + ); + + let _ = handle_page_bids( + &settings, + &services, + Some(&graph), + AuctionDispatch { + orchestrator: &orchestrator, + slots: &article_slot(), + registry: Some(®istry), + }, + &mut ec_context, + req, + ) + .await + .expect("should return ok response"); + + assert_auction_used_the_canonical_row(&captured, &ec_context); + } } } diff --git a/crates/trusted-server-core/src/redacted.rs b/crates/trusted-server-core/src/redacted.rs index 3389c4c817..df17f43708 100644 --- a/crates/trusted-server-core/src/redacted.rs +++ b/crates/trusted-server-core/src/redacted.rs @@ -1,14 +1,89 @@ -// NOTE: This file is also included in build.rs via #[path]. -// It must remain self-contained (no `crate::` imports). - -//! A wrapper type that redacts sensitive values in [`Debug`] and [`fmt::Display`] output. +//! A wrapper type that redacts sensitive values in [`Debug`] and [`fmt::Display`] output, +//! and the marking that hides sensitive settings from the configuration view. //! //! Use [`Redacted`] for secrets, passwords, API keys, and other sensitive values //! that must never appear in logs or error messages. +//! +//! A settings field holding a value that is sensitive but is not a +//! [`Redacted`] carries `#[serde(serialize_with = "crate::redacted::sensitive")]`. +use core::cell::Cell; use core::fmt; -use serde::{Deserialize, Serialize}; +use serde::ser::SerializeMap as _; +use serde::{Deserialize, Serialize, Serializer}; + +/// The key a sensitive value is wrapped under while [`to_marked_value`] +/// runs. It starts with a control character, so no setting can share it. +pub const SENSITIVE_MARKER: &str = "\u{1}sensitive"; + +std::thread_local! { + static MARKING: Cell = const { Cell::new(false) }; +} + +/// Restores the marking flag when a marked serialization ends, including +/// by a panic. +struct MarkingGuard { + previous: bool, +} + +impl MarkingGuard { + fn begin() -> Self { + Self { + previous: MARKING.with(|marking| marking.replace(true)), + } + } +} + +impl Drop for MarkingGuard { + fn drop(&mut self) { + MARKING.with(|marking| marking.set(self.previous)); + } +} + +/// Serializes `value` to JSON with every sensitive value wrapped as +/// `{SENSITIVE_MARKER: value}`. +/// +/// Only this call marks. Every other serialization, the configuration push +/// included, writes sensitive values as they are. +/// +/// # Errors +/// +/// When `value` cannot be represented as JSON. +pub fn to_marked_value(value: &T) -> Result +where + T: Serialize + ?Sized, +{ + let _guard = MarkingGuard::begin(); + serde_json::to_value(value) +} + +/// Whether a [`to_marked_value`] call is running on this thread. +#[must_use] +pub fn is_marking() -> bool { + MARKING.with(Cell::get) +} + +/// Serializes a sensitive settings field, for +/// `#[serde(serialize_with = "crate::redacted::sensitive")]`, marked as +/// [`to_marked_value`] says. +/// +/// # Errors +/// +/// As the serializer's own. +pub fn sensitive(value: &T, serializer: S) -> Result +where + T: Serialize + ?Sized, + S: Serializer, +{ + if is_marking() { + let mut map = serializer.serialize_map(Some(1))?; + map.serialize_entry(SENSITIVE_MARKER, value)?; + map.end() + } else { + value.serialize(serializer) + } +} /// Wraps a value so that [`Debug`] and [`fmt::Display`] print `[REDACTED]` /// instead of the inner contents. @@ -16,6 +91,8 @@ use serde::{Deserialize, Serialize}; /// Access the real value via [`expose`](Redacted::expose). Callers must /// never log or display the returned reference. /// +/// Serializes as the inner value, marked as [`to_marked_value`] says. +/// /// # Examples /// /// ``` @@ -25,16 +102,21 @@ use serde::{Deserialize, Serialize}; /// assert_eq!(format!("{:?}", secret), "[REDACTED]"); /// assert_eq!(secret.expose(), "my-secret-key"); /// ``` -#[derive(Clone, Serialize, Deserialize)] +#[derive(Clone, Deserialize)] #[serde(transparent)] pub struct Redacted(T); +impl Serialize for Redacted { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + sensitive(&self.0, serializer) + } +} + impl Redacted { /// Creates a new [`Redacted`] value. - #[allow( - dead_code, - reason = "used by the library crate but not by build.rs path inclusion" - )] pub fn new(value: T) -> Self { Self(value) } @@ -143,6 +225,55 @@ mod tests { ); } + #[test] + fn serialization_is_transparent_outside_a_marked_serialization() { + let secret = Redacted::new("serialize-me".to_owned()); + + let value = serde_json::to_value(&secret).expect("should serialize"); + + assert_eq!(value, serde_json::json!("serialize-me")); + assert!(!is_marking(), "should not be marking outside the call"); + } + + #[test] + fn a_marked_serialization_wraps_redacted_and_sensitive_values() { + #[derive(Serialize)] + struct Fixture { + plain: String, + #[serde(serialize_with = "sensitive")] + marked: String, + secret: Redacted, + } + let fixture = Fixture { + plain: "open".to_owned(), + marked: "hidden".to_owned(), + secret: Redacted::new("secret".to_owned()), + }; + + let marked = to_marked_value(&fixture).expect("should serialize"); + let normal = serde_json::to_value(&fixture).expect("should serialize"); + + assert_eq!( + marked, + serde_json::json!({ + "plain": "open", + "marked": { SENSITIVE_MARKER: "hidden" }, + "secret": { SENSITIVE_MARKER: "secret" }, + }), + "should wrap only the marked values" + ); + assert_eq!( + normal, + serde_json::json!({ + "plain": "open", + "marked": "hidden", + "secret": "secret", + }), + "should leave an ordinary serialization unchanged" + ); + assert!(!is_marking(), "should clear the flag when the call ends"); + } + #[test] fn struct_with_redacted_field_debug() { #[derive(Debug)] diff --git a/crates/trusted-server-core/src/request_signing/endpoints.rs b/crates/trusted-server-core/src/request_signing/endpoints.rs index d35e51a151..6308d6d049 100644 --- a/crates/trusted-server-core/src/request_signing/endpoints.rs +++ b/crates/trusted-server-core/src/request_signing/endpoints.rs @@ -1,18 +1,17 @@ //! HTTP endpoint handlers for request signing operations. //! -//! This module provides endpoint handlers for JWKS retrieval, signature verification, -//! key rotation, and key deactivation operations. +//! This module provides endpoint handlers for JWKS retrieval and signature +//! verification, and the key ID rules key creation follows. use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt}; use http::{Request, Response, StatusCode, header}; use serde::{Deserialize, Serialize}; -use crate::error::{IntoHttpResponse, TrustedServerError}; +use crate::error::TrustedServerError; use crate::http_util::enforce_max_body_size; use crate::platform::RuntimeServices; use crate::request_signing::discovery::TrustedServerDiscovery; -use crate::request_signing::rotation::KeyRotationManager; use crate::request_signing::signing; use crate::settings::Settings; @@ -99,7 +98,6 @@ pub struct VerifySignatureResponse { } const VERIFY_MAX_BODY_BYTES: usize = 4096; -const ADMIN_MAX_BODY_BYTES: usize = 4096; /// Will verify a signature given a payload and kid /// Useful for testing integration with signatures @@ -160,62 +158,11 @@ pub fn handle_verify_signature( Ok(json_response(StatusCode::OK, response_json)) } -/// JSON request body for the key-rotation endpoint. -#[derive(Debug, Deserialize, Serialize)] -pub struct RotateKeyRequest { - /// Optional explicit key identifier for the new signing key. - #[serde(skip_serializing_if = "Option::is_none")] - pub kid: Option, -} - -/// JSON response body for the key-rotation endpoint. -#[derive(Debug, Deserialize, Serialize)] -pub struct RotateKeyResponse { - /// Whether the rotation operation succeeded. - pub success: bool, - /// Human-readable summary of the rotation result. - pub message: String, - /// Newly generated or supplied key identifier. - pub new_kid: String, - /// Previously active key identifier, if one existed. - pub previous_kid: Option, - /// Active key identifiers after the rotation completes. - pub active_kids: Vec, - /// Public JWK associated with the newly active key. - pub jwk: serde_json::Value, - /// Error detail when rotation fails. - #[serde(skip_serializing_if = "Option::is_none")] - pub error: Option, -} - -struct SigningStoreIds<'a> { - config_store_id: &'a str, - secret_store_id: &'a str, -} - const MAX_KID_LENGTH: usize = 128; -fn signing_store_ids( - settings: &Settings, -) -> Result, Report> { - settings - .request_signing - .as_ref() - .map(|setting| SigningStoreIds { - config_store_id: setting.config_store_id.as_str(), - secret_store_id: setting.secret_store_id.as_str(), - }) - .ok_or_else(|| { - TrustedServerError::Configuration { - message: "missing signing storage configuration".to_string(), - } - .into() - }) -} - -/// Validates the structural constraints every kid must satisfy regardless of -/// operation: non-empty, length-bounded, and limited to a safe charset so a kid -/// cannot smuggle a CSV separator or other control characters into storage. +/// Validates the structural constraints every kid must satisfy: non-empty, +/// length-bounded, and limited to a safe charset so a kid cannot smuggle a CSV +/// separator or other control characters into storage. fn validate_kid_format(kid: &str) -> Result<(), Report> { if kid.is_empty() || kid.len() > MAX_KID_LENGTH { return Err(Report::new(TrustedServerError::BadRequest { @@ -235,27 +182,23 @@ fn validate_kid_format(kid: &str) -> Result<(), Report> { Ok(()) } -/// Validates a kid for creation/rotation. +/// Validates a kid for a new key. /// -/// Enforces the **portable-KID contract**: a kid created through any adapter must -/// be storable by every platform's key-name encoder, so create/rotate validates -/// against the strictest common denominator. Concretely the kid must start with a -/// lowercase ASCII letter, on top of the structural [`validate_kid_format`] -/// charset checks. +/// Enforces the **portable-KID contract**: a new kid must be storable by every +/// platform's key-name encoder, so creation validates against the strictest +/// common denominator. Concretely the kid must start with a lowercase ASCII +/// letter, on top of the structural [`validate_kid_format`] charset checks. /// /// That floor is currently set by the Fermyon Spin variable encoder /// (`spin_variable_name` in the Spin adapter's `platform.rs`), which requires a /// lowercase ASCII leading character: it would otherwise alias digit-leading keys /// (`1foo` and `n1foo` both map to `v_n1foo`) or reject uppercase/punctuation /// leading keys (`KidA`, `_kid`, `-kid`, `.kid`, `:kid`) at storage time. -/// Enforcing the floor here turns those into a client-correctable 400 on every -/// adapter instead of a runtime 5xx on Spin. [`kid_is_creatable`] exposes this -/// predicate so adapter crates can pin the contract with a test — core must never -/// accept a kid its encoder rejects. System-generated KIDs (`ts-YYYY-MM-DD`) -/// start with `t` and are unaffected. -/// -/// Deactivation/deletion deliberately uses the looser [`validate_kid_format`] so -/// legacy KIDs created under earlier validation rules remain removable. +/// Checking the floor here refuses such a kid for every platform rather than +/// leaving it to fail at storage time on Spin. [`kid_is_creatable`] exposes +/// this predicate so adapter crates can pin the contract with a test, because +/// core must never accept a kid its encoder rejects. System-generated KIDs +/// (`ts-YYYY-MM-DD`) start with `t` and are unaffected. fn validate_kid(kid: &str) -> Result<(), Report> { validate_kid_format(kid)?; @@ -268,233 +211,28 @@ fn validate_kid(kid: &str) -> Result<(), Report> { Ok(()) } -/// Returns whether `kid` satisfies the create/rotate portable-KID contract -/// enforced by [`validate_kid`]. +/// Returns whether `kid` satisfies the portable-KID contract that +/// [`validate_kid`] checks for a new key. /// -/// Exposed so platform adapter crates can assert their key-name encoder accepts -/// every kid this validation admits, pinning the cross-adapter contract against -/// silent drift between this validation floor and an adapter's storage encoder. +/// Exposed for the tool that creates keys, which asks before it writes +/// anything, and so platform adapter crates can assert their key-name encoder +/// accepts every kid this validation admits, pinning the cross-adapter contract +/// against silent drift between this validation floor and an adapter's storage +/// encoder. #[must_use] pub fn kid_is_creatable(kid: &str) -> bool { validate_kid(kid).is_ok() } -/// Rotates the current active kid by generating and saving a new one. -/// -/// # Response contract -/// -/// Returns `200 OK` with `success: true` on success, `400 Bad Request` for an -/// invalid operator-supplied `kid`, or `500 Internal Server Error` when rotation -/// fails. Failure responses include `success: false` and a populated `error` -/// field. Unlike [`handle_verify_signature`], the error field contains internal -/// detail — this is intentional because this endpoint is auth-gated and -/// operator-facing only. -/// -/// # Errors -/// -/// Returns an error if the request signing settings are missing or JSON parsing fails. -pub fn handle_rotate_key( - settings: &Settings, - services: &RuntimeServices, - req: Request, -) -> Result, Report> { - let SigningStoreIds { - config_store_id, - secret_store_id, - } = signing_store_ids(settings)?; - - let body = request_body_bytes(req.into_body(), "rotate-key")?; - enforce_max_body_size(&body, ADMIN_MAX_BODY_BYTES, "rotate-key")?; - let rotate_req: RotateKeyRequest = if body.is_empty() { - RotateKeyRequest { kid: None } - } else { - serde_json::from_slice(&body).change_context(TrustedServerError::Configuration { - message: "invalid JSON request body".into(), - })? - }; - - let manager = KeyRotationManager::new(config_store_id, secret_store_id); - let validation_result = if let Some(kid) = rotate_req.kid.as_deref() { - validate_kid(kid) - } else { - Ok(()) - }; - let result = validation_result.and_then(|()| manager.rotate_key(services, rotate_req.kid)); - - match result { - Ok(result) => { - let jwk_value = serde_json::to_value(&result.jwk).map_err(|e| { - Report::new(TrustedServerError::Configuration { - message: format!("failed to serialize JWK: {}", e), - }) - })?; - - let response = RotateKeyResponse { - success: true, - message: "Key rotated successfully".to_string(), - new_kid: result.new_kid, - previous_kid: result.previous_kid, - active_kids: result.active_kids, - jwk: jwk_value, - error: None, - }; - - let response_json = serde_json::to_string(&response).map_err(|e| { - Report::new(TrustedServerError::Configuration { - message: format!("failed to serialize response: {}", e), - }) - })?; - - Ok(json_response(StatusCode::OK, response_json)) - } - Err(e) => { - let status = e.current_context().status_code(); - let response = RotateKeyResponse { - success: false, - message: "Key rotation failed".to_string(), - new_kid: String::new(), - previous_kid: None, - active_kids: vec![], - jwk: serde_json::json!({}), - error: Some(format!("{}", e)), - }; - - let response_json = serde_json::to_string(&response).map_err(|e| { - Report::new(TrustedServerError::Configuration { - message: format!("failed to serialize response: {}", e), - }) - })?; - - Ok(json_response(status, response_json)) - } - } -} - -/// JSON request body for the key-deactivation endpoint. -#[derive(Debug, Deserialize, Serialize)] -pub struct DeactivateKeyRequest { - /// Key identifier to deactivate or delete. - pub kid: String, - /// Whether the key should be deleted from storage after deactivation. - #[serde(default)] - pub delete: bool, -} - -/// JSON response body for the key-deactivation endpoint. -#[derive(Debug, Deserialize, Serialize)] -pub struct DeactivateKeyResponse { - /// Whether the deactivation or deletion succeeded. - pub success: bool, - /// Human-readable summary of the operation result. - pub message: String, - /// Key identifier that was deactivated or deleted. - pub deactivated_kid: String, - /// Whether the key was deleted from storage. - pub deleted: bool, - /// Active key identifiers remaining after the operation. - pub remaining_active_kids: Vec, - /// Error detail when the operation fails. - #[serde(skip_serializing_if = "Option::is_none")] - pub error: Option, -} - -/// Deactivates or deletes an active signing key. -/// -/// # Response contract +/// Returns whether `kid` has the shape of a stored key id, which +/// [`validate_kid_format`] checks. /// -/// Returns `200 OK` with `success: true` on success, `400 Bad Request` for an -/// invalid operator-supplied `kid`, or `500 Internal Server Error` when -/// deactivation fails. Failure responses include `success: false` and a populated -/// `error` field. Like [`handle_rotate_key`] and unlike -/// [`handle_verify_signature`], the error field contains internal detail — this -/// is intentional because this endpoint is auth-gated and operator-facing only. -/// -/// # Errors -/// -/// Returns an error if the request signing settings are missing or JSON parsing fails. -pub fn handle_deactivate_key( - settings: &Settings, - services: &RuntimeServices, - req: Request, -) -> Result, Report> { - let SigningStoreIds { - config_store_id, - secret_store_id, - } = signing_store_ids(settings)?; - - let body = request_body_bytes(req.into_body(), "deactivate-key")?; - enforce_max_body_size(&body, ADMIN_MAX_BODY_BYTES, "deactivate-key")?; - let deactivate_req: DeactivateKeyRequest = - serde_json::from_slice(&body).change_context(TrustedServerError::Configuration { - message: "invalid JSON request body".into(), - })?; - - let manager = KeyRotationManager::new(config_store_id, secret_store_id); - - // Use the looser format validation here so legacy KIDs created under earlier - // validation rules (e.g. digit- or uppercase-leading) can still be - // deactivated or deleted. The stricter validate_kid only gates new key - // creation/rotation. - let result = validate_kid_format(&deactivate_req.kid).and_then(|()| { - if deactivate_req.delete { - manager.delete_key(services, &deactivate_req.kid) - } else { - manager.deactivate_key(services, &deactivate_req.kid) - } - }); - - match result { - Ok(()) => { - let remaining_keys = manager.list_active_keys(services).unwrap_or_else(|e| { - log::warn!("failed to list active keys after deactivation: {}", e); - vec![] - }); - - let response = DeactivateKeyResponse { - success: true, - message: if deactivate_req.delete { - "Key deleted successfully".to_string() - } else { - "Key deactivated successfully".to_string() - }, - deactivated_kid: deactivate_req.kid, - deleted: deactivate_req.delete, - remaining_active_kids: remaining_keys, - error: None, - }; - - let response_json = serde_json::to_string(&response).map_err(|e| { - Report::new(TrustedServerError::Configuration { - message: format!("failed to serialize response: {}", e), - }) - })?; - - Ok(json_response(StatusCode::OK, response_json)) - } - Err(e) => { - let status = e.current_context().status_code(); - let response = DeactivateKeyResponse { - success: false, - message: if deactivate_req.delete { - "Key deletion failed".to_string() - } else { - "Key deactivation failed".to_string() - }, - deactivated_kid: deactivate_req.kid.clone(), - deleted: false, - remaining_active_kids: vec![], - error: Some(format!("{}", e)), - }; - - let response_json = serde_json::to_string(&response).map_err(|e| { - Report::new(TrustedServerError::Configuration { - message: format!("failed to serialize response: {}", e), - }) - })?; - - Ok(json_response(status, response_json)) - } - } +/// Looser than [`kid_is_creatable`], so that a key created under an earlier +/// rule can still be deactivated and deleted. Exposed for the tool that +/// retires keys, which asks before it reads or writes anything. +#[must_use] +pub fn kid_is_well_formed(kid: &str) -> bool { + validate_kid_format(kid).is_ok() } #[cfg(test)] @@ -719,266 +457,6 @@ mod tests { assert!(result.is_err(), "Malformed JSON should error"); } - #[test] - fn test_handle_rotate_key_with_empty_body() { - let settings = crate::test_support::tests::create_test_settings(); - let req = build_request(Method::POST, "https://test.com/admin/keys/rotate", None); - - let resp = handle_rotate_key(&settings, &noop_services(), req) - .expect("should return a response even when stores are unavailable"); - - assert_eq!( - resp.status(), - StatusCode::INTERNAL_SERVER_ERROR, - "should return 500 when store writes fail" - ); - - let body = response_body_string(resp); - let response: RotateKeyResponse = - serde_json::from_str(&body).expect("should deserialize rotate response"); - - assert!( - !response.success, - "should report failure when store writes fail" - ); - assert!( - response.error.is_some(), - "should include error detail in failure response" - ); - } - - #[test] - fn test_handle_rotate_key_with_custom_kid() { - let settings = crate::test_support::tests::create_test_settings(); - - let req_body = RotateKeyRequest { - kid: Some("test-custom-key".to_string()), - }; - - let body_json = serde_json::to_string(&req_body).expect("should serialize rotate request"); - let req = build_request( - Method::POST, - "https://test.com/admin/keys/rotate", - Some(&body_json), - ); - - let resp = handle_rotate_key(&settings, &noop_services(), req) - .expect("should return a response even when stores are unavailable"); - - assert_eq!( - resp.status(), - StatusCode::INTERNAL_SERVER_ERROR, - "should return 500 when store writes fail" - ); - - let body = response_body_string(resp); - let response: RotateKeyResponse = - serde_json::from_str(&body).expect("should deserialize rotate response"); - - assert!( - !response.success, - "should report failure when store writes fail" - ); - assert!( - response.error.is_some(), - "should include error detail in failure response" - ); - } - - #[test] - fn test_handle_rotate_key_invalid_json() { - let settings = crate::test_support::tests::create_test_settings(); - let req = build_request( - Method::POST, - "https://test.com/admin/keys/rotate", - Some("invalid json"), - ); - - let result = handle_rotate_key(&settings, &noop_services(), req); - assert!(result.is_err(), "Invalid JSON should return error"); - } - - #[test] - fn test_handle_rotate_key_rejects_invalid_kid() { - let settings = crate::test_support::tests::create_test_settings(); - - let req_body = RotateKeyRequest { - kid: Some("bad,kid".to_string()), - }; - - let body_json = serde_json::to_string(&req_body).expect("should serialize rotate request"); - let req = build_request( - Method::POST, - "https://test.com/admin/keys/rotate", - Some(&body_json), - ); - - let resp = handle_rotate_key(&settings, &noop_services(), req) - .expect("should return a response for invalid kid"); - - assert_eq!( - resp.status(), - StatusCode::BAD_REQUEST, - "should reject malformed kid as a bad request" - ); - - let body = response_body_string(resp); - let response: RotateKeyResponse = - serde_json::from_str(&body).expect("should deserialize rotate response"); - - assert!( - !response.success, - "should report failure when supplied kid is invalid" - ); - assert!( - response - .error - .as_deref() - .is_some_and(|error| error.contains("kid must contain only")), - "should explain the kid character restrictions" - ); - } - - #[test] - fn test_handle_deactivate_key_request() { - let settings = crate::test_support::tests::create_test_settings(); - - let req_body = DeactivateKeyRequest { - kid: "test-old-key".to_string(), - delete: false, - }; - - let body_json = - serde_json::to_string(&req_body).expect("should serialize deactivate request"); - let req = build_request( - Method::POST, - "https://test.com/admin/keys/deactivate", - Some(&body_json), - ); - - let resp = handle_deactivate_key(&settings, &noop_services(), req) - .expect("should return a response even when stores are unavailable"); - - assert_eq!( - resp.status(), - StatusCode::INTERNAL_SERVER_ERROR, - "should return 500 when active-kids cannot be read" - ); - - let body = response_body_string(resp); - let response: DeactivateKeyResponse = - serde_json::from_str(&body).expect("should deserialize deactivate response"); - - assert!( - !response.success, - "should report failure when store reads fail" - ); - assert!( - response.error.is_some(), - "should include error detail in failure response" - ); - } - - #[test] - fn test_handle_deactivate_key_with_delete() { - let settings = crate::test_support::tests::create_test_settings(); - - let req_body = DeactivateKeyRequest { - kid: "test-old-key".to_string(), - delete: true, - }; - - let body_json = - serde_json::to_string(&req_body).expect("should serialize deactivate request"); - let req = build_request( - Method::POST, - "https://test.com/admin/keys/deactivate", - Some(&body_json), - ); - - let resp = handle_deactivate_key(&settings, &noop_services(), req) - .expect("should return a response even when stores are unavailable"); - - assert_eq!( - resp.status(), - StatusCode::INTERNAL_SERVER_ERROR, - "should return 500 when active-kids cannot be read" - ); - - let body = response_body_string(resp); - let response: DeactivateKeyResponse = - serde_json::from_str(&body).expect("should deserialize deactivate response"); - - assert!( - !response.success, - "should report failure when store reads fail" - ); - assert!( - !response.deleted, - "should not report deletion when the operation failed" - ); - assert!( - response.error.is_some(), - "should include error detail in failure response" - ); - } - - #[test] - fn test_handle_deactivate_key_invalid_json() { - let settings = crate::test_support::tests::create_test_settings(); - let req = build_request( - Method::POST, - "https://test.com/admin/keys/deactivate", - Some("invalid json"), - ); - - let result = handle_deactivate_key(&settings, &noop_services(), req); - assert!(result.is_err(), "Invalid JSON should return error"); - } - - #[test] - fn test_handle_deactivate_key_rejects_invalid_kid() { - let settings = crate::test_support::tests::create_test_settings(); - - let req_body = DeactivateKeyRequest { - kid: "bad kid".to_string(), - delete: false, - }; - - let body_json = - serde_json::to_string(&req_body).expect("should serialize deactivate request"); - let req = build_request( - Method::POST, - "https://test.com/admin/keys/deactivate", - Some(&body_json), - ); - - let resp = handle_deactivate_key(&settings, &noop_services(), req) - .expect("should return a response for invalid kid"); - - assert_eq!( - resp.status(), - StatusCode::BAD_REQUEST, - "should reject malformed kid as a bad request" - ); - - let body = response_body_string(resp); - let response: DeactivateKeyResponse = - serde_json::from_str(&body).expect("should deserialize deactivate response"); - - assert!( - !response.success, - "should report failure when supplied kid is invalid" - ); - assert!( - response - .error - .as_deref() - .is_some_and(|error| error.contains("kid must contain only")), - "should explain the kid character restrictions" - ); - } - #[test] fn verify_signature_rejects_oversized_body() { let settings = crate::test_support::tests::create_test_settings(); @@ -1018,63 +496,6 @@ mod tests { ); } - #[test] - fn rotate_key_rejects_oversized_body() { - let settings = crate::test_support::tests::create_test_settings(); - let oversized = "x".repeat(ADMIN_MAX_BODY_BYTES + 1); - let req = build_request( - Method::POST, - "https://test.com/admin/keys/rotate", - Some(&oversized), - ); - let err = handle_rotate_key(&settings, &noop_services(), req) - .expect_err("should reject oversized body"); - assert_eq!( - err.current_context().status_code(), - StatusCode::PAYLOAD_TOO_LARGE, - "should return 413 for rotate-key body over limit" - ); - } - - #[test] - fn rotate_key_rejects_streaming_body() { - let settings = crate::test_support::tests::create_test_settings(); - let req = build_streaming_request(Method::POST, "https://test.com/admin/keys/rotate"); - let err = handle_rotate_key(&settings, &noop_services(), req) - .expect_err("should reject streaming rotate body"); - assert_eq!( - err.current_context().status_code(), - StatusCode::BAD_REQUEST, - "should return 400 for streaming rotate body" - ); - assert!( - matches!( - err.current_context(), - TrustedServerError::BadRequest { message } - if message == "rotate-key request body must be buffered, not streamed" - ), - "should explain that rotate bodies must be buffered" - ); - } - - #[test] - fn deactivate_key_rejects_oversized_body() { - let settings = crate::test_support::tests::create_test_settings(); - let oversized = "x".repeat(ADMIN_MAX_BODY_BYTES + 1); - let req = build_request( - Method::POST, - "https://test.com/admin/keys/deactivate", - Some(&oversized), - ); - let err = handle_deactivate_key(&settings, &noop_services(), req) - .expect_err("should reject oversized body"); - assert_eq!( - err.current_context().status_code(), - StatusCode::PAYLOAD_TOO_LARGE, - "should return 413 for deactivate-key body over limit" - ); - } - #[test] fn validate_kid_accepts_allowed_operator_supplied_ids() { validate_kid("azAZ09-_.:").expect("should accept allowed kid characters"); @@ -1115,7 +536,7 @@ mod tests { fn validate_kid_rejects_non_lowercase_leading_ids() { // The Spin variable encoder requires a lowercase ASCII leading character, // so uppercase- and punctuation-leading KIDs that pass validate_kid_format - // must be rejected on the create/rotate path to avoid a runtime 5xx on Spin. + // must be rejected for a new key, which Spin could not store. for kid in &["KidA", "_kid", "-kid", ".kid", ":kid"] { assert!( validate_kid(kid).is_err(), @@ -1127,63 +548,32 @@ mod tests { } #[test] - fn validate_kid_format_allows_digit_leading_for_legacy_removal() { - // Deactivation/deletion uses validate_kid_format so digit-leading KIDs - // created under earlier rules stay removable, while the stricter - // validate_kid still blocks them on the create/rotate path. + fn validate_kid_format_allows_digit_leading_ids() { + // The leading-letter rule belongs to validate_kid alone, so the + // structural check accepts a digit-leading kid that a new key may not + // take. for kid in &["2026-key", "0abc", "9xyz", "1-key"] { validate_kid_format(kid) - .unwrap_or_else(|e| panic!("format check should accept legacy kid {kid}: {e:?}")); + .unwrap_or_else(|e| panic!("format check should accept kid {kid}: {e:?}")); assert!( validate_kid(kid).is_err(), - "create/rotate validation must still reject digit-leading kid: {kid}" + "new-key validation must still reject digit-leading kid: {kid}" ); } } #[test] - fn deactivate_allows_legacy_digit_leading_kid_past_validation() { - // A digit-leading legacy kid must pass validation and reach storage - // (which fails with the noop test stores -> 500) rather than being - // rejected up front as a 400 bad request. - let settings = crate::test_support::tests::create_test_settings(); - let req_body = DeactivateKeyRequest { - kid: "1legacy-key".to_string(), - delete: true, - }; - let body_json = - serde_json::to_string(&req_body).expect("should serialize deactivate request"); - let req = build_request( - Method::POST, - "https://test.com/admin/keys/deactivate", - Some(&body_json), - ); - - let resp = handle_deactivate_key(&settings, &noop_services(), req) - .expect("should return a response for a legacy digit-leading kid"); - - assert_ne!( - resp.status(), - StatusCode::BAD_REQUEST, - "legacy digit-leading kid must not be rejected as a bad request on deactivation" - ); - } - - #[test] - fn test_rotate_key_request_deserialization() { - let json = r#"{"kid":"custom-key"}"#; - let req: RotateKeyRequest = - serde_json::from_str(json).expect("should deserialize rotate key request"); - assert_eq!(req.kid, Some("custom-key".to_string())); - } - - #[test] - fn test_deactivate_key_request_deserialization() { - let json = r#"{"kid":"old-key","delete":true}"#; - let req: DeactivateKeyRequest = - serde_json::from_str(json).expect("should deserialize deactivate key request"); - assert_eq!(req.kid, "old-key"); - assert!(req.delete); + fn a_well_formed_kid_need_not_be_one_a_new_key_may_take() { + for kid in ["2026-key", "KidA", "-kid", "ts-2026-01-01"] { + assert!(kid_is_well_formed(kid), "should accept {kid}"); + } + for kid in ["2026-key", "KidA", "-kid"] { + assert!(!kid_is_creatable(kid), "should not create {kid}"); + } + let too_long = "a".repeat(129); + for kid in ["", "kid-a,kid-b", "kid a", "kid/a", &too_long] { + assert!(!kid_is_well_formed(kid), "should refuse {kid:?}"); + } } #[test] diff --git a/crates/trusted-server-core/src/request_signing/rotation.rs b/crates/trusted-server-core/src/request_signing/rotation.rs index 2d7222b705..e971ff6d2f 100644 --- a/crates/trusted-server-core/src/request_signing/rotation.rs +++ b/crates/trusted-server-core/src/request_signing/rotation.rs @@ -167,8 +167,7 @@ impl KeyRotationManager { signing_key: &SigningKey, ) -> Result<(), Report> { // The platform secret-store write interface is string-based, so signing - // keys are persisted as base64 text. The Fastly adapter applies its own - // transport-level base64 encoding when calling the management API. + // keys are persisted as base64 text. let key_b64 = general_purpose::STANDARD.encode(signing_key.as_bytes()); services diff --git a/crates/trusted-server-core/src/response_privacy.rs b/crates/trusted-server-core/src/response_privacy.rs index 46ff4bf5b4..301b8f799f 100644 --- a/crates/trusted-server-core/src/response_privacy.rs +++ b/crates/trusted-server-core/src/response_privacy.rs @@ -24,6 +24,17 @@ use crate::settings::Settings; #[derive(Clone, Copy, Debug)] pub struct TerminalPrivateResponse; +/// Request marker meaning the response is personalized to this request and +/// must not be shared through any cache or template. +/// +/// Any integration may set it from a request filter, and core acts on it +/// without knowing which integration asked. Whenever it is present the request +/// keeps to the origin path rather than a shared template. For an HTML +/// document core also buffers the full body, and it enforces private caching +/// on the HTML response that body produces. +#[derive(Debug, Clone, Copy)] +pub struct PersonalizedResponse; + const INACTIVE_AD_STACK_BROWSER_CACHE_CONTROL: &str = "private, max-age=60"; #[derive(Debug, Clone, Copy)] @@ -84,7 +95,10 @@ pub fn enforce_private_no_store(response: &mut Response) { } /// Marks a Trusted Server response as terminal-private and applies its cache policy. -pub(crate) fn enforce_terminal_private_cache_privacy(response: &mut Response) { +/// +/// Public so a response an integration crate synthesizes for one reader is +/// kept out of shared caches the same way. +pub fn enforce_terminal_private_cache_privacy(response: &mut Response) { enforce_private_no_store(response); response.extensions_mut().insert(TerminalPrivateResponse); } @@ -144,6 +158,12 @@ pub fn enforce_set_cookie_cache_privacy(response: &mut Response) { .remove::(); } +/// Headers a response must leave with, whatever `settings.response_headers` +/// say. [`apply_response_headers_with_cache_privacy`] sets them again after +/// the operator's headers. +#[derive(Debug, Clone)] +pub struct RequiredHeaders(pub HeaderMap); + /// Applies operator-configured `settings.response_headers` with cookie-privacy /// hardening. /// @@ -188,6 +208,12 @@ pub fn apply_response_headers_with_cache_privacy(settings: &Settings, response: response.headers_mut().insert(header_name, header_value); } + if let Some(RequiredHeaders(required)) = response.extensions_mut().remove::() { + for (name, value) in &required { + response.headers_mut().insert(name.clone(), value.clone()); + } + } + enforce_uncacheable_cache_privacy(response); // Operator headers can themselves introduce Set-Cookie (alongside public @@ -208,18 +234,19 @@ mod tests { fn settings_with_response_headers(headers: &[(&str, &str)]) -> Settings { let mut s = Settings::from_toml( r#" - [[handlers]] - path = "^/_ts/admin" - username = "admin" - password = "admin-pass" - [publisher] domain = "test-publisher.example.com" cookie_domain = ".test-publisher.example.com" origin_url = "https://origin.test-publisher.example.com" proxy_secret = "unit-test-proxy-secret" + [geo] + assume_single_jurisdiction = true + [ec] + module = "hmac" + + [ec.hmac] passphrase = "test-secret-key-32-bytes-minimum" "#, ) @@ -610,4 +637,30 @@ mod tests { "operator headers should still apply to cacheable responses" ); } + + #[test] + fn a_required_header_outlasts_an_operator_header_of_the_same_name() { + let settings = settings_with_response_headers(&[ + ("Content-Security-Policy", "default-src 'self'"), + ("X-Frame-Options", "SAMEORIGIN"), + ]); + let mut response = response_builder() + .header( + header::CONTENT_SECURITY_POLICY, + "sandbox; default-src 'none'", + ) + .header(header::X_FRAME_OPTIONS, "DENY") + .body(edgezero_core::body::Body::empty()) + .expect("should build response"); + let required = response.headers().clone(); + response.extensions_mut().insert(RequiredHeaders(required)); + + apply_response_headers_with_cache_privacy(&settings, &mut response); + + assert_eq!( + response.headers()[header::CONTENT_SECURITY_POLICY], + "sandbox; default-src 'none'" + ); + assert_eq!(response.headers()[header::X_FRAME_OPTIONS], "DENY"); + } } diff --git a/crates/trusted-server-core/src/robots_txt/contributor.rs b/crates/trusted-server-core/src/robots_txt/contributor.rs new file mode 100644 index 0000000000..c4ee61a4ea --- /dev/null +++ b/crates/trusted-server-core/src/robots_txt/contributor.rs @@ -0,0 +1,82 @@ +//! The extension point a module uses to add rules to `/robots.txt`. + +use core::time::Duration; + +use error_stack::Report; + +use super::Contribution; +use crate::error::TrustedServerError; +use crate::module_context::ModuleCall; +use crate::permissions::PermissionSet; + +/// The `[robots-txt] modules` entry that refuses every crawler. +/// +/// Core's own, and it names no vendor. Selecting it makes the file the refusal +/// and nothing else, whatever else is selected, and puts +/// [`REFUSE_ALL_ROBOTS_TAG`](super::REFUSE_ALL_ROBOTS_TAG) on every response. +pub const REFUSE_ALL: &str = "refuse_all"; + +/// The `[robots-txt] modules` entry that allows every crawler everywhere. +/// +/// Core's own, so permission is something the document says rather than +/// something that follows from a lookup coming back empty. +pub const ALLOW_ALL: &str = "allow_all"; + +/// Something that adds rules to `/robots.txt`. +/// +/// A module supplies one through +/// [`IntegrationRegistrationBuilder::with_robots_txt_contributor`](crate::integrations::IntegrationRegistrationBuilder::with_robots_txt_contributor), +/// and `[robots-txt] modules` selects it by the name it was declared under. +/// Core holds what a contributor returns, assembles the file from every +/// selected contributor in the order `modules` names them, and writes the +/// bytes. The contributor never does, so it cannot produce a malformed file. +/// +/// `Send + Sync` so the registry can share it between requests, while the +/// future it returns is pinned to one thread, because the host SDKs produce +/// `!Send` futures on wasm32. +#[async_trait::async_trait(?Send)] +pub trait RobotsTxtContributor: Send + Sync + core::fmt::Debug { + /// The longest one answer may be held before this contributor is asked + /// again. + /// + /// A ceiling, not a schedule. Each contributor's answer is held and + /// refreshed on its own, and the file is assembled from whatever each one + /// holds at the time, so two contributors never have to agree on anything. + fn refresh(&self) -> Duration; + + /// A value that changes whenever this contributor's answer would, such as a + /// hash of its settings. + /// + /// It keys the held answers, so an answer given for different settings is + /// never served for these. Core hashes it into the key an answer is held + /// under, so nothing of it is written to the store. + fn fingerprint(&self) -> String; + + /// The permissions this contributor declares, which decide whether a + /// gated value is passed to it. None by default. + fn required_permissions(&self) -> PermissionSet { + PermissionSet::none() + } + + /// This contributor's rules. + /// + /// Core holds the answer for the publisher and serves it to every + /// crawler. So the call carries the request the answer was asked for, + /// being its method, path, host and scheme without its query, and the + /// request's services, and nothing about who asked. An implementation + /// hands its own function to [`ModuleCall::inject`], naming what it + /// needs. Its answer must be the same for every request the publisher + /// receives, because it is served for all of them. + /// + /// A contribution with no groups is an answer, and is served as one. An + /// error is a failure, and core serves this contributor's held copy + /// instead. With no held copy the whole file fails closed with `503`. + /// + /// # Errors + /// + /// When this contributor cannot give its rules now. + async fn contribute( + &self, + call: ModuleCall<'_>, + ) -> Result>; +} diff --git a/crates/trusted-server-core/src/robots_txt/document.rs b/crates/trusted-server-core/src/robots_txt/document.rs new file mode 100644 index 0000000000..8973e61595 --- /dev/null +++ b/crates/trusted-server-core/src/robots_txt/document.rs @@ -0,0 +1,540 @@ +//! The file served at `/robots.txt`, held as a document rather than as text. +//! +//! Contributors hand core groups of records, and core writes every byte. So a +//! contributor cannot produce a malformed file, and the rules that have to hold +//! whatever a contributor says are applied here, in one place. +//! +//! 1. A refusal cannot be loosened. When `refuse_all` is selected the file is +//! the refusal and nothing else, see [`refusal`]. +//! 2. A path in `always_allow` stays open to every crawler a contribution +//! refuses the whole site to, see [`assemble`]. +//! 3. The publisher's own text and the `Sitemap` line go where the publisher +//! put them, around what the contributors gave. + +use std::borrow::Cow; + +use serde::{Deserialize, Serialize}; + +use super::RobotsTxtConfig; + +/// One record in a group, in the order it was given. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Record { + /// A path the group's crawlers may fetch. + Allow(String), + /// A path the group's crawlers may not fetch. + Disallow(String), + /// A line of explanation, without its leading `#`. + Comment(String), + /// Any other `name: value` record, kept where it was given, a Terms + /// Document Locator for example. + Other { + /// The record's name, as it was written. + name: String, + /// Its value. + value: String, + }, +} + +impl Record { + fn line(&self) -> String { + match self { + Self::Allow(path) => format!("Allow: {}", one_line(path)), + Self::Disallow(path) => format!("Disallow: {}", one_line(path)), + Self::Comment(text) => format!("#{}", one_line(text)), + Self::Other { name, value } => format!("{}: {}", one_line(name), one_line(value)), + } + } + + /// `Disallow: /`, or the same rule written with wildcards, such as + /// `Disallow: /*`, which RFC 9309 section 2.2.3 reads as matching every path. + fn refuses_the_whole_site(&self) -> bool { + matches!(self, Self::Disallow(path) + if one_line(path).strip_prefix('/').is_some_and(|rest| rest.bytes().all(|b| b == b'*'))) + } +} + +/// `value` with every CR and LF taken out, so that it cannot end its line and +/// start a record of its own. +fn one_line(value: &str) -> Cow<'_, str> { + if value.contains(['\r', '\n']) { + Cow::Owned(value.replace(['\r', '\n'], "")) + } else { + Cow::Borrowed(value) + } +} + +/// The crawlers a group is for, and the records that apply to them. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct Group { + /// Each written on its own `User-Agent` line. An empty name is kept as it + /// was given, because the file is written back as it was read. + pub user_agents: Vec, + /// The group's records, in order. + pub records: Vec, +} + +/// What one contributor adds to the file. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct Contribution { + /// Records before the first group, such as a comment introducing the rules. + pub preamble: Vec, + /// The groups, in order. + pub groups: Vec, +} + +impl Contribution { + /// A contribution read from `robots.txt` text, keeping every group, record + /// and comment in the order it came. + /// + /// A `User-Agent` line after a group's rules starts the next group, and + /// consecutive `User-Agent` lines name the crawlers of one group, as RFC + /// 9309 section 2.1 describes. A `#` starts a comment wherever it is, as + /// section 2.2 allows, and the comment is kept as a record of its own after + /// the rule it followed. Blank lines and lines with no `:` carry nothing + /// and are passed over. A record before the first group is kept in the + /// preamble rather than dropped, so nothing given is lost. + #[must_use] + pub fn parse(text: &str) -> Self { + let mut contribution = Self::default(); + let mut current: Option = None; + for raw in text.split('\n') { + let (rule, comment) = match raw.split_once('#') { + Some((rule, comment)) => (rule.trim(), Some(comment.trim_end())), + None => (raw.trim(), None), + }; + if let Some((name, value)) = rule.split_once(':') { + let (name, value) = (name.trim(), value.trim().to_owned()); + match name.to_ascii_lowercase().as_str() { + "user-agent" => { + if current.as_ref().is_some_and(|group| { + group + .records + .iter() + .any(|record| !matches!(record, Record::Comment(_))) + }) && let Some(finished) = current.take() + { + contribution.groups.push(finished); + } + current + .get_or_insert_with(Group::default) + .user_agents + .push(value); + } + "allow" => push(&mut contribution, &mut current, Record::Allow(value)), + "disallow" => push(&mut contribution, &mut current, Record::Disallow(value)), + _ => push( + &mut contribution, + &mut current, + Record::Other { + name: name.to_owned(), + value, + }, + ), + } + } + if let Some(comment) = comment { + push( + &mut contribution, + &mut current, + Record::Comment(comment.to_owned()), + ); + } + } + if let Some(group) = current { + contribution.groups.push(group); + } + contribution + } + + /// Whether this contribution has anything to write. + #[must_use] + pub fn is_empty(&self) -> bool { + self.preamble.is_empty() && self.groups.is_empty() + } + + /// The contribution as text, with a blank line between its parts and none + /// at either end. + fn write(&self) -> String { + let mut parts: Vec = Vec::new(); + if !self.preamble.is_empty() { + parts.push( + self.preamble + .iter() + .map(Record::line) + .collect::>() + .join("\n"), + ); + } + for group in &self.groups { + let mut lines: Vec = group + .user_agents + .iter() + .map(|agent| format!("User-Agent: {}", one_line(agent))) + .collect(); + lines.extend(group.records.iter().map(Record::line)); + parts.push(lines.join("\n")); + } + parts.join("\n\n") + } +} + +fn push(contribution: &mut Contribution, current: &mut Option, record: Record) { + match current { + Some(group) => group.records.push(record), + None => contribution.preamble.push(record), + } +} + +/// What `allow_all` contributes, which is every crawler allowed everywhere. +/// +/// Permission the document says, rather than permission that follows from a +/// lookup coming back empty. +#[must_use] +pub fn allow_all() -> Contribution { + Contribution { + preamble: Vec::new(), + groups: vec![Group { + user_agents: vec!["*".to_owned()], + records: vec![Record::Allow("/".to_owned())], + }], + } +} + +/// The file `refuse_all` serves, which refuses every crawler except for the +/// paths in `always_allow`. +/// +/// Nothing a contributor gives, and none of the publisher's own text, is +/// added, because a refusal cannot be loosened. `Allow` beats `Disallow` on +/// the longest match under RFC 9309 section 2.2.2, so each allowance lets that +/// one path through. +#[must_use] +pub fn refusal(always_allow: &[String]) -> String { + let mut file = String::from("User-agent: *"); + for path in always_allow { + file.push_str("\nAllow: "); + file.push_str(path); + } + file.push_str("\nDisallow: /\n"); + file +} + +/// The whole file, being the publisher's text above, each contribution in the +/// order `modules` names it, the group that keeps `always_allow` open, the +/// `Sitemap` line, and the publisher's text below. +#[must_use] +pub fn assemble(config: &RobotsTxtConfig, contributions: &[Contribution]) -> String { + let mut sections: Vec = Vec::new(); + if let Some(top) = config + .top_text + .as_deref() + .filter(|text| !text.trim().is_empty()) + { + sections.push(top.trim_end().to_owned()); + } + for contribution in contributions.iter().filter(|c| !c.is_empty()) { + sections.push(contribution.write()); + } + if let Some(segment) = kept_open(contributions, &config.always_allow) { + sections.push(segment); + } + if let Some(sitemap) = &config.sitemap { + sections.push(format!("Sitemap: {sitemap}")); + } + if let Some(bottom) = config + .bottom_text + .as_deref() + .filter(|text| !text.trim().is_empty()) + { + sections.push(bottom.trim_end().to_owned()); + } + let mut file = sections.join("\n\n"); + file.push('\n'); + file +} + +/// Every crawler the contributions refuse the whole site to, by the name on +/// its `User-Agent` line, in order and without repeats. `*` is included, so +/// the paths stay open to the crawlers no group names. A group with no name +/// is passed over, because there is nothing to address a rule to. +fn refused_entirely(contributions: &[Contribution]) -> Vec<&str> { + let mut names: Vec<&str> = Vec::new(); + for group in contributions.iter().flat_map(|c| c.groups.iter()) { + if !group.records.iter().any(Record::refuses_the_whole_site) { + continue; + } + for name in &group.user_agents { + if !name.is_empty() && !names.contains(&name.as_str()) { + names.push(name); + } + } + } + names +} + +/// A group of core's own that keeps each path in `always_allow` open to the +/// crawlers the contributions refuse the whole site to, or `None` when there is +/// nothing to keep open. +/// +/// **The contributions are never edited.** RFC 9309 section 2.2.1 has a +/// crawler combine every group that matches it, and section 2.2.2 gives the +/// match with the most octets, so a second group naming the same crawlers with +/// an `Allow` for the path beats `Disallow: /` for that path alone. +/// +/// It goes below the contributions. A crawler that obeys only the first group +/// it finds then still meets `Disallow: /` first, so the worst it can do is +/// fail to read the one page. Written above, it would read the whole site. +fn kept_open(contributions: &[Contribution], paths: &[String]) -> Option { + let names = refused_entirely(contributions); + if names.is_empty() || paths.is_empty() { + return None; + } + let mut lines: Vec = names + .iter() + .map(|name| format!("User-Agent: {}", one_line(name))) + .collect(); + lines.extend(paths.iter().map(|path| format!("Allow: {path}"))); + Some(lines.join("\n")) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn config() -> RobotsTxtConfig { + RobotsTxtConfig { + modules: vec!["a_contributor".to_owned()], + ..RobotsTxtConfig::refuse_all() + } + } + + /// Text a contributor might read from somewhere else, with the shapes the + /// file has to survive: a record inside a group, crawlers sharing a group, + /// and a group with no name. + const GIVEN: &str = "User-Agent: ExampleBot\nDisallow: /\n\nUser-Agent: \nDisallow: /\n\n\ + User-Agent: OtherBot\nDisallow: /private/\n\nUser-Agent: ThirdBot\n\ + User-Agent: FourthBot\nDisallow: /\n\nUser-Agent: *\n\ + TDL: https://terms.example.com/socw/2.txt\nAllow: /"; + + #[test] + fn a_contribution_is_written_back_exactly_as_it_was_read() { + assert_eq!(Contribution::parse(GIVEN).write(), GIVEN); + } + + #[test] + fn crawlers_named_together_share_one_group_and_a_record_keeps_its_place() { + let parsed = Contribution::parse(GIVEN); + + assert_eq!(parsed.groups.len(), 5); + assert_eq!(parsed.groups[3].user_agents, vec!["ThirdBot", "FourthBot"]); + assert_eq!( + parsed.groups[4].records, + vec![ + Record::Other { + name: "TDL".to_owned(), + value: "https://terms.example.com/socw/2.txt".to_owned(), + }, + Record::Allow("/".to_owned()), + ] + ); + } + + #[test] + fn blank_lines_at_either_end_are_not_part_of_the_file() { + let parsed = Contribution::parse("\nUser-Agent: ExampleBot\nDisallow: /\n\n\n"); + + assert_eq!(parsed.write(), "User-Agent: ExampleBot\nDisallow: /"); + } + + #[test] + fn a_comment_and_a_record_before_any_group_are_kept() { + let parsed = Contribution::parse("# rules for crawlers\nUser-Agent: *\nAllow: /"); + + assert_eq!( + parsed.preamble, + vec![Record::Comment(" rules for crawlers".to_owned())] + ); + assert_eq!( + parsed.write(), + "# rules for crawlers\n\nUser-Agent: *\nAllow: /" + ); + } + + #[test] + fn the_publishers_text_and_the_sitemap_go_around_the_contributions() { + let mut settings = config(); + settings.top_text = Some("# top".to_owned()); + settings.sitemap = Some("https://publisher.example.com/sitemap.xml".to_owned()); + settings.bottom_text = Some("# bottom".to_owned()); + + assert_eq!( + assemble( + &settings, + &[Contribution::parse("User-agent: *\nAllow: /\n")] + ), + "# top\n\nUser-Agent: *\nAllow: /\n\n\ + Sitemap: https://publisher.example.com/sitemap.xml\n\n# bottom\n", + ); + } + + /// The page a refused crawler is sent to stays open, by a group of core's + /// own below the contributions, which are not edited. + #[test] + fn an_always_allowed_path_is_kept_open_to_every_crawler_refused_the_site() { + let mut settings = config(); + settings.always_allow = vec!["/ai-notice/".to_owned()]; + + let file = assemble(&settings, &[Contribution::parse(GIVEN)]); + + assert!( + file.starts_with(GIVEN), + "the contribution is not edited: {file}" + ); + assert!( + file.ends_with( + "\n\nUser-Agent: ExampleBot\nUser-Agent: ThirdBot\nUser-Agent: FourthBot\n\ + Allow: /ai-notice/\n" + ), + "the group naming every crawler refused the whole site goes last: {file}" + ); + } + + /// The path is kept open to `*` as it is to a crawler refused by name, so + /// the crawlers no group names can still read it. + #[test] + fn an_always_allowed_path_is_kept_open_when_every_crawler_is_refused() { + let mut settings = config(); + settings.always_allow = vec!["/ai-notice/".to_owned()]; + + assert_eq!( + assemble( + &settings, + &[Contribution::parse("User-Agent: *\nDisallow: /")] + ), + "User-Agent: *\nDisallow: /\n\nUser-Agent: *\nAllow: /ai-notice/\n", + "a group for `*` below the contribution keeps the path open" + ); + } + + #[test] + fn a_wildcard_that_matches_every_path_refuses_the_whole_site() { + let mut settings = config(); + settings.always_allow = vec!["/ai-notice/".to_owned()]; + + assert_eq!( + assemble( + &settings, + &[Contribution::parse("User-Agent: ExampleBot\nDisallow: /*")] + ), + "User-Agent: ExampleBot\nDisallow: /*\n\n\ + User-Agent: ExampleBot\nAllow: /ai-notice/\n", + "`Disallow: /*` is the same rule as `Disallow: /`" + ); + } + + #[test] + fn a_comment_after_a_rule_does_not_hide_a_whole_site_refusal() { + let mut settings = config(); + settings.always_allow = vec!["/ads.txt".to_owned()]; + let parsed = Contribution::parse("User-Agent: *\nDisallow: / # temporarily closed"); + + assert_eq!( + parsed.groups[0].records, + vec![ + Record::Disallow("/".to_owned()), + Record::Comment(" temporarily closed".to_owned()), + ] + ); + assert!( + assemble(&settings, &[parsed]).ends_with("\n\nUser-Agent: *\nAllow: /ads.txt\n"), + "the path stays open to the crawlers refused the site" + ); + } + + #[test] + fn a_comment_between_user_agent_lines_keeps_their_group() { + let mut settings = config(); + settings.always_allow = vec!["/ads.txt".to_owned()]; + let parsed = Contribution::parse( + "User-Agent: ExampleBot\n# the same rules for its sibling\nUser-Agent: OtherBot\nDisallow: /", + ); + + assert_eq!(parsed.groups.len(), 1); + assert_eq!(parsed.groups[0].user_agents, vec!["ExampleBot", "OtherBot"]); + assert!( + assemble(&settings, &[parsed]) + .ends_with("\n\nUser-Agent: ExampleBot\nUser-Agent: OtherBot\nAllow: /ads.txt\n"), + "every crawler of the group is kept open to the path" + ); + } + + #[test] + fn a_line_break_in_a_value_cannot_start_a_record() { + let contribution = Contribution { + preamble: vec![Record::Comment(" note\nAllow: /".to_owned())], + groups: vec![Group { + user_agents: vec!["ExampleBot\r\nUser-Agent: *".to_owned()], + records: vec![ + Record::Disallow("/x\nUser-Agent: *\nAllow: /".to_owned()), + Record::Other { + name: "TDL\r".to_owned(), + value: "a\rAllow: /".to_owned(), + }, + ], + }], + }; + + assert_eq!( + assemble(&config(), &[contribution]), + "# noteAllow: /\n\n\ + User-Agent: ExampleBotUser-Agent: *\n\ + Disallow: /xUser-Agent: *Allow: /\n\ + TDL: aAllow: /\n", + "every value stays on the line its record starts" + ); + } + + #[test] + fn nothing_is_kept_open_when_nothing_is_refused() { + let mut settings = config(); + settings.always_allow = vec!["/ai-notice/".to_owned()]; + + assert_eq!( + assemble(&settings, &[allow_all()]), + "User-Agent: *\nAllow: /\n" + ); + } + + #[test] + fn contributions_appear_in_the_order_given() { + let first = Contribution::parse("User-Agent: FirstBot\nDisallow: /"); + let second = Contribution::parse("User-Agent: SecondBot\nDisallow: /"); + + assert_eq!( + assemble(&config(), &[first, second]), + "User-Agent: FirstBot\nDisallow: /\n\nUser-Agent: SecondBot\nDisallow: /\n" + ); + } + + #[test] + fn an_empty_contribution_leaves_no_gap() { + assert_eq!( + assemble(&config(), &[Contribution::default(), allow_all()]), + "User-Agent: *\nAllow: /\n" + ); + } + + #[test] + fn refusing_with_no_allowances_refuses_everything() { + assert_eq!(refusal(&[]), "User-agent: *\nDisallow: /\n"); + } + + #[test] + fn each_allowance_is_written_in_order_ahead_of_the_refusal() { + assert_eq!( + refusal(&["/ads.txt".to_owned(), "/ai-notice/".to_owned()]), + "User-agent: *\nAllow: /ads.txt\nAllow: /ai-notice/\nDisallow: /\n" + ); + } +} diff --git a/crates/trusted-server-core/src/robots_txt/holding.rs b/crates/trusted-server-core/src/robots_txt/holding.rs new file mode 100644 index 0000000000..f0ffeddcbf --- /dev/null +++ b/crates/trusted-server-core/src/robots_txt/holding.rs @@ -0,0 +1,943 @@ +//! Each contributor's answer, held between requests by the loading cache. +//! +//! A contributor's answer is loaded through +//! [`RuntimeServices::load_cached`], so the contributor is asked once however +//! many requests want its rules at the same moment, and the copy in the host's +//! shared store answers every instance until the contributor's refresh is due. +//! +//! The last answer a contributor gave is also kept apart from the cache, for +//! far longer than any refresh, and read only when the contributor fails. It is +//! then served past its age, because yesterday's rules are a better answer than +//! none. It is kept in the host's value cache when the host supplies one, and +//! in the key-value store otherwise, because a host that starts an instance +//! for each request may give its requests a value cache and no key-value store. +//! A value cache may drop it sooner, so on such a host a copy past its age is +//! likely to be there rather than certain, and a key-value store that cannot +//! give an entry a lifetime keeps it until it is deleted there. + +use core::time::Duration; +use std::sync::Arc; + +use error_stack::Report; +use sha2::{Digest as _, Sha256}; + +use super::{Contribution, RobotsTxtContributor}; +use crate::error::TrustedServerError; +use crate::module_context::{ModuleContext, ResolvedRequest}; +use crate::platform::{ + CacheLifetimes, CacheSpec, KvError, LoadError, LoadedValue, Loader, RuntimeServices, StoreKey, +}; +use crate::settings::Settings; + +/// The cache contributors' answers are held in. +pub(crate) const CACHE: &str = crate::settings::ROBOTS_TXT_CACHE; + +/// Where each contributor's last answer is kept apart from the cache. +const KEPT: &str = "robots-txt-kept"; + +/// How long a contributor's last answer is kept, far past any refresh, so it +/// is there to serve when the contributor cannot answer. +const KEPT_LIFETIME: Duration = Duration::from_secs(30 * 24 * 3600); + +/// The lifetimes `[cache.robots-txt]` leaves unset. A contributor's own +/// refresh ends each copy sooner than the store lifetime. +const DEFAULT_LIFETIMES: CacheLifetimes = CacheLifetimes { + memory: Duration::from_secs(60), + store: Duration::from_secs(7 * 24 * 3600), + store_idle: Duration::from_secs(30 * 24 * 3600), +}; + +/// The cache contributors' answers are held in, with the lifetimes +/// `[cache.robots-txt]` sets. +/// +/// Its loads run in the caller, because a contributor may block the thread it +/// runs on while it asks its service. +pub(crate) fn cache_spec(settings: &Settings) -> CacheSpec { + CacheSpec { + name: CACHE, + lifetimes: settings.cache.robots_txt.lifetimes(DEFAULT_LIFETIMES), + memory_entries: 64, + detachable: false, + } +} + +/// The contribution of `contributor`, selected as `id`, from a copy young +/// enough to serve, and from the contributor otherwise, which is asked for +/// `request`. +/// +/// # Errors +/// +/// When the contributor fails and no answer of its is kept. A kept answer is +/// returned past its age rather than an error. +pub(crate) async fn contribution_for( + id: &'static str, + contributor: &Arc, + request: &ResolvedRequest, + services: &RuntimeServices, + settings: &Settings, +) -> Result> { + let prefix = services.server_key_prefix(); + let held = held_key(id, &contributor.fingerprint()); + let key = StoreKey::new(prefix, CACHE, held.clone()); + let kept = StoreKey::new(prefix, KEPT, held); + let spec = cache_spec(settings); + let ask = || { + load( + Ask { + id, + contributor: Arc::clone(contributor), + request: request.clone(), + services: services.clone(), + }, + kept.clone(), + ) + }; + let loaded = match held_answer(services, &spec, &key, ask()).await { + Err(Held::Unreadable(error)) => { + // A copy that no longer reads would answer every request until + // it ends, so it is removed and the contributor asked again. + log::warn!( + "Asking `{id}` again for robots.txt rules, the held copy is unreadable: {error}" + ); + services + .loading_cache_for(CACHE) + .remove(services, &spec, &key) + .await; + held_answer(services, &spec, &key, ask()).await + } + loaded => loaded, + }; + match loaded { + Ok(contribution) => Ok(contribution), + Err(Held::Failed(error) | Held::Unreadable(error)) => { + match kept_answer(services, &kept).await { + Some(contribution) => { + log::warn!( + "Serving the kept robots.txt rules of `{id}` past their age: {error}" + ); + Ok(contribution) + } + None => Err(Report::new(TrustedServerError::Integration { + integration: id.to_owned(), + message: format!("the robots.txt contributor did not answer: {error}"), + })), + } + } + } +} + +/// Why no held or fresh answer was read. +enum Held { + /// The load failed. + Failed(LoadError), + /// The cache answered with a copy that does not read as a contribution. + Unreadable(LoadError), +} + +/// The answer the cache holds for `key`, loaded with `load` when it holds +/// none. +async fn held_answer( + services: &RuntimeServices, + spec: &CacheSpec, + key: &StoreKey, + load: Loader, +) -> Result { + let bytes = services + .load_cached(spec, key, load) + .await + .map_err(Held::Failed)?; + serde_json::from_slice::(&bytes).map_err(|error| { + Held::Unreadable(LoadError::new(format!( + "the held answer is unreadable: {error}" + ))) + }) +} + +/// What a load owns to ask one contributor, because it may outlive the +/// request that started it. +struct Ask { + /// The name the contributor was declared under. + id: &'static str, + contributor: Arc, + request: ResolvedRequest, + services: RuntimeServices, +} + +/// Asks the contributor, keeps its answer apart for when it next fails, and +/// gives the answer the contributor's refresh as its lifetime. +fn load(ask: Ask, kept: StoreKey) -> Loader { + Box::new(move || { + Box::pin(async move { + let Ask { + id, + contributor, + request, + services, + } = ask; + let context = ModuleContext::new(request.view()).with_services(&services); + let contribution = contributor + .contribute(context.call(id, contributor.required_permissions())) + .await + .map_err(|report| LoadError::new(format!("{report:?}")))?; + let bytes = serde_json::to_vec(&contribution) + .map_err(|error| LoadError::new(format!("the answer cannot be held: {error}")))?; + keep_answer(&services, &kept, &bytes).await; + Ok(LoadedValue::new(bytes).with_lifetime(contributor.refresh())) + }) + }) +} + +/// The key one contributor's answer for one fingerprint is held under, a hash +/// so the fingerprint's contents, whatever they are, never appear in a store. +fn held_key(id: &str, fingerprint: &str) -> String { + let mut hasher = Sha256::new(); + hasher.update(id.as_bytes()); + hasher.update(b"\n"); + hasher.update(fingerprint.as_bytes()); + let mut key = String::with_capacity(64); + for byte in hasher.finalize() { + key.push_str(&format!("{byte:02x}")); + } + key +} + +/// The answer kept for when the contributor fails, from the host's value +/// cache when it supplies one and from the key-value store otherwise. +/// +/// Nothing read, for whatever reason, is no kept answer, and the failure that +/// led here is reported instead. +async fn kept_answer(services: &RuntimeServices, kept: &StoreKey) -> Option { + let key = kept.to_string(); + let bytes = match services.value_cache() { + Some(cache) => cache.get(&key).await?, + None => match services.kv_store().get_bytes(&key).await { + Ok(bytes) => bytes?, + Err(error) => { + log::debug!("No kept robots.txt answer could be read: {error}"); + return None; + } + }, + }; + serde_json::from_slice(&bytes) + .inspect_err(|error| log::debug!("The kept robots.txt answer is unreadable: {error}")) + .ok() +} + +/// Keeps an answer for when the contributor next fails, in the host's value +/// cache when it supplies one and in the key-value store otherwise. +async fn keep_answer(services: &RuntimeServices, kept: &StoreKey, bytes: &[u8]) { + let key = kept.to_string(); + let bytes = bytes::Bytes::copy_from_slice(bytes); + if let Some(cache) = services.value_cache() { + cache.put(&key, bytes, KEPT_LIFETIME).await; + return; + } + let store = services.kv_store(); + let written = match store + .put_bytes_with_ttl(&key, bytes.clone(), KEPT_LIFETIME) + .await + { + // A store that cannot give an entry a lifetime keeps it until it is + // replaced, so the answer is still there when the contributor fails. + Err(KvError::Unsupported { .. }) => store.put_bytes(&key, bytes).await, + written => written, + }; + if let Err(error) = written { + log::debug!("The robots.txt answer was not kept for a later failure: {error}"); + } +} + +#[cfg(test)] +pub(crate) mod tests { + use std::sync::Mutex; + + use edgezero_core::body::Body as EdgeBody; + + use super::*; + use crate::evidence::RequestInfo; + use crate::module_context::{ModuleCall, ModuleRequest}; + use crate::platform::test_support::{MemoryLoadingCache, StubHttpClient}; + use crate::prefix::test_support::{assert_keys_follow_the_publisher, trusted_server_for}; + + /// A contributor whose answers a test decides, counting how often it is + /// asked. + #[derive(Debug)] + pub(crate) struct Scripted { + answers: Mutex>>, + pub(crate) asked: Mutex, + pub(crate) refresh: Duration, + pub(crate) fingerprint: Mutex, + } + + impl Scripted { + pub(crate) fn new(answers: Vec>) -> Self { + Self { + answers: Mutex::new(answers), + asked: Mutex::new(0), + refresh: Duration::from_secs(3600), + fingerprint: Mutex::new("first settings".to_owned()), + } + } + + pub(crate) fn times_asked(&self) -> usize { + *self.asked.lock().expect("should lock the count") + } + } + + #[async_trait::async_trait(?Send)] + impl RobotsTxtContributor for Scripted { + fn refresh(&self) -> Duration { + self.refresh + } + + fn fingerprint(&self) -> String { + self.fingerprint.lock().expect("should lock").clone() + } + + async fn contribute( + &self, + _call: ModuleCall<'_>, + ) -> Result> { + *self.asked.lock().expect("should lock the count") += 1; + let next = self + .answers + .lock() + .expect("should lock the answers") + .remove(0); + next.map_err(|message| Report::new(TrustedServerError::Proxy { message })) + } + } + + pub(crate) fn rules(text: &str) -> Contribution { + Contribution::parse(text) + } + + /// A request for `/robots.txt` as a crawler makes it, with a query of its + /// own. + pub(crate) fn robots_request() -> http::Request { + http::Request::builder() + .uri("https://www.publisher.example/robots.txt?crawler=one") + .header(http::header::HOST, "www.publisher.example") + .body(EdgeBody::empty()) + .expect("should build the request") + } + + /// The request the contributors are asked for, as the handler resolves it. + pub(crate) fn request() -> ResolvedRequest { + ResolvedRequest::of(&robots_request(), &crate::platform::ClientInfo::default()) + .without_query() + } + + /// A contributor whose rules name what it was handed, so a test can read + /// back the request, whether the reader's evidence reached it, and + /// whether the services did. + #[derive(Debug)] + pub(crate) struct Echoing; + + impl Echoing { + fn rules( + &self, + request: ModuleRequest<'_>, + evidence: Option<&dyn RequestInfo>, + services: Option<&RuntimeServices>, + ) -> Contribution { + rules(&format!( + "User-Agent: EchoBot\nDisallow: /{}{}?{}/{}/{}", + request.host(), + request.path(), + request.query(), + if evidence.is_some() { + "evidence" + } else { + "no-evidence" + }, + if services.is_some() { + "services" + } else { + "no-services" + }, + )) + } + } + + #[async_trait::async_trait(?Send)] + impl RobotsTxtContributor for Echoing { + fn refresh(&self) -> Duration { + Duration::from_secs(60) + } + + fn fingerprint(&self) -> String { + "echo".to_owned() + } + + async fn contribute( + &self, + call: ModuleCall<'_>, + ) -> Result> { + Ok(call.inject(self, Self::rules)?) + } + } + + /// What [`Echoing`] answers for [`request`]. + pub(crate) const ECHOED: &str = + "User-Agent: EchoBot\nDisallow: /www.publisher.example/robots.txt?/no-evidence/services"; + + /// The answer is shared by every crawler, so the contributor is handed + /// the request it was asked for and its services, and nothing about who + /// asked, not even the query. + #[tokio::test] + async fn the_contributor_is_asked_for_the_request_and_not_who_asked() { + let echoing: Arc = Arc::new(Echoing); + let services = services_with(Arc::default(), Arc::default()); + + let answer = contribution_for("echo", &echoing, &request(), &services, &settings()) + .await + .expect("should ask the contributor"); + + assert_eq!(answer, rules(ECHOED)); + } + + /// A store held in memory for the life of one test, so that what one + /// instance wrote is what the next one reads. + #[derive(Debug, Default)] + pub(crate) struct MemoryKvStore { + pub(crate) entries: Mutex>, + /// The lifetime each entry was last written with. + pub(crate) lifetimes: Mutex>, + /// Whether a write with a lifetime is refused, as it is by a store + /// that has none to give. + pub(crate) refuses_lifetimes: bool, + } + + #[async_trait::async_trait(?Send)] + impl crate::platform::PlatformKvStore for MemoryKvStore { + async fn get_bytes( + &self, + key: &str, + ) -> Result, crate::platform::KvError> { + Ok(self + .entries + .lock() + .expect("should lock the test store") + .get(key) + .cloned()) + } + + async fn put_bytes( + &self, + key: &str, + value: bytes::Bytes, + ) -> Result<(), crate::platform::KvError> { + self.entries + .lock() + .expect("should lock the test store") + .insert(key.to_owned(), value); + Ok(()) + } + + async fn put_bytes_with_ttl( + &self, + key: &str, + value: bytes::Bytes, + ttl: Duration, + ) -> Result<(), crate::platform::KvError> { + if self.refuses_lifetimes { + return Err(KvError::Unsupported { + operation: "put_bytes_with_ttl".to_owned(), + }); + } + self.lifetimes + .lock() + .expect("should lock the test store") + .insert(key.to_owned(), ttl); + self.put_bytes(key, value).await + } + + async fn delete(&self, key: &str) -> Result<(), crate::platform::KvError> { + self.entries + .lock() + .expect("should lock the test store") + .remove(key); + Ok(()) + } + + async fn list_keys_page( + &self, + _prefix: &str, + _cursor: Option<&str>, + _limit: usize, + ) -> Result { + Ok(edgezero_core::key_value_store::KvPage::default()) + } + } + + /// Services keeping answers in `cache` and kept answers in `store`, as one + /// instance of a host with no value cache, for the publisher [`settings`] + /// describes. + pub(crate) fn services_with( + cache: Arc, + store: Arc, + ) -> RuntimeServices { + crate::platform::test_support::build_services_with_http_client(Arc::new( + StubHttpClient::new(), + )) + .with_kv_store(store) + .with_loading_cache(cache) + .with_prefixes(trusted_server_for("www.publisher.example")) + } + + pub(crate) fn settings() -> Settings { + let mut settings = crate::test_support::tests::create_test_settings(); + settings.publisher.domain = "www.publisher.example".to_owned(); + settings + } + + fn contributor(scripted: &Arc) -> Arc { + Arc::clone(scripted) as Arc + } + + #[tokio::test] + async fn a_contributor_is_asked_once_and_its_answer_is_held_after_that() { + let scripted = Arc::new(Scripted::new(vec![Ok(rules("User-Agent: *\nAllow: /"))])); + let cache = Arc::new(MemoryLoadingCache::default()); + let services = services_with(Arc::clone(&cache), Arc::default()); + + let first = contribution_for( + "a", + &contributor(&scripted), + &request(), + &services, + &settings(), + ) + .await + .expect("should ask the contributor"); + let second = contribution_for( + "a", + &contributor(&scripted), + &request(), + &services, + &settings(), + ) + .await + .expect("should serve the held copy"); + + assert_eq!(first, second); + assert_eq!(scripted.times_asked(), 1); + assert!( + cache.calls().iter().all(|(name, _)| *name == CACHE), + "every call should be to the robots.txt cache" + ); + } + + #[tokio::test] + async fn the_held_answer_is_keyed_by_the_publisher() { + let scripted = Arc::new(Scripted::new(vec![ + Ok(rules("User-Agent: OneBot\nDisallow: /")), + Ok(rules("User-Agent: OtherBot\nDisallow: /")), + ])); + let cache = Arc::new(MemoryLoadingCache::default()); + let services = services_with(Arc::clone(&cache), Arc::default()); + + contribution_for( + "a", + &contributor(&scripted), + &request(), + &services, + &settings(), + ) + .await + .expect("should answer the first publisher"); + let answer = contribution_for( + "a", + &contributor(&scripted), + &request(), + &services.with_prefixes(trusted_server_for("other.example")), + &settings(), + ) + .await + .expect("should answer the second publisher"); + + assert_eq!(answer, rules("User-Agent: OtherBot\nDisallow: /")); + assert_eq!(scripted.times_asked(), 2, "each publisher is asked for"); + let keys = cache.keys(); + assert!( + keys[0].starts_with("ts.other.example/robots-txt/"), + "{keys:?}" + ); + assert!( + keys[1].starts_with("ts.publisher.example/robots-txt/"), + "{keys:?}" + ); + } + + /// Both the held answer and the one kept for a failure are keyed by the + /// publisher the request's services carry. + #[tokio::test] + async fn the_held_and_kept_keys_follow_the_publisher() { + let keys_for = |domain: &str| { + let scripted = Arc::new(Scripted::new(vec![Ok(rules("User-Agent: *\nAllow: /"))])); + let cache = Arc::new(MemoryLoadingCache::default()); + let store = Arc::new(MemoryKvStore::default()); + let services = services_with(Arc::clone(&cache), Arc::clone(&store)) + .with_prefixes(trusted_server_for(domain)); + futures::executor::block_on(contribution_for( + "a", + &contributor(&scripted), + &request(), + &services, + &settings(), + )) + .expect("should ask the contributor"); + let kept: Vec = store + .entries + .lock() + .expect("should lock the test store") + .keys() + .cloned() + .collect(); + format!("{:?} {kept:?}", cache.keys()) + }; + + assert_keys_follow_the_publisher(keys_for); + } + + #[tokio::test] + async fn changed_settings_are_not_answered_from_the_old_copy() { + let scripted = Arc::new(Scripted::new(vec![ + Ok(rules("User-Agent: FirstBot\nDisallow: /")), + Ok(rules("User-Agent: SecondBot\nDisallow: /")), + ])); + let services = services_with(Arc::default(), Arc::default()); + + contribution_for( + "a", + &contributor(&scripted), + &request(), + &services, + &settings(), + ) + .await + .expect("should ask for the first settings"); + *scripted.fingerprint.lock().expect("should lock") = "second settings".to_owned(); + let answer = contribution_for( + "a", + &contributor(&scripted), + &request(), + &services, + &settings(), + ) + .await + .expect("should ask again for the new settings"); + + assert_eq!(answer, rules("User-Agent: SecondBot\nDisallow: /")); + } + + #[tokio::test] + async fn the_kept_answer_is_served_when_the_contributor_fails() { + let mut scripted = Scripted::new(vec![ + Ok(rules("User-Agent: HeldBot\nDisallow: /")), + Err("down".to_owned()), + ]); + // A refresh of zero gives each answer no time in the cache, so the + // second request asks again on any machine at any moment. + scripted.refresh = Duration::ZERO; + let scripted = Arc::new(scripted); + let services = services_with(Arc::default(), Arc::default()); + + contribution_for( + "a", + &contributor(&scripted), + &request(), + &services, + &settings(), + ) + .await + .expect("should ask the contributor"); + let answer = contribution_for( + "a", + &contributor(&scripted), + &request(), + &services, + &settings(), + ) + .await + .expect("should serve yesterday's rules"); + + assert_eq!(answer, rules("User-Agent: HeldBot\nDisallow: /")); + assert_eq!(scripted.times_asked(), 2, "should have asked again first"); + } + + #[tokio::test] + async fn the_kept_answer_outlives_the_instance_that_kept_it() { + let scripted = Arc::new(Scripted::new(vec![ + Ok(rules("User-Agent: KeptBot\nDisallow: /")), + Err("down".to_owned()), + ])); + let store = Arc::new(MemoryKvStore::default()); + + contribution_for( + "a", + &contributor(&scripted), + &request(), + &services_with(Arc::default(), Arc::clone(&store)), + &settings(), + ) + .await + .expect("should ask the contributor"); + // A second instance, with a cache of its own and the same store. + let answer = contribution_for( + "a", + &contributor(&scripted), + &request(), + &services_with(Arc::default(), Arc::clone(&store)), + &settings(), + ) + .await + .expect("should serve the kept answer"); + + assert_eq!(answer, rules("User-Agent: KeptBot\nDisallow: /")); + assert_eq!(scripted.times_asked(), 2); + let keys: Vec = store + .entries + .lock() + .expect("should lock the test store") + .keys() + .cloned() + .collect(); + assert_eq!(keys.len(), 1, "{keys:?}"); + assert!( + keys[0].starts_with("ts.publisher.example/robots-txt-kept/"), + "{keys:?}" + ); + let lifetimes = store.lifetimes.lock().expect("should lock the test store"); + assert_eq!( + lifetimes.values().copied().collect::>(), + vec![KEPT_LIFETIME], + "the answer should be kept far longer than any refresh" + ); + } + + /// A store with no lifetime to give still keeps the answer, so it is + /// there to serve when the contributor fails. + #[tokio::test] + async fn a_store_without_lifetimes_still_keeps_the_answer() { + let mut scripted = Scripted::new(vec![ + Ok(rules("User-Agent: OneBot\nDisallow: /")), + Err("cannot answer".to_owned()), + ]); + scripted.refresh = Duration::ZERO; + let scripted = Arc::new(scripted); + let store = Arc::new(MemoryKvStore { + refuses_lifetimes: true, + ..MemoryKvStore::default() + }); + let services = services_with(Arc::default(), Arc::clone(&store)); + + contribution_for( + "a", + &contributor(&scripted), + &request(), + &services, + &settings(), + ) + .await + .expect("should ask the contributor"); + let later = contribution_for( + "a", + &contributor(&scripted), + &request(), + &services, + &settings(), + ) + .await + .expect("should serve the kept answer"); + + assert_eq!(later, rules("User-Agent: OneBot\nDisallow: /")); + assert!( + store + .lifetimes + .lock() + .expect("should lock the test store") + .is_empty(), + "the entry was written without a lifetime" + ); + } + + #[tokio::test] + async fn with_nothing_kept_a_failure_is_an_error_and_is_not_held() { + let scripted = Arc::new(Scripted::new(vec![ + Err("down".to_owned()), + Ok(rules("User-Agent: LateBot\nDisallow: /")), + ])); + let services = services_with(Arc::default(), Arc::default()); + + assert!( + contribution_for( + "a", + &contributor(&scripted), + &request(), + &services, + &settings() + ) + .await + .is_err() + ); + let answer = contribution_for( + "a", + &contributor(&scripted), + &request(), + &services, + &settings(), + ) + .await + .expect("the next request should ask again"); + + assert_eq!(answer, rules("User-Agent: LateBot\nDisallow: /")); + assert_eq!(scripted.times_asked(), 2); + } + + /// A held copy that no longer reads is removed and the contributor asked + /// again, rather than every request failing until the copy ends. + #[tokio::test] + async fn an_unreadable_held_copy_is_removed_and_the_contributor_asked_again() { + let scripted = Arc::new(Scripted::new(vec![Ok(rules( + "User-Agent: FreshBot\nDisallow: /", + ))])); + let cache = Arc::new(MemoryLoadingCache::default()); + let services = services_with(Arc::clone(&cache), Arc::default()); + let settings = settings(); + let key = StoreKey::new( + services.server_key_prefix(), + CACHE, + held_key("a", &scripted.fingerprint()), + ); + services + .load_cached( + &cache_spec(&settings), + &key, + Box::new(|| { + Box::pin(async { + Ok(LoadedValue::new(bytes::Bytes::from_static( + b"not a contribution", + ))) + }) + }), + ) + .await + .expect("should hold the unreadable copy"); + + let answer = contribution_for( + "a", + &contributor(&scripted), + &request(), + &services, + &settings, + ) + .await + .expect("should ask the contributor again"); + let again = contribution_for( + "a", + &contributor(&scripted), + &request(), + &services, + &settings, + ) + .await + .expect("should serve the fresh copy"); + + assert_eq!(answer, rules("User-Agent: FreshBot\nDisallow: /")); + assert_eq!(again, answer); + assert_eq!(scripted.times_asked(), 1, "the fresh answer should be held"); + } + + /// A value cache held in memory for the life of one test. + #[derive(Debug, Default)] + struct MemoryValueCache { + entries: Mutex>, + } + + #[async_trait::async_trait(?Send)] + impl crate::platform::PlatformValueCache for MemoryValueCache { + async fn get(&self, key: &str) -> Option { + self.entries + .lock() + .expect("should lock the test cache") + .get(key) + .cloned() + } + + async fn put(&self, key: &str, value: bytes::Bytes, _lifetime: Duration) { + self.entries + .lock() + .expect("should lock the test cache") + .insert(key.to_owned(), value); + } + } + + /// A host that gives its requests a value cache and no key-value store + /// keeps the answer in the value cache, and serves it from there when the + /// contributor fails. + #[tokio::test] + async fn the_kept_answer_is_in_the_value_cache_where_the_host_has_one() { + let mut scripted = Scripted::new(vec![ + Ok(rules("User-Agent: CachedBot\nDisallow: /")), + Err("down".to_owned()), + ]); + scripted.refresh = Duration::ZERO; + let scripted = Arc::new(scripted); + let value_cache = Arc::new(MemoryValueCache::default()); + let services = crate::platform::RuntimeServices::builder() + .config_store(Arc::new(crate::platform::test_support::NoopConfigStore)) + .secret_store(Arc::new(crate::platform::test_support::NoopSecretStore)) + .kv_store(Arc::new(crate::platform::UnavailableKvStore)) + .backend(Arc::new(crate::platform::test_support::NoopBackend)) + .http_client(Arc::new(StubHttpClient::new())) + .geo(Arc::new(crate::platform::test_support::NoopGeo)) + .value_cache(Arc::clone(&value_cache) as Arc) + .loading_caches(Arc::new(crate::platform::LoadingCaches::every(Arc::new( + MemoryLoadingCache::default(), + )))) + .client_info(crate::platform::ClientInfo::default()) + .prefixes(trusted_server_for("www.publisher.example")) + .build(); + + contribution_for( + "a", + &contributor(&scripted), + &request(), + &services, + &settings(), + ) + .await + .expect("should ask the contributor"); + let answer = contribution_for( + "a", + &contributor(&scripted), + &request(), + &services, + &settings(), + ) + .await + .expect("should serve the kept answer from the value cache"); + + assert_eq!(answer, rules("User-Agent: CachedBot\nDisallow: /")); + assert_eq!( + value_cache + .entries + .lock() + .expect("should lock the test cache") + .len(), + 1, + "the answer should be kept in the value cache" + ); + } + + #[test] + fn the_key_carries_nothing_of_the_fingerprint() { + let key = held_key("a", "license=secret-value"); + + assert_eq!(key.len(), 64); + assert!(!key.contains("secret"), "{key}"); + assert_ne!(key, held_key("b", "license=secret-value")); + } +} diff --git a/crates/trusted-server-core/src/robots_txt/middleware.rs b/crates/trusted-server-core/src/robots_txt/middleware.rs new file mode 100644 index 0000000000..a51bc37c5c --- /dev/null +++ b/crates/trusted-server-core/src/robots_txt/middleware.rs @@ -0,0 +1,228 @@ +//! `/robots.txt` as a middleware of core's own, named `robots_txt` in a +//! `[[fetch]]` entry on the file's path. Its preparation makes the file from +//! the selected contributors, through their holding, and its action replaces +//! the origin's file with it. Where the origin has no file, or answers with a +//! page, the creating rule runs it on an empty one. + +use std::any::Any; +use std::time::Duration; + +use error_stack::Report; + +use super::{RETRY_AFTER_SECONDS, RobotsTxtConfig, Selected, document, file_for}; +use crate::error::TrustedServerError; +use crate::middleware::{ + Middleware, MiddlewareAction, MiddlewareContext, MiddlewarePhase, Preparation, Refusal, +}; +use crate::module_context::{ModuleCall, ModuleRequest, ResolvedRequest}; +use crate::platform::RuntimeServices; +use crate::settings::Settings; +use crate::streaming_processor::StreamProcessor; + +/// The name an entry writes, and the integration the registry lists the +/// middleware under. +pub const MODULE: &str = "robots_txt"; + +/// The media type the middleware works on. +pub const MEDIA_TYPE: &str = "text/plain"; + +/// What a request is told when no file can be made. +const UNAVAILABLE: &str = "robots.txt is temporarily unavailable\n"; + +/// The middleware, holding the section it was made for and the contributors +/// the registry resolved for it, in the order `modules` names them. +#[derive(Debug)] +pub struct RobotsTxt { + config: RobotsTxtConfig, + selected: Vec, +} + +impl RobotsTxt { + /// The middleware for `config`, with `selected` the contributors in the + /// order `modules` names them. + #[must_use] + pub(crate) fn new(config: RobotsTxtConfig, selected: Vec) -> Self { + Self { config, selected } + } + + /// The file for the request, which is the refusal alone or every selected + /// contributor's held answer assembled. + async fn file( + &self, + request: ModuleRequest<'_>, + settings: &Settings, + services: &RuntimeServices, + ) -> Result> { + if self.config.refuses_all() { + return Ok(document::refusal(&self.config.always_allow)); + } + // The answers are held and served to every crawler, so the + // contributors are handed the request with nothing one crawler put in + // the address. + let request = ResolvedRequest::from_view(&request).without_query(); + file_for(&self.config, &self.selected, &request, services, settings).await + } +} + +impl Middleware for RobotsTxt { + fn middleware_id(&self) -> &'static str { + MODULE + } + + fn handles_media_type(&self, media_type: &str) -> bool { + media_type == MEDIA_TYPE + } + + fn phases(&self) -> &[MiddlewarePhase] { + &MiddlewarePhase::ALL + } + + fn prepare<'a>( + &'a self, + call: ModuleCall<'a>, + _context: &'a MiddlewareContext<'a>, + ) -> Option> { + Some(Box::pin(async move { + let file = match call.inject(self, Self::file) { + Ok(file) => file.await, + Err(withheld) => Err(Report::new(TrustedServerError::Configuration { + message: format!("robots.txt cannot be made for this request: {withheld:?}"), + })), + }; + match file { + Ok(file) => Ok(Box::new(file) as Box), + Err(report) => { + log::error!("Could not assemble robots.txt: {report:?}"); + Err(Refusal::service_unavailable( + Duration::from_secs(u64::from(RETRY_AFTER_SECONDS)), + UNAVAILABLE, + )) + } + } + })) + } + + fn create(&self, context: &MiddlewareContext<'_>) -> MiddlewareAction { + match context.take_prepared::(MODULE) { + Some(file) => MiddlewareAction { + stream: Some(Box::new(ReplaceBody { file: Some(file) })), + ..MiddlewareAction::pass() + }, + // Nothing prepared means the preparation did not run, which no request can cause. + None => { + log::error!("robots.txt was not prepared, so the origin's file is served as it is"); + MiddlewareAction::pass() + } + } + } +} + +/// Drops the body it is given and writes the file in its place. +struct ReplaceBody { + file: Option, +} + +impl StreamProcessor for ReplaceBody { + fn process_chunk(&mut self, _chunk: &[u8], is_last: bool) -> Result, std::io::Error> { + Ok(if is_last { + self.file.take().unwrap_or_default().into_bytes() + } else { + Vec::new() + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::integrations::IntegrationDocumentState; + use crate::middleware::test_support::context; + use crate::middleware::{MiddlewareChain, Prepared}; + + fn middleware() -> RobotsTxt { + RobotsTxt::new(RobotsTxtConfig::refuse_all(), Vec::new()) + } + + /// What the middleware is told in `phase`, with `prepared` as what its + /// preparation left. + fn context_with<'a>( + phase: MiddlewarePhase, + state: &'a IntegrationDocumentState, + prepared: &'a Prepared, + ) -> MiddlewareContext<'a> { + MiddlewareContext { + prepared, + ..context(phase, state) + } + } + + #[test] + fn the_middleware_works_on_text_in_either_phase() { + let middleware = middleware(); + + assert!(middleware.handles_media_type(MEDIA_TYPE)); + assert!(!middleware.handles_media_type(crate::middleware::HTML_MEDIA_TYPE)); + assert_eq!(middleware.phases(), &MiddlewarePhase::ALL); + assert_eq!(middleware.middleware_id(), MODULE); + } + + #[test] + fn the_prepared_file_replaces_whatever_the_origin_sent() { + let state = IntegrationDocumentState::default(); + let prepared = Prepared::default(); + prepared.insert( + MiddlewarePhase::Fetch, + MODULE, + Box::new(String::from("User-agent: *\nDisallow: /\n")), + ); + let context = context_with(MiddlewarePhase::Fetch, &state, &prepared); + + let mut processor = middleware() + .create(&context) + .stream + .expect("a prepared file should be written as a stream processor"); + let mut output = processor + .process_chunk(b"User-agent: *\nAllow: /\n", false) + .expect("should take the origin's file"); + output.extend( + processor + .process_chunk(b"", true) + .expect("should write the file at the end"), + ); + + assert_eq!(output, b"User-agent: *\nDisallow: /\n"); + } + + #[test] + fn with_nothing_prepared_the_origins_file_is_left_as_it_is() { + let state = IntegrationDocumentState::default(); + let prepared = Prepared::default(); + let context = context_with(MiddlewarePhase::Fetch, &state, &prepared); + + let action = middleware().create(&context); + + assert!(action.stream.is_none(), "should pass the body through"); + assert!(action.head_inserts.is_empty()); + } + + #[test] + fn a_value_of_another_type_under_the_name_is_not_a_file() { + let state = IntegrationDocumentState::default(); + let prepared = Prepared::default(); + prepared.insert(MiddlewarePhase::Serve, MODULE, Box::new(7_u32)); + let context = context_with(MiddlewarePhase::Serve, &state, &prepared); + + assert!(middleware().create(&context).stream.is_none()); + } + + #[test] + fn a_chain_plans_the_middleware_on_text() { + let chain = MiddlewareChain::new( + MiddlewarePhase::Fetch, + MEDIA_TYPE, + vec![std::sync::Arc::new(middleware()) as std::sync::Arc], + ); + + assert_eq!(chain.ids(), [MODULE]); + } +} diff --git a/crates/trusted-server-core/src/robots_txt/mod.rs b/crates/trusted-server-core/src/robots_txt/mod.rs new file mode 100644 index 0000000000..ea6f030794 --- /dev/null +++ b/crates/trusted-server-core/src/robots_txt/mod.rs @@ -0,0 +1,767 @@ +//! A publisher's `robots.txt`, assembled from the rules the selected +//! contributors give. +//! +//! `[robots-txt] modules` names, in order, what makes the file. Two names are +//! core's own and need nothing else: [`REFUSE_ALL`] refuses every crawler, and +//! [`ALLOW_ALL`] allows every crawler everywhere. Any other name is the name a +//! module declared a [`RobotsTxtContributor`] under. Core never names a +//! module here and knows nothing of what any contributor asks or of whom. +//! +//! # Contributors add rules, core writes the file +//! +//! A contributor returns a [`Contribution`], being groups of records, and core +//! writes every byte in [`document`]. That is what lets these hold whatever a +//! contributor says. +//! +//! 1. A refusal cannot be loosened. With `refuse_all` selected the file is the +//! refusal and nothing else. +//! 2. A path in `always_allow` stays open to every crawler a contribution +//! refuses the whole site to, `*` included. A narrower rule is kept as +//! given, so `Disallow: /ads` still closes off `/ads.txt`. +//! 3. The `Sitemap` line, the publisher's own text, and the `X-Robots-Tag` +//! header on every page agree with the file served. +//! 4. Several contributors combine in the order `modules` gives, and none of +//! them owns the file. +//! 5. A contributor never writes bytes, so it cannot produce a malformed file. +//! 6. A contributor that fails with no held answer fails the whole file closed +//! with `503`, because leaving out a contributor's rules would allow by +//! omission whatever those rules refused. A `404` or an empty file reads to +//! a crawler as permission to crawl everything. +//! +//! # Settings +//! +//! `[robots-txt]` holds `modules` and the document's own settings, being +//! `always_allow`, `sitemap`, `top_text` and `bottom_text`. Settings belonging +//! to one contributor go in its own block, `[robots-txt.]`, which that +//! contributor reads and core does not. + +mod contributor; +pub mod document; +mod holding; +pub mod middleware; + +use std::collections::BTreeMap; +use std::sync::Arc; + +use edgezero_core::body::Body as EdgeBody; +use error_stack::Report; +use http::Response; +use http::header::{self, HeaderValue}; +use serde::{Deserialize, Deserializer, Serialize}; +use validator::{Validate, ValidationError}; + +pub use contributor::{ALLOW_ALL, REFUSE_ALL, RobotsTxtContributor}; +pub use document::{Contribution, Group, Record}; + +use crate::error::TrustedServerError; +use crate::module_context::ResolvedRequest; +use crate::platform::RuntimeServices; +use crate::settings::Settings; + +/// The path this module answers. +pub const ROBOTS_TXT_PATH: &str = "/robots.txt"; + +/// How long a crawler is asked to wait when there is no file to give it. +const RETRY_AFTER_SECONDS: u32 = 600; + +/// The `X-Robots-Tag` a response carries when `refuse_all` is selected. +/// +/// A robots.txt asks a crawler not to fetch. This tells one that has already +/// fetched not to index, and it is the only one of the two that works on a page +/// a crawler learned about from somewhere else, or reached before reading +/// robots.txt, or chose to fetch anyway. They do different jobs and a site that +/// is not ready to be seen needs both. +/// +/// Set by the same selection so the two cannot drift apart. +/// +/// `nofollow` as well as `noindex`, so a crawler does not walk on from a page +/// it should not have indexed into the rest of a site that is not ready either. +pub const REFUSE_ALL_ROBOTS_TAG: &str = "noindex, nofollow"; + +/// The type folder of every robots.txt crate, which a name written in +/// `[robots-txt] modules` may leave off. +pub const MODULE_TYPE: &str = "robots-txt"; + +/// Settings for serving `robots.txt`. +/// +/// Mapped from the `[robots-txt]` TOML section. With the section absent the +/// publisher keeps their own file and `/robots.txt` reaches their origin like +/// any other path. +#[derive(Debug, Clone, Deserialize, Serialize, Validate)] +pub struct RobotsTxtConfig { + /// What makes the file, in order. One name or a list, so + /// `modules = "refuse_all"` and `modules = ["refuse_all"]` are the same. + /// + /// Required, and refused at load when absent or empty. There is no default, + /// because a default would decide for a publisher what crawlers may do. + #[serde(default, deserialize_with = "one_or_many")] + pub modules: Vec, + + /// Paths kept open to every crawler refused the whole site, the page a + /// refused crawler is sent to for example. A narrower rule, such as + /// `Disallow: /ads`, still closes off `/ads.txt`. `/robots.txt` needs no + /// entry, because RFC 9309 section 2.2.2 says it is implicitly allowed. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[validate(custom(function = validate_always_allow))] + pub always_allow: Vec, + + /// The site's sitemap, added as a `Sitemap:` line. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[validate(url)] + pub sitemap: Option, + + /// The publisher's own text, placed above the contributors' rules. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub top_text: Option, + + /// The publisher's own text, placed below the contributors' rules. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub bottom_text: Option, + + /// Each contributor's own settings, `[robots-txt.]`, keyed by its + /// name. + /// + /// Read by that contributor and by nothing in core, which only checks that + /// each is a block and belongs to a contributor `modules` names. + #[serde(flatten)] + pub contributors: BTreeMap, +} + +fn one_or_many<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + #[derive(Deserialize)] + #[serde(untagged)] + enum OneOrMany { + One(String), + Many(Vec), + } + Ok(match OneOrMany::deserialize(deserializer)? { + OneOrMany::One(name) => vec![name], + OneOrMany::Many(names) => names, + }) +} + +fn validate_always_allow(paths: &[String]) -> Result<(), ValidationError> { + // A path is written into the file as it stands, so it must be a path and + // must not be able to start a second line. + if paths + .iter() + .any(|path| !path.starts_with('/') || path.chars().any(char::is_whitespace)) + { + return Err(ValidationError::new("robots_txt_always_allow_not_a_path")); + } + Ok(()) +} + +impl RobotsTxtConfig { + /// The configuration that refuses every crawler with no exception. + /// + /// The starting point for a publisher who wants the refusal with + /// allowances of their own. Needs no contributor, no key and no network. + #[must_use] + pub fn refuse_all() -> Self { + Self { + modules: vec![REFUSE_ALL.to_owned()], + always_allow: Vec::new(), + sitemap: None, + top_text: None, + bottom_text: None, + contributors: BTreeMap::new(), + } + } + + /// Whether `modules` names `id`, by its full name or by its short form, + /// whichever of the two `id` and the name in `modules` are written in. + #[must_use] + pub fn selects(&self, id: &str) -> bool { + self.modules.iter().any(|selected| { + crate::module_name::resolve(MODULE_TYPE, selected, &[id]).is_some() + || crate::module_name::short_form(MODULE_TYPE, selected) == id + }) + } + + /// Whether the file is the refusal, which it is whenever `refuse_all` is + /// selected, whatever else is. + #[must_use] + pub fn refuses_all(&self) -> bool { + self.selects(REFUSE_ALL) + } + + /// The `X-Robots-Tag` every response should carry, or `None` when the + /// publisher's crawler policy places no blanket rule on pages. + #[must_use] + pub fn response_robots_tag(&self) -> Option<&'static str> { + self.refuses_all().then_some(REFUSE_ALL_ROBOTS_TAG) + } + + /// The settings block of the contributor named `id`, when there is one. + #[must_use] + pub fn contributor_settings(&self, id: &str) -> Option<&serde_json::Value> { + self.contributors.get(id).or_else(|| { + self.contributors + .get(crate::module_name::short_form(MODULE_TYPE, id)) + }) + } + + /// Checks what the type cannot. + /// + /// # Errors + /// + /// When `modules` names nothing or names one thing twice, when a key is + /// not a setting, and when a contributor's block belongs to one `modules` + /// does not name. + pub fn prepare_runtime(&self) -> Result<(), Report> { + if self.modules.is_empty() { + return Err(Report::new(configuration( + "[robots-txt] names no module. Name what makes the file, in order: \ + `refuse_all` to refuse every crawler, `allow_all` to allow every crawler, \ + or the name of each module whose rules make the file. There is no default, \ + because a default would decide for the publisher what crawlers may do." + .to_owned(), + ))); + } + // Compared on the short form, so the full name and the short form of + // one module count as naming it twice. + for (index, name) in self.modules.iter().enumerate() { + let short_name = crate::module_name::short_form(MODULE_TYPE, name); + if self.modules[..index] + .iter() + .any(|earlier| crate::module_name::short_form(MODULE_TYPE, earlier) == short_name) + { + return Err(Report::new(configuration(format!( + "[robots-txt] modules names `{name}` twice." + )))); + } + } + for (key, value) in &self.contributors { + if !value.is_object() { + return Err(Report::new(configuration(format!( + "robots-txt.{key} is not a setting. [robots-txt] holds modules, \ + always_allow, sitemap, top_text, bottom_text, and a block for each \ + contributor that has settings of its own." + )))); + } + if !self.selects(key) { + return Err(Report::new(configuration(format!( + "[robots-txt.{key}] configures a contributor that `modules` does not \ + name, so nothing would read it. Add `{key}` to modules or remove the \ + block." + )))); + } + } + Ok(()) + } +} + +fn configuration(message: String) -> TrustedServerError { + TrustedServerError::Configuration { message } +} + +/// Puts [`REFUSE_ALL_ROBOTS_TAG`] on an outgoing response when `refuse_all` is +/// selected, in place of any `X-Robots-Tag` it carried, and does nothing +/// otherwise. Called once where an adapter finalizes a response. +pub fn apply_response_robots_tag(settings: &Settings, response: &mut Response) { + if settings + .robots_txt + .as_ref() + .and_then(RobotsTxtConfig::response_robots_tag) + .is_none() + { + return; + } + response.headers_mut().insert( + header::HeaderName::from_static("x-robots-tag"), + HeaderValue::from_static(REFUSE_ALL_ROBOTS_TAG), + ); +} + +/// One entry of `modules` as the registry resolved it: the name a +/// contributor was declared under and the contributor, or `allow_all` and +/// none. +pub(crate) type Selected = (&'static str, Option>); + +/// The entries of `config.modules` resolved against `contributors`, the +/// contributors a deployment runs by the names they were declared under. +/// +/// # Errors +/// +/// When an entry names a contributor `contributors` does not carry, which +/// the registry refuses before it asks, so this is a wiring fault. +pub(crate) fn select( + config: &RobotsTxtConfig, + contributors: &[(&'static str, Arc)], +) -> Result, Report> { + let mut selected = Vec::with_capacity(config.modules.len()); + for written in &config.modules { + if written == REFUSE_ALL { + // The refusal needs no contributor, and the middleware answers it + // before asking any. + continue; + } + if written == ALLOW_ALL { + selected.push((ALLOW_ALL, None)); + continue; + } + let names: Vec<&str> = contributors.iter().map(|(name, _)| *name).collect(); + let Some((declared, contributor)) = + crate::module_name::resolve(MODULE_TYPE, written, &names) + .and_then(|declared| contributors.iter().find(|(name, _)| *name == declared)) + else { + return Err(Report::new(configuration(format!( + "[robots-txt] modules names `{written}`, which no module in this deployment supplies" + )))); + }; + selected.push((declared, Some(Arc::clone(contributor)))); + } + Ok(selected) +} + +/// The file for `config`, from the contributions of `selected` in order, +/// each asked for `request`. +pub(crate) async fn file_for( + config: &RobotsTxtConfig, + selected: &[Selected], + request: &ResolvedRequest, + services: &RuntimeServices, + settings: &Settings, +) -> Result> { + let mut contributions = Vec::with_capacity(selected.len()); + for (id, contributor) in selected { + contributions.push(match contributor { + None => document::allow_all(), + Some(contributor) => { + holding::contribution_for(id, contributor, request, services, settings).await? + } + }); + } + Ok(document::assemble(config, &contributions)) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use super::holding::tests::{ + ECHOED, Echoing, MemoryKvStore, Scripted, request, robots_request, rules, services_with, + settings, + }; + use super::*; + use crate::integrations::{IntegrationBuilder, IntegrationRegistration, IntegrationRegistry}; + use crate::middleware::Middleware; + use crate::module_context::{ModuleContext, ModuleRequest}; + use http::StatusCode; + + fn parsed(document: serde_json::Value) -> RobotsTxtConfig { + serde_json::from_value(document).expect("should read the robots.txt settings") + } + + #[test] + fn a_single_module_reads_as_a_list_of_one() { + let config = parsed(serde_json::json!({ "modules": "refuse_all" })); + + assert_eq!(config.modules, vec!["refuse_all"]); + assert!(config.refuses_all()); + config + .prepare_runtime() + .expect("a single module written as a string must load"); + } + + #[test] + fn a_list_of_modules_keeps_its_order() { + let config = parsed(serde_json::json!({ + "modules": ["a_module", "allow_all"], + "a_module": { "anything": "the module reads" }, + })); + + assert_eq!(config.modules, vec!["a_module", "allow_all"]); + config.prepare_runtime().expect("should load"); + } + + #[test] + fn no_module_is_refused_rather_than_defaulted() { + let error = parsed(serde_json::json!({ "sitemap": "https://example.com/sitemap.xml" })) + .prepare_runtime() + .expect_err("there is no default"); + let message = format!("{error:?}"); + + assert!(message.contains("refuse_all"), "{message}"); + assert!(message.contains("allow_all"), "{message}"); + } + + #[test] + fn an_empty_list_is_refused_the_same_way() { + assert!( + parsed(serde_json::json!({ "modules": [] })) + .prepare_runtime() + .is_err() + ); + } + + #[test] + fn a_name_given_twice_is_refused() { + assert!( + parsed(serde_json::json!({ "modules": ["allow_all", "allow_all"] })) + .prepare_runtime() + .is_err() + ); + } + + /// A section may write a contributor's full name or leave the type folder + /// off, so both spellings select the block, and naming both is naming the + /// contributor twice. + #[test] + fn a_contributor_named_in_full_selects_its_block() { + parsed(serde_json::json!({ + "modules": "robots-txt.a_module", + "a_module": { "anything": "the module reads" }, + })) + .prepare_runtime() + .expect("the full name should select the block written at the short name"); + + let error = parsed(serde_json::json!({ + "modules": ["a_module", "robots-txt.a_module"], + "a_module": { "anything": "the module reads" }, + })) + .prepare_runtime() + .expect_err("one contributor named in two spellings is named twice"); + assert!(format!("{error:?}").contains("twice"), "{error:?}"); + } + + #[test] + fn a_key_that_is_not_a_setting_is_refused() { + let message = format!( + "{:?}", + parsed(serde_json::json!({ "modules": "allow_all", "sitmap": "a typo" })) + .prepare_runtime() + .expect_err("a misspelled setting must not load") + ); + + assert!(message.contains("is not a setting"), "{message}"); + } + + #[test] + fn a_block_for_a_contributor_not_selected_is_refused() { + let message = format!( + "{:?}", + parsed(serde_json::json!({ "modules": "allow_all", "a_module": {} })) + .prepare_runtime() + .expect_err("nothing would read the block") + ); + + assert!(message.contains("does not name"), "{message}"); + } + + #[test] + fn a_path_kept_open_must_be_a_path_on_one_line() { + for path in ["ads.txt", "/ads.txt\nDisallow: /", "/a path"] { + let config = parsed(serde_json::json!({ + "modules": "refuse_all", + "always_allow": [path], + })); + + assert!(config.validate().is_err(), "should refuse {path:?}"); + } + parsed(serde_json::json!({ "modules": "refuse_all", "always_allow": ["/ads.txt"] })) + .validate() + .expect("should accept a path"); + } + + #[test] + fn refusing_sets_the_response_header_and_a_contributor_does_not() { + assert_eq!( + RobotsTxtConfig::refuse_all().response_robots_tag(), + Some(REFUSE_ALL_ROBOTS_TAG) + ); + assert_eq!( + parsed(serde_json::json!({ "modules": "allow_all" })).response_robots_tag(), + None + ); + } + + #[test] + fn the_header_replaces_one_the_origin_set() { + let mut settings = settings(); + settings.robots_txt = Some(RobotsTxtConfig::refuse_all()); + let mut response = Response::builder() + .header("x-robots-tag", "all") + .body(EdgeBody::empty()) + .expect("should build a response"); + + apply_response_robots_tag(&settings, &mut response); + + let tags: Vec<_> = response.headers().get_all("x-robots-tag").iter().collect(); + assert_eq!(tags, vec![REFUSE_ALL_ROBOTS_TAG]); + } + + #[test] + fn a_response_is_left_alone_when_nothing_refuses_every_crawler() { + let mut allowing = settings(); + allowing.robots_txt = Some(parsed(serde_json::json!({ "modules": "allow_all" }))); + let mut without_a_section = settings(); + without_a_section.robots_txt = None; + + for settings in [allowing, without_a_section] { + let mut response = Response::builder() + .header("x-robots-tag", "noarchive") + .body(EdgeBody::empty()) + .expect("should build a response"); + + apply_response_robots_tag(&settings, &mut response); + + let tags: Vec<_> = response.headers().get_all("x-robots-tag").iter().collect(); + assert_eq!(tags, vec!["noarchive"]); + } + } + + #[tokio::test] + async fn contributions_are_assembled_in_the_order_selected() { + let first: Arc = Arc::new(Scripted::new(vec![Ok(rules( + "User-Agent: FirstBot\nDisallow: /", + ))])); + let config = parsed(serde_json::json!({ + "modules": ["first", "allow_all"], + "sitemap": "https://publisher.example.com/sitemap.xml", + })); + + let file = file_for( + &config, + &[("first", Some(first)), (ALLOW_ALL, None)], + &request(), + &services_with(Arc::default(), Arc::default()), + &settings(), + ) + .await + .expect("should assemble the file"); + + assert_eq!( + file, + "User-Agent: FirstBot\nDisallow: /\n\nUser-Agent: *\nAllow: /\n\n\ + Sitemap: https://publisher.example.com/sitemap.xml\n" + ); + } + + /// What the middleware prepares for `settings`, with the services given, + /// through the registry that made it from `[robots-txt]`, as the handler + /// runs it for `/robots.txt` in the fetch phase. + async fn prepared_by( + settings: &Settings, + extra: &[IntegrationBuilder], + services: &RuntimeServices, + ) -> Result, crate::middleware::Refusal> { + use crate::middleware::{MiddlewarePhase, Prepared}; + + let registry = IntegrationRegistry::with_registrations(settings, extra) + .expect("should build the registry"); + let chain = registry.middleware_chain( + &settings.fetch, + MiddlewarePhase::Fetch, + middleware::MEDIA_TYPE, + ROBOTS_TXT_PATH, + ); + assert_eq!( + chain.ids(), + [middleware::MODULE], + "the entry names the middleware" + ); + let request = robots_request(); + let module_context = + ModuleContext::new(ModuleRequest::of(&request, services.client_info())) + .with_settings(settings) + .with_services(services); + let state = crate::integrations::IntegrationDocumentState::default(); + let prepared = Prepared::default(); + let context = crate::middleware::MiddlewareContext { + prepared: &prepared, + ..crate::middleware::test_support::context(MiddlewarePhase::Fetch, &state) + }; + chain.prepare(&module_context, &context).await?; + Ok(prepared.take::(MiddlewarePhase::Fetch, middleware::MODULE)) + } + + /// `settings` with `config` as its `[robots-txt]` section and the entry + /// that places the middleware on the file. + fn settings_placing(config: RobotsTxtConfig) -> Settings { + let mut settings = settings(); + settings.robots_txt = Some(config); + settings.fetch = + crate::middleware::PhaseEntries::new(vec![crate::middleware::PhaseEntry { + media_type: middleware::MEDIA_TYPE.to_owned(), + path: Some(ROBOTS_TXT_PATH.to_owned()), + middleware: vec![middleware::MODULE.to_owned()], + }]); + settings + } + + /// Leaving out the rules of a contributor that failed would allow by + /// omission what those rules refused, so the whole file fails closed. + #[tokio::test] + async fn with_nothing_held_a_failure_refuses_the_request() { + let failing: Arc = + Arc::new(Scripted::new(vec![Err("cannot answer".to_owned())])); + let config = parsed(serde_json::json!({ "modules": ["failing", "allow_all"] })); + let middleware = + middleware::RobotsTxt::new(config, vec![("failing", Some(failing)), (ALLOW_ALL, None)]); + let settings = settings(); + let services = services_with(Arc::default(), Arc::default()); + let request = robots_request(); + let module_context = + ModuleContext::new(ModuleRequest::of(&request, services.client_info())) + .with_settings(&settings) + .with_services(&services); + let state = crate::integrations::IntegrationDocumentState::default(); + let context = crate::middleware::test_support::context( + crate::middleware::MiddlewarePhase::Fetch, + &state, + ); + + let refusal = middleware + .prepare( + module_context.call( + middleware::MODULE, + crate::permissions::PermissionSet::none(), + ), + &context, + ) + .expect("the middleware prepares") + .await + .expect_err("a crawler reads an empty or allow everything file as permission"); + + assert_eq!(refusal.status, StatusCode::SERVICE_UNAVAILABLE); + assert_eq!( + refusal.retry_after, + Some(std::time::Duration::from_secs(u64::from( + RETRY_AFTER_SECONDS + ))) + ); + assert_eq!(refusal.message, "robots.txt is temporarily unavailable\n"); + } + + /// The refusal asks nothing of anything, and dominates whatever else is + /// selected, because a refusal cannot be loosened. + #[tokio::test] + async fn refusing_every_crawler_asks_no_contributor_and_reads_no_store() { + let mut refusing = parsed(serde_json::json!({ "modules": ["allow_all", "refuse_all"] })); + refusing.always_allow = vec!["/ads.txt".to_owned()]; + let settings = settings_placing(refusing); + let store = Arc::new(MemoryKvStore::default()); + + let file = prepared_by( + &settings, + &[], + &services_with(Arc::default(), Arc::clone(&store)), + ) + .await + .expect("the refusal answers without asking") + .expect("the refusal is prepared"); + + assert_eq!( + file, "User-agent: *\nAllow: /ads.txt\nDisallow: /\n", + "the refusal, whatever else is selected" + ); + assert!( + store + .entries + .lock() + .expect("should lock the test store") + .is_empty(), + "nothing is held for a refusal" + ); + } + + /// The registry makes the middleware only for settings that carry the + /// section, so a publisher keeping their own file is left alone. + #[test] + fn the_middleware_exists_only_with_the_section() { + let without = IntegrationRegistry::new(&settings()).expect("should build the registry"); + assert!( + !without.middleware_ids().contains(&middleware::MODULE), + "{:?}", + without.middleware_ids() + ); + + let with = IntegrationRegistry::new(&settings_placing(RobotsTxtConfig::refuse_all())) + .expect("should build the registry"); + assert!( + with.middleware_ids().contains(&middleware::MODULE), + "{:?}", + with.middleware_ids() + ); + } + + /// The id of the builder that registers the echoing contributor. + const ECHO_BUILDER: &str = "robots-txt-echo"; + + /// The module name that builder is selected by. + const ECHO_BUILDER_MODULE: &str = "testing.robots-txt-echo"; + + fn register_echo( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder(ECHO_BUILDER) + .with_robots_txt_contributor("robots-txt.echo", Arc::new(Echoing)) + .build(), + )) + } + + fn accept(_settings: &Settings) -> Result> { + Ok(true) + } + + fn settings_selecting_echo() -> Settings { + let mut settings = settings(); + settings.robots_txt = Some(parsed(serde_json::json!({ "modules": ["echo"] }))); + settings + .insert_module_config("testing", ECHO_BUILDER_MODULE, &serde_json::json!({})) + .expect("should select the test builder"); + settings + } + + fn echo_builder() -> IntegrationBuilder { + IntegrationBuilder::new(ECHO_BUILDER, ECHO_BUILDER, register_echo, accept) + .with_module_name(ECHO_BUILDER_MODULE) + } + + /// The middleware asks a contributor for the request it is answering, + /// being its host and path, without the query and without the reader's + /// evidence. + #[tokio::test] + async fn the_middleware_asks_the_contributors_for_the_request_it_answers() { + let mut settings = settings_selecting_echo(); + settings.fetch = settings_placing(RobotsTxtConfig::refuse_all()).fetch; + + let file = prepared_by( + &settings, + &[echo_builder()], + &services_with(Arc::default(), Arc::default()), + ) + .await + .expect("should answer") + .expect("the file is prepared"); + + let config = settings + .robots_txt + .as_ref() + .expect("the settings carry a [robots-txt] section"); + assert_eq!(file, document::assemble(config, &[rules(ECHOED)])); + } + + /// A name nothing supplies is refused where the registry is built, so the + /// file does not fail on the first request for it. + #[test] + fn a_selection_nothing_supplies_is_refused_when_the_registry_is_built() { + let mut settings = settings(); + settings.robots_txt = Some(parsed(serde_json::json!({ "modules": ["nobody"] }))); + + let error = IntegrationRegistry::new(&settings) + .err() + .expect("should refuse a contributor nothing supplies"); + let message = format!("{error:?}"); + + assert!(message.contains("`nobody`"), "{message}"); + assert!(message.contains("refuse_all"), "{message}"); + assert!(message.contains("allow_all"), "{message}"); + } +} diff --git a/crates/trusted-server-core/src/rsc_flight.rs b/crates/trusted-server-core/src/rsc_flight.rs index a9e3a40cec..dac4a4b6ae 100644 --- a/crates/trusted-server-core/src/rsc_flight.rs +++ b/crates/trusted-server-core/src/rsc_flight.rs @@ -29,7 +29,7 @@ enum RowState { /// URLs like `\/\/origin.example.com` (JSON-escaped slashes) will not be rewritten. This is /// acceptable because Flight responses from client-side navigation typically contain plain URLs, /// not doubly-escaped JSON-in-JS content. For inlined `__next_f` data in HTML (which can have -/// escape sequences), the HTML post-processor in `integrations/nextjs/` handles those cases. +/// escape sequences), the HTML post-processor in `crates/framework/nextjs` handles those cases. pub struct RscFlightUrlRewriter { origin_url: String, origin_http_url: Option, diff --git a/crates/trusted-server-core/src/secret_resolution.rs b/crates/trusted-server-core/src/secret_resolution.rs index f362a0b7b2..6bf4afa67f 100644 --- a/crates/trusted-server-core/src/secret_resolution.rs +++ b/crates/trusted-server-core/src/secret_resolution.rs @@ -1,18 +1,91 @@ //! Runtime resolution of `EdgeZero` app-config secret references. //! //! Config blobs carry secret-store key names at rest. This module walks the -//! public `EdgeZero` metadata contract and replaces those names only in the -//! in-memory value used to build runtime [`crate::settings::Settings`]. +//! public `EdgeZero` metadata contract, together with the leaves +//! [`ConfiguredSecretFields`] reads out of the configuration itself, and +//! replaces those names only in the in-memory value used to build runtime +//! [`crate::settings::Settings`]. + +use std::fmt; use edgezero_core::app_config::{AppConfigMeta, SecretField, SecretKind, SecretPathSegment}; use error_stack::Report; use serde_json::Value; use crate::error::TrustedServerError; +use crate::inspect::config::{PathPattern, PathStep}; use crate::platform::{PlatformSecretStore, StoreName}; +/// Where one resolution pass wrote secrets, what it wrote, and every leaf it +/// looked for, which the configuration view masks. +/// +/// Never serialized, and [`fmt::Debug`] prints only how many there are. +#[derive(Clone, Default)] +pub struct ResolvedSecrets { + paths: Vec>, + values: Vec, + patterns: Vec, +} + +impl ResolvedSecrets { + /// The concrete path of every leaf a secret was written into. + pub(crate) fn paths(&self) -> &[Vec] { + &self.paths + } + + /// The secret values written. + pub(crate) fn values(&self) -> &[String] { + &self.values + } + + /// Every secret leaf the pass looked for, found or not. + pub(crate) fn patterns(&self) -> &[PathPattern] { + &self.patterns + } + + /// A record of `paths` and `values`, for a test of what uses one. + #[cfg(test)] + pub(crate) fn recorded(paths: Vec>, values: Vec) -> Self { + Self { + paths, + values, + patterns: Vec::new(), + } + } +} + +impl fmt::Debug for ResolvedSecrets { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("ResolvedSecrets") + .field("paths", &self.paths.len()) + .field("values", &self.values.len()) + .field("patterns", &self.patterns.len()) + .finish() + } +} + +/// Secret leaves whose paths come from the configuration rather than the type. +/// +/// [`AppConfigMeta::secret_fields`] is an associated function of the type, so +/// every path it can return is fixed, and `EdgeZero`'s path segments have no +/// way to say "whatever name the operator chose". A configuration can hold +/// secrets under such a name, as an Edge Cookie module block under a label +/// does, and an implementation of this trait finds those by reading the +/// configuration being loaded. +pub trait ConfiguredSecretFields: AppConfigMeta { + /// The secret leaves `data` holds under names the operator chose, each + /// with its full path from the configuration root. + fn configured_secret_fields(data: &Value) -> Vec; +} + /// Resolve all secret references in a serialized Trusted Server app config. /// +/// Both the fixed leaves [`AppConfigMeta::secret_fields`] lists and the ones +/// [`ConfiguredSecretFields::configured_secret_fields`] reads out of `data` +/// are resolved in one pass, so a failure in either leaves the configuration +/// as it was. +/// /// The input is mutated in memory; the verified envelope is never rewritten. /// Secret values are not included in structural or platform errors. /// @@ -21,167 +94,220 @@ use crate::platform::{PlatformSecretStore, StoreName}; /// Returns [`TrustedServerError::Configuration`] when a required path or key /// is malformed, a secret is unavailable, is not valid UTF-8, or resolves to an /// empty value. -pub fn resolve_secret_references( +pub fn resolve_secret_references( data: &mut Value, secret_store: &dyn PlatformSecretStore, default_store_name: &StoreName, ) -> Result<(), Report> { + resolve_secret_references_with::(data, secret_store, default_store_name, Vec::new()) + .map(|_| ()) +} + +/// Resolve all secret references in a serialized Trusted Server app config, +/// together with `extra`, the leaves known only where the configuration is +/// loaded, such as the ones a deployment's modules declare in their own +/// tables. +/// +/// A leaf listed more than once is looked up once, because the second lookup +/// would read the secret itself as the name of a key. What comes back says +/// where the secrets went, which the configuration view masks. +/// +/// # Errors +/// +/// As [`resolve_secret_references`]. +pub fn resolve_secret_references_with( + data: &mut Value, + secret_store: &dyn PlatformSecretStore, + default_store_name: &StoreName, + extra: Vec, +) -> Result> { + let fields = listed_fields::(data, extra); let mut resolved_data = data.clone(); - for field in C::secret_fields() { + let mut resolver = Resolver { + secret_store, + default_store_name, + resolved: ResolvedSecrets::default(), + }; + for field in fields { if matches!(field.kind, SecretKind::StoreRef) { continue; } - resolve_field( - &mut resolved_data, - &field, - &field.path, - "", - secret_store, - default_store_name, - )?; + resolver + .resolved + .patterns + .extend(PathPattern::from_segments(&field.path)); + resolver.resolve_field(&mut resolved_data, &field, &field.path, "", &[])?; } *data = resolved_data; - Ok(()) + Ok(resolver.resolved) } -fn resolve_field( - node: &mut Value, - field: &SecretField, - remaining: &[SecretPathSegment], - rendered_path: &str, - secret_store: &dyn PlatformSecretStore, - default_store_name: &StoreName, -) -> Result<(), Report> { - match remaining.split_first() { - Some((SecretPathSegment::Field(name), [])) => resolve_leaf( - node, - field, - name.as_ref(), - rendered_path, - secret_store, - default_store_name, - ), - Some((SecretPathSegment::OptionalField(name), [])) => { - if matches!(node.get(name.as_ref()), None | Some(Value::Null)) { - return Ok(()); +/// The secret leaves a resolution pass over `data` with `extra` looks for, +/// with nothing resolved, for a check that runs while the secret fields still +/// hold key names. +pub(crate) fn unresolved_secrets( + data: &Value, + extra: Vec, +) -> ResolvedSecrets { + ResolvedSecrets { + patterns: listed_fields::(data, extra) + .iter() + .filter(|field| !matches!(field.kind, SecretKind::StoreRef)) + .filter_map(|field| PathPattern::from_segments(&field.path)) + .collect(), + ..ResolvedSecrets::default() + } +} + +/// Every secret leaf of `C` and `extra` in `data`, each listed once. +fn listed_fields( + data: &Value, + extra: Vec, +) -> Vec { + let mut listed = std::collections::BTreeSet::new(); + C::secret_fields() + .into_iter() + .chain(C::configured_secret_fields(data)) + .chain(extra) + .filter(|field| listed.insert(field.dotted_path())) + .collect() +} + +/// One resolution pass, recording each secret it writes. +struct Resolver<'a> { + secret_store: &'a dyn PlatformSecretStore, + default_store_name: &'a StoreName, + resolved: ResolvedSecrets, +} + +impl Resolver<'_> { + /// `rendered_path` names the node for messages, and `steps` is the same + /// path as the configuration view addresses it. + fn resolve_field( + &mut self, + node: &mut Value, + field: &SecretField, + remaining: &[SecretPathSegment], + rendered_path: &str, + steps: &[PathStep], + ) -> Result<(), Report> { + match remaining.split_first() { + Some((SecretPathSegment::Field(name), [])) => { + self.resolve_leaf(node, field, name.as_ref(), rendered_path, steps) } - resolve_leaf( - node, - field, - name.as_ref(), - rendered_path, - secret_store, - default_store_name, - ) - } - Some((SecretPathSegment::Field(name), rest)) => { - let next_path = join_field(rendered_path, name.as_ref()); - let child = node - .as_object_mut() - .and_then(|object| object.get_mut(name.as_ref())) - .ok_or_else(|| missing_path(&next_path))?; - if child.is_null() { - return Err(missing_path(&next_path)); + Some((SecretPathSegment::OptionalField(name), [])) => { + if matches!(node.get(name.as_ref()), None | Some(Value::Null)) { + return Ok(()); + } + self.resolve_leaf(node, field, name.as_ref(), rendered_path, steps) } - resolve_field( - child, - field, - rest, - &next_path, - secret_store, - default_store_name, - ) - } - Some((SecretPathSegment::OptionalField(name), rest)) => { - let next_path = join_field(rendered_path, name.as_ref()); - let Some(child) = node - .as_object_mut() - .and_then(|object| object.get_mut(name.as_ref())) - else { - return Ok(()); - }; - if child.is_null() { - return Ok(()); + Some((SecretPathSegment::Field(name), rest)) => { + let next_path = join_field(rendered_path, name.as_ref()); + let child = node + .as_object_mut() + .and_then(|object| object.get_mut(name.as_ref())) + .ok_or_else(|| missing_path(&next_path))?; + if child.is_null() { + return Err(missing_path(&next_path)); + } + let next_steps = with_step(steps, PathStep::Key(name.to_string())); + self.resolve_field(child, field, rest, &next_path, &next_steps) } - resolve_field( - child, - field, - rest, - &next_path, - secret_store, - default_store_name, - ) - } - Some((SecretPathSegment::ArrayEach, rest)) => { - let items = node.as_array_mut().ok_or_else(|| { - configuration_error(format!("expected an array at `{rendered_path}`")) - })?; - for (index, item) in items.iter_mut().enumerate() { - let indexed_path = format!("{rendered_path}[{index}]"); - resolve_field( - item, - field, - rest, - &indexed_path, - secret_store, - default_store_name, - )?; + Some((SecretPathSegment::OptionalField(name), rest)) => { + let next_path = join_field(rendered_path, name.as_ref()); + let Some(child) = node + .as_object_mut() + .and_then(|object| object.get_mut(name.as_ref())) + else { + return Ok(()); + }; + if child.is_null() { + return Ok(()); + } + let next_steps = with_step(steps, PathStep::Key(name.to_string())); + self.resolve_field(child, field, rest, &next_path, &next_steps) } - Ok(()) + Some((SecretPathSegment::ArrayEach, rest)) => { + let items = node.as_array_mut().ok_or_else(|| { + configuration_error(format!("expected an array at `{rendered_path}`")) + })?; + for (index, item) in items.iter_mut().enumerate() { + let indexed_path = format!("{rendered_path}[{index}]"); + let next_steps = with_step(steps, PathStep::Index(index)); + self.resolve_field(item, field, rest, &indexed_path, &next_steps)?; + } + Ok(()) + } + Some(_) => Err(configuration_error(format!( + "unsupported secret path segment in `{}`", + field.dotted_path() + ))), + None => Ok(()), } - Some(_) => Err(configuration_error(format!( - "unsupported secret path segment in `{}`", - field.dotted_path() - ))), - None => Ok(()), } -} -fn resolve_leaf( - parent: &mut Value, - field: &SecretField, - key: &str, - rendered_parent: &str, - secret_store: &dyn PlatformSecretStore, - default_store_name: &StoreName, -) -> Result<(), Report> { - let leaf_path = join_field(rendered_parent, key); - let object = parent.as_object_mut().ok_or_else(|| { - configuration_error(format!("expected an object containing `{leaf_path}`")) - })?; - - let key_name = match object.get(key) { - Some(Value::String(value)) if !value.is_empty() => value.clone(), - Some(Value::Null) | None if field.optional => return Ok(()), - Some(Value::Null) | None => return Err(missing_path(&leaf_path)), - Some(Value::String(_)) => { - return Err(configuration_error(format!( - "secret key reference at `{leaf_path}` must not be empty" - ))); - } - _ => { + fn resolve_leaf( + &mut self, + parent: &mut Value, + field: &SecretField, + key: &str, + rendered_parent: &str, + steps: &[PathStep], + ) -> Result<(), Report> { + let leaf_path = join_field(rendered_parent, key); + let object = parent.as_object_mut().ok_or_else(|| { + configuration_error(format!("expected an object containing `{leaf_path}`")) + })?; + + let key_name = match object.get(key) { + Some(Value::String(value)) if !value.is_empty() => value.clone(), + Some(Value::Null) | None if field.optional => return Ok(()), + Some(Value::Null) | None => return Err(missing_path(&leaf_path)), + Some(Value::String(_)) => { + return Err(configuration_error(format!( + "secret key reference at `{leaf_path}` must not be empty" + ))); + } + _ => { + return Err(configuration_error(format!( + "secret key reference at `{leaf_path}` must be a string" + ))); + } + }; + + let resolved = self + .secret_store + .get_string(self.default_store_name, &key_name) + .map_err(|_| { + configuration_error(format!( + "failed to resolve secret reference at `{leaf_path}` from secret store \ + `{}`", + self.default_store_name + )) + })?; + if resolved.is_empty() { return Err(configuration_error(format!( - "secret key reference at `{leaf_path}` must be a string" + "resolved secret at `{leaf_path}` must not be empty" ))); } - }; - let resolved = secret_store - .get_string(default_store_name, &key_name) - .map_err(|_| { - configuration_error(format!( - "failed to resolve secret reference at `{leaf_path}` from secret store \ - `{default_store_name}`" - )) - })?; - if resolved.is_empty() { - return Err(configuration_error(format!( - "resolved secret at `{leaf_path}` must not be empty" - ))); + self.resolved + .paths + .push(with_step(steps, PathStep::Key(key.to_owned()))); + if !self.resolved.values.contains(&resolved) { + self.resolved.values.push(resolved.clone()); + } + object.insert(key.to_owned(), Value::String(resolved)); + Ok(()) } +} - object.insert(key.to_owned(), Value::String(resolved)); - Ok(()) +/// `steps` with `step` added to the end. +fn with_step(steps: &[PathStep], step: PathStep) -> Vec { + let mut next = Vec::with_capacity(steps.len() + 1); + next.extend_from_slice(steps); + next.push(step); + next } fn join_field(prefix: &str, field: &str) -> String { @@ -238,6 +364,28 @@ mod tests { struct Fixture; + impl ConfiguredSecretFields for Fixture { + /// Every key under `labeled` holds its secret at + /// `labeled..secret`, which is the shape a fixed path cannot + /// name. + fn configured_secret_fields(data: &Value) -> Vec { + data.get("labeled") + .and_then(Value::as_object) + .into_iter() + .flatten() + .map(|(name, _)| SecretField { + kind: SecretKind::KeyInDefault, + optional: false, + path: vec![ + SecretPathSegment::Field("labeled".into()), + SecretPathSegment::Field(name.clone().into()), + SecretPathSegment::Field("secret".into()), + ], + }) + .collect() + } + } + impl AppConfigMeta for Fixture { fn secret_fields() -> Vec { vec![ @@ -298,6 +446,61 @@ mod tests { assert!(data["outer"][0]["optional"].is_null()); } + #[test] + fn a_leaf_listed_twice_is_looked_up_once() { + // The second lookup would read the secret itself as the name of a + // key, which this store does not hold. + let mut data = serde_json::json!({ + "outer": [{"token": "token-a"}], + "feature": {"credential": "feature-key"} + }); + let listed_again = vec![SecretField { + kind: SecretKind::KeyInDefault, + optional: true, + path: vec![ + SecretPathSegment::OptionalField("feature".into()), + SecretPathSegment::Field("credential".into()), + ], + }]; + + resolve_secret_references_with::( + &mut data, + &store(), + &StoreName::from("secrets"), + listed_again, + ) + .expect("should look a leaf up once however often it is listed"); + + assert_eq!(data["feature"]["credential"], "resolved-feature"); + } + + #[test] + fn resolves_a_leaf_only_the_caller_lists() { + let mut data = serde_json::json!({ + "outer": [{"token": "token-a"}], + "added": {"key_name": "token-b"} + }); + let added = vec![SecretField { + kind: SecretKind::KeyInDefault, + optional: true, + path: vec![ + SecretPathSegment::OptionalField("added".into()), + SecretPathSegment::Field("key_name".into()), + ], + }]; + + resolve_secret_references_with::( + &mut data, + &store(), + &StoreName::from("secrets"), + added, + ) + .expect("should resolve the leaf the caller listed"); + + assert_eq!(data["added"]["key_name"], "resolved-b"); + assert_eq!(data["outer"][0]["token"], "resolved-a"); + } + #[test] fn resolves_present_and_skips_absent_optional_intermediate() { let mut absent = serde_json::json!({ @@ -316,6 +519,38 @@ mod tests { assert_eq!(present["feature"]["credential"], "resolved-feature"); } + #[test] + fn resolves_a_secret_under_a_name_only_the_configuration_holds() { + let mut data = serde_json::json!({ + "outer": [{"token": "token-a"}], + "labeled": {"chosen": {"secret": "feature-key"}}, + }); + + resolve_secret_references::(&mut data, &store(), &StoreName::from("secrets")) + .expect("should resolve a secret the fixed paths cannot name"); + + assert_eq!(data["outer"][0]["token"], "resolved-a"); + assert_eq!(data["labeled"]["chosen"]["secret"], "resolved-feature"); + } + + #[test] + fn a_failed_configured_field_leaves_the_fixed_ones_unresolved() { + // Fixed and configured leaves resolve in one pass, so a failure in + // either hands back the configuration exactly as it arrived. + let mut data = serde_json::json!({ + "outer": [{"token": "token-a"}], + "labeled": {"chosen": {"secret": "missing"}}, + }); + let original = data.clone(); + + let err = + resolve_secret_references::(&mut data, &store(), &StoreName::from("secrets")) + .expect_err("should reject a missing configured secret"); + + assert!(err.to_string().contains("labeled.chosen.secret")); + assert_eq!(data, original, "should preserve unresolved data on failure"); + } + #[test] fn rejects_missing_required_path_without_secret_values() { for mut data in [ diff --git a/crates/trusted-server-core/src/service_version.rs b/crates/trusted-server-core/src/service_version.rs new file mode 100644 index 0000000000..3f6aa84268 --- /dev/null +++ b/crates/trusted-server-core/src/service_version.rs @@ -0,0 +1,170 @@ +//! The deployment's versions as response headers, a middleware of core's +//! own named `service_version` in a `[[serve]]` entry. A deployment that +//! names it writes `X-TS-Version`, the git version the build compiled in, +//! when the build knows one, and `X-TS-Service-Version`, the version the +//! adapter reports for the service, when it reports one, on every page the +//! entry covers. A deployment that does not name it sends nothing about its +//! versions on a page, because no adapter writes either header. What the +//! build was made from and the platform's version are always in the +//! attestation document, under `build` and `platform.serviceVersion`. + +use http::HeaderValue; + +use crate::attestation::UNKNOWN; +use crate::constants::{HEADER_X_TS_SERVICE_VERSION, HEADER_X_TS_VERSION, TS_GIT_VERSION}; +use crate::middleware::{Middleware, MiddlewareAction, MiddlewareContext, MiddlewarePhase}; + +/// The name an entry writes, and the integration the registry lists the +/// middleware under. +pub const MODULE: &str = "service_version"; + +/// The middleware. It holds the git version the build compiled in, and reads +/// the platform's version from the context. +#[derive(Debug)] +pub struct ServiceVersion { + /// The deployed git version, `None` when the build knew none. + pub git_version: Option<&'static str>, +} + +impl Default for ServiceVersion { + fn default() -> Self { + Self { + git_version: TS_GIT_VERSION, + } + } +} + +impl Middleware for ServiceVersion { + fn middleware_id(&self) -> &'static str { + MODULE + } + + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Serve] + } + + fn create(&self, context: &MiddlewareContext<'_>) -> MiddlewareAction { + let mut response_headers = Vec::new(); + if let Some(value) = header_value("git version", self.git_version) { + response_headers.push((HEADER_X_TS_VERSION, value)); + } + let service_version = + Some(context.platform.service_version.as_str()).filter(|version| *version != UNKNOWN); + if let Some(value) = header_value("service version", service_version) { + response_headers.push((HEADER_X_TS_SERVICE_VERSION, value)); + } + MiddlewareAction { + response_headers, + ..MiddlewareAction::pass() + } + } +} + +/// `version` as a header value, or `None`, with a warning, when a header +/// cannot carry it. +fn header_value(what: &str, version: Option<&str>) -> Option { + let version = version?; + match HeaderValue::from_str(version) { + Ok(value) => Some(value), + Err(_) => { + log::warn!("the {what} {version:?} is not a header value, so it is not written"); + None + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::attestation::PlatformIdentity; + use crate::integrations::IntegrationDocumentState; + use crate::middleware::test_support; + + fn action_for(git_version: Option<&'static str>, service_version: &str) -> MiddlewareAction { + let platform = PlatformIdentity { + service_version: service_version.to_owned(), + ..PlatformIdentity::named("example-platform") + }; + let document_state = IntegrationDocumentState::default(); + ServiceVersion { git_version }.create(&test_support::context_on( + MiddlewarePhase::Serve, + &document_state, + &platform, + )) + } + + fn headers_of(action: &MiddlewareAction) -> Vec<(&str, &[u8])> { + action + .response_headers + .iter() + .map(|(name, value)| (name.as_str(), value.as_bytes())) + .collect() + } + + #[test] + fn the_git_version_the_build_compiled_in_is_written_as_x_ts_version() { + let action = action_for(Some("v1.3.0"), UNKNOWN); + + assert_eq!( + headers_of(&action), + [("x-ts-version", b"v1.3.0".as_slice())], + "should write the git version alone on a platform reporting no version" + ); + } + + #[test] + fn the_version_the_platform_reports_is_written_as_x_ts_service_version() { + let action = action_for(None, "52"); + + assert_eq!( + headers_of(&action), + [("x-ts-service-version", b"52".as_slice())], + "should write the platform's version alone when the build knew no git version" + ); + } + + #[test] + fn both_versions_are_written_when_both_are_known() { + let action = action_for(Some("v1.3.0"), "52"); + + assert_eq!( + headers_of(&action), + [ + ("x-ts-version", b"v1.3.0".as_slice()), + ("x-ts-service-version", b"52".as_slice()), + ], + "should write the git version and then the platform's" + ); + } + + #[test] + fn a_build_knowing_no_git_version_on_a_platform_reporting_none_writes_nothing() { + assert!( + action_for(None, UNKNOWN).is_pass(), + "should write nothing when neither version is known" + ); + } + + #[test] + fn a_version_that_is_not_a_header_value_is_not_written() { + assert!( + action_for(Some("v1\n"), "52\n").is_pass(), + "should write nothing for versions a header cannot carry" + ); + } + + #[test] + fn the_default_carries_the_git_version_the_build_compiled_in() { + assert_eq!( + ServiceVersion::default().git_version, + TS_GIT_VERSION, + "should carry the compiled-in TS_GIT_VERSION" + ); + } + + #[test] + fn it_runs_in_the_serve_phase_under_its_name() { + assert_eq!(ServiceVersion::default().phases(), [MiddlewarePhase::Serve]); + assert_eq!(ServiceVersion::default().middleware_id(), "service_version"); + } +} diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 3efc157701..d34944f173 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -6,23 +6,29 @@ use regex::Regex; use serde::{Deserialize, Deserializer, Serialize, de::DeserializeOwned}; use serde_json::Value as JsonValue; use sha2::{Digest as _, Sha256}; -use std::collections::{HashMap, HashSet}; +use std::borrow::Cow; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::ops::{Deref, DerefMut}; use std::str::FromStr; use std::sync::OnceLock; use std::time::Duration; use subtle::ConstantTimeEq as _; use url::Url; -use validator::{Validate, ValidationError}; +use validator::{Validate, ValidationError, ValidationErrors, ValidationErrorsKind}; use crate::auction_config_types::AuctionConfig; use crate::cache_policy::{CachePolicy, CacheVisibility}; use crate::consent_config::ConsentConfig; use crate::constants::INTERNAL_HEADERS; use crate::creative_opportunities::CreativeOpportunitiesConfig; +use crate::ec::module::{ + EcModuleSelection, HMAC_MODULE_KEY, HOST_SIGNALS_MODULE_KEY, check_named_module_configuration, +}; use crate::error::TrustedServerError; use crate::host_header::validate_host_header_override_value; +use crate::middleware::{MiddlewarePhase, PhaseEntries}; use crate::platform::PlatformImageOptimizerRegion; +use crate::provider_table::{ProviderChoice, ProviderList, SectionModules}; use crate::redacted::Redacted; #[cfg(test)] @@ -39,10 +45,11 @@ pub struct Publisher { /// (see [`ec_cookie_domain`](Self::ec_cookie_domain)). #[validate(custom(function = validate_cookie_domain))] pub cookie_domain: String, + #[serde(serialize_with = "crate::redacted::sensitive")] #[validate(custom(function = validate_no_trailing_slash))] pub origin_url: String, /// Optional outbound Host header to send while connecting to `origin_url`. - #[serde(default)] + #[serde(default, serialize_with = "crate::redacted::sensitive")] #[validate(custom(function = validate_host_header_override))] pub origin_host_header_override: Option, /// Secret used to encrypt/decrypt proxied URLs in `/first-party/proxy`. @@ -215,165 +222,200 @@ impl Publisher { } } -#[derive(Default, Clone, Deserialize, Serialize)] -pub struct IntegrationSettings { - #[serde(flatten)] - entries: HashMap, +/// Which integrations run, and the settings each one is given. +/// +/// The sections of module types core does not read itself, such as `[cmp]` or +/// `[tag]`, each named for the type of module it selects. +/// +/// Every top-level table that is not one of Trusted Server's own settings is +/// read as one of these. Whether its type is one a module on offer has is +/// only known where the registry is built, so it is checked there. +#[derive(Clone, Default, PartialEq, Serialize)] +#[serde(transparent)] +pub struct TypeSections(BTreeMap); + +impl std::fmt::Debug for TypeSections { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.debug_map().entries(self.0.iter()).finish() + } } -impl std::fmt::Debug for IntegrationSettings { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let mut integration_ids = self.entries.keys().collect::>(); - integration_ids.sort_unstable(); - formatter - .debug_struct("IntegrationSettings") - .field("integration_ids", &integration_ids) - .finish() +impl<'de> Deserialize<'de> for TypeSections { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let entries = serde_json::Map::::deserialize(deserializer)?; + let mut sections = BTreeMap::new(); + for (name, value) in entries { + let JsonValue::Object(table) = value else { + return Err(serde::de::Error::custom(format!( + "unknown field `{name}`, which is neither a setting Trusted Server reads nor \ + the section of a module type, which is a table" + ))); + }; + if name.contains('.') || !crate::module_name::is_valid(&name) { + return Err(serde::de::Error::custom(format!( + "unknown field `{name}`. The section of a module type is named for \ + the type, in lower case letters, digits, `_` or `-`" + ))); + } + let section = SectionModules::from_entries(table).map_err(|message| { + serde::de::Error::custom(format!( + "[{name}] is not a section Trusted Server reads itself, so it is read as \ + the section of a module type, and it {message}" + )) + })?; + sections.insert(name, section); + } + Ok(Self(sections)) } } -pub trait IntegrationConfig: DeserializeOwned + Validate { - fn is_enabled(&self) -> bool; +impl TypeSections { + /// Whether no type section is present. + #[must_use] + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } - /// Validate the public field schema for an explicitly disabled config. - /// - /// The default deserializes the integration's normal schema, except it - /// permits omitted enabled-only required fields. Override this only when a - /// disabled integration has a distinct public schema. + /// Each type section, by name. + pub fn iter(&self) -> impl Iterator { + self.0 + .iter() + .map(|(name, section)| (name.as_str(), section)) + } + + /// The section of `type_name`, when present. + #[must_use] + pub fn section(&self, type_name: &str) -> Option<&SectionModules> { + self.0.get(type_name) + } + + /// The section of `type_name`, created empty when absent. + pub fn section_mut(&mut self, type_name: &str) -> &mut SectionModules { + self.0.entry(type_name.to_owned()).or_default() + } + + /// Refuses a section that selects nothing, and checks each selection. /// /// # Errors /// - /// Returns a deserialization error when the disabled public field schema is invalid. - fn validate_disabled_schema(raw: &JsonValue) -> Result<(), serde_json::Error> { - match serde_json::from_value::(raw.clone()) { - Ok(_) => Ok(()), - Err(error) if error.to_string().starts_with("missing field ") => Ok(()), - Err(error) => Err(error), + /// Naming the first section at fault. + pub fn validate(&self) -> Result<(), Report> { + for (name, section) in &self.0 { + if section.selected().is_empty() { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "[{name}] selects no module. A module type's section names what runs \ + with `module` or `modules`, so select one or remove the section" + ), + })); + } + section + .validate(name) + .map_err(|message| Report::new(TrustedServerError::Configuration { message }))?; } + Ok(()) } } -impl IntegrationSettings { - /// Inserts a configuration value for an integration. - /// - /// # Errors - /// - /// Returns an error if the configuration cannot be serialized to JSON. - #[cfg_attr(not(test), allow(dead_code))] - pub fn insert_config( - &mut self, - integration_id: impl Into, - value: &T, - ) -> Result<(), Report> - where - T: Serialize, - { - let json = - serde_json::to_value(value).change_context(TrustedServerError::Configuration { - message: "Failed to serialize integration configuration".to_string(), - })?; - self.entries.insert(integration_id.into(), json); - Ok(()) +/// The section that holds what one vendor's modules share. +pub const SHARED_TYPE: &str = "shared"; + +/// The `[shared]` section, one `[shared.]` table per vendor, holding +/// the settings every module of that vendor reads, such as an endpoint and a +/// license key, read through [`Settings::shared_config`]. +#[derive(Clone, Default, Deserialize, Serialize)] +#[serde(transparent)] +pub struct SharedTables(BTreeMap>); + +impl std::fmt::Debug for SharedTables { + // The tables hold secrets, so only the vendors are shown. + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("SharedTables") + .field("vendors", &self.vendors().collect::>()) + .finish() } +} - fn is_explicitly_disabled(raw: &JsonValue) -> bool { - raw.as_object() - .and_then(|map| map.get("enabled")) - .and_then(JsonValue::as_bool) - == Some(false) +impl SharedTables { + /// Whether no vendor has a table. + #[must_use] + pub fn is_empty(&self) -> bool { + self.0.is_empty() } - fn remove_legacy_static_secret_store_selectors(&mut self) { - let Some(datadome) = self - .entries - .get_mut("datadome") - .and_then(JsonValue::as_object_mut) - else { - return; - }; + /// The vendors with a table, in name order. + pub fn vendors(&self) -> impl Iterator { + self.0.keys().map(String::as_str) + } - let mut removed = datadome.remove("server_side_key_secret_store").is_some(); - if let Some(bypass) = datadome - .get_mut("protection_test_bypass") - .and_then(JsonValue::as_object_mut) - { - removed |= bypass.remove("credential_secret_store").is_some(); - } - if removed { - log::warn!( - "DataDome secret-store selectors are deprecated and ignored; static credentials resolve through the default app-config secret store" - ); - } + /// Whether `[shared.]` is present. + #[must_use] + pub fn contains(&self, vendor: &str) -> bool { + self.0.contains_key(vendor) } - /// Retrieves and validates a typed configuration for an integration. + /// Reads and validates `[shared.]` as `T`, or `None` when the table + /// is absent. /// /// # Errors /// - /// Returns an error if the configuration cannot be parsed from JSON or fails validation. - pub fn get_typed( - &self, - integration_id: &str, - ) -> Result, Report> + /// When the table cannot be read as `T` or fails its validation, naming + /// the table and the underlying error. + pub fn get_typed(&self, vendor: &str) -> Result, Report> where - T: IntegrationConfig, + T: DeserializeOwned + Validate, { - let raw = match self.entries.get(integration_id) { - Some(value) => value, - None => return Ok(None), + let Some(table) = self.0.get(vendor) else { + return Ok(None); }; - - if Self::is_explicitly_disabled(raw) { - T::validate_disabled_schema(raw).change_context(TrustedServerError::Configuration { - message: format!( - "Integration '{integration_id}' configuration could not be parsed" - ), + let config: T = + serde_json::from_value(JsonValue::Object(table.clone())).map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: format!("[{SHARED_TYPE}.{vendor}] could not be read: {error}"), + }) })?; - return Ok(None); - } - - let config: T = serde_json::from_value(raw.clone()).change_context( - TrustedServerError::Configuration { - message: format!( - "Integration '{integration_id}' configuration could not be parsed" - ), - }, - )?; - - // Field validation runs only for integrations that resolve to enabled. - // An integration whose `enabled` flag is omitted falls back to its - // serde default, which the explicit-`false` fast path above cannot - // observe. Validating before this check would reject documented - // template placeholders in sections that are not actually turned on. - if !config.is_enabled() { - return Ok(None); - } - - config.validate().map_err(|err| { + config.validate().map_err(|error| { Report::new(TrustedServerError::Configuration { - message: format!( - "Integration '{integration_id}' configuration failed validation: {err}" - ), + message: format!("[{SHARED_TYPE}.{vendor}] failed validation: {error}"), }) })?; - Ok(Some(config)) } -} - -impl Deref for IntegrationSettings { - type Target = HashMap; - fn deref(&self) -> &Self::Target { - &self.entries + /// Refuses a table whose name is not a vendor's name, being lower case + /// letters, digits, `_` or `-`. + /// + /// # Errors + /// + /// Naming the first table at fault. + pub fn validate_names(&self) -> Result<(), Report> { + match self + .vendors() + .find(|vendor| vendor.contains('.') || !crate::module_name::is_valid(vendor)) + { + Some(vendor) => Err(Report::new(TrustedServerError::Configuration { + message: format!( + "[{SHARED_TYPE}] has a table named `{vendor}`, which is not a vendor's \ + name. A vendor's shared table is named in lower case letters, \ + digits, `_` or `-`" + ), + })), + None => Ok(()), + } } } -impl DerefMut for IntegrationSettings { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.entries - } -} +/// The settings type a module reads from its table, beneath the section that +/// selects it. +/// +/// The type states which settings the module takes and how they are +/// validated, and nothing else. Whether the module runs is not its business, +/// because the section's selection names what runs. +pub trait IntegrationConfig: DeserializeOwned + Validate {} /// A partner (SSP, DSP, identity vendor) configured in `[[ec.partners]]`. /// @@ -537,15 +579,65 @@ impl EcPartner { /// /// Mapped from the `[ec]` TOML section. Controls EC identity generation, /// KV store names, and partner registry. -#[derive(Debug, Default, Clone, Deserialize, Serialize, Validate)] -#[serde(deny_unknown_fields)] +/// +/// Every key in the section other than the fields below is one module's +/// `[ec.]` settings table, held in +/// [`module_blocks`](Self::module_blocks), so the field names below are +/// reserved and cannot name a module. +#[derive(Debug, Default, Clone, Deserialize, Serialize)] pub struct Ec { - /// Publisher passphrase used as HMAC key for EC generation. - #[validate(custom(function = Ec::validate_passphrase))] - pub passphrase: Redacted, + /// The name of the Edge Cookie identity module to activate. + /// + /// Set it in the `[ec]` TOML section, for example `"hmac"`. The name is + /// the module's implementation unless its `[ec.]` block names a + /// different one, in which case the name is a label of the operator's + /// choosing (see [`module_blocks`](Self::module_blocks)). Deployment + /// tooling can merge a `TRUSTED_SERVER__EC__MODULE` environment value + /// into the published configuration before it is loaded, so the same + /// compiled WebAssembly can switch modules at deployment. The running + /// server reads its settings from the platform config store, not the + /// environment. When absent, no Edge Cookie is generated and Trusted + /// Server runs statelessly, and the explicit `"none"` spells the same + /// choice. A selection whose implementation needs settings it has no block + /// for is rejected at startup by + /// [`validate_module_selection`](Self::validate_module_selection). + /// + /// Typed as [`EcModuleSelection`], which reads and writes the same + /// string, so every check that asks which module is selected matches on + /// one vocabulary rather than comparing string literals. + #[serde(default)] + pub module: Option, - /// Fastly KV store name for the EC identity graph. + /// Deprecated location of the HMAC passphrase, read so a configuration + /// written for the previous release still starts. + /// + /// [`migrate_legacy_ec_layout`](Self::migrate_legacy_ec_layout) maps it + /// to `module = "hmac"` with the passphrase in the `[ec.hmac]` block and + /// logs a deprecation warning, so a fleet can move configuration and + /// binaries independently. A configuration carrying both the old and the + /// new form is rejected rather than guessed at. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub passphrase: Option>, + + /// Extra exact origins allowed to POST the client resolve endpoint. + /// + /// The endpoint always accepts `https://{publisher.domain}` and nothing + /// else by default. A publisher whose pages are served from another origin, + /// `www` being the common case, lists those origins here. Each entry is a + /// serialized origin (RFC 6454 §6.1), being `http://` or `https://` and a + /// host with an optional port and nothing after it, and + /// [`validate_resolve_allowed_origins`](Self::validate_resolve_allowed_origins) + /// refuses any other entry when the settings load. Each is compared with + /// the request's `Origin` by the same-origin test of RFC 6454 §5, so the + /// scheme, the host and the port all have to match, with a missing port + /// meaning the scheme's default. A suffix or subdomain match is never + /// performed, because control of a DNS namespace does not make every host + /// under it a trusted identity-setting origin. #[serde(default)] + pub resolve_allowed_origins: Vec, + + /// Fastly KV store name for the EC identity graph. + #[serde(default, serialize_with = "crate::redacted::sensitive")] pub ec_store: Option, /// Maximum number of concurrent pull-sync requests. @@ -568,8 +660,20 @@ pub struct Ec { /// Partners (SSPs, DSPs, identity vendors) for EC identity sync. #[serde(default, deserialize_with = "vec_from_seq_or_map")] - #[validate(nested)] pub partners: Vec, + + /// The settings tables of the Edge Cookie identity modules, keyed by the + /// name each is written under. + /// + /// A module has a block only when it has settings of its own, so + /// `[ec] module = "hmac"` needs an `[ec.hmac]` block for its required + /// passphrase while a module with no settings needs none. The + /// [`module`](Self::module) selector names the active module and + /// [`validate_module_selection`](Self::validate_module_selection) + /// rejects a block it does not name, so at most one block survives + /// startup. + #[serde(flatten)] + pub module_blocks: EcModuleBlocks, } impl Ec { @@ -631,145 +735,914 @@ impl Ec { } Ok(()) } -} - -#[derive(Debug, Default, Clone, Deserialize, Serialize, Validate)] -#[serde(deny_unknown_fields)] -pub struct Rewrite { - /// List of domains to exclude from rewriting. Supports wildcards (e.g., "*.example.com"). - /// URLs from these domains will not be proxied through first-party endpoints. - #[serde(default)] - pub exclude_domains: Vec, -} -impl Rewrite { - /// Checks if a URL should be excluded from rewriting based on domain matching - #[allow(dead_code)] - #[must_use] - pub fn is_excluded(&self, url: &str) -> bool { - // Parse URL to extract host - let Ok(parsed) = url::Url::parse(url) else { - return false; - }; - - let host = parsed.host_str().unwrap_or(""); - - // Check exact domain matches (with wildcard support) - for domain in &self.exclude_domains { - if let Some(suffix) = domain.strip_prefix("*.") { - // Wildcard: *.example.com matches both example.com and sub.example.com - if host == suffix || host.ends_with(&format!(".{}", suffix)) { - return true; - } - } else if host == domain { - return true; + /// Refuses a [`resolve_allowed_origins`](Self::resolve_allowed_origins) + /// entry that is not a serialized origin. + /// + /// The resolve endpoint reads each entry with + /// [`parse_serialized_origin`](crate::ec::resolve::parse_serialized_origin), + /// and an entry that parse rejects matches no request, so every resolve + /// from the origin it was meant to allow would answer `403`. Checking with + /// the same parse here means an entry that loads is one a request can + /// match. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::Configuration`] naming the first entry + /// that is not a serialized origin and what is wrong with it. + pub fn validate_resolve_allowed_origins(&self) -> Result<(), Report> { + for entry in &self.resolve_allowed_origins { + if let Err(reason) = crate::ec::resolve::parse_serialized_origin(entry) { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "[ec] resolve_allowed_origins entry `{entry}` {reason}. \ + Each entry is a bare origin, being `http://` or `https://` and a \ + host with an optional port, such as `https://www.example.com`" + ), + })); } } - - false + Ok(()) } -} -#[derive(Debug, Default, Clone, Deserialize, Serialize, Validate)] -#[serde(deny_unknown_fields)] -pub struct Handler { - #[validate(length(min = 1), custom(function = validate_path))] - pub path: String, - #[validate(custom(function = validate_redacted_not_empty))] - pub username: Redacted, - #[validate(custom(function = validate_redacted_not_empty))] - pub password: Redacted, - #[serde(skip, default)] - #[validate(skip)] - regex: OnceLock>, -} + /// Validates the module selection against the configured blocks. + /// + /// When [`module`](Self::module) is set, the selection has to resolve + /// to an implementation this deployment can configure, and every block in + /// [`module_blocks`](Self::module_blocks) has to be the one the + /// selector names, so a deployment that selects a module (in TOML or via + /// the environment override) but has not configured it fails fast at + /// startup rather than silently running stateless. When no module is + /// selected, Trusted Server runs statelessly and this check passes. + /// + /// Whether the named implementation exists at all is settled by + /// [`build_module`](crate::ec::module::build_module), which is the + /// one place that knows both the implementations built into core and the + /// one this deployment's adapter injects. + /// + /// Whether this build compiles an implementation in at all is answered by + /// `check_named_module_configuration` in [`crate::ec::module`], beside + /// the resolution it belongs to, rather than here, because the settings + /// cannot know which implementations are compiled out of this build. Only + /// the resolution knows that. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::Configuration`] when a module name or + /// implementation is not a module name, when a block is configured with no + /// selector or alongside `"none"`, when the selector names a key `[ec]` + /// reads as its own setting, when the selected implementation is not + /// compiled into this build, when a block the selector does not name is + /// configured, or when the selected module resolves to an implementation + /// that needs settings and has no block. + pub fn validate_module_selection(&self) -> Result<(), Report> { + for (name, block) in self.module_blocks.iter() { + Self::validate_module_name(name)?; + if let Some(implementation) = &block.implementation { + Self::validate_module_name(implementation)?; + } + } -impl Handler { - /// Known handler password placeholders that must not be used in deployments. - pub const PASSWORD_PLACEHOLDERS: &[&str] = &[ - "replace-with-admin-password-32-bytes", - "replace-with-admin-password", - "change-me-admin-password", - ]; + let Some(selection) = self.module.as_ref() else { + if !self.module_blocks.is_empty() { + return Err(Report::new(TrustedServerError::Configuration { + message: "[ec.] module blocks are configured but no [ec] module \ + is selected. Set [ec] module = \"\" to activate one, or \ + remove the blocks to run statelessly" + .to_owned(), + })); + } + return Ok(()); + }; - /// Returns `true` if `password` matches a known placeholder value - /// (case-insensitive). - #[must_use] - pub fn is_placeholder_password(password: &str) -> bool { - let password = password.trim(); - Self::PASSWORD_PLACEHOLDERS - .iter() - .any(|placeholder| placeholder.eq_ignore_ascii_case(password)) - } + // `"none"` is explicit statelessness, the same meaning as omitting the + // selector, spelled out. It is subject to the same rule that no + // module blocks may be left configured. + let EcModuleSelection::Named(name) = selection else { + if !self.module_blocks.is_empty() { + return Err(Report::new(TrustedServerError::Configuration { + message: "[ec] module = \"none\" selects stateless operation, but \ + [ec.] module blocks are configured. Remove the blocks, \ + or select the module they configure" + .to_owned(), + })); + } + return Ok(()); + }; - fn compiled_regex(&self) -> Result<&Regex, Report> { - match self - .regex - .get_or_init(|| Regex::new(&self.path).map_err(|err| err.to_string())) + let name = name.as_str(); + Self::validate_module_name(name)?; + if EC_SECTION_KEYS.contains(&name) || name == REMOVED_EC_PROVIDERS_TABLE { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "[ec] module = \"{name}\" names a key the `[ec]` section reads as its \ + own setting, so no module can be configured under it. Give the \ + module a name of its own" + ), + })); + } + + // Whether this build compiles the implementation in at all is the + // resolution's question, not the settings', so it is asked there. + let implementation = self.module_blocks.implementation(name); + check_named_module_configuration(implementation)?; + + // A module has a block only when it has settings, and core knows + // which of its own implementations need them. Both modules that + // derive an identifier at the edge take a passphrase, so both need one. + // The demonstration module is built from nothing and needs none. A + // module an adapter injects has the contents of its block read by + // that adapter when it builds the module, so core cannot say whether + // it needs one. + if (implementation == HMAC_MODULE_KEY || implementation == HOST_SIGNALS_MODULE_KEY) + && !self.module_blocks.contains_key(name) { - Ok(regex) => Ok(regex), - Err(message) => Err(Report::new(TrustedServerError::Configuration { + return Err(Report::new(TrustedServerError::Configuration { message: format!( - "Handler path regex `{}` failed to compile: {message}", - self.path + "Edge Cookie module `{name}` is selected but has no `[ec.{name}]` configuration" ), - })), + })); + } + + // Every configured block must be the selected one. An unreferenced + // block is almost always a mistake (a mistyped selector or a stale + // block), and accepting it silently invites configuration drift. + let unreferenced: Vec<&str> = self + .module_blocks + .keys() + .map(String::as_str) + .filter(|configured| *configured != name) + .collect(); + if unreferenced.is_empty() { + Ok(()) + } else { + Err(Report::new(TrustedServerError::Configuration { + message: format!( + "[ec.{}] is configured but `{name}` is selected. Remove the unselected \ + block, or correct the selector", + unreferenced.join("], [ec.") + ), + })) } } - /// Eagerly compile the handler regex to fail fast during startup. + /// Validates one module name or implementation id. /// /// # Errors /// - /// Returns a configuration error if the handler path regex does not compile. - pub fn prepare_runtime(&self) -> Result<(), Report> { - self.compiled_regex().map(|_| ()) + /// Returns [`TrustedServerError::Configuration`] when `name` is not a + /// module name, see [`crate::module_name::is_valid`]. + fn validate_module_name(name: &str) -> Result<(), Report> { + if crate::module_name::is_valid(name) { + return Ok(()); + } + Err(Report::new(TrustedServerError::Configuration { + message: format!( + "Edge Cookie module name `{name}` is not a module name, which is parts \ + joined by `.`, each of lower case letters, digits, `_` or `-`" + ), + })) } - /// Determine whether this handler applies to the request path. + /// Migrates the deprecated `[ec] passphrase` form to the module layout. + /// + /// A configuration still carrying the old key keeps working for one + /// release cycle, mapping to `module = "hmac"` with the passphrase in + /// the `[ec.hmac]` block, and a deprecation warning names the new + /// location. A configuration carrying both forms is rejected so a + /// half-edited file fails loudly instead of one form silently winning. + /// + /// The deprecated key is held to the same passphrase rules as the new + /// `[ec.hmac]` block. Validation runs before this migration and the + /// deprecated field carries no check of its own, so without the check here + /// a short or empty passphrase in the old location would start a + /// deployment that the new location rejects. /// /// # Errors /// - /// Returns a configuration error if the handler path regex does not compile. - pub fn matches_path(&self, path: &str) -> Result> { - self.compiled_regex().map(|regex| regex.is_match(path)) + /// Returns [`TrustedServerError::Configuration`] when both the deprecated + /// key and any part of the module configuration are present, or when the + /// deprecated passphrase fails [`Self::validate_passphrase`]. + pub fn migrate_legacy_ec_layout(&mut self) -> Result<(), Report> { + let Some(passphrase) = self.passphrase.take() else { + return Ok(()); + }; + if self.module.is_some() || !self.module_blocks.is_empty() { + return Err(Report::new(TrustedServerError::Configuration { + message: "[ec] passphrase (deprecated) and the [ec] module configuration \ + are both present. Keep exactly one form, moving the passphrase to \ + [ec.hmac] and deleting the old key" + .to_owned(), + })); + } + Self::validate_passphrase(&passphrase).map_err(|err| { + Report::new(TrustedServerError::Configuration { + message: format!( + "[ec] passphrase (deprecated) is invalid ({err}). Use a random secret \ + of at least {} bytes, placed in [ec.hmac]", + Self::MIN_PASSPHRASE_LENGTH, + ), + }) + })?; + log::warn!( + "[ec] passphrase is deprecated. Move it to [ec.hmac] passphrase and set \ + [ec] module = \"hmac\"" + ); + self.module = Some(EcModuleSelection::from(HMAC_MODULE_KEY)); + self.module_blocks.insert( + HMAC_MODULE_KEY.to_owned(), + EcModuleBlock::from(HmacModuleConfig { passphrase }), + ); + Ok(()) } } -#[derive(Debug, Default, Clone, Deserialize, Serialize)] -#[serde(deny_unknown_fields)] -pub struct RequestSigning { - #[serde(default = "default_request_signing_enabled")] - pub enabled: bool, - pub config_store_id: String, - pub secret_store_id: String, +impl Validate for Ec { + /// Validates the partner entries and the settings of every block that + /// configures a module built into core. + /// + /// Written out rather than derived because each block's errors are keyed + /// by the name the operator wrote the block under, so a passphrase + /// `[ec.primary]` rejects is reported at `ec.primary.passphrase`. A + /// derived nested validation would key them under this struct's own field + /// name, which is a path no configuration has. + fn validate(&self) -> Result<(), ValidationErrors> { + let mut errors = ValidationErrors::new(); + errors.merge_self("partners", self.partners.validate()); + let blocks = self + .module_blocks + .hmac_blocks() + .map(|(name, config)| (name, config.validate())) + .chain( + self.module_blocks + .host_signals_blocks() + .map(|(name, config)| (name, config.validate())), + ); + for (name, result) in blocks { + if let Err(block_errors) = result { + errors.errors_mut().insert( + Cow::Owned(name.to_owned()), + ValidationErrorsKind::Struct(Box::new(block_errors)), + ); + } + } + if errors.is_empty() { + Ok(()) + } else { + Err(errors) + } + } } -impl RequestSigning { - /// Reserved example store-id values from the config template, plus the - /// empty string, that must not be deployed while request signing is enabled. - pub const STORE_ID_PLACEHOLDERS: &[&str] = &[ - "", - "", - ]; +/// The keys the `[ec]` section reads as its own settings. +/// +/// Every other key in the section is one module's `[ec.]` settings +/// table, so these names are reserved and cannot name a module. They are +/// written out because serde reads the [`Ec`] fields by name and has no way to +/// report the list back for an error message. +const EC_SECTION_KEYS: &[&str] = &[ + "module", + "passphrase", + "resolve_allowed_origins", + "ec_store", + "pull_sync_concurrency", + "cluster_trust_threshold", + "cluster_recheck_secs", + "partners", +]; - /// Returns `true` if `store_id` is empty or a known template placeholder - /// (case-insensitive). - #[must_use] - pub fn is_placeholder_store_id(store_id: &str) -> bool { - let store_id = store_id.trim(); - store_id.is_empty() - || Self::STORE_ID_PLACEHOLDERS - .iter() - .any(|p| p.eq_ignore_ascii_case(store_id)) - } +/// The removed table that used to hold every module's settings. +const REMOVED_EC_PROVIDERS_TABLE: &str = "providers"; - /// Returns `true` if `store_id` cannot be deployed as-is: a placeholder, or - /// a value with surrounding whitespace that the key-management routes would - /// forward to the management API verbatim. +/// The key in a module block that names the implementation it configures. +/// +/// Anything reading a module block out of a serialized configuration rather +/// than out of [`EcModuleBlock`] reads the same key from here, so the two +/// cannot drift apart. +pub(crate) const MODULE_IMPLEMENTATION_KEY: &str = "implementation"; + +/// The `[ec.]` settings tables of the Edge Cookie identity modules, +/// keyed by the name each is written under. +/// +/// A module that has settings is configured in its own table, for example: +/// +/// ```toml +/// [ec] +/// module = "hmac" +/// +/// [ec.hmac] +/// passphrase = "replace-with-32-plus-byte-random-secret" +/// ``` +/// +/// A table may name the implementation it configures, which makes its own name +/// a label of the operator's choosing, so the same configuration can also be +/// written as: +/// +/// ```toml +/// [ec] +/// module = "primary" +/// +/// [ec.primary] +/// implementation = "hmac" +/// passphrase = "replace-with-32-plus-byte-random-secret" +/// ``` +/// +/// The active module is chosen by the [`Ec::module`] selector, and the one +/// table present must be the one it names (see +/// [`Ec::validate_module_selection`]). +#[derive(Debug, Default, Clone, Serialize)] +#[serde(transparent)] +pub struct EcModuleBlocks(BTreeMap); + +impl EcModuleBlocks { + /// The implementation the module `name` resolves to. + /// + /// A block may name the implementation it configures. Without one, and + /// when the module has no block at all, the module's name is its + /// implementation. + #[must_use] + pub fn implementation<'a>(&'a self, name: &'a str) -> &'a str { + self.0 + .get(name) + .and_then(|block| block.implementation.as_deref()) + .unwrap_or(name) + } + + /// Every configured block that sets the built-in HMAC module up, with + /// the name it is written under. + /// + /// The name is the label the operator chose, so a caller reporting one of + /// these settings names the path the operator wrote rather than the + /// implementation's own. + pub fn hmac_blocks(&self) -> impl Iterator { + self.0 + .iter() + .filter_map(|(name, block)| block.hmac_settings().map(|config| (name.as_str(), config))) + } + + /// Every configured block that sets the built-in host-signal module up, + /// with the name it is written under. + /// + /// The same contract as [`hmac_blocks`](Self::hmac_blocks), for the other + /// module core builds itself. + pub fn host_signals_blocks(&self) -> impl Iterator { + self.0.iter().filter_map(|(name, block)| { + block + .host_signals_settings() + .map(|config| (name.as_str(), config)) + }) + } +} + +impl Deref for EcModuleBlocks { + type Target = BTreeMap; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for EcModuleBlocks { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl<'de> Deserialize<'de> for EcModuleBlocks { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct BlocksVisitor; + + impl<'de> serde::de::Visitor<'de> for BlocksVisitor { + type Value = EcModuleBlocks; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("the `[ec.]` module settings tables") + } + + fn visit_map(self, mut map: A) -> Result + where + A: serde::de::MapAccess<'de>, + { + let mut blocks = BTreeMap::new(); + while let Some(name) = map.next_key::()? { + let table = map.next_value::()?; + blocks.insert(name.clone(), read_module_block::(&name, table)?); + } + Ok(EcModuleBlocks(blocks)) + } + } + + deserializer.deserialize_map(BlocksVisitor) + } +} + +/// Reads one key left over from the `[ec]` section as the module block it +/// names. +/// +/// This is where the `[ec]` section gets the unknown-key check that the fixed +/// fields lose by holding the module blocks alongside them. A key that is +/// not a table cannot be a module, so it is reported as the mistyped setting +/// it almost certainly is. +fn read_module_block(name: &str, table: JsonValue) -> Result +where + E: serde::de::Error, +{ + if name == REMOVED_EC_PROVIDERS_TABLE { + return Err(E::custom( + "[ec.providers] is no longer read. Each module's settings moved to a table of \ + its own under [ec], so a block written as [ec.providers.hmac] is now [ec.hmac]", + )); + } + let JsonValue::Object(mut table) = table else { + let known = EC_SECTION_KEYS + .iter() + .map(|key| format!("`{key}`")) + .collect::>() + .join(", "); + return Err(E::custom(format!( + "unknown field `{name}`, expected one of {known}, or an [ec.] module \ + settings table" + ))); + }; + let implementation = match table.remove(MODULE_IMPLEMENTATION_KEY) { + None => None, + Some(JsonValue::String(implementation)) => Some(implementation), + Some(_) => { + return Err(E::custom(format!( + "`implementation` in [ec.{name}] must be a string naming the module \ + implementation the block configures" + ))); + } + }; + + // The implementation decides how the rest of the table is read. Core reads + // its own modules' settings here, so a mistyped key fails where the + // configuration is read rather than at the request that needed it, and + // keeps the settings of a module an adapter injects as the raw values + // that adapter deserializes for itself. + let invalid = |err| E::custom(format!("[ec.{name}] is invalid ({err})")); + let settings = match implementation.as_deref().unwrap_or(name) { + HMAC_MODULE_KEY => EcModuleSettings::Hmac( + serde_json::from_value(JsonValue::Object(table)).map_err(invalid)?, + ), + HOST_SIGNALS_MODULE_KEY => EcModuleSettings::HostSignals( + serde_json::from_value(JsonValue::Object(table)).map_err(invalid)?, + ), + _ => EcModuleSettings::Injected(table), + }; + Ok(EcModuleBlock { + implementation, + settings, + }) +} + +/// One module's `[ec.]` settings table. +#[derive(Debug, Clone, Serialize)] +pub struct EcModuleBlock { + /// The implementation this block configures, written as + /// `implementation = ""`, when the block's own name is not it. + /// + /// Setting it makes the block name a label of the operator's choosing, so + /// one implementation can be configured under any name. Everything that + /// resolves the selected module reads the implementation rather than the + /// label. + #[serde(skip_serializing_if = "Option::is_none")] + pub implementation: Option, + + /// Everything the block holds other than `implementation`. + #[serde(flatten)] + pub settings: EcModuleSettings, +} + +impl EcModuleBlock { + /// The built-in HMAC module's settings, when this block configures it. + #[must_use] + pub fn hmac_settings(&self) -> Option<&HmacModuleConfig> { + match &self.settings { + EcModuleSettings::Hmac(config) => Some(config), + EcModuleSettings::HostSignals(_) | EcModuleSettings::Injected(_) => None, + } + } + + /// The built-in host-signal module's settings, when this block + /// configures it. + #[must_use] + pub fn host_signals_settings(&self) -> Option<&HostSignalsModuleConfig> { + match &self.settings { + EcModuleSettings::HostSignals(config) => Some(config), + EcModuleSettings::Hmac(_) | EcModuleSettings::Injected(_) => None, + } + } +} + +impl From for EcModuleBlock { + /// Builds the `[ec.hmac]` block, the built-in HMAC module configured + /// under its own name. A block under a label names its implementation + /// instead. + fn from(config: HmacModuleConfig) -> Self { + Self { + implementation: None, + settings: EcModuleSettings::Hmac(config), + } + } +} + +impl From for EcModuleBlock { + /// Builds the `[ec.host_signals]` block, the built-in host-signal module + /// configured under its own name. A block under a label names its + /// implementation instead. + fn from(config: HostSignalsModuleConfig) -> Self { + Self { + implementation: None, + settings: EcModuleSettings::HostSignals(config), + } + } +} + +/// The settings one module block holds, read according to the +/// implementation the block configures. +#[derive(Debug, Clone, Serialize)] +#[serde(untagged)] +pub enum EcModuleSettings { + /// The built-in HMAC module's settings. + Hmac(HmacModuleConfig), + + /// The built-in host-signal module's settings. + HostSignals(HostSignalsModuleConfig), + + /// The settings of a module an adapter injects, kept as the raw values + /// the block held. The adapter that builds the module deserializes them + /// into the vendor crate's own config type, so core never names a vendor + /// and a new module adds nothing here. + Injected(serde_json::Map), +} + +/// Configuration for the built-in HMAC Edge Cookie module. +/// +/// Mapped from the `[ec.hmac]` TOML block, or from a block under a label whose +/// `implementation` is `hmac`. Unknown keys are rejected, so a mistyped +/// setting fails at startup instead of being accepted silently and leaving the +/// intended setting at its default. +#[derive(Debug, Default, Clone, Deserialize, Serialize, Validate)] +#[serde(deny_unknown_fields)] +pub struct HmacModuleConfig { + /// Publisher passphrase used as the HMAC key for EC generation. + #[validate(custom(function = Ec::validate_passphrase))] + pub passphrase: Redacted, +} + +/// Configuration for the built-in host-signal Edge Cookie module. +/// +/// Mapped from the `[ec.host_signals]` TOML block, or from a block under a +/// label whose `implementation` is `host_signals`. Unknown keys are rejected, +/// so a mistyped setting fails at startup instead of being accepted silently +/// and leaving the intended setting at its default. +#[derive(Debug, Default, Clone, Deserialize, Serialize, Validate)] +#[serde(deny_unknown_fields)] +pub struct HostSignalsModuleConfig { + /// Passphrase used as the HMAC key over the host signals and client IP. + #[validate(custom(function = Ec::validate_passphrase))] + pub passphrase: Redacted, +} + +/// Device-detection configuration. +/// +/// Mapped from the `[device]` TOML section. Selects which device-detection +/// module classifies a request into device signals, mirroring the Edge +/// Cookie module selection in [`Ec`]. +#[derive(Debug, Default, Clone, PartialEq, Eq, Deserialize, Serialize, Validate)] +#[serde(deny_unknown_fields)] +pub struct DeviceConfig { + /// The key of the device-detection module to activate. + /// + /// Defaults to the built-in `builtin` module when absent, which classifies + /// from the User-Agent alone, so device classification itself makes no + /// host-specific call. The opt-in `fastly` module strengthens the + /// browser/bot gate with the host's TLS and HTTP/2 signals, which the Fastly + /// entry point reads on every request regardless of this selector. Override + /// it with the + /// `TRUSTED_SERVER__device__module` environment variable so the same + /// compiled WebAssembly can switch modules at deployment. An unknown key is + /// rejected at startup by + /// [`validate_module_selection`](Self::validate_module_selection). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub module: Option, +} + +impl DeviceConfig { + /// Returns the active device-detection module key, defaulting to the + /// built-in heuristic. + #[must_use] + pub fn module_key(&self) -> &str { + self.module.as_deref().unwrap_or("builtin") + } + + /// Validates the selected device-detection module. + /// + /// `builtin` and `fastly` are resolved by the core and the Fastly adapter. + /// Any other key names an integration module that declares a device + /// module, and the registry rejects a key no module supplies, so this + /// cannot be a closed list without shutting modules out of device detection + /// entirely. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::Configuration`] never, today. The error + /// type is kept because the caller treats module validation uniformly + /// across the three capabilities. + pub fn validate_module_selection(&self) -> Result<(), Report> { + Ok(()) + } +} + +/// Which permission signal modules run, and in what order. +/// +/// Mapped from the `[permission-signal]` TOML section, where `modules` +/// selects. `[ec]`, `[geo]` and `[device]` each name one module with +/// `module`, whereas signals compose, because a request can carry a TCF string +/// and a Global Privacy Control header at once and both have something to say. +/// So here `modules` names a list, and the order is the policy, because the +/// last module with an opinion decides. +/// +/// A module that gains settings will take them in a +/// `[permission-signal.]` block named for it. None of the modules that +/// ship has settings, so `modules` is the only key accepted, and any other key +/// is refused as an unknown field rather than silently ignored. +/// +/// See `crates/trusted-server-core/src/permission_signal/README.md`. +#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Validate)] +pub struct PermissionSignalConfig { + /// The modules to run, in order, named by their crate folders below + /// `crates/permission-signal`, for example `gpc`, `gpp`, `us-privacy`, + /// `tcf` and `mtm` for the five that ship. + /// + /// Absent means every module the adapter offers, in the order it offers + /// them. A publisher who does not want to act on one removes it from the + /// list, and there is no separate switch, because a module that is not + /// listed does not run. An empty list runs none of them, leaving every + /// permission at its country and region baseline. + /// + /// Which names are valid is only known where the module crates are + /// linked, so the check that each name matches an available module and + /// none is repeated happens at the adapter's composition root, through + /// [`build_permission_signal_modules`], and refuses startup rather than + /// silently ignoring a typo. + /// + /// [`build_permission_signal_modules`]: + /// crate::permission_signal::build_permission_signal_modules + #[serde(skip_serializing_if = "Option::is_none")] + pub modules: Option>, +} + +/// Read by hand rather than derived, so that `sources` and `module`, the keys +/// `modules` replaced, are refused with a message saying what to write instead. A derived +/// struct could only refuse it by name by declaring it as a field, and would +/// then list it among the keys it expects whenever it refused any other. +impl<'de> Deserialize<'de> for PermissionSignalConfig { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let mut section = serde_json::Map::::deserialize(deserializer)?; + if section.contains_key("sources") { + return Err(serde::de::Error::custom( + "[permission-signal] sources is no longer accepted. Name the modules \ + to run, in order, in [permission-signal] modules instead", + )); + } + if section.contains_key("module") { + return Err(serde::de::Error::custom( + "[permission-signal] module is `modules` here, a list, because signals \ + compose. Name the modules to run, in order, in [permission-signal] \ + modules instead", + )); + } + if let Some(key) = section.keys().find(|key| key.as_str() != "modules") { + return Err(serde::de::Error::custom(format!( + "unknown field `{key}` in [permission-signal], expected `modules`. No \ + permission signal module takes settings yet, so a \ + [permission-signal.] block is not accepted" + ))); + } + // Read as an option, so an explicit JSON null is the same as leaving + // the key out. + let modules = match section.remove("modules") { + Some(value) => serde_json::from_value(value).map_err(serde::de::Error::custom)?, + None => None, + }; + Ok(Self { modules }) + } +} + +/// Which prefix module gives the start of every name the browser holds and +/// every key in a shared store. +/// +/// Mapped from the `[prefix]` TOML section. See [`crate::prefix`]. +#[derive(Debug, Default, Clone, PartialEq, Eq, Deserialize, Serialize, Validate)] +#[serde(deny_unknown_fields)] +pub struct PrefixConfig { + /// The prefix module to run, named as any module is, by its crate folder. + /// + /// Core's own module runs when this is absent or names `ts`, giving `ts` + /// for the names the browser holds and `ts.` for shared + /// store keys. A name no module supplies is refused when the registry is + /// built, in + /// [`IntegrationRegistry::with_plan_and_registrations`](crate::integrations::IntegrationRegistry::with_plan_and_registrations). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub module: Option, +} + +/// Geo / IP intelligence configuration. +/// +/// Mapped from the `[geo]` TOML section. Selects which module resolves a +/// client IP into [`GeoInfo`](crate::platform::GeoInfo), mirroring the Edge +/// Cookie module selection in [`Ec`]. +#[derive(Debug, Default, Clone, PartialEq, Eq, Deserialize, Serialize, Validate)] +#[serde(deny_unknown_fields)] +pub struct GeoConfig { + /// The key of the geo module to activate. + /// + /// No module is the default: Trusted Server resolves no geolocation and + /// makes no host geo call, so a default deployment is not tied to any host + /// geo service, and the permission baseline comes from the top of the + /// `rules` tree in `permissions.yaml`. `module = "none"` spells + /// the same choice explicitly. The host platform's own geo lookup is + /// opt-in via `module = "platform"`. Override it with the + /// `TRUSTED_SERVER__geo__module` environment variable so the same compiled + /// WebAssembly can switch modules at deployment. An unknown key is rejected + /// at startup by + /// [`validate_module_selection`](Self::validate_module_selection). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub module: Option, + + /// Acknowledges that, with no geo module, every request is treated as + /// coming from the place at the top of the `permissions.yaml` `rules` tree. + /// + /// With geolocation off, a visitor from any other jurisdiction silently + /// receives that top node's permission rules. A deployment that + /// runs an Edge Cookie module without a geo module must set this to + /// `true`, checked at startup by + /// [`validate_jurisdiction_acknowledgment`](Self::validate_jurisdiction_acknowledgment), + /// so serving a single jurisdiction is an explicit operator decision rather + /// than an accident of the default configuration. + #[serde(default)] + pub assume_single_jurisdiction: bool, +} + +impl GeoConfig { + /// Validates that the selected geo module is available in this build. + /// + /// No selector is valid and is the default, running without geolocation, + /// the same way the Edge Cookie module runs statelessly when none is + /// selected. The explicit `"none"` spells the same choice. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::Configuration`] when the selected module + /// key is not one this build provides. + pub fn validate_module_selection(&self) -> Result<(), Report> { + // Unset, `none` and `platform` are resolved by the core. Any other key + // names an integration module that declares a geo module, and the + // registry rejects one that no module supplies, so this check cannot be + // a closed list without shutting modules out of geo entirely. + Ok(()) + } + + /// Validates that the compiled `permissions.yaml` parses and declares its + /// top node. + /// + /// The top node is required: its `group` is the permission baseline for a + /// request the geo module leaves unmatched, and its `jurisdiction` is the + /// consent handling for that same request, so there must always be one. + /// Checking it here turns a malformed policy into a configuration error at + /// startup rather than a panic on the first lookup. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::Configuration`] when the compiled policy + /// fails to parse, most usefully when its top node omits `group` or + /// `jurisdiction`. + pub fn validate_permission_policy() -> Result<(), Report> { + crate::permissions::validate_default_policy().map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: format!("permissions.yaml is not usable: {error}"), + }) + }) + } + + /// Validates that running jurisdiction consumers without geolocation is + /// explicitly acknowledged. + /// + /// With no geo module, every request resolves to the permission baseline + /// at the top of the `permissions.yaml` `rules` tree, so a visitor from any + /// other jurisdiction silently receives that node's rules. That is + /// acceptable only as an explicit operator + /// decision. When an Edge Cookie module is configured (the permission + /// model gates it by jurisdiction) and no geo module is selected, + /// [`assume_single_jurisdiction`](Self::assume_single_jurisdiction) must be + /// `true`. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::Configuration`] when an Edge Cookie + /// module is configured, no geo module is selected, and + /// `assume_single_jurisdiction` is not set. + pub fn validate_jurisdiction_acknowledgment( + &self, + ec: &Ec, + ) -> Result<(), Report> { + // Location is resolved by the host lookup (`platform`) or by any + // integration module that declares a geo module. Only an unset + // selector and the explicit `none` resolve nothing, so only those + // two leave every request on the declared jurisdiction. + let geo_disabled = matches!(self.module.as_deref(), None | Some("none")); + // The selector is a typed enum, so statelessness is the absent + // selector or the explicit `none`, matched rather than compared as a + // string. + let ec_active = !matches!(ec.module, None | Some(EcModuleSelection::None)); + if geo_disabled && ec_active && !self.assume_single_jurisdiction { + return Err(Report::new(TrustedServerError::Configuration { + message: "[ec] module is configured but no [geo] module is selected, so \ + every request would be treated as the top of the permissions.yaml \ + rules tree. Set [geo] assume_single_jurisdiction = true to \ + acknowledge single-jurisdiction operation, or select a geo module" + .to_owned(), + })); + } + Ok(()) + } +} + +#[derive(Debug, Default, Clone, Deserialize, Serialize, Validate)] +#[serde(deny_unknown_fields)] +pub struct Rewrite { + /// List of domains to exclude from rewriting. Supports wildcards (e.g., "*.example.com"). + /// URLs from these domains will not be proxied through first-party endpoints. + #[serde(default)] + pub exclude_domains: Vec, +} + +impl Rewrite { + /// Checks if a URL should be excluded from rewriting based on domain matching + #[allow(dead_code)] + #[must_use] + pub fn is_excluded(&self, url: &str) -> bool { + // Parse URL to extract host + let Ok(parsed) = url::Url::parse(url) else { + return false; + }; + + let host = parsed.host_str().unwrap_or(""); + + // Check exact domain matches (with wildcard support) + for domain in &self.exclude_domains { + if let Some(suffix) = domain.strip_prefix("*.") { + // Wildcard: *.example.com matches both example.com and sub.example.com + if host == suffix || host.ends_with(&format!(".{}", suffix)) { + return true; + } + } else if host == domain { + return true; + } + } + + false + } +} + +#[derive(Debug, Default, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct RequestSigning { + #[serde(default = "default_request_signing_enabled")] + pub enabled: bool, + /// The removed `config_store_id`. The field never holds a value, because + /// reading one always fails. + #[serde(default, skip_serializing)] + #[allow( + dead_code, + reason = "the field exists so that reading the removed key fails with directions" + )] + config_store_id: RemovedSigningStoreId, + /// The removed `secret_store_id`, refused as `config_store_id` is. + #[serde(default, skip_serializing)] + #[allow( + dead_code, + reason = "the field exists so that reading the removed key fails with directions" + )] + secret_store_id: RemovedSigningStoreId, +} + +impl RequestSigning { + /// Request signing switched on or off. #[must_use] - pub fn is_unusable_store_id(store_id: &str) -> bool { - Self::is_placeholder_store_id(store_id) || store_id != store_id.trim() + pub fn new(enabled: bool) -> Self { + Self { + enabled, + ..Self::default() + } } } @@ -1528,6 +2401,7 @@ pub struct ProxyAssetRoute { /// Only the scheme, host, and port are used. Any path or query configured on /// this URL is rejected because the incoming request path/query, or the /// configured rewrite result, replaces them at runtime. + #[serde(serialize_with = "crate::redacted::sensitive")] pub origin_url: String, /// Optional regex matched against the incoming request path before proxying. pub path_pattern: Option, @@ -1760,7 +2634,6 @@ impl ProxyAssetRoute { } #[derive(Debug, Clone, Deserialize, Serialize)] -#[serde(deny_unknown_fields)] pub struct Proxy { /// Enable TLS certificate verification when proxying to HTTPS origins. /// Defaults to true for secure production use. @@ -1776,33 +2649,30 @@ pub struct Proxy { /// /// When empty (the default), proxy hosts are not restricted. Configure this /// in production to constrain signed and fetched first-party proxy targets. - /// When `integrations.prebid.external_bundle_url` is configured, this list + /// When `auction.prebid.external_bundle_url` is configured, this list /// must include its host and any HTTPS redirect targets. #[serde(default, deserialize_with = "vec_from_seq_or_map")] pub allowed_domains: Vec, /// Path-prefix-based asset proxy routes evaluated before publisher fallback. #[serde(default, deserialize_with = "vec_from_seq_or_map")] pub asset_routes: Vec, + /// The modules this section selects, with each one's settings in the + /// table at its name. + #[serde(flatten)] + pub modules: SectionModules, } fn default_certificate_check() -> bool { true } -fn is_admin_placeholder_password(password: &str) -> bool { - Handler::is_placeholder_password(password) - || matches!( - password.trim().to_ascii_lowercase().as_str(), - "changeme" | "password" | "admin" - ) -} - impl Default for Proxy { fn default() -> Self { Self { certificate_check: default_certificate_check(), allowed_domains: Vec::new(), asset_routes: Vec::new(), + modules: SectionModules::default(), } } } @@ -1897,12 +2767,15 @@ impl Proxy { } } -/// Direct Tinybird Events API telemetry configuration. +/// The analytics module's name in full, which is also the table holding its +/// settings and which every message about them names. +const ANALYTICS_MODULE: &str = "analytics.tinybird"; + +/// Direct Tinybird Events API telemetry configuration, the settings of the +/// module `[analytics]` selects. #[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct TinybirdSettings { - /// Master enablement for auction telemetry ingestion. - #[serde(default)] - pub enabled: bool, /// Regional Tinybird API host, without scheme or path. #[serde(default)] pub api_host: String, @@ -1950,7 +2823,6 @@ fn default_tinybird_max_body_bytes() -> usize { impl Default for TinybirdSettings { fn default() -> Self { Self { - enabled: false, api_host: String::new(), secret_store: None, auction_dataset: default_tinybird_auction_dataset(), @@ -1969,7 +2841,7 @@ impl TinybirdSettings { self.api_host = self.api_host.trim().to_ascii_lowercase(); if self.secret_store.take().is_some() { log::warn!( - "tinybird.secret_store is deprecated and ignored; static credentials resolve through the default app-config secret store" + "{ANALYTICS_MODULE}.secret_store is deprecated and ignored; static credentials resolve through the default app-config secret store" ); } self.auction_dataset = self.auction_dataset.trim().to_owned(); @@ -1985,32 +2857,39 @@ impl TinybirdSettings { self.normalize(); if !(0.0..=1.0).contains(&self.access_sample_rate) { return Err(Report::new(TrustedServerError::Configuration { - message: "tinybird.access_sample_rate must be between 0.0 and 1.0".to_owned(), + message: format!( + "{ANALYTICS_MODULE}.access_sample_rate must be between 0.0 and 1.0" + ), })); } if self.max_body_bytes < 1024 { return Err(Report::new(TrustedServerError::Configuration { - message: "tinybird.max_body_bytes must be at least 1024".to_owned(), + message: format!("{ANALYTICS_MODULE}.max_body_bytes must be at least 1024"), })); } if self.access_enabled { return Err(Report::new(TrustedServerError::Configuration { - message: "tinybird.access_enabled is reserved for future access-log telemetry; no emitter is currently wired".to_owned(), + message: format!( + "{ANALYTICS_MODULE}.access_enabled is reserved for future access-log telemetry; no emitter is currently wired" + ), })); } - if !self.enabled { - return Ok(()); - } validate_tinybird_api_host(&self.api_host)?; - validate_tinybird_dataset(&self.auction_dataset, "tinybird.auction_dataset")?; + validate_tinybird_dataset( + &self.auction_dataset, + &format!("{ANALYTICS_MODULE}.auction_dataset"), + )?; let token = self.auction_token_secret.as_ref().ok_or_else(|| { Report::new(TrustedServerError::Configuration { - message: - "tinybird.auction_token_secret is required when Tinybird telemetry is enabled" - .to_owned(), + message: format!( + "{ANALYTICS_MODULE}.auction_token_secret is required when [analytics] selects the module" + ), }) })?; - validate_tinybird_secret(token.expose(), "tinybird.auction_token_secret") + validate_tinybird_secret( + token.expose(), + &format!("{ANALYTICS_MODULE}.auction_token_secret"), + ) } } @@ -2023,13 +2902,14 @@ fn validate_tinybird_api_host(host: &str) -> Result<(), Report Result<(), Report, + /// The analytics module's settings. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tinybird: Option, +} + +/// Whether `written`, selected in `[analytics]`, names the analytics module, +/// in full or by its name within the section. +fn names_analytics_module(written: &str) -> bool { + written == ANALYTICS_MODULE + || written == crate::module_name::short_form("analytics", ANALYTICS_MODULE) +} + +/// Whether the document `data` selects the analytics module in `[analytics]`, +/// read before the settings are, to tell which secret references are live. +pub(crate) fn document_selects_analytics_module(data: &JsonValue) -> bool { + data.pointer("/analytics/module") + .and_then(JsonValue::as_str) + .is_some_and(names_analytics_module) +} + +impl AnalyticsConfig { + /// Whether `module` selects the analytics module. + fn selects_analytics_module(&self) -> bool { + self.module.as_deref().is_some_and(names_analytics_module) + } + + /// The analytics module's settings, when `module` selects it. + #[must_use] + pub fn analytics_module(&self) -> Option<&TinybirdSettings> { + if self.selects_analytics_module() { + self.tinybird.as_ref() + } else { + None + } + } + + fn normalize(&mut self) { + if let Some(settings) = &mut self.tinybird { + settings.normalize(); + } + } + + /// Refuses a section that selects nothing or a module this build does + /// not have, and checks the selected module's settings. + fn prepare_runtime(&mut self) -> Result<(), Report> { + let short = crate::module_name::short_form("analytics", ANALYTICS_MODULE); + match self.module.as_deref() { + None => { + return Err(Report::new(TrustedServerError::Configuration { + message: "[analytics] selects no module. A module type's section names \ + what runs with `module` or `modules`, so select one or remove \ + the section" + .to_owned(), + })); + } + Some(written) if !self.selects_analytics_module() => { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "[analytics] selects `{written}`, which no module this deployment \ + supplies. The analytics modules it supplies are [{short}]" + ), + })); + } + Some(_) => {} + } + match &mut self.tinybird { + Some(settings) => settings.prepare_runtime(), + // Checking the defaults reports the first setting the module + // needs and the table does not have. + None => TinybirdSettings::default().prepare_runtime(), + } + } +} + +/// Cache behavior configuration. +#[derive(Debug, Default, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct CacheSettings { + /// Ordered static/rehosted asset rules. The first enabled matching rule wins. #[serde(default)] pub asset_rules: Vec, + /// How long the copies of each robots.txt contributor's answer last, + /// `[cache.robots-txt]`. + #[serde( + rename = "robots-txt", + default, + skip_serializing_if = "LoadingCacheConfig::is_default" + )] + pub robots_txt: LoadingCacheConfig, +} + +/// The most seconds a loading cache lifetime setting accepts, a year. +pub const MAX_LOADING_CACHE_SECONDS: u32 = 365 * 24 * 60 * 60; + +/// The name of the cache that holds each robots.txt module's answer, whose +/// settings are `[cache.robots-txt]`. +pub const ROBOTS_TXT_CACHE: &str = "robots-txt"; + +/// How long one loading cache's copies last, each field defaulting to the +/// cache's own default when absent. +/// +/// See [`crate::platform::PlatformLoadingCache`]. The memory copy is the brief +/// one a process keeps in front of the shared store, and the shared store's +/// copy is the one every instance reads. +#[derive(Debug, Default, Clone, PartialEq, Eq, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct LoadingCacheConfig { + /// The module whose loading cache holds this cache's copies, as its short + /// name or in full, for example `51degrees` or `loading-cache.51degrees`. + /// Absent, no copy is kept and every call loads. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub module: Option, + /// How long the brief copy in process memory is used, in seconds. Zero + /// keeps no memory copy, and a missing value is still loaded once. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub memory_seconds: Option, + /// How long the copy in the shared store is used at most, in seconds. An + /// answer with a shorter lifetime of its own ends sooner. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub store_seconds: Option, + /// How long the copy in the shared store lasts unused, in seconds, after + /// which it leaves the store. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub store_idle_seconds: Option, +} + +impl LoadingCacheConfig { + /// Whether nothing is set, so every lifetime is the cache's default. + #[must_use] + pub fn is_default(&self) -> bool { + *self == Self::default() + } + + /// The lifetimes these settings give, each from `defaults` where unset. + #[must_use] + pub fn lifetimes( + &self, + defaults: crate::platform::CacheLifetimes, + ) -> crate::platform::CacheLifetimes { + let seconds = |value: Option, default: Duration| { + value.map_or(default, |value| Duration::from_secs(u64::from(value))) + }; + crate::platform::CacheLifetimes { + memory: seconds(self.memory_seconds, defaults.memory), + store: seconds(self.store_seconds, defaults.store), + store_idle: seconds(self.store_idle_seconds, defaults.store_idle), + } + } + + /// Refuses a lifetime out of range, naming the setting by `table`. + /// + /// # Errors + /// + /// When the memory lifetime is over [`MAX_LOADING_CACHE_SECONDS`], or the + /// store lifetime or idle time is zero or over it. + pub fn check(&self, table: &str) -> Result<(), Report> { + check_loading_cache_seconds(table, "memory_seconds", self.memory_seconds, 0)?; + check_loading_cache_seconds(table, "store_seconds", self.store_seconds, 1)?; + check_loading_cache_seconds(table, "store_idle_seconds", self.store_idle_seconds, 1) + } +} + +fn check_loading_cache_seconds( + table: &str, + field: &str, + value: Option, + least: u32, +) -> Result<(), Report> { + match value { + Some(value) if value < least || value > MAX_LOADING_CACHE_SECONDS => { + Err(Report::new(TrustedServerError::Configuration { + message: format!( + "[{table}] {field} is {value}, and must be from {least} to \ + {MAX_LOADING_CACHE_SECONDS}" + ), + })) + } + _ => Ok(()), + } } impl CacheSettings { + /// Each cache a loading cache may hold, by its name, with the settings + /// its `[cache.]` table carries. + #[must_use] + pub fn loading_caches(&self) -> [(&'static str, &LoadingCacheConfig); 1] { + [(ROBOTS_TXT_CACHE, &self.robots_txt)] + } + fn normalize(&mut self) { for rule in &mut self.asset_rules { rule.normalize(); @@ -2077,9 +3143,11 @@ impl CacheSettings { /// /// # Errors /// - /// Returns a configuration error if any rule ID is duplicate, or if an - /// enabled rule has an invalid policy/matcher or cannot compile its regex/glob. + /// Returns a configuration error if a loading cache lifetime is out of + /// range, if any rule ID is duplicate, or if an enabled rule has an + /// invalid policy/matcher or cannot compile its regex/glob. pub fn prepare_runtime(&self) -> Result<(), Report> { + self.robots_txt.check("cache.robots-txt")?; let mut seen_ids = HashSet::new(); for rule in &self.asset_rules { if rule.id.is_empty() { @@ -2672,6 +3740,126 @@ fn is_default_auction_debug_comment_options(value: &AuctionDebugCommentOptions) *value == AuctionDebugCommentOptions::default() } +// The module selectors are new sections, so a serialized blob that carries +// them is rejected by a base-revision binary that has never heard of them. +// Omitting the default table keeps an unchanged `ts config push` readable +// across a rollout or a rollback. +fn is_default_device_config(value: &DeviceConfig) -> bool { + *value == DeviceConfig::default() +} + +fn is_default_geo_config(value: &GeoConfig) -> bool { + *value == GeoConfig::default() +} + +fn is_default_prefix_config(value: &PrefixConfig) -> bool { + *value == PrefixConfig::default() +} + +fn is_default_permission_signal_config(value: &PermissionSignalConfig) -> bool { + *value == PermissionSignalConfig::default() +} + +/// Message a configuration still carrying the removed `[integrations]` table +/// is rejected with. +const REMOVED_INTEGRATIONS_TABLE_MESSAGE: &str = "Configuration table `[integrations]` was removed. Each module is selected in the \ + section of its type, as [] module = \"\" or modules = [\"\", ...], with its \ + settings in the table at its name, [.], as described in the CHANGELOG.md \ + breaking migration"; + +/// The removed `[integrations]` table. +/// +/// Reading one always fails, with [`REMOVED_INTEGRATIONS_TABLE_MESSAGE`], so +/// the value is never held and the type carries no data. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct RemovedIntegrationsTable; + +impl<'de> Deserialize<'de> for RemovedIntegrationsTable { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + // The value is read and discarded first so that a table, a string or + // anything else all reach the same message. Reporting a type error + // instead would send an operator looking for a type problem in a table + // that has simply moved. + serde::de::IgnoredAny::deserialize(deserializer)?; + Err(serde::de::Error::custom(REMOVED_INTEGRATIONS_TABLE_MESSAGE)) + } +} + +/// A table that is no longer read under its old name, refused with directions +/// to its new one when a configuration still carries it. +macro_rules! refused_table { + ($(#[$doc:meta])* $name:ident => $message:expr) => { + $(#[$doc])* + #[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] + pub struct $name; + + impl<'de> Deserialize<'de> for $name { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + // The value is read and discarded first so that a table, a + // string or anything else all reach the same message. A type + // error instead would send an operator looking for a type + // problem in a table that has simply moved. + serde::de::IgnoredAny::deserialize(deserializer)?; + Err(serde::de::Error::custom($message)) + } + } + }; +} + +refused_table! { + /// The `[adserver]` table, renamed `[ad-server]`. + RenamedAdServerTable => "Configuration table `[adserver]` is now `[ad-server]`. Select the \ + ad server with `[ad-server] module` and move its settings to `[ad-server.]` \ + unchanged" +} + +refused_table! { + /// The `[integration]` table, which is no longer read. + RemovedIntegrationTable => "Configuration table `[integration]` is no longer read. Each \ + module is selected in the section of its type, as [] module = \"\" or \ + modules = [\"\", ...], with its settings in the table at its name, \ + [.]" +} + +refused_table! { + /// The `[[handlers]]` list, which gated paths behind HTTP Basic + /// authentication. + RemovedHandlers => "Configuration list `[[handlers]]` is not read, and no request is \ + asked for a password. Remove every `[[handlers]]` entry. Nothing is served beneath \ + /_ts/admin, so rotate signing keys with `ts keys`, purge the cache with \ + `ts cache purge`, and put a password on a path of the publisher's own at its origin \ + or in the hosting platform" +} + +refused_table! { + /// A store id under `[request_signing]`, which the service does not read. + RemovedSigningStoreId => "`[request_signing]` does not take `config_store_id` or \ + `secret_store_id`. The service reads its signing keys from the stores linked as \ + `jwks_store` and `signing_keys` and writes neither. Remove both keys, and give the \ + ids to `ts keys rotate` and `ts keys deactivate` as `--config-store-id` and \ + `--secret-store-id`" +} + +refused_table! { + /// The `[tinybird]` table, now `[analytics.tinybird]`. + MovedTinybirdTable => "Configuration table `[tinybird]` is now `[analytics.tinybird]`. \ + Select it with `[analytics] module = \"tinybird\"`, which replaces its `enabled` \ + line, and move its other settings there unchanged" +} + +refused_table! { + /// The `[permission_signal]` table, renamed `[permission-signal]`. + RenamedPermissionSignalTable => "Configuration table `[permission_signal]` is now \ + `[permission-signal]`, named exactly as its folder crates/permission-signal. Move its \ + settings there unchanged" +} + /// Behavior of the `` auction dump. Only consulted when /// [`DebugConfig::auction_html_comment`] is true. /// @@ -2686,9 +3874,9 @@ pub struct AuctionDebugCommentOptions { #[serde(default = "default_true")] pub include_provider_responses: bool, - /// Include `mediator_response` when a mediator ran. + /// Include `adserver_response` when an ad server ran. #[serde(default = "default_true")] - pub include_mediator_response: bool, + pub include_adserver_response: bool, /// Include each provider's `bids` array (vs. status/metadata only). #[serde(default = "default_true")] @@ -2730,7 +3918,7 @@ impl Default for AuctionDebugCommentOptions { fn default() -> Self { Self { include_provider_responses: true, - include_mediator_response: true, + include_adserver_response: true, include_bids: true, metadata_keys: default_auction_debug_metadata_keys(), verbosity: AuctionDebugCommentVerbosity::Redacted, @@ -2955,7 +4143,6 @@ fn validate_trusted_client_ip_shared_secret( } #[derive(Debug, Default, Clone, Deserialize, Serialize, Validate)] -#[serde(deny_unknown_fields)] pub struct Settings { #[validate(nested)] pub publisher: Publisher, @@ -2973,14 +4160,54 @@ pub struct Settings { #[serde(default, skip_serializing_if = "Option::is_none")] #[validate(nested)] pub trusted_client_ip: Option, - #[serde(default)] + /// Optional `robots.txt`, assembled by core from the contributors + /// `[robots-txt] modules` selects. + /// + /// `None` leaves `/robots.txt` to the publisher's origin, and must stay + /// omitted from serialized config blobs for the reason given on + /// `trusted_client_ip`. + #[serde( + rename = "robots-txt", + default, + skip_serializing_if = "Option::is_none" + )] #[validate(nested)] - pub ec: Ec, + pub robots_txt: Option, #[serde(default)] - pub integrations: IntegrationSettings, - #[serde(default, deserialize_with = "vec_from_seq_or_map")] #[validate(nested)] - pub handlers: Vec, + pub ec: Ec, + /// The removed `[integrations]` table. + /// + /// The name is kept so a configuration written for the previous release is + /// told where its blocks moved, rather than being handed a bare + /// unknown-field error listing every table Trusted Server accepts. The + /// field never holds a value, because reading one always fails. + #[serde(default, skip_serializing)] + #[allow( + dead_code, + reason = "the field exists so that reading the removed table fails with directions" + )] + integrations: RemovedIntegrationsTable, + /// The `[integration]` table, kept so a configuration carrying it is + /// told where modules are selected now. + #[serde(default, skip_serializing)] + #[allow( + dead_code, + reason = "the field exists so that reading the removed table fails with directions" + )] + integration: RemovedIntegrationTable, + /// The sections of module types core does not read itself, such as + /// `[cmp]` or `[tag]`, each named for the type of module it selects. + #[serde(flatten)] + pub sections: TypeSections, + /// The `[[handlers]]` list, kept so a configuration carrying one is + /// told that no path is gated by a password. + #[serde(default, skip_serializing)] + #[allow( + dead_code, + reason = "the field exists so that reading the removed list fails with directions" + )] + handlers: RemovedHandlers, #[serde(default, deserialize_with = "map_from_obj_or_str")] pub response_headers: HashMap, pub request_signing: Option, @@ -2990,23 +4217,123 @@ pub struct Settings { #[serde(default)] #[validate(nested)] pub auction: AuctionConfig, + /// The auction's demand sources. `[demand] modules` selects them and each + /// `[demand.]` table holds one source's settings. + #[serde(default, skip_serializing_if = "ProviderList::is_unset")] + pub demand: ProviderList, + /// The ad server that picks the auction winner. `[ad-server] module` + /// selects it and `[ad-server.]` holds its settings. + #[serde( + rename = "ad-server", + default, + skip_serializing_if = "ProviderChoice::is_unset" + )] + pub adserver: ProviderChoice, + /// The `[adserver]` table, kept so a configuration carrying it is told + /// its new name. + #[serde(rename = "adserver", default, skip_serializing)] + #[allow( + dead_code, + reason = "the field exists so that reading the renamed table fails with directions" + )] + renamed_adserver: RenamedAdServerTable, #[serde(default)] pub consent: ConsentConfig, #[serde(default)] pub cache: CacheSettings, + /// What one vendor's modules share, a table per vendor, `[shared.]`. + #[serde(default, skip_serializing_if = "SharedTables::is_empty")] + pub shared: SharedTables, #[serde(default)] pub proxy: Proxy, #[serde(default)] pub creative_opportunities: Option, #[serde(default)] pub image_optimizer: ImageOptimizerSettings, - #[serde(default)] - pub tinybird: TinybirdSettings, + /// Analytics. `[analytics] module` selects the module that runs, with its + /// settings in the table at its name. Omitted from a serialized + /// configuration when absent, for the reason given on `trusted_client_ip`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub analytics: Option, + /// The `[tinybird]` table, kept so a configuration carrying it is told + /// where it moved. + #[serde(rename = "tinybird", default, skip_serializing)] + #[allow( + dead_code, + reason = "the field exists so that reading the moved table fails with directions" + )] + moved_tinybird: MovedTinybirdTable, #[serde(default)] pub debug: DebugConfig, + #[serde(default, skip_serializing_if = "is_default_device_config")] + #[validate(nested)] + pub device: DeviceConfig, + #[serde(default, skip_serializing_if = "is_default_geo_config")] + #[validate(nested)] + pub geo: GeoConfig, + /// The prefix module, `[prefix] module`, see [`crate::prefix`]. + #[serde(default, skip_serializing_if = "is_default_prefix_config")] + #[validate(nested)] + pub prefix: PrefixConfig, + #[serde( + rename = "permission-signal", + default, + skip_serializing_if = "is_default_permission_signal_config" + )] + #[validate(nested)] + pub permission_signal: PermissionSignalConfig, + /// The `[permission_signal]` table, kept so a configuration carrying it is + /// told its new name. + #[serde(rename = "permission_signal", default, skip_serializing)] + #[allow( + dead_code, + reason = "the field exists so that reading the renamed table fails with directions" + )] + renamed_permission_signal: RenamedPermissionSignalTable, + /// What the configuration page at `/_ts/config` shows. + #[serde( + default, + skip_serializing_if = "crate::inspect::config::InspectConfig::is_default" + )] + pub inspect: crate::inspect::config::InspectConfig, + /// The attestation endpoint, which serves evidence signed with the + /// operator's key at the address its `endpoint` names. See + /// [`crate::attestation`]. + /// + /// `None` leaves the address to the publisher's origin, and stays out of + /// serialized config blobs for the reason given on `trusted_client_ip`. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[validate(nested)] + pub attestation: Option, + /// The page changes run on a document fetched from the origin, written + /// as `[[fetch]]` entries. See [`crate::middleware`]. + #[serde(default, skip_serializing_if = "PhaseEntries::is_empty")] + pub fetch: PhaseEntries, + /// The page changes run on each reader's copy of a document, written as + /// `[[serve]]` entries. + #[serde(default, skip_serializing_if = "PhaseEntries::is_empty")] + pub serve: PhaseEntries, + /// Where the loader wrote secrets, which the configuration view masks. + /// Never read from a document and never written to one. + #[serde(skip)] + resolved_secrets: crate::secret_resolution::ResolvedSecrets, } impl Settings { + /// Where the loader wrote secrets, for the configuration view. + #[must_use] + pub(crate) fn resolved_secrets(&self) -> &crate::secret_resolution::ResolvedSecrets { + &self.resolved_secrets + } + + /// Records where the loader wrote secrets. + pub(crate) fn set_resolved_secrets( + &mut self, + secrets: crate::secret_resolution::ResolvedSecrets, + ) { + self.resolved_secrets = secrets; + } + /// Creates a new [`Settings`] instance from a TOML string. /// /// # Errors @@ -3051,9 +4378,15 @@ impl Settings { /// - [`TrustedServerError::Configuration`] if the TOML is invalid or missing required fields #[cfg(test)] pub fn from_toml_and_env(toml_str: &str) -> Result> { + // The build script compiles the git version in through this variable, + // and cargo sets it on a test process too. It is not a setting. + let variables = std::env::vars() + .filter(|(name, _)| name != "TRUSTED_SERVER__GIT_VERSION") + .collect(); let environment = Environment::default() .prefix(ENVIRONMENT_VARIABLE_PREFIX) - .separator(ENVIRONMENT_VARIABLE_SEPARATOR); + .separator(ENVIRONMENT_VARIABLE_SEPARATOR) + .source(Some(variables)); let toml = File::from_str(toml_str, FileFormat::Toml); let config = Config::builder() @@ -3073,6 +4406,14 @@ impl Settings { Self::finalize_deserialized(settings, "Build-time configuration") } + /// The analytics module's settings, when `[analytics]` selects it. + #[must_use] + pub fn analytics_module(&self) -> Option<&TinybirdSettings> { + self.analytics + .as_ref() + .and_then(AnalyticsConfig::analytics_module) + } + /// Normalize settings after deserialization. /// /// # Errors @@ -3083,9 +4424,9 @@ impl Settings { self.proxy.normalize(); self.image_optimizer.normalize()?; self.debug.auction_html_comment_options.normalize(); - self.tinybird.normalize(); - self.integrations - .remove_legacy_static_secret_store_selectors(); + if let Some(analytics) = &mut self.analytics { + analytics.normalize(); + } self.consent.validate(); Ok(()) } @@ -3106,8 +4447,33 @@ impl Settings { }) })?; - settings.validate_admin_coverage()?; - settings.validate_admin_handler_passwords()?; + settings.ec.migrate_legacy_ec_layout()?; + settings.ec.validate_module_selection()?; + settings.ec.validate_resolve_allowed_origins()?; + settings.validate_module_sections()?; + settings.validate_phase_entries()?; + settings.device.validate_module_selection()?; + settings.geo.validate_module_selection()?; + GeoConfig::validate_permission_policy()?; + settings + .geo + .validate_jurisdiction_acknowledgment(&settings.ec)?; + + // Log the policy's declared default once per settings load, so an + // operator can see which permissions an unmatched request is granted + // without a signal, and which jurisdiction its consent gates apply. + let maps = crate::permissions::PermissionMaps::standard(); + let granted: Vec = maps + .baseline(None, None) + .permissions() + .iter() + .map(|permission| permission.to_string()) + .collect(); + log::debug!( + "Permission baseline: permissions.yaml top node, jurisdiction {}; granted without a signal: [{}]", + maps.default_jurisdiction(), + granted.join(", ") + ); Ok(settings) } @@ -3124,14 +4490,17 @@ impl Settings { self.image_optimizer.prepare_runtime()?; self.cache.prepare_runtime()?; self.proxy.prepare_runtime()?; - self.tinybird.prepare_runtime()?; + if let Some(analytics) = &mut self.analytics { + analytics.prepare_runtime()?; + } self.debug .auction_html_comment_options .validate_metadata_keys()?; self.validate_asset_image_optimizer_profile_sets()?; - - for handler in &self.handlers { - handler.prepare_runtime()?; + self.shared.validate_names()?; + if let Some(robots_txt) = &self.robots_txt { + robots_txt.prepare_runtime()?; + self.require_robots_txt_placed()?; } if let Some(co) = &mut self.creative_opportunities { @@ -3169,6 +4538,59 @@ impl Settings { Ok(()) } + /// Refuses a `[robots-txt]` section unless a `[[fetch]]` entry for + /// `text/plain` with `path = "/robots.txt"` names the `robots_txt` + /// middleware, and refuses any entry naming it on another path or media + /// type. Only a `[[fetch]]` entry makes the file where the origin has + /// none, and an entry on another path would replace that file with the + /// robots.txt rules. + fn require_robots_txt_placed(&self) -> Result<(), Report> { + use crate::robots_txt::ROBOTS_TXT_PATH; + use crate::robots_txt::middleware::{MEDIA_TYPE, MODULE}; + + let names = |entry: &crate::middleware::PhaseEntry| { + entry.middleware.iter().any(|name| name == MODULE) + }; + let on_the_file = |entry: &crate::middleware::PhaseEntry| { + entry.media_type == MEDIA_TYPE && entry.path.as_deref() == Some(ROBOTS_TXT_PATH) + }; + for (phase, entries) in [("fetch", &self.fetch), ("serve", &self.serve)] { + for (index, entry) in entries.entries().iter().enumerate() { + if names(entry) && !on_the_file(entry) { + return Err(Report::new(TrustedServerError::Settings { + message: format!( + "[[{phase}]] entry {} names `{MODULE}` for `{}` on {}, where it \ + would replace each file it covers with robots.txt. Name it only \ + in an entry with media_type = `{MEDIA_TYPE}` and path = \ + `{ROBOTS_TXT_PATH}`", + index + 1, + entry.media_type, + entry.path.as_deref().map_or_else( + || "every path".to_owned(), + |path| format!("`{path}`") + ), + ), + })); + } + } + } + if self + .fetch + .entries() + .iter() + .any(|entry| names(entry) && on_the_file(entry)) + { + return Ok(()); + } + Err(Report::new(TrustedServerError::Settings { + message: format!( + "[robots-txt] is set, but no [[fetch]] entry with media_type = `{MEDIA_TYPE}` and \ + path = `{ROBOTS_TXT_PATH}` names `{MODULE}`, so the file would never be made \ + where the origin has none. Add that entry" + ), + })) + } + /// Returns compiled creative opportunity slots when template delivery is enabled. #[must_use] pub fn creative_opportunity_slots( @@ -3190,8 +4612,15 @@ impl Settings { pub fn reject_placeholder_secrets(&self) -> Result<(), Report> { let mut insecure_fields: Vec = Vec::new(); - if Ec::is_placeholder_passphrase(self.ec.passphrase.expose()) { - insecure_fields.push("ec.passphrase".to_owned()); + for (name, hmac) in self.ec.module_blocks.hmac_blocks() { + if Ec::is_placeholder_passphrase(hmac.passphrase.expose()) { + insecure_fields.push(format!("ec.{name}.passphrase")); + } + } + for (name, host_signals) in self.ec.module_blocks.host_signals_blocks() { + if Ec::is_placeholder_passphrase(host_signals.passphrase.expose()) { + insecure_fields.push(format!("ec.{name}.passphrase")); + } } if Publisher::is_placeholder_proxy_secret(self.publisher.proxy_secret.expose()) { insecure_fields.push("publisher.proxy_secret".to_owned()); @@ -3222,11 +4651,6 @@ impl Settings { )); } } - for handler in &self.handlers { - if Handler::is_placeholder_password(handler.password.expose()) { - insecure_fields.push(format!("handlers[{}].password", handler.path)); - } - } if Publisher::is_placeholder_domain(&self.publisher.domain) { insecure_fields.push("publisher.domain".to_owned()); } @@ -3236,22 +4660,6 @@ impl Settings { if Publisher::is_placeholder_origin_url(&self.publisher.origin_url) { insecure_fields.push("publisher.origin_url".to_owned()); } - // Checked whenever the block is present, not just when it is enabled: - // the key rotate/deactivate admin routes are registered unconditionally - // and read these store IDs without consulting `enabled`, so placeholder - // IDs behind a disabled block would still reach key management at - // runtime. Surrounding whitespace is rejected too: the placeholder check - // trims for comparison but the raw value is what `signing_store_ids` - // forwards to `KeyRotationManager`, so a padded id would validate yet - // reach the management API unusable. - if let Some(request_signing) = &self.request_signing { - if RequestSigning::is_unusable_store_id(&request_signing.config_store_id) { - insecure_fields.push("request_signing.config_store_id".to_owned()); - } - if RequestSigning::is_unusable_store_id(&request_signing.secret_store_id) { - insecure_fields.push("request_signing.secret_store_id".to_owned()); - } - } if insecure_fields.is_empty() { return Ok(()); @@ -3306,172 +4714,223 @@ impl Settings { self.proxy.asset_route_for_path(path) } - /// Resolve the first handler whose regex matches the request path. - /// - /// # Errors - /// - /// Returns a configuration error if any handler regex does not compile. - pub fn handler_for_path( - &self, - path: &str, - ) -> Result, Report> { - for handler in &self.handlers { - if handler.matches_path(path)? { - return Ok(Some(handler)); - } - } - - Ok(None) + /// Every section that selects modules, with its name: `[proxy]`, + /// `[auction]` and each module type's own section. + pub fn module_sections(&self) -> impl Iterator { + [ + ("proxy", &self.proxy.modules), + ("auction", &self.auction.modules), + ] + .into_iter() + .chain(self.sections.iter()) } - /// Returns whether `path` is within the reserved Trusted Server admin - /// namespace. + /// The section that selects the module `name`, and the name as written + /// there, whether in full or with the section's type folder left off. #[must_use] - pub(crate) fn is_admin_path(path: &str) -> bool { - path == "/_ts/admin" || path.starts_with("/_ts/admin/") - } - - /// Known admin endpoint paths that must be covered by a handler. - /// - /// [`from_toml`](Self::from_toml) rejects configurations - /// where any of these paths lack a matching handler, ensuring admin - /// endpoints are always protected by authentication. - /// Update [`ADMIN_ENDPOINTS`](Self::ADMIN_ENDPOINTS) when adding new - /// admin routes to `crates/trusted-server-adapter-fastly/src/app.rs`. - /// - /// The `/_ts/admin/ec/{id}` entry is the canonical router pattern. Its - /// coverage is checked via [`admin_auth_probes`](Self::admin_auth_probes), - /// while validation errors continue to report this operator-facing route - /// template. - pub(crate) const ADMIN_ENDPOINTS: &[&str] = &[ - "/_ts/admin/keys/rotate", - "/_ts/admin/keys/deactivate", - "/_ts/admin/ec", - "/_ts/admin/ec/{id}", - "/_ts/admin/eids", - "/_ts/admin/cache/purge", - ]; - - /// Probes that establish handler coverage for the dynamic - /// `/_ts/admin/ec/{id}` route. - /// - /// Coverage cannot be sampled: the router accepts any single segment after - /// `/_ts/admin/ec/` and basic auth runs on the raw path before routing, so - /// a handler that matches only some ID shapes leaves the rest of the route - /// surface — including malformed IDs, which still reach the admin handler — - /// unauthenticated at configuration time and fail-closed at runtime. - /// - /// Both probes must match the same configuration for the route to count as - /// covered. The bare prefix rejects handlers anchored to specific ID - /// shapes; the concrete ID rejects handlers anchored to the prefix itself - /// (`^/_ts/admin/ec/$`). Together they admit only prefix-level matchers - /// such as `^/_ts/admin` or `^/_ts/admin/ec/`. - const ADMIN_EC_ID_AUTH_PROBES: [&str; 2] = [ - "/_ts/admin/ec/", - concat!( - "/_ts/admin/ec/", - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - ".Ab12Z9", - ), - ]; + pub fn module_selection(&self, name: &str) -> Option<(&str, &str)> { + self.module_sections().find_map(|(section, modules)| { + modules + .selected() + .iter() + .find(|written| crate::module_name::resolve(section, written, &[name]).is_some()) + .map(|written| (section, written.as_str())) + }) + } - fn admin_auth_probes(path: &'static str) -> [&'static str; 2] { - match path { - "/_ts/admin/ec/{id}" => Self::ADMIN_EC_ID_AUTH_PROBES, - path => [path, path], - } + /// Whether a section selects the module `name`. + #[must_use] + pub fn selects_module(&self, name: &str) -> bool { + self.module_selection(name).is_some() } - /// Returns admin endpoint paths that no configured handler covers. + /// The table `section` holds at the name `written`, as it was written, + /// or an empty one when the section holds none. + #[must_use] + pub fn section_table( + &self, + section: &str, + written: &str, + ) -> serde_json::Map { + self.module_sections() + .find(|(candidate, _)| *candidate == section) + .and_then(|(_, modules)| modules.settings_of(written)) + .cloned() + .unwrap_or_default() + } + + /// Reads and validates a selected module's settings, from the table at the + /// name it is written under beneath the section that selects it, or + /// returns `None` when no section selects it. /// - /// Called during settings finalization to enforce that every admin endpoint - /// has a handler. An empty return - /// value means all admin endpoints are properly covered. + /// A selected module with no table reads an empty one, so a module that + /// takes no settings runs on its selection alone and one that requires a + /// setting reports the setting it is missing. The parse and validation + /// messages carry the underlying error, because a report renders only its + /// outermost message in `ts config validate`. /// /// # Errors /// - /// Returns [`TrustedServerError::Configuration`] if any handler has an invalid path regex. - pub(crate) fn uncovered_admin_endpoints( - &self, - ) -> Result, Report> { - let mut uncovered = Vec::new(); - for &path in Self::ADMIN_ENDPOINTS { - let mut covered = true; - for probe in Self::admin_auth_probes(path) { - let mut probe_covered = false; - for handler in &self.handlers { - if handler.matches_path(probe)? { - probe_covered = true; - break; - } - } - covered &= probe_covered; - } - if !covered { - uncovered.push(path); - } - } - Ok(uncovered) + /// When the table cannot be read as `T` or fails its validation, naming the + /// table. + pub fn module_config(&self, name: &str) -> Result, Report> + where + T: IntegrationConfig, + { + let Some((section, written)) = self.module_selection(name) else { + return Ok(None); + }; + let table = self + .module_sections() + .find(|(candidate, _)| *candidate == section) + .and_then(|(_, modules)| modules.settings_of(written)) + .cloned() + .unwrap_or_default(); + let config: T = serde_json::from_value(JsonValue::Object(table)).map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: format!("[{section}.{written}] could not be read: {error}"), + }) + })?; + config.validate().map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: format!("[{section}.{written}] failed validation: {error}"), + }) + })?; + Ok(Some(config)) } - /// Validates that every admin endpoint is covered by at least one handler. + /// Reads `[shared.]`, the settings every module of `vendor` + /// shares, or `None` when the table is absent. /// /// # Errors /// - /// Returns [`TrustedServerError::Configuration`] listing any uncovered - /// admin endpoints. - pub(crate) fn validate_admin_coverage(&self) -> Result<(), Report> { - let uncovered = self.uncovered_admin_endpoints()?; - if uncovered.is_empty() { - return Ok(()); - } - Err(Report::new(TrustedServerError::Configuration { - message: format!( - "No handler covers admin endpoint(s): {}. \ - Add a [[handlers]] entry with a path regex matching /_ts/admin/ \ - to protect admin access.", - uncovered.join(", ") - ), - })) + /// When the table cannot be read as `T` or fails its validation, naming + /// the table. + pub fn shared_config(&self, vendor: &str) -> Result, Report> + where + T: DeserializeOwned + Validate, + { + self.shared.get_typed(vendor) } - /// Rejects placeholder and well-known weak handler passwords. - /// - /// Applies to every handler rather than to handlers inferred to cover an - /// admin endpoint: handler selection is first-match-wins over operator - /// regexes, so a narrow handler can shadow the admin namespace for paths no - /// probe enumerates. Handlers are Trusted Server's own basic-auth gates, so - /// a placeholder password is never valid on any of them. - pub(crate) fn validate_admin_handler_passwords( - &self, - ) -> Result<(), Report> { - for handler in &self.handlers { - if is_admin_placeholder_password(handler.password.expose()) { - return Err(Report::new(TrustedServerError::Configuration { - message: format!( - "Handler `{}` uses a placeholder password; configure a strong secret", - handler.path - ), - })); - } + /// The modules `section` selects, to change, with the section created + /// empty when it is a module type's and absent. + fn section_modules_mut(&mut self, section: &str) -> &mut SectionModules { + match section { + "proxy" => &mut self.proxy.modules, + "auction" => &mut self.auction.modules, + _ => self.sections.section_mut(section), } + } - Ok(()) + /// Selects the module `name` in `section`, written with the section's + /// type folder left off, with no table of its own. + pub fn select_module(&mut self, section: &str, name: &str) { + let written = crate::module_name::short_form(section, name).to_owned(); + self.section_modules_mut(section).select(&written); } - /// Retrieves the integration configuration of a specific type. + /// Stops selecting the module `name` in `section` and drops its table. + pub fn remove_module(&mut self, section: &str, name: &str) { + let written = crate::module_name::short_form(section, name).to_owned(); + self.section_modules_mut(section).remove(&written); + } + + /// Selects the module `name` in `section`, with `settings` as its table, + /// which is what writing both in a document does. + /// + /// The section is the name's type folder, except for the sections that + /// Trusted Server reads itself and that also select modules, where the + /// name is written in full. /// /// # Errors /// - /// Returns an error if the integration configuration exists but cannot be deserialized as the requested type. - pub fn integration_config( - &self, - integration_id: &str, - ) -> Result, Report> + /// Returns an error if `settings` cannot be serialized to a table. + pub fn insert_module_config( + &mut self, + section: &str, + name: &str, + settings: &T, + ) -> Result<(), Report> where - T: IntegrationConfig, + T: Serialize, { - self.integrations.get_typed(integration_id) + let JsonValue::Object(table) = + serde_json::to_value(settings).change_context(TrustedServerError::Configuration { + message: "Failed to serialize module configuration".to_string(), + })? + else { + return Err(Report::new(TrustedServerError::Configuration { + message: format!("the settings of `{name}` are not a table"), + })); + }; + let written = crate::module_name::short_form(section, name).to_owned(); + self.section_modules_mut(section).insert(&written, table); + Ok(()) + } + + /// The entries of `phase`, in the order written. + #[must_use] + pub fn phase_entries(&self, phase: MiddlewarePhase) -> &PhaseEntries { + match phase { + MiddlewarePhase::Fetch => &self.fetch, + MiddlewarePhase::Serve => &self.serve, + } + } + + /// Checks the shape of each phase's entries, see + /// [`PhaseEntries::validate`]. Whether a name is one a running module + /// supplies is only knowable where the registry is built, so it is + /// checked there. + /// + /// # Errors + /// + /// Naming the entry at fault. + pub fn validate_phase_entries(&self) -> Result<(), Report> { + for phase in MiddlewarePhase::ALL { + self.phase_entries(phase) + .validate(phase) + .map_err(|message| Report::new(TrustedServerError::Configuration { message }))?; + } + Ok(()) + } + + /// Checks every section's selection and tables, and that no module is + /// selected in two sections. + /// + /// # Errors + /// + /// Naming the section at fault. + pub fn validate_module_sections(&self) -> Result<(), Report> { + self.sections.validate()?; + for (section, modules) in [ + ("proxy", &self.proxy.modules), + ("auction", &self.auction.modules), + ] { + modules + .validate(section) + .map_err(|message| Report::new(TrustedServerError::Configuration { message }))?; + } + let mut seen: Vec<(&str, String)> = Vec::new(); + for (section, modules) in self.module_sections() { + for written in modules.selected() { + let full = if written.contains('.') { + written.clone() + } else { + format!("{section}.{written}") + }; + if let Some((earlier, _)) = seen.iter().find(|(_, name)| *name == full) { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "`{written}` is selected in both [{earlier}] and [{section}]. A \ + module is selected in one section" + ), + })); + } + seen.push((section, full)); + } + } + Ok(()) } } @@ -3559,8 +5018,14 @@ fn validation_error_summary(errors: &validator::ValidationErrors) -> String { }; match kind { validator::ValidationErrorsKind::Field(validations) => { + // A validator's message, where it gives one, says what the + // code cannot, such as which address an endpoint clashed + // with. for validation in validations { - messages.push(format!("{path}: {}", validation.code)); + messages.push(match &validation.message { + Some(message) => format!("{path}: {} ({message})", validation.code), + None => format!("{path}: {}", validation.code), + }); } } validator::ValidationErrorsKind::Struct(inner) => { @@ -3655,14 +5120,6 @@ fn validate_proxy_origin_url(value: &str) -> Result<(), ValidationError> { Ok(()) } -fn validate_path(value: &str) -> Result<(), ValidationError> { - Regex::new(value).map(|_| ()).map_err(|err| { - let mut validation_error = ValidationError::new("invalid_regex"); - validation_error.add_param("value".into(), &value); - validation_error.add_param("message".into(), &err.to_string()); - validation_error - }) -} fn from_value_or_str<'de, D, T>(deserializer: D) -> Result where D: Deserializer<'de>, @@ -3678,7 +5135,7 @@ where // Helper: allow Vec fields to deserialize from either a JSON array or a map of numeric indices. // This lets env vars such as -// TRUSTED_SERVER__INTEGRATIONS__PREBID__CLIENT_SIDE_BIDDERS__0=example-browser work; +// TRUSTED_SERVER__AUCTION__PREBID__CLIENT_SIDE_BIDDERS__0=example-browser work; // the config env source represents the value as an object rather than a sequence. // String inputs may also be JSON arrays or comma-separated values. /// Deserializes a `HashMap` from either: @@ -3740,7 +5197,18 @@ where } } -pub(crate) fn vec_from_seq_or_map<'de, D, T>(deserializer: D) -> Result, D::Error> +/// Reads a list setting written as a sequence, as a map keyed by position, +/// or as a string holding a JSON array or comma-separated values. +/// +/// The map and string forms are what an environment overlay produces, so a +/// list reads the same from a file and from the environment. Public so a +/// module crate's own list settings read the same way. +/// +/// # Errors +/// +/// When the value is none of those forms, a map key is not a position, or an +/// item cannot be read as `T`. +pub fn vec_from_seq_or_map<'de, D, T>(deserializer: D) -> Result, D::Error> where D: Deserializer<'de>, T: DeserializeOwned, @@ -3816,7 +5284,8 @@ mod tests { use regex::Regex; use serde_json::json; use std::collections::BTreeSet; - use std::sync::Arc; + + use crate::ec::resolve::NotAnOrigin; /// `DebugConfig` denies unknown fields, so a binary built before /// `sandbox_metrics_enabled` existed must still accept a default blob. @@ -3869,13 +5338,12 @@ mod tests { ); } - use crate::auction::build_orchestrator; - use crate::integrations::{ - IntegrationRegistry, gpt::GptConfig, nextjs::NextJsIntegrationConfig, - prebid::PrebidIntegrationConfig, - }; + use crate::integrations::IntegrationRegistry; use crate::redacted::Redacted; - use crate::test_support::tests::{crate_test_settings_str, create_test_settings}; + use crate::test_support::tests::{ + crate_test_settings_str, crate_test_settings_str_with_ec_section, create_test_settings, + hmac_passphrase, select_hmac_module, + }; fn trusted_client_ip_toml(ip_header: &str, auth_header: &str, shared_secret: &str) -> String { format!( @@ -3895,6 +5363,166 @@ mod tests { ); } + fn settings_str_with_robots_txt(section: &str) -> String { + format!( + "{}\n[robots-txt]\n{section}\n\n[[fetch]]\nmedia_type = \"text/plain\"\npath = \"/robots.txt\"\nmiddleware = [\"robots_txt\"]\n", + crate_test_settings_str() + ) + } + + #[test] + fn a_robots_txt_section_with_no_entry_placing_the_middleware_is_refused() { + let toml = format!( + "{}\n[robots-txt]\nmodules = \"refuse_all\"\n", + crate_test_settings_str() + ); + + let error = Settings::from_toml(&toml).expect_err("the file would never be made"); + + let message = format!("{error:?}"); + assert!( + message.contains( + "no [[fetch]] entry with media_type = `text/plain` and path = `/robots.txt` names \ + `robots_txt`" + ), + "{message}" + ); + } + + /// The `[[fetch]]` or `[[serve]]` entry placing the middleware on + /// `/robots.txt`. + const ROBOTS_TXT_ENTRY: &str = + "media_type = \"text/plain\"\npath = \"/robots.txt\"\nmiddleware = [\"robots_txt\"]\n"; + + /// Settings with `[robots-txt]` refusing every crawler and `entries`. + fn settings_str_placing_robots_txt(entries: &str) -> String { + format!( + "{}\n[robots-txt]\nmodules = \"refuse_all\"\n\n{entries}\n", + crate_test_settings_str() + ) + } + + #[test] + fn a_fetch_entry_on_the_files_path_places_robots_txt() { + let toml = settings_str_placing_robots_txt(&format!( + "[[fetch]]\n{ROBOTS_TXT_ENTRY}\n[[serve]]\n{ROBOTS_TXT_ENTRY}" + )); + + Settings::from_toml(&toml).expect("an entry on the file's path places the middleware"); + } + + /// An entry with no path would replace every text file, `ads.txt` + /// included, with the robots.txt rules. + #[test] + fn an_entry_naming_robots_txt_on_another_path_is_refused() { + let toml = settings_str_placing_robots_txt(&format!( + "[[fetch]]\n{ROBOTS_TXT_ENTRY}\n[[fetch]]\nmedia_type = \"text/plain\"\nmiddleware = [\"robots_txt\"]\n" + )); + + let error = Settings::from_toml(&toml).expect_err("every text file would be robots.txt"); + + let message = format!("{error:?}"); + assert!( + message.contains("[[fetch]] entry 2 names `robots_txt` for `text/plain` on every path"), + "{message}" + ); + } + + /// Only a `[[fetch]]` entry makes the file where the origin has none, so + /// a `[[serve]]` placement alone would leave an origin's `404` standing, + /// which a crawler reads as permission to crawl everything. + #[test] + fn robots_txt_placed_only_in_serve_is_refused() { + let toml = settings_str_placing_robots_txt(&format!("[[serve]]\n{ROBOTS_TXT_ENTRY}")); + + let error = Settings::from_toml(&toml).expect_err("the file would never be made"); + + assert!( + format!("{error:?}").contains("no [[fetch]] entry with media_type = `text/plain`"), + "{error:?}" + ); + } + + #[test] + fn the_robots_txt_section_is_read_by_its_hyphenated_name() { + let toml = settings_str_with_robots_txt( + "modules = [\"refuse_all\"]\nalways_allow = [\"/ads.txt\"]", + ); + + let settings = Settings::from_toml(&toml).expect("should read the section"); + + let robots_txt = settings + .robots_txt + .as_ref() + .expect("should carry the section"); + assert_eq!(robots_txt.modules, vec!["refuse_all"]); + assert_eq!(robots_txt.always_allow, vec!["/ads.txt"]); + assert!( + settings + .module_sections() + .all(|(name, _)| name != "robots-txt"), + "should not also be read as the section of a module type" + ); + } + + #[test] + fn a_robots_txt_section_naming_no_module_is_refused_when_the_settings_load() { + let toml = settings_str_with_robots_txt("sitemap = \"https://example.com/sitemap.xml\""); + + let error = Settings::from_toml(&toml).expect_err("there is no default"); + + assert!( + format!("{error:?}").contains("[robots-txt] names no module"), + "should say a module must be named: {error:?}" + ); + } + + #[test] + fn a_robots_txt_path_that_could_start_a_second_line_is_refused() { + let toml = settings_str_with_robots_txt( + "modules = \"refuse_all\"\nalways_allow = [\"/ads.txt\\nDisallow:\"]", + ); + + assert!( + Settings::from_toml(&toml).is_err(), + "should refuse a path that holds white space" + ); + } + + #[test] + fn settings_without_a_robots_txt_section_are_written_without_one() { + let settings = Settings::from_toml(&crate_test_settings_str()) + .expect("should load the test settings fixture"); + + let value = serde_json::to_value(&settings).expect("should serialize the settings"); + + assert!( + value.get("robots-txt").is_none(), + "a build that predates the section refuses a key it does not know" + ); + } + + #[test] + fn a_robots_txt_section_survives_the_stored_form() { + let toml = settings_str_with_robots_txt( + "modules = [\"allow_all\"]\nsitemap = \"https://example.com/sitemap.xml\"", + ); + let settings = Settings::from_toml(&toml).expect("should read the section"); + + let stored = serde_json::to_value(&settings).expect("should serialize the settings"); + let reloaded = Settings::from_json_value(stored).expect("should read the section back"); + + let robots_txt = reloaded + .robots_txt + .as_ref() + .expect("should still carry the section"); + assert_eq!(robots_txt.modules, vec!["allow_all"]); + assert_eq!( + robots_txt.sitemap.as_deref(), + Some("https://example.com/sitemap.xml") + ); + } + /// Mirrors the `Settings` schema of the revision that predates /// `trusted_client_ip`: every key that revision knew, and /// `deny_unknown_fields` so an extra key fails deserialization exactly as an @@ -3956,15 +5584,79 @@ mod tests { #[test] fn serialized_default_config_stays_readable_by_the_base_revision_schema() { - let settings = Settings::from_toml(&crate_test_settings_str()) + let mut settings = Settings::from_toml(&crate_test_settings_str()) .expect("should parse settings without trusted client IP configuration"); + // The guarantee covers a config that configures none of the sections + // added since the base revision. A configured section is serialized and, + // like a configured `trusted_client_ip`, needs a compatible blob restored + // before rolling back to a binary that predates it. The shared test + // config sets `[geo]` because the permission model requires a default + // country, so both selector tables are reset to unset here. + settings.geo = GeoConfig::default(); + settings.device = DeviceConfig::default(); + settings.sections = TypeSections::default(); + settings.auction.modules.clear(); + let value = serde_json::to_value(&settings).expect("should serialize settings"); serde_json::from_value::(value) .expect("base revision schema should accept a config blob with no trusted client IP"); } + #[test] + fn geo_selector_is_omitted_from_serialized_config_when_unset() { + // `ts config push` serializes `Settings` verbatim, so a selector nobody + // set must not appear in the blob. A `deny_unknown_fields` binary that + // predates the selector rejects the key during rollout or rollback. + // + // The shared fixture writes a `[geo]` table (it acknowledges running + // with no geo module), so the assertion is about the selector key + // rather than the table, which is what the blob's compatibility + // actually turns on. + let settings = Settings::from_toml(&crate_test_settings_str()) + .expect("should parse settings without a geo selector"); + + let value = serde_json::to_value(&settings).expect("should serialize settings"); + + let geo = value + .get("geo") + .expect("the geo table is written because the fixture acknowledges no geo module"); + assert!( + geo.get("module").is_none(), + "an unset geo selector should not be serialized, got {geo}" + ); + assert!( + value + .get("device") + .is_none_or(|device| device.get("module").is_none()), + "an unset device selector should not be serialized, got {value}" + ); + } + + #[test] + fn a_selected_geo_module_stays_in_the_serialized_config() { + // The shared test settings already carry a `[geo]` table, so the + // selector is set inside that table rather than in a second one, which + // TOML rejects as a duplicate key. + let settings = Settings::from_toml(&crate_test_settings_str().replace( + "[geo]", + "[geo] +module = \"none\"", + )) + .expect("should parse settings with a geo selector"); + + let value = serde_json::to_value(&settings).expect("should serialize settings"); + + assert_eq!( + value + .pointer("/geo/module") + .and_then(serde_json::Value::as_str), + Some("none"), + "a selected geo module should survive serialization" + ); + } + #[test] fn trusted_client_ip_parses_and_redacts_shared_secret_in_debug_output() { let settings = Settings::from_toml(&trusted_client_ip_toml( @@ -4001,7 +5693,7 @@ mod tests { // hand-written `Debug` impl prints only integration IDs, never values. // // Do not use `..Struct::default()` anywhere in this function. A default - // spread would let a new secret field be added to `Handler`, + // spread would let a new secret field be added to `EcPartner`, // `TinybirdSettings`, or any other struct built here without forcing // anyone to consider it. The compile break is the prompt; the canary // list below is still maintained by hand. List every field explicitly. @@ -4009,8 +5701,6 @@ mod tests { fn settings_debug_output_redacts_every_secret_field() { const CANARY_PROXY_SECRET: &str = "CANARY-PROXY-SECRET-0123456789"; const CANARY_EC_PASSPHRASE: &str = "CANARY-EC-PASSPHRASE-0123456789"; - const CANARY_HANDLER_USERNAME: &str = "CANARY-HANDLER-USERNAME-0123456789"; - const CANARY_HANDLER_PASSWORD: &str = "CANARY-HANDLER-PASSWORD-0123456789"; const CANARY_EC_PARTNER_API_TOKEN: &str = "CANARY-EC-PARTNER-API-TOKEN-0123456789"; const CANARY_EC_PARTNER_TS_PULL_TOKEN: &str = "CANARY-EC-PARTNER-TS-PULL-TOKEN-0123456789"; const CANARY_TRUSTED_CLIENT_IP_SHARED_SECRET: &str = @@ -4020,19 +5710,12 @@ mod tests { const CANARY_S3_SESSION_TOKEN: &str = "CANARY-S3-SESSION-TOKEN-0123456789"; const CANARY_TINYBIRD_AUCTION_TOKEN: &str = "CANARY-TINYBIRD-AUCTION-TOKEN-0123456789"; const CANARY_TINYBIRD_ACCESS_TOKEN: &str = "CANARY-TINYBIRD-ACCESS-TOKEN-0123456789"; - const CANARY_DATADOME_SERVER_SIDE_KEY: &str = "CANARY-DATADOME-SERVER-SIDE-KEY-0123456789"; + const CANARY_MODULE_KEY: &str = "CANARY-MODULE-TABLE-KEY-0123456789"; let mut settings = create_test_settings(); settings.publisher.proxy_secret = Redacted::new(CANARY_PROXY_SECRET.to_string()); - settings.ec.passphrase = Redacted::new(CANARY_EC_PASSPHRASE.to_string()); - - settings.handlers = vec![Handler { - path: "^/secure".to_string(), - username: Redacted::new(CANARY_HANDLER_USERNAME.to_string()), - password: Redacted::new(CANARY_HANDLER_PASSWORD.to_string()), - regex: OnceLock::new(), - }]; + select_hmac_module(&mut settings.ec, HMAC_MODULE_KEY, CANARY_EC_PASSPHRASE); settings.ec.partners = vec![EcPartner { name: "canary-partner".to_string(), @@ -4066,33 +5749,35 @@ mod tests { })); settings.proxy.asset_routes = vec![asset_route]; - settings.tinybird = TinybirdSettings { - auction_token_secret: Some(Redacted::new(CANARY_TINYBIRD_AUCTION_TOKEN.to_string())), - access_token_secret: Some(Redacted::new(CANARY_TINYBIRD_ACCESS_TOKEN.to_string())), - enabled: false, - api_host: String::new(), - secret_store: None, - auction_dataset: String::new(), - access_enabled: false, - access_dataset: String::new(), - access_sample_rate: 0.0f64, - max_body_bytes: 0, - }; + settings.analytics = Some(AnalyticsConfig { + module: Some(ANALYTICS_MODULE.to_string()), + tinybird: Some(TinybirdSettings { + auction_token_secret: Some(Redacted::new( + CANARY_TINYBIRD_AUCTION_TOKEN.to_string(), + )), + access_token_secret: Some(Redacted::new(CANARY_TINYBIRD_ACCESS_TOKEN.to_string())), + api_host: String::new(), + secret_store: None, + auction_dataset: String::new(), + access_enabled: false, + access_dataset: String::new(), + access_sample_rate: 0.0f64, + max_body_bytes: 0, + }), + }); - // `IntegrationSettings` stores integration configs as opaque JSON and - // relies on a hand-written `Debug` impl to suppress their values. That - // impl is the only thing keeping resolved DataDome credentials out of - // this output, so pin it here. + // A module's table is opaque JSON, and the section's hand-written + // `Debug` impl is the only thing keeping a secret a module's table + // holds out of this output, so pin it here. settings - .integrations - .insert_config( - "datadome", + .insert_module_config( + "testing", + "testing.example", &json!({ - "enabled": true, - "server_side_key_secret_name": CANARY_DATADOME_SERVER_SIDE_KEY, + "key_name": CANARY_MODULE_KEY, }), ) - .expect("should insert datadome integration config"); + .expect("should insert a module's table"); let debug = format!("{settings:?}"); @@ -4101,15 +5786,13 @@ mod tests { "should redact secret fields in Settings debug output" ); assert!( - debug.contains("^/secure"), - "should leave non-secret handler path visible in debug output" + debug.contains("canary-partner.example"), + "should leave a non-secret value visible in debug output" ); let canaries = [ ("publisher.proxy_secret", CANARY_PROXY_SECRET), - ("ec.passphrase", CANARY_EC_PASSPHRASE), - ("handlers[].username", CANARY_HANDLER_USERNAME), - ("handlers[].password", CANARY_HANDLER_PASSWORD), + ("ec.hmac.passphrase", CANARY_EC_PASSPHRASE), ("ec.partners[].api_token", CANARY_EC_PARTNER_API_TOKEN), ( "ec.partners[].ts_pull_token", @@ -4132,14 +5815,14 @@ mod tests { CANARY_S3_SESSION_TOKEN, ), ( - "tinybird.auction_token_secret", + "analytics.tinybird.auction_token_secret", CANARY_TINYBIRD_AUCTION_TOKEN, ), - ("tinybird.access_token_secret", CANARY_TINYBIRD_ACCESS_TOKEN), ( - "integrations.datadome.server_side_key_secret_name", - CANARY_DATADOME_SERVER_SIDE_KEY, + "analytics.tinybird.access_token_secret", + CANARY_TINYBIRD_ACCESS_TOKEN, ), + ("testing.example.key_name", CANARY_MODULE_KEY), ]; for (field, canary) in canaries { @@ -4550,13 +6233,147 @@ mod tests { } } + /// The shared fixture with one `[[handlers]]` entry ahead of everything + /// else, as a configuration written for a build that asked for passwords + /// carries it. + fn settings_str_with_a_handler() -> String { + format!( + "[[handlers]]\npath = \"^/_ts/admin\"\nusername = \"admin\"\n\ + password = \"admin-pass\"\n{}", + crate_test_settings_str() + ) + } + + #[test] + fn toml_settings_refuse_a_handlers_list_with_directions() { + let error = Settings::from_toml(&settings_str_with_a_handler()) + .expect_err("should refuse a configuration that still gates a path"); + + let rendered = format!("{error:?}"); + assert!( + rendered.contains("`[[handlers]]` is not read"), + "should say the list is not read: {rendered}" + ); + assert!( + rendered.contains("ts keys") && rendered.contains("ts cache purge"), + "should say where administration went: {rendered}" + ); + } + + #[test] + fn json_settings_refuse_a_handlers_list_with_directions() { + let settings = Settings::from_toml(&crate_test_settings_str()) + .expect("should load the test settings fixture"); + let mut value = serde_json::to_value(settings) + .expect("should serialize the test settings fixture to JSON"); + value["handlers"] = json!([ + { "path": "^/secure", "username": "user", "password": "handler_password" }, + ]); + + let error = Settings::from_json_value(value) + .expect_err("should refuse a stored document that still gates a path"); + + assert!( + format!("{error:?}").contains("`[[handlers]]` is not read"), + "should say the list is not read: {error:?}" + ); + } + + #[test] + fn an_empty_handlers_list_is_refused_too() { + let toml = format!("handlers = []\n{}", crate_test_settings_str()); + + let error = Settings::from_toml(&toml).expect_err("should refuse the key itself"); + + assert!( + format!("{error:?}").contains("`[[handlers]]` is not read"), + "should say the list is not read: {error:?}" + ); + } + + #[test] + fn settings_are_written_without_a_handlers_list() { + let settings = Settings::from_toml(&crate_test_settings_str()) + .expect("should load the test settings fixture"); + + let value = serde_json::to_value(&settings).expect("should serialize the settings"); + + assert!( + value.get("handlers").is_none(), + "should write nothing a later read would refuse" + ); + } + + #[test] + fn toml_settings_refuse_a_signing_store_id_with_directions() { + for key in ["config_store_id", "secret_store_id"] { + let fixture = crate_test_settings_str(); + assert!( + fixture.contains("[request_signing]\n"), + "the fixture should have the section the key is added to" + ); + let toml = fixture.replace( + "[request_signing]\n", + &format!("[request_signing]\n{key} = \"01GEXAMPLE\"\n"), + ); + + let error = Settings::from_toml(&toml).expect_err("should refuse the removed key"); + + let rendered = format!("{error:?}"); + assert!( + rendered.contains("does not take `config_store_id` or `secret_store_id`"), + "should say the key is not read, for {key}: {rendered}" + ); + assert!( + rendered.contains("ts keys rotate"), + "should say where the ids go, for {key}: {rendered}" + ); + } + } + + #[test] + fn json_settings_refuse_a_signing_store_id_with_directions() { + let settings = Settings::from_toml(&crate_test_settings_str()) + .expect("should load the test settings fixture"); + let mut value = serde_json::to_value(settings) + .expect("should serialize the test settings fixture to JSON"); + value["request_signing"]["secret_store_id"] = json!("01GEXAMPLE"); + + let error = Settings::from_json_value(value).expect_err("should refuse the removed key"); + + assert!( + format!("{error:?}").contains("--secret-store-id"), + "should say where the id goes: {error:?}" + ); + } + + #[test] + fn request_signing_is_written_as_enabled_alone() { + let mut settings = Settings::from_toml(&crate_test_settings_str()) + .expect("should load the test settings fixture"); + settings.request_signing = Some(RequestSigning::new(true)); + + let value = serde_json::to_value(&settings).expect("should serialize the settings"); + + assert_eq!( + value["request_signing"], + json!({ "enabled": true }), + "should write nothing a later read would refuse" + ); + let read = Settings::from_json_value(value).expect("should read what was written"); + assert!( + read.request_signing.is_some_and(|signing| signing.enabled), + "should keep request signing on across the round trip" + ); + } + #[test] fn json_settings_rejects_legacy_auction_provider_list_with_migration_guidance() { let settings = Settings::from_toml(&crate_test_settings_str()) .expect("should load the test settings fixture"); let mut value = serde_json::to_value(settings) .expect("should serialize the test settings fixture to JSON"); - value["auction"]["providers"] = json!(["prebid"]); + value["auction"]["providers"] = json!(["example"]); let error = Settings::from_json_value(value) .expect_err("should reject the removed auction provider list schema"); @@ -4566,16 +6383,17 @@ mod tests { "error should identify the removed field, got {rendered}" ); assert!( - rendered.contains("CHANGELOG.md"), - "error should direct operators to the migration guidance, got {rendered}" + rendered.contains("[demand] modules"), + "error should name where the setting moved to, got {rendered}" ); } #[test] fn toml_settings_reject_legacy_auction_provider_list_with_migration_guidance() { let toml = format!( - "{}\n[auction]\nproviders = [\"prebid\"]\n", + "{}\n", crate_test_settings_str() + .replace("[auction]\n", "[auction]\nproviders = [\"example\"]\n") ); let error = Settings::from_toml(&toml) @@ -4586,8 +6404,8 @@ mod tests { "error should identify the removed field, got {rendered}" ); assert!( - rendered.contains("CHANGELOG.md"), - "error should direct operators to the migration guidance, got {rendered}" + rendered.contains("[demand] modules"), + "error should name where the setting moved to, got {rendered}" ); } @@ -4595,7 +6413,7 @@ mod tests { fn auction_debug_comment_options_default_matches_serde_defaults() { let opts = AuctionDebugCommentOptions::default(); assert!(opts.include_provider_responses, "should default to true"); - assert!(opts.include_mediator_response, "should default to true"); + assert!(opts.include_adserver_response, "should default to true"); assert!(opts.include_bids, "should default to true"); assert_eq!( opts.metadata_keys, @@ -4758,61 +6576,235 @@ mod tests { ); } + /// `[analytics]` selecting the analytics module, with `table` as the + /// lines of its settings table. + fn settings_str_with_analytics_module(table: &str) -> String { + format!( + "{}\n[analytics]\nmodule = \"tinybird\"\n\n[analytics.tinybird]\n{table}\n", + crate_test_settings_str() + ) + } + + /// The lines of a telemetry table that passes every check. + const ANALYTICS_MODULE_LINES: &str = + "api_host = \"api.us-east.aws.tinybird.co\"\nauction_token_secret = \"test-auction-token\""; + #[test] - fn tinybird_defaults_to_disabled_placeholders() { + fn settings_without_an_analytics_section_select_no_module() { let settings = Settings::from_toml(&crate_test_settings_str()) - .expect("should parse settings without tinybird block"); + .expect("should parse settings without an analytics section"); + assert!(settings.analytics.is_none(), "should hold no section"); assert!( - !settings.tinybird.enabled, - "Tinybird should default disabled" - ); - assert_eq!(settings.tinybird.secret_store, None); - assert_eq!(settings.tinybird.auction_dataset, "auction_events_raw"); - assert!(settings.tinybird.auction_token_secret.is_none()); - } - - #[test] - fn tinybird_enabled_requires_host_dataset_and_token() { - let toml = format!( - "{}\n[tinybird]\nenabled = true\napi_host = \"https://api.example.com/path\"\n", - crate_test_settings_str() + settings.analytics_module().is_none(), + "should select no module" ); - - let err = Settings::from_toml(&toml).expect_err("should reject invalid api host"); + let value = serde_json::to_value(&settings).expect("should serialize the settings"); assert!( - format!("{err:?}").contains("tinybird.api_host"), - "should report tinybird.api_host validation error: {err:?}" + value.get("analytics").is_none() && value.get("tinybird").is_none(), + "should write neither the section nor the table it replaced: {value}" ); } #[test] - fn tinybird_accepts_region_host_without_scheme() { - let toml = format!( - "{}\n[tinybird]\nenabled = true\napi_host = \"api.us-east.aws.tinybird.co\"\nauction_token_secret = \"test-auction-token\"\n", - crate_test_settings_str() - ); + fn the_analytics_module_is_selected_by_its_short_or_full_name() { + for name in ["tinybird", "analytics.tinybird"] { + let toml = format!( + "{}\n[analytics]\nmodule = \"{name}\"\n\n[analytics.tinybird]\n{ANALYTICS_MODULE_LINES}\n", + crate_test_settings_str() + ); + + let settings = Settings::from_toml(&toml).expect("should accept the selection"); + + let telemetry = settings + .analytics_module() + .expect("should select the analytics module"); + assert_eq!(telemetry.api_host, "api.us-east.aws.tinybird.co"); + assert_eq!(telemetry.auction_dataset, "auction_events_raw"); + } + } + + #[test] + fn a_selected_analytics_module_survives_the_stored_form() { + let settings = + Settings::from_toml(&settings_str_with_analytics_module(ANALYTICS_MODULE_LINES)) + .expect("should accept the selection"); + + let stored = serde_json::to_value(&settings).expect("should serialize the settings"); + + assert_eq!(stored["analytics"]["module"], "tinybird"); + assert_eq!( + stored["analytics"]["tinybird"]["api_host"], + "api.us-east.aws.tinybird.co" + ); + assert!( + stored.get("tinybird").is_none(), + "should not write the table that moved: {stored}" + ); + } + + #[test] + fn the_analytics_module_needs_a_regional_host() { + let toml = + settings_str_with_analytics_module("api_host = \"https://api.example.com/path\""); + + let err = Settings::from_toml(&toml).expect_err("should reject invalid api host"); + + assert!( + format!("{err:?}").contains("analytics.tinybird.api_host"), + "should name the setting by its table: {err:?}" + ); + } + + #[test] + fn the_analytics_module_needs_its_token() { + let toml = settings_str_with_analytics_module("api_host = \"api.us-east.aws.tinybird.co\""); + + let err = Settings::from_toml(&toml).expect_err("should require the token"); - let settings = Settings::from_toml(&toml).expect("should accept Tinybird region host"); - assert!(settings.tinybird.enabled); - assert_eq!(settings.tinybird.api_host, "api.us-east.aws.tinybird.co"); + assert!( + format!("{err:?}").contains( + "analytics.tinybird.auction_token_secret is required when [analytics] selects \ + the module" + ), + "should say what is missing and why: {err:?}" + ); } #[test] - fn tinybird_access_enabled_is_rejected_until_emitter_is_wired() { + fn a_selected_analytics_module_with_no_table_is_told_what_it_needs() { let toml = format!( - "{}\n[tinybird]\naccess_enabled = true\n", + "{}\n[analytics]\nmodule = \"tinybird\"\n", crate_test_settings_str() ); + let err = Settings::from_toml(&toml).expect_err("should need the table's settings"); + + assert!( + format!("{err:?}").contains("analytics.tinybird.api_host"), + "should name the first setting the module needs: {err:?}" + ); + } + + #[test] + fn access_telemetry_is_rejected_until_an_emitter_is_wired() { + let toml = settings_str_with_analytics_module(&format!( + "{ANALYTICS_MODULE_LINES}\naccess_enabled = true" + )); + let err = Settings::from_toml(&toml) - .expect_err("should reject access telemetry before emitter exists"); + .expect_err("should reject access telemetry before an emitter exists"); + + assert!( + format!("{err:?}").contains("analytics.tinybird.access_enabled"), + "should name the unsupported setting: {err:?}" + ); + } + + #[test] + fn an_analytics_section_selecting_nothing_is_refused() { + let toml = format!( + "{}\n[analytics.tinybird]\n{ANALYTICS_MODULE_LINES}\n", + crate_test_settings_str() + ); + + let err = Settings::from_toml(&toml).expect_err("nothing would read the table"); + + assert!( + format!("{err:?}").contains("[analytics] selects no module"), + "should say the section selects nothing: {err:?}" + ); + } + + #[test] + fn an_analytics_module_this_build_does_not_have_is_refused() { + let toml = format!( + "{}\n[analytics]\nmodule = \"another\"\n", + crate_test_settings_str() + ); + + let err = Settings::from_toml(&toml).expect_err("no such module"); + let message = format!("{err:?}"); + + assert!( + message.contains( + "[analytics] selects `another`, which no module this deployment supplies" + ), + "should name what was written: {message}" + ); assert!( - format!("{err:?}").contains("tinybird.access_enabled"), - "should report unsupported tinybird.access_enabled setting: {err:?}" + message.contains("The analytics modules it supplies are [tinybird]"), + "should list what can be selected: {message}" ); } + /// The token is a secret reference, and an empty or absent one is refused + /// as the settings load, before any deploy step sees the document. + #[test] + fn an_analytics_token_reference_that_is_empty_or_absent_is_refused_when_the_settings_load() { + for (case, table) in [ + ( + "an empty token", + "api_host = \"api.us-east.aws.tinybird.co\"\nauction_token_secret = \"\"", + ), + ("no token", "api_host = \"api.us-east.aws.tinybird.co\""), + ] { + let err = Settings::from_toml(&settings_str_with_analytics_module(table)) + .err() + .unwrap_or_else(|| panic!("{case}: the settings should not load")); + + assert!( + format!("{err:?}").contains("auction_token_secret"), + "{case}: should name the token: {err:?}" + ); + } + } + + #[test] + fn the_analytics_module_table_refuses_a_key_it_does_not_know() { + let toml = settings_str_with_analytics_module(&format!( + "{ANALYTICS_MODULE_LINES}\nenabled = true" + )); + + let err = Settings::from_toml(&toml).expect_err("the enabled line is gone"); + + assert!( + format!("{err:?}").contains("enabled"), + "should name the key it does not know: {err:?}" + ); + } + + #[test] + fn a_table_the_section_does_not_select_gives_no_analytics_module() { + let analytics = AnalyticsConfig { + module: None, + tinybird: Some(TinybirdSettings::default()), + }; + + assert!( + analytics.analytics_module().is_none(), + "a table nothing selects should give no settings" + ); + } + + #[test] + fn the_tinybird_table_is_refused_with_directions() { + for table in [ + "enabled = true\napi_host = \"api.us-east.aws.tinybird.co\"", + "enabled = false", + ] { + let toml = format!("{}\n[tinybird]\n{table}\n", crate_test_settings_str()); + + let err = Settings::from_toml(&toml).expect_err("the table has moved"); + + assert!( + format!("{err:?}") + .contains("Configuration table `[tinybird]` is now `[analytics.tinybird]`"), + "should say where the table went: {err:?}" + ); + } + } + #[test] fn settings_rejects_removed_consent_store_toml() { let toml = format!( @@ -4853,27 +6845,16 @@ mod tests { assert!(settings.is_ok()); let settings = settings.expect("should parse valid TOML"); - let prebid_cfg = settings - .integration_config::("prebid") - .expect("Prebid config query should succeed") - .expect("Prebid config should load from test settings"); - assert_eq!(prebid_cfg.timeout_ms, 1000); assert!( settings - .integration_config::("nextjs") - .expect("Next.js config query should succeed") + .module_config::(ENDPOINT_MODULE) + .expect("a query for a module no section selects should succeed") .is_none(), - "Next.js integration should default to disabled" + "a module no section selects should not run" ); - let raw_nextjs = settings - .integrations - .get("nextjs") - .expect("test settings should include nextjs block"); - assert_eq!(raw_nextjs["enabled"], json!(false)); - assert_eq!( - raw_nextjs["rewrite_attributes"], - json!(["href", "link", "url"]), - "Next.js rewrite attributes should default to href/link/url" + assert!( + settings.auction.modules.selected().is_empty(), + "the fixture should select no auction module" ); assert_eq!(settings.publisher.domain, "test-publisher.com"); assert_eq!(settings.publisher.cookie_domain, ".test-publisher.com"); @@ -4892,7 +6873,12 @@ mod tests { ); assert_eq!(settings.publisher.origin_host_header_override, None); assert_eq!( - settings.ec.passphrase.expose(), + settings.ec.module.as_ref(), + Some(&EcModuleSelection::from(HMAC_MODULE_KEY)), + "test settings should select the hmac EC module" + ); + assert_eq!( + hmac_passphrase(&settings.ec, HMAC_MODULE_KEY), "test-secret-key-32-bytes-minimum" ); @@ -4977,6 +6963,99 @@ mod tests { ); } + #[test] + fn loading_cache_lifetimes_are_read_with_defaults_for_what_is_unset() { + let toml_str = format!( + r#"{} + + [cache.robots-txt] + store_seconds = 600 + store_idle_seconds = 3600 + "#, + crate_test_settings_str() + ); + let settings = Settings::from_toml(&toml_str).expect("should parse the lifetimes"); + let defaults = crate::platform::CacheLifetimes { + memory: Duration::from_secs(5), + store: Duration::from_secs(60), + store_idle: Duration::from_secs(70), + }; + + assert_eq!( + settings.cache.robots_txt.lifetimes(defaults), + crate::platform::CacheLifetimes { + memory: Duration::from_secs(5), + store: Duration::from_secs(600), + store_idle: Duration::from_secs(3600), + } + ); + } + + #[test] + fn a_loading_cache_lifetime_out_of_range_is_refused_naming_it() { + for (field, value, least) in [ + ("store_seconds", "0", 1), + ("store_idle_seconds", "0", 1), + ("memory_seconds", "31536001", 0), + ] { + let toml_str = format!( + "{} + +[cache.robots-txt] +{field} = {value} +", + crate_test_settings_str() + ); + let error = Settings::from_toml(&toml_str) + .expect_err("a lifetime out of range should not load"); + let message = format!("{error:?}"); + assert!( + message.contains(&format!( + "[cache.robots-txt] {field} is {value}, and must be from {least} to 31536000" + )), + "{field}: {message}" + ); + } + } + + #[test] + fn unset_loading_cache_lifetimes_are_not_written_back() { + let settings = + Settings::from_toml(&crate_test_settings_str()).expect("should parse the settings"); + let written = serde_json::to_value(&settings.cache).expect("should serialize"); + + assert_eq!(written, serde_json::json!({ "asset_rules": [] })); + } + + #[test] + fn a_loading_cache_module_is_read_and_written_back() { + let toml_str = format!( + r#"{} + + [cache.robots-txt] + module = "51degrees" + "#, + crate_test_settings_str() + ); + let settings = Settings::from_toml(&toml_str).expect("should parse the selection"); + + assert_eq!( + settings.cache.robots_txt.module.as_deref(), + Some("51degrees") + ); + assert!(!settings.cache.robots_txt.is_default()); + assert_eq!( + settings.cache.loading_caches(), + [(super::ROBOTS_TXT_CACHE, &settings.cache.robots_txt)], + "should list the robots.txt cache by its name" + ); + let written = serde_json::to_value(&settings.cache).expect("should serialize"); + assert_eq!( + written, + serde_json::json!({ "asset_rules": [], "robots-txt": { "module": "51degrees" } }) + ); + } + #[test] fn cache_asset_rule_nextjs_preset_is_operator_controlled() { let toml_str = format!( @@ -5120,6 +7199,140 @@ mod tests { ); } + #[test] + fn module_selection_allows_no_module_for_stateless_operation() { + let ec = Ec::default(); + assert!(ec.module.is_none(), "default Ec selects no module"); + ec.validate_module_selection() + .expect("should allow no module selected and run statelessly"); + } + + #[test] + fn selecting_an_implementation_that_needs_settings_without_its_block_is_rejected() { + // The built-in HMAC module has a required passphrase, so selecting + // it with no `[ec.hmac]` block is a deployment that would run + // stateless under a selector saying otherwise. + let toml_str = crate_test_settings_str_with_ec_section("[ec]\nmodule = \"hmac\"\n"); + + let err = Settings::from_toml(&toml_str) + .expect_err("an implementation with no settings block should fail at startup"); + assert!( + format!("{err:?}").contains("`hmac` is selected but has no `[ec.hmac]` configuration"), + "should name the missing block: {err:?}" + ); + } + + #[test] + fn selecting_a_module_with_no_settings_needs_no_block() { + // A block exists only when a module has settings, and only the + // adapter that injects a module knows whether it has any, so a + // selector naming one core does not supply is left to + // `build_module`, which is where the injected modules are known. + let toml_str = crate_test_settings_str_with_ec_section("[ec]\nmodule = \"acme\"\n"); + + let settings = + Settings::from_toml(&toml_str).expect("a module with no settings should need no block"); + assert!( + settings.ec.module_blocks.is_empty(), + "the selection should stand on its own with no block configured" + ); + } + + /// The fixture settings with `line` added to `[ec]`. + fn settings_toml_with_ec_line(line: &str) -> String { + crate_test_settings_str_with_ec_section(&format!( + "[ec]\nmodule = \"hmac\"\n{line}\n\n[ec.hmac]\n\ + passphrase = \"test-secret-key-32-bytes-minimum\"\n" + )) + } + + /// The `resolve_allowed_origins` line listing `entries`, each written as a + /// TOML basic string. + fn resolve_allowed_origins_line(entries: &[&str]) -> String { + let written: Vec = entries + .iter() + .map(|entry| serde_json::to_string(entry).expect("should write a string")) + .collect(); + format!("resolve_allowed_origins = [{}]", written.join(", ")) + } + + #[test] + fn resolve_allowed_origins_load_only_as_bare_origins() { + let accepted: [(&str, Option<&[&str]>); 7] = [ + ("an absent list", None), + ("an empty list", Some(&[])), + ("https with no port", Some(&["https://www.example.com"])), + ("https with :443", Some(&["https://www.example.com:443"])), + ("http with :80", Some(&["http://www.example.com:80"])), + ( + "a non-default port", + Some(&["https://www.example.com:8443"]), + ), + ( + "the generator's https://{domain} shape", + Some(&["https://www.example.com", "https://m.example.com"]), + ), + ]; + for (label, entries) in accepted { + let line = entries + .map(resolve_allowed_origins_line) + .unwrap_or_default(); + + let settings = Settings::from_toml(&settings_toml_with_ec_line(&line)) + .unwrap_or_else(|err| panic!("should load {label}: {err:?}")); + + assert_eq!( + settings.ec.resolve_allowed_origins, + entries.unwrap_or_default(), + "should keep the entries of {label} as written" + ); + } + + let refused = [ + ("https://www.example.com/", NotAnOrigin::Path), + ("https://www.example.com/path", NotAnOrigin::Path), + ("https://www.example.com\\", NotAnOrigin::Path), + ("https://www.example.com?a=1", NotAnOrigin::Query), + ("https://www.example.com#top", NotAnOrigin::Fragment), + ("https://user@www.example.com", NotAnOrigin::Userinfo), + ("https://user:pass@www.example.com", NotAnOrigin::Userinfo), + ("www.example.com", NotAnOrigin::Scheme), + ("www.example.com:443", NotAnOrigin::Scheme), + ("ftp://www.example.com", NotAnOrigin::Scheme), + ("wss://www.example.com", NotAnOrigin::Scheme), + ("null", NotAnOrigin::Scheme), + ("", NotAnOrigin::Empty), + ("https://", NotAnOrigin::Host), + ("https://www.example.com ", NotAnOrigin::Host), + ]; + for (entry, reason) in refused { + let line = resolve_allowed_origins_line(&["https://www.example.com", entry]); + + let Err(err) = Settings::from_toml(&settings_toml_with_ec_line(&line)) else { + panic!("should refuse `{entry}` at load"); + }; + + assert!( + format!("{err:?}") + .contains(&format!("resolve_allowed_origins entry `{entry}` {reason}")), + "should name `{entry}` and say it {reason}: {err:?}" + ); + } + } + + #[test] + fn a_resolve_allowed_origin_with_a_trailing_slash_is_refused_at_load() { + let line = resolve_allowed_origins_line(&["https://www.example.com/"]); + + let err = Settings::from_toml(&settings_toml_with_ec_line(&line)) + .expect_err("should refuse an entry that would never match a request's Origin"); + + assert!( + format!("{err:?}").contains("`https://www.example.com/`"), + "should name the entry: {err:?}" + ); + } + #[test] fn cache_asset_rule_globs_respect_path_separators() { let toml_str = format!( @@ -5226,6 +7439,25 @@ mod tests { ); } + #[test] + fn module_blocks_without_a_selector_are_rejected() { + // A half-migrated configuration that carries an [ec.hmac] block but + // never selects it would silently run stateless, so it is rejected at + // startup instead. + let toml_str = crate_test_settings_str().replace("module = \"hmac\"\n", ""); + + let err = Settings::from_toml(&toml_str) + .expect_err("a module block with no selector should fail at startup"); + assert!( + matches!( + err.current_context(), + TrustedServerError::Configuration { .. } + ), + "should be a configuration error, got: {:?}", + err.current_context() + ); + } + #[test] fn cache_asset_rule_policy_validation_rejects_unsafe_config() { let missing_ttl = format!( @@ -5339,6 +7571,78 @@ mod tests { ); } + #[test] + fn legacy_passphrase_migrates_to_the_hmac_module() { + let mut ec = Ec { + passphrase: Some(Redacted::new("test-secret-key-32-bytes-minimum".to_owned())), + ..Ec::default() + }; + ec.migrate_legacy_ec_layout() + .expect("should migrate the deprecated form"); + assert_eq!( + ec.module.as_ref(), + Some(&EcModuleSelection::from(HMAC_MODULE_KEY)), + "the deprecated passphrase should select the hmac module" + ); + assert_eq!( + hmac_passphrase(&ec, HMAC_MODULE_KEY), + "test-secret-key-32-bytes-minimum", + "the passphrase should move into the hmac block" + ); + assert!( + ec.passphrase.is_none(), + "the deprecated field should be consumed by the migration" + ); + } + + /// The crate test configuration with its `[ec]` section rewritten to the + /// deprecated single-passphrase form. + fn legacy_ec_settings_str(passphrase: &str) -> String { + let legacy = crate_test_settings_str_with_ec_section(&format!( + "[ec]\npassphrase = \"{passphrase}\"\n" + )); + assert!( + !legacy.contains("[ec.hmac]"), + "the legacy configuration should carry no module block" + ); + legacy + } + + #[test] + fn a_legacy_passphrase_is_held_to_the_passphrase_rules() { + // Validation runs before the migration and the deprecated field + // carries no check of its own, so the migration itself has to apply + // the passphrase rules. Without that, a value the new `[ec.hmac]` + // block rejects would still start a deployment from the old location. + let short = Settings::from_toml(&legacy_ec_settings_str("short")) + .expect_err("a short legacy passphrase should be rejected"); + assert!( + format!("{short:?}").contains("passphrase (deprecated) is invalid"), + "should name the deprecated passphrase as the fault: {short:?}" + ); + + let empty = Settings::from_toml(&legacy_ec_settings_str("")) + .expect_err("an empty legacy passphrase should be rejected"); + assert!( + format!("{empty:?}").contains("passphrase (deprecated) is invalid"), + "should name the deprecated passphrase as the fault: {empty:?}" + ); + + let settings = + Settings::from_toml(&legacy_ec_settings_str("test-secret-key-32-bytes-minimum")) + .expect("a legacy passphrase of adequate length should still start"); + assert_eq!( + settings.ec.module.as_ref(), + Some(&EcModuleSelection::from(HMAC_MODULE_KEY)), + "an adequate legacy passphrase should still select the hmac module" + ); + assert_eq!( + hmac_passphrase(&settings.ec, HMAC_MODULE_KEY), + "test-secret-key-32-bytes-minimum", + "an adequate legacy passphrase should still move into the hmac block" + ); + } + #[test] fn cache_asset_rule_validation_rejects_invalid_config() { let duplicate_ids = format!( @@ -5417,22 +7721,504 @@ mod tests { } #[test] - fn validate_rejects_trailing_slash_in_origin_url() { + fn legacy_passphrase_alongside_module_config_is_rejected() { + let mut ec = Ec { + passphrase: Some(Redacted::new("test-secret-key-32-bytes-minimum".to_owned())), + module: Some(EcModuleSelection::from(HMAC_MODULE_KEY)), + ..Ec::default() + }; + let err = ec + .migrate_legacy_ec_layout() + .expect_err("both forms present should be rejected"); + assert!( + matches!( + err.current_context(), + TrustedServerError::Configuration { .. } + ), + "should be a configuration error, got: {:?}", + err.current_context() + ); + } + + #[test] + fn an_unknown_key_in_the_hmac_module_block_is_rejected() { + // A mistyped key dropped silently would leave the setting the + // operator meant to change at its default. let toml_str = crate_test_settings_str().replace( - r#"origin_url = "https://origin.test-publisher.com""#, - r#"origin_url = "https://origin.test-publisher.com/""#, + "passphrase = \"test-secret-key-32-bytes-minimum\"", + "passphrase = \"test-secret-key-32-bytes-minimum\"\n typo_key = \"x\"", + ); + assert!( + toml_str.contains("typo_key"), + "the test configuration should carry the unknown key" ); - let result = Settings::from_toml(&toml_str); + let err = Settings::from_toml(&toml_str) + .expect_err("an unknown key in [ec.hmac] should be rejected"); assert!( - result.is_err(), - "origin_url ending with '/' should fail validation" + format!("{err:?}").contains("typo_key"), + "should name the unknown key: {err:?}" ); } #[test] - fn validate_rejects_invalid_publisher_domains() { - for domain in [ + fn module_none_is_explicit_stateless() { + let ec = Ec { + module: Some(EcModuleSelection::None), + ..Ec::default() + }; + ec.validate_module_selection() + .expect("explicit none with no blocks should be valid"); + } + + #[test] + fn module_none_with_configured_blocks_is_rejected() { + let mut ec = Ec::default(); + select_hmac_module(&mut ec, HMAC_MODULE_KEY, "test-secret-key-32-bytes-minimum"); + ec.module = Some(EcModuleSelection::None); + assert!( + ec.validate_module_selection().is_err(), + "none alongside configured blocks should be rejected" + ); + } + + #[test] + fn device_module_defaults_to_builtin_and_rejects_unknown() { + let config = DeviceConfig::default(); + assert_eq!( + config.module_key(), + "builtin", + "no selector should default to the built-in module" + ); + config + .validate_module_selection() + .expect("should validate the built-in default"); + + let fastly = DeviceConfig { + module: Some("fastly".to_owned()), + }; + fastly + .validate_module_selection() + .expect("should validate the fastly opt-in"); + + // As with geo, a key core does not know is not a settings error, + // because a module's name is a legitimate value here too. + let module_key = DeviceConfig { + module: Some("acme".to_owned()), + }; + module_key + .validate_module_selection() + .expect("a module id should be accepted by settings validation"); + + // And as with geo, the rejection happens at registry build, so a + // mistyped selector cannot fall back to the built-in module in + // silence. + let mut settings = crate::test_support::tests::create_test_settings(); + settings.device.module = Some("acme".to_owned()); + let error = match crate::integrations::IntegrationRegistry::new(&settings) { + Ok(_) => { + panic!("a device module no module supplies should be rejected at registry build") + } + Err(error) => error, + }; + let message = error.to_string(); + assert!( + message.contains("acme") && message.contains("[device] module"), + "the error should name the selector and the module, got: {message}" + ); + } + + #[test] + fn an_unselected_module_block_is_rejected() { + // A vendor selector with the vendor block present, plus a stray hmac + // block, is almost always a stale or mistyped configuration. + let toml_str = crate_test_settings_str().replace( + "module = \"hmac\"", + "module = \"acme\"\n\n [ec.acme]\n api_key = \"example\"", + ); + let err = Settings::from_toml(&toml_str) + .expect_err("a configured but unselected block should fail at startup"); + assert!( + matches!( + err.current_context(), + TrustedServerError::Configuration { .. } + ), + "should be a configuration error, got: {:?}", + err.current_context() + ); + } + + #[test] + fn a_host_signals_block_is_read_as_the_built_in_modules_settings() { + // The block is the built-in module's own settings, rather than kept as + // the raw values of a module an adapter injects. + let settings = Settings::from_toml(&crate_test_settings_str_with_ec_section(&format!( + "[ec] +module = \"{HOST_SIGNALS_MODULE_KEY}\" + +\ + [ec.{HOST_SIGNALS_MODULE_KEY}] +\ + passphrase = \"test-secret-key-32-bytes-minimum\" +" + ))) + .expect("a host_signals selection with its block should load"); + assert!( + settings + .ec + .module_blocks + .get(HOST_SIGNALS_MODULE_KEY) + .and_then(EcModuleBlock::host_signals_settings) + .is_some(), + "the block should be read as the host-signal module's settings" + ); + } + #[test] + fn a_labeled_block_the_selector_does_not_name_is_rejected() { + // A block under a label is still a module block, so it is held to + // the rule every block is held to, which is that the selector names + // it. + let toml_str = crate_test_settings_str_with_ec_section( + r#"[ec] +module = "hmac" + +[ec.hmac] +passphrase = "test-secret-key-32-bytes-minimum" + +[ec.primary] +implementation = "hmac" +passphrase = "another-test-secret-key-32-bytes" +"#, + ); + + let err = Settings::from_toml(&toml_str) + .expect_err("a labeled block the selector does not name should fail at startup"); + let message = format!("{err:?}"); + assert!( + message.contains("[ec.primary] is configured but `hmac` is selected"), + "should name the unselected labeled block, got: {message}" + ); + } + + #[test] + fn the_removed_providers_table_is_rejected_with_its_new_location() { + let toml_str = crate_test_settings_str_with_ec_section( + "[ec]\nmodule = \"hmac\"\n\n[ec.providers.hmac]\npassphrase = \"test-secret-key-32-bytes-minimum\"\n", + ); + + let err = Settings::from_toml(&toml_str) + .expect_err("the removed [ec.providers] table should be rejected"); + let message = format!("{err:?}"); + assert!( + message.contains("[ec.providers] is no longer read") && message.contains("[ec.hmac]"), + "should send the operator to the new location, got: {message}" + ); + } + + #[test] + fn a_key_under_ec_that_is_not_a_table_is_an_unknown_field() { + // Holding the module blocks alongside the fixed keys costs the + // section serde's own unknown-key check, so a mistyped setting has to + // be caught where the blocks are read. + let toml_str = crate_test_settings_str_with_ec_section( + "[ec]\nmodule = \"hmac\"\nec_stor = \"ec_identity_store\"\n\n[ec.hmac]\npassphrase = \"test-secret-key-32-bytes-minimum\"\n", + ); + + let err = + Settings::from_toml(&toml_str).expect_err("a mistyped [ec] key should be rejected"); + let message = format!("{err:?}"); + assert!( + message.contains("unknown field `ec_stor`") && message.contains("`ec_store`"), + "should name the mistyped key and the keys it could have been, got: {message}" + ); + } + + #[test] + fn geo_module_accepts_default_platform_and_none_and_rejects_unknown() { + let config = GeoConfig::default(); + assert!( + config.module.is_none(), + "geo should default to no selector, which selects no geo module" + ); + config + .validate_module_selection() + .expect("should validate the default of running without geolocation"); + + let platform = GeoConfig { + module: Some("platform".to_owned()), + assume_single_jurisdiction: false, + }; + platform + .validate_module_selection() + .expect("should validate the explicit platform selection"); + + let none = GeoConfig { + module: Some("none".to_owned()), + assume_single_jurisdiction: false, + }; + none.validate_module_selection() + .expect("should validate the explicit opt-out of geolocation"); + + // A key core does not know is not a settings error, because a + // module's name is a legitimate value and a closed list here would + // shut every module out of geo. Settings accepts it and the registry + // decides. + let module_key = GeoConfig { + module: Some("acme".to_owned()), + assume_single_jurisdiction: false, + }; + module_key + .validate_module_selection() + .expect("a module id should be accepted by settings validation"); + + // The rejection moved to registry build, where it is known whether any + // module supplies the name. With no module supplying `acme`, building + // the registry fails and the error names the selector and the module. + let mut settings = crate::test_support::tests::create_test_settings(); + settings.geo.module = Some("acme".to_owned()); + // `IntegrationRegistry` is not `Debug`, so the error is taken by match + // rather than `expect_err`. + let error = match crate::integrations::IntegrationRegistry::new(&settings) { + Ok(_) => { + panic!("a geo module no module supplies should be rejected at registry build") + } + Err(error) => error, + }; + let message = error.to_string(); + assert!( + message.contains("acme") && message.contains("[geo] module"), + "the error should name the selector and the module, got: {message}" + ); + } + + #[test] + fn the_reserved_ec_keys_are_the_ones_the_section_reads() { + // The list is written out for the unknown-field message and the + // reserved-name check, so it has to stay level with the struct. + let ec = Ec { + passphrase: Some(Redacted::new("test-secret-key-32-bytes-minimum".to_owned())), + ..Ec::default() + }; + let value = serde_json::to_value(&ec).expect("should serialize the [ec] section"); + let written = value + .as_object() + .expect("the section should serialize as a table") + .keys() + .map(String::as_str) + .collect::>(); + + assert_eq!( + written, + EC_SECTION_KEYS.iter().copied().collect::>(), + "EC_SECTION_KEYS should name every key the [ec] section reads as its own" + ); + } + + #[test] + fn unknown_keys_in_module_sections_are_rejected() { + // A mistyped key must fail at startup rather than silently selecting + // a default behind the operator's back. + for (section, bad_key) in [ + ("[geo]", "providr = \"platform\""), + ("[device]", "providr = \"builtin\""), + ] { + let toml_str = format!( + "{}\n\n {section}\n {bad_key}\n", + crate_test_settings_str() + ); + assert!( + Settings::from_toml(&toml_str).is_err(), + "an unknown key in {section} should be rejected" + ); + } + + let toml_str = crate_test_settings_str() + .replace("[ec.hmac]", "[ec.hmac]\n unexpected = \"value\""); + assert!( + Settings::from_toml(&toml_str).is_err(), + "an unknown key in [ec.hmac] should be rejected" + ); + } + + #[test] + fn a_reserved_key_cannot_name_a_module() { + let toml_str = crate_test_settings_str_with_ec_section("[ec]\nmodule = \"ec_store\"\n"); + + let err = + Settings::from_toml(&toml_str).expect_err("a reserved key should not name a module"); + assert!( + format!("{err:?}").contains("names a key the `[ec]` section reads as its own setting"), + "should say why the name cannot be used: {err:?}" + ); + } + + #[test] + fn module_names_and_implementations_follow_the_module_name_rule() { + for ec_section in [ + "[ec]\nmodule = \"Primary\"\n", + "[ec]\nmodule = \"primary\"\n\n[ec.primary]\nimplementation = \"Hmac\"\n", + "[ec]\nmodule = \"primary\"\n\n[ec.Primary]\nimplementation = \"hmac\"\npassphrase = \"test-secret-key-32-bytes-minimum\"\n", + ] { + let err = Settings::from_toml(&crate_test_settings_str_with_ec_section(ec_section)) + .expect_err("a name that is not a module name should be rejected"); + assert!( + format!("{err:?}").contains("is not a module name"), + "should hold `{ec_section}` to the module name rule: {err:?}" + ); + } + } + + #[test] + fn an_hmac_block_without_its_passphrase_names_the_block() { + // The implementation the block names decides how its settings are + // read, so the block the operator wrote is what the error names. + let toml_str = crate_test_settings_str_with_ec_section( + "[ec]\nmodule = \"primary\"\n\n[ec.primary]\nimplementation = \"hmac\"\n", + ); + + let err = Settings::from_toml(&toml_str) + .expect_err("an hmac block without its passphrase should be rejected"); + let message = format!("{err:?}"); + assert!( + message.contains("[ec.primary] is invalid") && message.contains("passphrase"), + "should name the block and the setting it lacks, got: {message}" + ); + } + + #[test] + fn a_labeled_block_round_trips_through_serialization() { + // The pushed configuration is the serialized settings, so a label and + // the implementation it names have to survive being written back. + let toml_str = crate_test_settings_str_with_ec_section( + "[ec]\nmodule = \"primary\"\n\n[ec.primary]\nimplementation = \"hmac\"\npassphrase = \"test-secret-key-32-bytes-minimum\"\n", + ); + let settings = Settings::from_toml(&toml_str).expect("should parse a labeled module block"); + + let value = serde_json::to_value(&settings).expect("should serialize settings"); + assert_eq!( + value["ec"]["primary"]["implementation"], "hmac", + "the label should keep the implementation it names" + ); + + let reparsed = + Settings::from_json_value(value).expect("should reparse the serialized settings"); + assert_eq!( + reparsed.ec.module_blocks.implementation("primary"), + HMAC_MODULE_KEY, + "the implementation should survive the round trip" + ); + assert_eq!( + hmac_passphrase(&reparsed.ec, "primary"), + "test-secret-key-32-bytes-minimum" + ); + + // A block under the implementation's own name is written back as it + // was, without gaining a key the operator never wrote. + let written = serde_json::to_value(create_test_settings()) + .expect("should serialize the test settings"); + assert!( + written["ec"]["hmac"].get("implementation").is_none(), + "an unlabeled block should not gain an implementation key: {}", + written["ec"]["hmac"] + ); + } + + #[test] + fn an_injected_modules_settings_are_kept_as_written() { + // Core never names a vendor, so the block of a module an adapter + // injects is kept as the values it held for that adapter to read. + let toml_str = crate_test_settings_str_with_ec_section( + "[ec]\nmodule = \"acme\"\n\n[ec.acme]\nendpoint = \"https://ec.acme.example.com\"\n", + ); + let settings = Settings::from_toml(&toml_str).expect("should parse a vendor module block"); + + let block = settings + .ec + .module_blocks + .get("acme") + .expect("should configure the acme block"); + assert_eq!( + settings.ec.module_blocks.implementation("acme"), + "acme", + "a block that names no implementation is its own name's" + ); + let EcModuleSettings::Injected(injected) = &block.settings else { + panic!("a vendor block should keep its settings as the raw values it held"); + }; + assert_eq!(injected["endpoint"], "https://ec.acme.example.com"); + + // They survive being written back into the pushed configuration, so + // the adapter reads what the operator wrote. + let written = serde_json::to_value(&settings).expect("should serialize settings"); + assert_eq!( + written["ec"]["acme"]["endpoint"], + "https://ec.acme.example.com" + ); + } + + #[test] + fn the_compiled_permission_policy_validates_at_startup() { + // The compiled-in sample declares its top node, so startup + // accepts it. A policy that omitted `group` or `jurisdiction` would be + // rejected here rather than panicking on the first lookup, which the + // parser tests in `permissions` cover directly. + GeoConfig::validate_permission_policy() + .expect("the compiled-in sample should validate at startup"); + } + + #[test] + fn ec_without_geo_requires_the_single_jurisdiction_acknowledgment() { + // The base test settings acknowledge single-jurisdiction operation. + // Removing the acknowledgment while an EC module is configured and + // no geo module is selected must fail at startup. + let toml_str = crate_test_settings_str().replace("assume_single_jurisdiction = true\n", ""); + let err = Settings::from_toml(&toml_str) + .expect_err("an EC module with no geo module needs the acknowledgment"); + assert!( + matches!( + err.current_context(), + TrustedServerError::Configuration { .. } + ), + "should be a configuration error, got: {:?}", + err.current_context() + ); + + // Selecting a geo module removes the requirement. + let toml_str = crate_test_settings_str() + .replace("assume_single_jurisdiction = true\n", "") + .replace("[geo]", "[geo]\n module = \"platform\""); + Settings::from_toml(&toml_str) + .expect("a geo module resolves jurisdictions, so no acknowledgment is needed"); + + // With no EC module there is no jurisdiction consumer to protect. + let toml_str = crate_test_settings_str() + .replace("assume_single_jurisdiction = true\n", "") + .replace("module = \"hmac\"", "") + .replace( + "[ec.hmac]\n passphrase = \"test-secret-key-32-bytes-minimum\"", + "", + ); + Settings::from_toml(&toml_str) + .expect("stateless operation needs no jurisdiction acknowledgment"); + } + + #[test] + fn validate_rejects_trailing_slash_in_origin_url() { + let toml_str = crate_test_settings_str().replace( + r#"origin_url = "https://origin.test-publisher.com""#, + r#"origin_url = "https://origin.test-publisher.com/""#, + ); + + let result = Settings::from_toml(&toml_str); + assert!( + result.is_err(), + "origin_url ending with '/' should fail validation" + ); + } + + #[test] + fn validate_rejects_invalid_publisher_domains() { + for domain in [ "", ".example.com", "example.com.", @@ -5596,18 +8382,6 @@ origin_host_header_overide = "www.example.com""#, } } - #[test] - fn prepare_runtime_rejects_invalid_handler_regex() { - let toml_str = crate_test_settings_str().replace(r#"path = "^/secure""#, r#"path = "(""#); - - let err = Settings::from_toml(&toml_str).expect_err("should reject invalid handler regex"); - assert!( - err.to_string() - .contains("Handler path regex `(` failed to compile"), - "should describe the invalid handler regex" - ); - } - #[test] fn test_settings_missing_required_fields() { let re = Regex::new(r"origin_url = .*").expect("regex should compile"); @@ -5815,32 +8589,6 @@ source_domain = "partner.example.com" ); } - #[test] - fn is_placeholder_handler_password_rejects_known_template_value() { - assert!( - Handler::is_placeholder_password("replace-with-admin-password-32-bytes"), - "init-template handler password should be rejected" - ); - } - - #[test] - fn reject_placeholder_secrets_includes_handler_passwords() { - let mut settings = - Settings::from_toml(&crate_test_settings_str()).expect("should parse test settings"); - settings.publisher.proxy_secret = Redacted::new("unit-test-proxy-secret".to_owned()); - settings.ec.passphrase = Redacted::new("test-secret-key-32-bytes-minimum".to_owned()); - settings.handlers[0].password = - Redacted::new("replace-with-admin-password-32-bytes".to_owned()); - - let err = settings - .reject_placeholder_secrets() - .expect_err("should reject placeholder handler password"); - assert!( - format!("{err:?}").contains("handlers"), - "error should mention handler password field" - ); - } - fn test_partner_with_pull_token(ts_pull_token: &str) -> EcPartner { test_partner_with_tokens(None, ts_pull_token) } @@ -5867,7 +8615,6 @@ source_domain = "partner.example.com" let mut settings = Settings::from_toml(&crate_test_settings_str()).expect("should parse test settings"); settings.publisher.proxy_secret = Redacted::new("unit-test-proxy-secret".to_owned()); - settings.ec.passphrase = Redacted::new("test-secret-key-32-bytes-minimum".to_owned()); settings.ec.partners = vec![test_partner_with_pull_token( "partner-api-token-32-bytes-minimum", )]; @@ -5886,7 +8633,6 @@ source_domain = "partner.example.com" let mut settings = Settings::from_toml(&crate_test_settings_str()).expect("should parse test settings"); settings.publisher.proxy_secret = Redacted::new("unit-test-proxy-secret".to_owned()); - settings.ec.passphrase = Redacted::new("test-secret-key-32-bytes-minimum".to_owned()); settings.ec.partners = vec![test_partner_with_pull_token( "unit-test-realistic-pull-sync-token-32-bytes-min", )]; @@ -5901,7 +8647,6 @@ source_domain = "partner.example.com" let mut settings = Settings::from_toml(&crate_test_settings_str()).expect("should parse test settings"); settings.publisher.proxy_secret = Redacted::new("unit-test-proxy-secret".to_owned()); - settings.ec.passphrase = Redacted::new("test-secret-key-32-bytes-minimum".to_owned()); settings.ec.partners = vec![test_partner_with_tokens( Some("partner-api-token-32-bytes-minimum"), "replace-with-partner-api-token-32-bytes-minimum", @@ -5921,26 +8666,6 @@ source_domain = "partner.example.com" ); } - #[test] - fn is_unusable_store_id_rejects_placeholders_empty_and_padded_values() { - for placeholder in RequestSigning::STORE_ID_PLACEHOLDERS { - assert!( - RequestSigning::is_unusable_store_id(placeholder), - "should reject placeholder store id '{placeholder}'" - ); - } - for bad in ["", " ", " 01GCFG ", "01GCFG "] { - assert!( - RequestSigning::is_unusable_store_id(bad), - "should reject unusable store id '{bad}'" - ); - } - assert!( - !RequestSigning::is_unusable_store_id("01GCFG"), - "should accept a clean store id" - ); - } - #[test] fn test_settings_empty_toml() { let toml_str = ""; @@ -5970,7 +8695,7 @@ source_domain = "partner.example.com" } #[test] - fn test_handlers_override_with_env() { + fn test_ec_partners_override_with_indexed_env() { let toml_str = crate_test_settings_str(); let origin_key = format!( @@ -5979,127 +8704,50 @@ source_domain = "partner.example.com" ENVIRONMENT_VARIABLE_SEPARATOR, ENVIRONMENT_VARIABLE_SEPARATOR ); - // Override handler 0 via env vars - let path_key_0 = format!( - "{}{}HANDLERS{}0{}PATH", + let partner_0_name_key = format!( + "{}{}EC{}PARTNERS{}0{}NAME", ENVIRONMENT_VARIABLE_PREFIX, ENVIRONMENT_VARIABLE_SEPARATOR, ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, ENVIRONMENT_VARIABLE_SEPARATOR ); - let username_key_0 = format!( - "{}{}HANDLERS{}0{}USERNAME", + let partner_0_source_domain_key = format!( + "{}{}EC{}PARTNERS{}0{}SOURCE_DOMAIN", ENVIRONMENT_VARIABLE_PREFIX, ENVIRONMENT_VARIABLE_SEPARATOR, ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, ENVIRONMENT_VARIABLE_SEPARATOR ); - let password_key_0 = format!( - "{}{}HANDLERS{}0{}PASSWORD", + let partner_0_openrtb_atype_key = format!( + "{}{}EC{}PARTNERS{}0{}OPENRTB_ATYPE", ENVIRONMENT_VARIABLE_PREFIX, ENVIRONMENT_VARIABLE_SEPARATOR, ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, ENVIRONMENT_VARIABLE_SEPARATOR ); - // Admin handler at index 1 (required for admin endpoint coverage) - let path_key_1 = format!( - "{}{}HANDLERS{}1{}PATH", + let partner_0_bidstream_enabled_key = format!( + "{}{}EC{}PARTNERS{}0{}BIDSTREAM_ENABLED", ENVIRONMENT_VARIABLE_PREFIX, ENVIRONMENT_VARIABLE_SEPARATOR, ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, ENVIRONMENT_VARIABLE_SEPARATOR ); - let username_key_1 = format!( - "{}{}HANDLERS{}1{}USERNAME", + let partner_0_api_token_key = format!( + "{}{}EC{}PARTNERS{}0{}API_TOKEN", ENVIRONMENT_VARIABLE_PREFIX, ENVIRONMENT_VARIABLE_SEPARATOR, ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, ENVIRONMENT_VARIABLE_SEPARATOR ); - let password_key_1 = format!( - "{}{}HANDLERS{}1{}PASSWORD", + let partner_1_name_key = format!( + "{}{}EC{}PARTNERS{}1{}NAME", ENVIRONMENT_VARIABLE_PREFIX, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR - ); - - temp_env::with_vars( - [ - (origin_key, Some("https://origin.test-publisher.com")), - (path_key_0, Some("^/env-handler")), - (username_key_0, Some("env-user")), - (password_key_0, Some("env-pass")), - (path_key_1, Some("^/_ts/admin")), - (username_key_1, Some("admin")), - (password_key_1, Some("admin-pass")), - ], - || { - let settings = - Settings::from_toml_and_env(&toml_str).expect("Settings should load from env"); - assert_eq!(settings.handlers.len(), 2); - let handler = &settings.handlers[0]; - assert_eq!(handler.path, "^/env-handler"); - assert_eq!(handler.username.expose(), "env-user"); - assert_eq!(handler.password.expose(), "env-pass"); - }, - ); - } - - #[test] - fn test_ec_partners_override_with_indexed_env() { - let toml_str = crate_test_settings_str(); - - let origin_key = format!( - "{}{}PUBLISHER{}ORIGIN_URL", - ENVIRONMENT_VARIABLE_PREFIX, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR - ); - let partner_0_name_key = format!( - "{}{}EC{}PARTNERS{}0{}NAME", - ENVIRONMENT_VARIABLE_PREFIX, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR - ); - let partner_0_source_domain_key = format!( - "{}{}EC{}PARTNERS{}0{}SOURCE_DOMAIN", - ENVIRONMENT_VARIABLE_PREFIX, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR - ); - let partner_0_openrtb_atype_key = format!( - "{}{}EC{}PARTNERS{}0{}OPENRTB_ATYPE", - ENVIRONMENT_VARIABLE_PREFIX, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR - ); - let partner_0_bidstream_enabled_key = format!( - "{}{}EC{}PARTNERS{}0{}BIDSTREAM_ENABLED", - ENVIRONMENT_VARIABLE_PREFIX, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR - ); - let partner_0_api_token_key = format!( - "{}{}EC{}PARTNERS{}0{}API_TOKEN", - ENVIRONMENT_VARIABLE_PREFIX, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR - ); - let partner_1_name_key = format!( - "{}{}EC{}PARTNERS{}1{}NAME", - ENVIRONMENT_VARIABLE_PREFIX, - ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, ENVIRONMENT_VARIABLE_SEPARATOR, ENVIRONMENT_VARIABLE_SEPARATOR, ENVIRONMENT_VARIABLE_SEPARATOR @@ -6190,8 +8838,10 @@ source_domain = "partner.example.com" ); } + /// An environment variable can give a handler to a configuration that + /// carries none, and that is refused as the list itself is. #[test] - fn test_invalid_handler_override_fails_during_runtime_preparation() { + fn a_handler_given_by_an_environment_variable_is_refused_too() { let toml_str = crate_test_settings_str(); let origin_key = format!( @@ -6212,9 +8862,14 @@ source_domain = "partner.example.com" origin_key, Some("https://origin.test-publisher.com"), || { - temp_env::with_var(path_key, Some("("), || { - let _ = Settings::from_toml_and_env(&toml_str) - .expect_err("should reject invalid handler regex override"); + temp_env::with_var(path_key, Some("^/secure"), || { + let error = Settings::from_toml_and_env(&toml_str) + .expect_err("should refuse a handler an environment variable gives"); + + assert!( + format!("{error:?}").contains("`[[handlers]]` is not read"), + "should say the list is not read: {error:?}" + ); }); }, ); @@ -6454,11 +9109,6 @@ source_domain = "partner.example.com" let from_toml = Settings::from_toml( r#" - [[handlers]] - path = "^/_ts/admin" - username = "admin" - password = "admin-pass" - [publisher] domain = "example.com" cookie_domain = ".example.com" @@ -6466,7 +9116,13 @@ source_domain = "partner.example.com" proxy_secret = "unit-test-proxy-secret" [ec] + module = "hmac" + + [ec.hmac] passphrase = "test-secret-key-32-bytes-minimum" + + [geo] + assume_single_jurisdiction = true "#, ) .expect("should parse settings without max_buffered_body_bytes"); @@ -6484,11 +9140,6 @@ source_domain = "partner.example.com" // silently breaking traffic. let result = Settings::from_toml( r#" - [[handlers]] - path = "^/_ts/admin" - username = "admin" - password = "admin-pass" - [publisher] domain = "example.com" cookie_domain = ".example.com" @@ -6496,155 +9147,261 @@ source_domain = "partner.example.com" proxy_secret = "unit-test-proxy-secret" max_buffered_body_bytes = 0 + [geo] + assume_single_jurisdiction = true + [ec] + module = "hmac" + + [ec.hmac] passphrase = "test-secret-key-32-bytes-minimum" "#, ); + let error = result.expect_err("should reject a zero buffered-body cap"); assert!( - result.is_err(), - "publisher.max_buffered_body_bytes = 0 must fail config validation" + error.to_string().contains("max_buffered_body_bytes"), + "the rejection should be for the zero cap, not another validation, got: {error}" ); } - #[test] - fn test_disabled_integration_does_not_register() { - use crate::integrations::testlight::TestlightConfig; - use serde_json::json; + /// The settings of an example module that takes none. + #[derive(Debug, Deserialize, Validate)] + #[serde(deny_unknown_fields)] + struct NoSettings {} - let mut settings = create_test_settings(); - settings - .integrations - .insert_config( - "testlight", - &json!({ - "enabled": false, - "endpoint": "https://testlight.test/auction", - "rewrite_scripts": true, - }), - ) - .expect("should insert integration config"); + impl IntegrationConfig for NoSettings {} - let config = settings - .integration_config::("testlight") - .expect("integration parsing should succeed"); + /// The settings of an example module that requires one. + #[derive(Debug, Deserialize, Validate)] + #[serde(deny_unknown_fields)] + struct OneRequiredSetting { + #[validate(url)] + endpoint: String, + } + + impl IntegrationConfig for OneRequiredSetting {} + + /// The example modules' names, in a section of their own type. + const EXAMPLE_SECTION: &str = "example"; + const PLAIN_MODULE: &str = "example.plain"; + const ENDPOINT_MODULE: &str = "example.endpoint"; + + /// A module no section selects has no configuration, whatever else the + /// settings hold. + #[test] + fn a_module_that_is_not_selected_has_no_configuration() { + let settings = create_test_settings(); - assert!(config.is_none(), "Disabled integrations should be skipped"); + assert!( + !settings.selects_module(ENDPOINT_MODULE), + "the shared fixture should not select the example module" + ); + assert!( + settings + .module_config::(ENDPOINT_MODULE) + .expect("reading an unselected module should succeed") + .is_none(), + "a module that is not selected should have no configuration" + ); } + /// A selected module with no table of its own is read from an empty one, + /// so one that takes no settings runs on its selection alone. #[test] - fn disabled_integration_can_omit_enabled_required_fields_and_skip_semantic_validation() { + fn a_selected_module_with_no_table_is_read_from_an_empty_one() { let mut settings = create_test_settings(); - settings - .integrations - .insert_config( - "gpt", - &json!({ - "enabled": false, - }), - ) - .expect("should insert GPT config"); + settings.select_module(EXAMPLE_SECTION, PLAIN_MODULE); - let config = settings - .integration_config::("gpt") - .expect("minimal disabled GPT config should be ignored"); - assert!(config.is_none(), "disabled GPT config should be skipped"); - IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) - .expect("disabled invalid integration config should not fail registry startup"); + assert!( + settings + .module_config::(PLAIN_MODULE) + .expect("a module that takes no settings should read from an empty table") + .is_some(), + "selecting the module should be the whole configuration" + ); } + /// The same empty table makes a module that requires a setting report the + /// setting it is missing, rather than starting without it. #[test] - fn minimal_disabled_prebid_deserializes_without_enabled_only_validation() { + fn a_selected_module_without_a_required_setting_names_it() { let mut settings = create_test_settings(); - settings - .integrations - .insert_config( - "prebid", - &json!({ - "enabled": false, - }), - ) - .expect("should insert prebid config"); + settings.select_module(EXAMPLE_SECTION, ENDPOINT_MODULE); - let config = settings - .integration_config::("prebid") - .expect("disabled prebid config should be ignored"); - assert!(config.is_none(), "disabled prebid config should be skipped"); - IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), + let error = settings + .module_config::(ENDPOINT_MODULE) + .expect_err("should reject a selected module with no endpoint"); + + let rendered = error.to_string(); + assert!( + rendered.contains("[example.endpoint]") && rendered.contains("endpoint"), + "should name the module's table and the missing setting: {rendered}" + ); + } + + /// A table written for a module its section does not select is refused, + /// rather than sitting in the configuration doing nothing. + #[test] + fn a_table_for_a_module_that_is_not_selected_is_refused() { + let toml = format!( + "{}\n[testing]\nmodule = \"example\"\n\n[testing.another]\n", + crate_test_settings_str() + ); + + let error = Settings::from_toml(&toml).expect_err("should reject a table nothing selects"); + let rendered = format!("{error:?}"); + + assert!( + rendered.contains("[testing.another] is configured") + && rendered.contains("does not select"), + "should name the table and the selection it is missing from: {rendered}" + ); + } + + /// A module type's section that selects nothing is refused, which is also + /// what a misspelt section of Trusted Server's own meets. + #[test] + fn a_section_that_selects_no_module_is_refused() { + let toml = format!( + "{}\n[framework.nextjs]\nrewrite_attributes = [\"href\"]\n", + crate_test_settings_str() + ); + + let error = + Settings::from_toml(&toml).expect_err("should reject a section selecting nothing"); + + assert!( + format!("{error:?}").contains("[framework] selects no module"), + "should name the section: {error:?}" + ); + } + + /// A key left in a module's table that its settings do not have, such as + /// the removed `enabled`, is refused when the module reads its table, so a + /// configuration cannot read as switched off while the module runs. + #[test] + fn an_enabled_key_left_in_a_table_is_refused() { + let toml = format!( + "{}\n[example]\nmodules = [\"endpoint\"]\n\n[example.endpoint]\nendpoint = \"https://endpoint.example\"\nenabled = false\n", + crate_test_settings_str() + ); + let settings = Settings::from_toml(&toml).expect("core reads no module's table itself"); + + let error = settings + .module_config::(ENDPOINT_MODULE) + .expect_err("should reject a leftover enabled key"); + let rendered = format!("{error:?}"); + + assert!( + rendered.contains("[example.endpoint]") && rendered.contains("enabled"), + "should name the table and the key: {rendered}" + ); + } + + /// Naming one module twice is a mistake rather than a way of running it + /// twice, so it is refused. + #[test] + fn naming_a_module_twice_is_refused() { + let toml = format!( + "{}\n[example]\nmodules = [\"endpoint\", \"endpoint\"]\n", + crate_test_settings_str() + ); + + let error = Settings::from_toml(&toml).expect_err("should reject a repeated name"); + + assert!( + format!("{error:?}").contains("more than once"), + "should report the repeated name: {error:?}" + ); + } + + /// The tables removed from the configuration are refused with the move + /// spelled out, rather than with a bare unknown-field error. + #[test] + fn the_removed_integration_tables_are_refused_with_directions() { + for (table, expected) in [ + ("[integrations.example]", "CHANGELOG.md"), + ( + "[integration]\nmodules = [\"example\"]", + "is no longer read", ), - ) - .expect("disabled default-enabled prebid config should not fail registry startup"); - build_orchestrator(&settings) - .expect("minimal disabled prebid config should not fail orchestrator startup"); + ] { + let toml = format!("{}\n{table}\n", crate_test_settings_str()); + + let error = Settings::from_toml(&toml).expect_err("should reject the removed table"); + let rendered = format!("{error:?}"); + + assert!( + rendered.contains("the section of its type") && rendered.contains(expected), + "should say where modules are selected now: {rendered}" + ); + } } + /// The same refusal reaches a configuration blob, which is the shape the + /// runtime loads rather than TOML. #[test] - fn disabled_removed_prebid_and_aps_fields_are_rejected() { - for (integration_id, removed_field) in [("prebid", "server_url"), ("aps", "account_id")] { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config( - integration_id, - &json!({ - "enabled": false, - (removed_field): "removed-value", - }), - ) - .expect("should insert removed integration config field"); - - let error = match integration_id { - "prebid" => settings - .integration_config::(integration_id) - .expect_err("should reject removed disabled Prebid field"), - "aps" => settings - .integration_config::(integration_id) - .expect_err("should reject removed disabled APS field"), - _ => unreachable!("test integration ID should be known"), - }; + fn json_settings_refuse_the_removed_integration_tables() { + for table in ["integrations", "integration"] { + let mut value = serde_json::to_value(create_test_settings()) + .expect("should serialize the test settings fixture to JSON"); + value + .as_object_mut() + .expect("settings should serialize as an object") + .insert(table.to_owned(), json!({ "example": {} })); + + let error = Settings::from_json_value(value) + .expect_err("should reject the removed table in JSON"); + assert!( - format!("{error:?}").contains(removed_field), - "should identify removed field `{removed_field}`: {error:?}" + format!("{error:?}").contains("the section of its type"), + "should say where modules are selected now: {error:?}" ); } } #[test] - fn enabled_invalid_integration_fails_registry_startup() { + fn invalid_settings_for_a_selected_module_fail_registry_startup() { + fn build( + settings: &Settings, + ) -> Result, Report> + { + Ok(settings + .module_config::(ENDPOINT_MODULE)? + .map(|_| crate::integrations::IntegrationRegistration::builder("endpoint").build())) + } + + fn validate(settings: &Settings) -> Result> { + settings + .module_config::(ENDPOINT_MODULE) + .map(|config| config.is_some()) + } + let mut settings = create_test_settings(); settings - .integrations - .insert_config( - "gpt", + .insert_module_config( + EXAMPLE_SECTION, + ENDPOINT_MODULE, &json!({ - "enabled": true, - "script_url": "not a url", + "endpoint": "not a url", }), ) - .expect("should insert GPT config"); + .expect("should insert the example module's table"); + let extra = [crate::integrations::IntegrationBuilder::new( + "endpoint", + "settings-tests", + build, + validate, + ) + .with_module_name(ENDPOINT_MODULE)]; - let err = match IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) { - Ok(_) => panic!("enabled invalid integration should fail registry startup"), + let err = match IntegrationRegistry::with_registrations(&settings, &extra) { + Ok(_) => panic!("a selected module with invalid settings should fail startup"), Err(err) => err, }; assert!( - err.to_string().contains("Integration 'gpt'"), - "should identify the invalid integration config" + format!("{err:?}").contains("[example.endpoint]"), + "should identify the invalid module table: {err:?}" ); } @@ -6700,11 +9457,8 @@ source_domain = "partner.example.com" #[test] fn test_auction_creative_processing_defaults_when_omitted() { - let toml_str = crate_test_settings_str() - + r#" - [auction] - enabled = true - "#; + let toml_str = + crate_test_settings_str().replace("[auction]\n", "[auction]\nenabled = true\n"); let settings = Settings::from_toml(&toml_str).expect("should parse valid TOML"); @@ -6720,12 +9474,10 @@ source_domain = "partner.example.com" #[test] fn test_auction_rewrite_creatives_accepts_explicit_false() { - let toml_str = crate_test_settings_str() - + r#" - [auction] - enabled = true - rewrite_creatives = false - "#; + let toml_str = crate_test_settings_str().replace( + "[auction]\n", + "[auction]\nenabled = true\nrewrite_creatives = false\n", + ); let settings = Settings::from_toml(&toml_str).expect("should parse valid TOML"); @@ -6746,12 +9498,7 @@ source_domain = "partner.example.com" #[test] fn test_auction_allowed_context_keys_from_toml() { - let toml_str = crate_test_settings_str() - + r#" - [auction] - enabled = true - allowed_context_keys = ["permutive_segments", "lockr_ids"] - "#; + let toml_str = crate_test_settings_str().replace("[auction]\n", "[auction]\nenabled = true\nallowed_context_keys = [\"permutive_segments\", \"lockr_ids\"]\n"); let settings = Settings::from_toml(&toml_str).expect("should parse valid TOML"); assert_eq!( settings.auction.allowed_context_keys, @@ -6761,12 +9508,10 @@ source_domain = "partner.example.com" #[test] fn test_auction_empty_allowed_context_keys_blocks_all() { - let toml_str = crate_test_settings_str() - + r#" - [auction] - enabled = true - allowed_context_keys = [] - "#; + let toml_str = crate_test_settings_str().replace( + "[auction]\n", + "[auction]\nenabled = true\nallowed_context_keys = []\n", + ); let settings = Settings::from_toml(&toml_str).expect("should parse valid TOML"); assert!( settings.auction.allowed_context_keys.is_empty(), @@ -6785,6 +9530,7 @@ source_domain = "partner.example.com" "*.Example.Org".to_string(), ], asset_routes: vec![], + modules: SectionModules::default(), }; proxy.normalize(); assert_eq!( @@ -6805,6 +9551,7 @@ source_domain = "partner.example.com" "cdn.example.com".to_string(), ], asset_routes: vec![], + modules: SectionModules::default(), }; proxy.normalize(); assert_eq!( @@ -6820,6 +9567,7 @@ source_domain = "partner.example.com" certificate_check: true, allowed_domains: vec!["*".to_string(), "tracker.com".to_string()], asset_routes: vec![], + modules: SectionModules::default(), }; proxy.normalize(); assert_eq!( @@ -6835,6 +9583,7 @@ source_domain = "partner.example.com" certificate_check: true, allowed_domains: vec!["*".to_string()], asset_routes: vec![], + modules: SectionModules::default(), }; proxy.normalize(); assert!( @@ -6849,6 +9598,7 @@ source_domain = "partner.example.com" certificate_check: true, allowed_domains: vec![" ".to_string(), "\t".to_string()], asset_routes: vec![], + modules: SectionModules::default(), }; proxy.normalize(); assert!( @@ -6867,6 +9617,7 @@ source_domain = "partner.example.com" origin_url: " https://assets.example.com ".to_string(), ..Default::default() }], + modules: SectionModules::default(), }; proxy.normalize(); assert_eq!( @@ -6891,6 +9642,7 @@ source_domain = "partner.example.com" target_path: Some(" /rewritten/$1 ".to_string()), ..Default::default() }], + modules: SectionModules::default(), }; proxy.normalize(); @@ -7615,6 +10367,7 @@ source_domain = "partner.example.com" ..Default::default() }, ], + modules: SectionModules::default(), }; let route = proxy @@ -7643,6 +10396,7 @@ source_domain = "partner.example.com" ..Default::default() }, ], + modules: SectionModules::default(), }; let route = proxy @@ -7829,8 +10583,6 @@ source_domain = "partner.example.com" ); } - // --- admin endpoint coverage --- - #[test] fn test_publisher_rejects_cookie_domain_with_metacharacters() { for bad_domain in [ @@ -7857,357 +10609,22 @@ source_domain = "partner.example.com" ); } - /// Helper that returns a settings TOML string WITHOUT any admin handler, - /// for tests that need to verify uncovered-admin-endpoint behaviour. - fn settings_str_without_admin_handler() -> String { - r#" - [[handlers]] - path = "^/secure" - username = "user" - password = "pass" - - [publisher] - domain = "test-publisher.com" - cookie_domain = ".test-publisher.com" - origin_url = "https://origin.test-publisher.com" - proxy_secret = "unit-test-proxy-secret" - - [ec] - passphrase = "test-secret-key-32-bytes-minimum" - - [request_signing] - config_store_id = "test-config-store-id" - secret_store_id = "test-secret-store-id" - "# - .to_string() - } - - #[test] - fn uncovered_admin_endpoints_returns_all_when_no_handler_covers_admin() { - // Deserialize directly to bypass from_toml's admin validation, - // since this test exercises uncovered_admin_endpoints itself. - let settings: Settings = - toml::from_str(&settings_str_without_admin_handler()).expect("should deserialize TOML"); - let uncovered = settings - .uncovered_admin_endpoints() - .expect("should check admin coverage"); - assert_eq!( - uncovered, - vec![ - "/_ts/admin/keys/rotate", - "/_ts/admin/keys/deactivate", - "/_ts/admin/ec", - "/_ts/admin/ec/{id}", - "/_ts/admin/eids", - "/_ts/admin/cache/purge", - ], - "should report every admin endpoint as uncovered" - ); - } - - #[test] - fn uncovered_admin_endpoints_returns_empty_when_handler_covers_admin() { - let settings = create_test_settings(); - let uncovered = settings - .uncovered_admin_endpoints() - .expect("should check admin coverage"); - assert!( - uncovered.is_empty(), - "should report no uncovered admin endpoints when handler covers /_ts/admin" - ); - } - - #[test] - fn uncovered_admin_endpoints_detects_partial_coverage() { - let toml_str = settings_str_without_admin_handler() - + r#" - [[handlers]] - path = "^/_ts/admin/keys/rotate$" - username = "admin" - password = "secret" - "#; - // Deserialize directly to bypass from_toml's admin validation, - // since this test exercises uncovered_admin_endpoints itself. - let settings: Settings = toml::from_str(&toml_str).expect("should deserialize TOML"); - let uncovered = settings - .uncovered_admin_endpoints() - .expect("should check admin coverage"); - assert_eq!( - uncovered, - vec![ - "/_ts/admin/keys/deactivate", - "/_ts/admin/ec", - "/_ts/admin/ec/{id}", - "/_ts/admin/eids", - "/_ts/admin/cache/purge", - ], - "should detect the admin endpoints not covered by the narrow handler" - ); - } - - #[test] - fn from_toml_rejects_literal_parameter_template_auth_coverage() { - let toml_str = crate_test_settings_str().replace( - r#"path = "^/_ts/admin" - username = "admin" - password = "admin-pass""#, - r#"path = "^/_ts/admin/(keys/rotate|keys/deactivate|ec|eids|cache/purge)$" - username = "admin" - password = "strong-test-password" - - [[handlers]] - path = "^/_ts/admin/ec/[{]id[}]$" - username = "admin" - password = "strong-test-password""#, - ); - - let error = Settings::from_toml(&toml_str) - .expect_err("should reject literal parameter-template auth coverage"); - let message = format!("{error:?}"); - assert!( - message.contains("/_ts/admin/ec/{id}"), - "should identify the concrete EC route as uncovered, got: {message}" - ); - } - - #[test] - fn from_toml_rejects_lowercase_only_dynamic_admin_ec_auth_coverage() { - let toml_str = crate_test_settings_str().replace( - r#"path = "^/_ts/admin" - username = "admin" - password = "admin-pass""#, - r#"path = "^/_ts/admin/(keys/rotate|keys/deactivate|ec|eids|cache/purge)$" - username = "admin" - password = "strong-test-password" - - [[handlers]] - path = "^/_ts/admin/ec/[a-f0-9]{64}[.][a-z0-9]{6}$" - username = "admin" - password = "strong-test-password""#, - ); - - let error = Settings::from_toml(&toml_str) - .expect_err("should reject lowercase-only dynamic EC auth coverage"); - let message = format!("{error:?}"); - assert!( - message.contains("/_ts/admin/ec/{id}"), - "should identify the mixed-case EC route as uncovered, got: {message}" - ); - } - - #[test] - fn from_toml_rejects_placeholder_password_on_shadowing_admin_handler() { - // Handler selection is first-match-wins, so a narrow handler placed - // ahead of the admin matcher governs the EC IDs it matches. No probe - // enumerates those IDs, so the placeholder check cannot be limited to - // handlers inferred to cover an admin endpoint. - let toml_str = crate_test_settings_str().replace( - r#"path = "^/_ts/admin" - username = "admin" - password = "admin-pass""#, - r#"path = "^/_ts/admin/ec/[a-f0-9]{64}[.]zzzzzz$" - username = "admin" - password = "change-me-admin-password" - - [[handlers]] - path = "^/_ts/admin" - username = "admin" - password = "strong-test-password""#, - ); - - let error = Settings::from_toml(&toml_str) - .expect_err("should reject placeholder password on shadowing admin handler"); - let message = format!("{error:?}"); - assert!( - message.contains("placeholder password"), - "should identify the placeholder handler password, got: {message}" - ); - } - - #[test] - fn from_toml_rejects_weak_password_on_non_admin_handler() { - let toml_str = crate_test_settings_str().replace( - r#"path = "^/_ts/admin" - username = "admin" - password = "admin-pass""#, - r#"path = "^/_ts/admin" - username = "admin" - password = "strong-test-password" - - [[handlers]] - path = "^/private" - username = "admin" - password = "changeme""#, - ); - - let error = Settings::from_toml(&toml_str) - .expect_err("should reject a weak password on any handler"); - let message = format!("{error:?}"); - assert!( - message.contains("placeholder password"), - "should identify the weak handler password, got: {message}" - ); - } - - #[test] - fn from_toml_rejects_sampled_id_only_dynamic_admin_ec_auth_coverage() { - // A handler anchored to the full EC ID grammar still leaves the rest of - // the route surface (malformed IDs, which the router accepts and the - // admin handler rejects with 400) unauthenticated, so coverage must not - // be inferred from ID-shaped samples. - let toml_str = crate_test_settings_str().replace( - r#"path = "^/_ts/admin" - username = "admin" - password = "admin-pass""#, - r#"path = "^/_ts/admin/(keys/rotate|keys/deactivate|ec|eids|cache/purge)$" - username = "admin" - password = "strong-test-password" - - [[handlers]] - path = "^/_ts/admin/ec/[a-f0-9]{64}[.][A-Za-z0-9]{6}$" - username = "admin" - password = "strong-test-password""#, - ); - - let error = Settings::from_toml(&toml_str) - .expect_err("should reject ID-sampled dynamic EC auth coverage"); - let message = format!("{error:?}"); - assert!( - message.contains("/_ts/admin/ec/{id}"), - "should identify the dynamic EC route as uncovered, got: {message}" - ); - } - - #[test] - fn from_toml_rejects_prefix_anchored_admin_ec_auth_coverage() { - // `^/_ts/admin/ec/$` matches the prefix probe but no actual lookup. - let toml_str = crate_test_settings_str().replace( - r#"path = "^/_ts/admin" - username = "admin" - password = "admin-pass""#, - r#"path = "^/_ts/admin/(keys/rotate|keys/deactivate|ec|eids|cache/purge)$" - username = "admin" - password = "strong-test-password" - - [[handlers]] - path = "^/_ts/admin/ec/$" - username = "admin" - password = "strong-test-password""#, - ); - - let error = Settings::from_toml(&toml_str) - .expect_err("should reject prefix-anchored dynamic EC auth coverage"); - let message = format!("{error:?}"); - assert!( - message.contains("/_ts/admin/ec/{id}"), - "should identify the dynamic EC route as uncovered, got: {message}" - ); - } - - #[test] - fn from_toml_accepts_prefix_matcher_admin_ec_auth_coverage() { - let toml_str = crate_test_settings_str().replace( - r#"path = "^/_ts/admin" - username = "admin" - password = "admin-pass""#, - r#"path = "^/_ts/admin/(keys/rotate|keys/deactivate|ec|eids|cache/purge)$" - username = "admin" - password = "strong-test-password" - - [[handlers]] - path = "^/_ts/admin/ec/" - username = "admin" - password = "strong-test-password""#, - ); - - Settings::from_toml(&toml_str) - .expect("should accept a prefix-level matcher for the dynamic EC route"); - } - - #[test] - fn from_toml_and_env_rejects_config_without_admin_handler() { - let origin_key = format!( - "{}{}PUBLISHER{}ORIGIN_URL", - ENVIRONMENT_VARIABLE_PREFIX, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR - ); - temp_env::with_var( - origin_key, - Some("https://origin.test-publisher.com"), - || { - let result = Settings::from_toml_and_env(&settings_str_without_admin_handler()); - assert!( - result.is_err(), - "should reject configuration when admin endpoints are not covered" - ); - let err = format!("{:?}", result.unwrap_err()); - assert!( - err.contains("No handler covers admin endpoint"), - "error should mention uncovered admin endpoints, got: {err}" - ); - }, - ); - } - - #[test] - fn from_toml_rejects_admin_handler_placeholder_password() { - let toml_str = crate_test_settings_str() - .replace(r#"password = "admin-pass""#, r#"password = "changeme""#); - - let result = Settings::from_toml(&toml_str); - assert!( - result.is_err(), - "should reject placeholder password on admin handler" - ); - let err = format!("{:?}", result.expect_err("should reject placeholder")); - assert!( - err.contains("placeholder password"), - "error should mention placeholder admin password, got: {err}" - ); - } - - #[test] - fn from_toml_accepts_non_placeholder_admin_password() { - let settings = Settings::from_toml(&crate_test_settings_str()) - .expect("should accept non-placeholder admin password"); - assert_eq!(settings.handlers.len(), 2, "should parse handlers"); - } - - #[test] - fn from_toml_rejects_config_without_admin_handler() { - let result = Settings::from_toml(&settings_str_without_admin_handler()); - assert!( - result.is_err(), - "should reject configuration when admin endpoints are not covered" - ); - let err = format!("{:?}", result.expect_err("should be an error")); - assert!( - err.contains("No handler covers admin endpoint"), - "error should mention uncovered admin endpoints, got: {err}" - ); - } - - /// Verifies that [`Settings::ADMIN_ENDPOINTS`] stays in sync with the - /// admin route table in `crates/trusted-server-adapter-fastly/src/app.rs`. - /// - /// If this test fails, a route was added or removed in the Fastly - /// router without updating `ADMIN_ENDPOINTS` (or vice versa). #[test] fn settings_parses_creative_opportunities_section() { let toml = r#" -[[handlers]] -path = "^/_ts/admin" -username = "admin" -password = "unit-test-admin-secret" - [publisher] domain = "example.com" cookie_domain = ".example.com" origin_url = "https://origin.example.com" proxy_secret = "secret" +[geo] +assume_single_jurisdiction = true + [ec] +module = "hmac" + +[ec.hmac] passphrase = "test-secret-key-32-bytes-minimum" [creative_opportunities] @@ -8280,18 +10697,19 @@ formats = [{ width = 300, height = 250 }] #[test] fn settings_rejects_invalid_creative_opportunity_slot_id() { let toml = r#" -[[handlers]] -path = "^/_ts/admin" -username = "admin" -password = "unit-test-admin-secret" - [publisher] domain = "example.com" cookie_domain = ".example.com" origin_url = "https://origin.example.com" proxy_secret = "secret" +[geo] +assume_single_jurisdiction = true + [ec] +module = "hmac" + +[ec.hmac] passphrase = "test-secret-key-32-bytes-minimum" [creative_opportunities] @@ -8316,18 +10734,19 @@ formats = [{ width = 300, height = 250 }] // id injected via env is rejected by from_toml_and_env (the build-time // path uses the same validation against the merged config). let toml = r#" -[[handlers]] -path = "^/_ts/admin" -username = "admin" -password = "unit-test-admin-secret" - [publisher] domain = "example.com" cookie_domain = ".example.com" origin_url = "https://origin.example.com" proxy_secret = "secret" +[geo] +assume_single_jurisdiction = true + [ec] +module = "hmac" + +[ec.hmac] passphrase = "test-secret-key-32-bytes-minimum" [creative_opportunities] @@ -8358,18 +10777,19 @@ gam_network_id = "21765378893" fn creative_opportunity_settings_toml(slot_body: &str) -> String { format!( r#" -[[handlers]] -path = "^/_ts/admin" -username = "admin" -password = "unit-test-admin-secret" - [publisher] domain = "example.com" cookie_domain = ".example.com" origin_url = "https://origin.example.com" proxy_secret = "secret" +[geo] +assume_single_jurisdiction = true + [ec] +module = "hmac" + +[ec.hmac] passphrase = "test-secret-key-32-bytes-minimum" [creative_opportunities] @@ -8470,45 +10890,351 @@ formats = [{{ width = 300, height = 250 }}] ); } + /// An unset selector must not serialize as `"module": null`. + /// + /// The section as a whole is skipped while every field is default, so this + /// serializes the struct directly. Once a later change makes another field + /// required, the section is always emitted and a null selector would then + /// reach a config blob, where a binary that predates the field rejects it. + #[test] + fn an_unset_module_selector_is_omitted_from_the_serialized_section() { + let geo = GeoConfig::default(); + let json = serde_json::to_string(&geo).expect("should serialize the geo section"); + assert!( + !json.contains("module"), + "an unset geo selector should be omitted rather than serialized as null, got {json}" + ); + + let device = DeviceConfig::default(); + let json = serde_json::to_string(&device).expect("should serialize the device section"); + assert!( + !json.contains("module"), + "an unset device selector should be omitted rather than serialized as null, got {json}" + ); + } +} + +#[cfg(test)] +mod permission_signal_config_tests { + use super::*; + use serde_json::json; + + use crate::config::TrustedServerAppConfig; + use crate::test_support::tests::crate_test_settings_str; + + // Which names are valid is only known where the scheme crates are linked, + // so the checks that a name matches an available module, and that none + // is repeated, live with the seam in `permission_signal::select`. What is + // tested here is the shape of the section itself. + + /// The test fixture's configuration with `section` written as its + /// `[permission-signal]` section. + fn settings_toml_with(section: &str) -> String { + format!( + "{}\n[permission-signal]\n{section}\n", + crate_test_settings_str() + ) + } + + #[test] + fn no_section_is_allowed_and_means_every_module() { + let config = PermissionSignalConfig::default(); + assert!( + config.modules.is_none(), + "absent rather than empty, because the two mean opposite things" + ); + } + + #[test] + fn the_section_round_trips_through_toml() { + let parsed: PermissionSignalConfig = + toml::from_str(r#"modules = ["gpc", "tcf"]"#).expect("should parse the section"); + assert_eq!( + parsed.modules.as_deref(), + Some(["gpc".to_owned(), "tcf".to_owned()].as_slice()), + "the order written is the order read, because the order is the policy" + ); + } + #[test] - fn admin_endpoints_match_fastly_router() { - let router_source = include_str!("../../trusted-server-adapter-fastly/src/app.rs"); + fn the_section_round_trips_through_a_config_blob() { + // The section is written by derive and read by hand, so what a push + // writes into a blob must be what a deployment reads back from it. + let written = PermissionSignalConfig { + modules: Some(vec!["tcf".to_owned(), "gpc".to_owned()]), + }; + let blob = serde_json::to_value(&written).expect("should write the section"); + let read: PermissionSignalConfig = + serde_json::from_value(blob).expect("should read back what was written"); + assert_eq!(read, written, "the list and its order survive the blob"); + + let null: PermissionSignalConfig = serde_json::from_value(json!({ "modules": null })) + .expect("should read an explicit null"); + assert_eq!( + null, + PermissionSignalConfig::default(), + "an explicit null is the same as leaving the key out" + ); + } + + #[test] + fn an_empty_list_is_kept_apart_from_no_list() { + let parsed: PermissionSignalConfig = + toml::from_str("modules = []").expect("should parse an empty list"); + assert_eq!( + parsed.modules.as_deref(), + Some(&[][..]), + "a publisher acting on no signal at all writes an empty list, and it must \ + not read back as having written nothing" + ); + } + + #[test] + fn an_unknown_key_is_refused() { + let error = toml::from_str::(r#"module_list = ["gpc"]"#) + .expect_err("should refuse a misspelled key rather than silently ignore it"); + assert!( + error + .to_string() + .contains("unknown field `module_list` in [permission-signal], expected `modules`"), + "the refusal names the key it did not recognize and the one it accepts: {error}" + ); + } + + #[test] + fn the_singular_key_is_refused_naming_modules() { + let error = toml::from_str::(r#"module = ["gpc"]"#) + .expect_err("should refuse the singular key the other sections use"); + assert!( + error + .to_string() + .contains("[permission-signal] module is `modules` here"), + "the refusal says which key to write instead: {error}" + ); + } - for endpoint in Settings::ADMIN_ENDPOINTS { + #[test] + fn the_removed_sources_key_is_refused_naming_modules() { + for written in [ + r#"sources = ["gpc", "tcf"]"#, + "modules = [\"gpc\", \"tcf\"]\nsources = [\"gpc\", \"tcf\"]", + ] { + let error = toml::from_str::(written) + .expect_err("should refuse the removed key, alone or beside its replacement"); assert!( - router_source.contains(endpoint), - "ADMIN_ENDPOINTS lists \"{endpoint}\" but it was not found in \ - crates/trusted-server-adapter-fastly/src/app.rs — remove it from ADMIN_ENDPOINTS or \ - add the route back to the router" + error.to_string().contains( + "[permission-signal] sources is no longer accepted. Name the modules \ + to run, in order, in [permission-signal] modules instead" + ), + "the refusal says which key to write instead: {error}" ); } + } - // Also verify we haven't missed any admin routes in the router. - // Best-effort: only detects string-literal routes in the NamedRoute - // table. If you define admin routes differently (e.g. via constants), - // add them to ADMIN_ENDPOINTS manually. - let admin_routes_in_router: Vec<&str> = router_source - .lines() - .filter_map(|line| { - let trimmed = line.trim(); - // Route entries look like: path: "/_ts/admin/...", - if trimmed.starts_with("path: ") && trimmed.contains("\"/_ts/admin/") { - let start = trimmed.find("\"/_ts/admin/")?; - let rest = &trimmed[start + 1..]; - let end = rest.find('"')?; - Some(&rest[..end]) - } else { - None - } - }) - .collect(); + #[test] + fn every_way_settings_are_read_refuses_the_removed_sources_key() { + let written = settings_toml_with(r#"sources = ["gpc", "tcf"]"#); + + let error = Settings::from_toml(&written).expect_err("should refuse the removed key"); + assert!( + format!("{error:?}").contains("[permission-signal] modules"), + "reading a TOML file names the key that replaced it: {error:?}" + ); + + // `ts config push` parses the file into a TOML value before reading the + // settings from it. + let value: toml::Value = toml::from_str(&written).expect("should parse as TOML"); + let error = value + .try_into::() + .expect_err("should refuse the removed key before a push"); + assert!( + error.to_string().contains("[permission-signal] modules"), + "a push names the key that replaced it: {error}" + ); + + // A deployment reads its settings from a JSON config blob. + let settings = Settings::from_toml(&crate_test_settings_str()) + .expect("should load the test settings fixture"); + let mut blob = serde_json::to_value(settings).expect("should serialize the fixture"); + blob["permission-signal"] = json!({ "sources": ["gpc", "tcf"] }); + let error = + Settings::from_json_value(blob).expect_err("should refuse the removed key at startup"); + assert!( + format!("{error:?}").contains("[permission-signal] modules"), + "startup names the key that replaced it: {error:?}" + ); + } + + #[test] + fn the_old_table_name_is_refused_with_its_new_name() { + let written = format!( + "{}\n[permission_signal]\nmodules = [\"gpc\"]\n", + crate_test_settings_str() + ); + let error = Settings::from_toml(&written).expect_err("should refuse the old table name"); + let message = format!("{error:?}"); + assert!( + message.contains("[permission_signal]") && message.contains("[permission-signal]"), + "the refusal names the old table and the new one: {message}" + ); + } - for route in &admin_routes_in_router { + #[test] + fn the_old_ad_server_table_name_is_refused_with_its_new_name() { + let written = format!( + "{}\n[adserver]\nmodule = \"adserver_mock\"\n", + crate_test_settings_str() + ); + let error = Settings::from_toml(&written).expect_err("should refuse the old table name"); + let message = format!("{error:?}"); + assert!( + message.contains("[adserver]") && message.contains("[ad-server] module"), + "the refusal names the old table and the new selector: {message}" + ); + } + + #[test] + fn a_block_of_module_settings_is_refused_as_an_unknown_field() { + // No module takes settings yet, so a block for one is refused rather + // than read and then ignored. + let written = + settings_toml_with("modules = [\"gpc\"]\n\n[permission-signal.gpc]\nenabled = true"); + let error = + Settings::from_toml(&written).expect_err("should refuse settings no module takes"); + let message = format!("{error:?}"); + assert!( + message.contains("unknown field `gpc` in [permission-signal]") + && message.contains("[permission-signal.] block is not accepted"), + "the refusal names the block and says why it is refused: {message}" + ); + } + + mod shared_tables { + use super::*; + + #[derive(Debug, Deserialize, Validate)] + #[serde(deny_unknown_fields)] + struct ExampleShared { + #[validate(url)] + endpoint: String, + } + + fn with(toml: &str) -> Result> { + Settings::from_toml(&format!("{}\n{toml}", crate_test_settings_str())) + } + + #[test] + fn a_vendor_reads_its_own_shared_table_and_nothing_else() { + let settings = with( + r#" + [shared.example] + endpoint = "https://api.example.com/v1" + "#, + ) + .expect("should load"); + + let read: ExampleShared = settings + .shared_config("example") + .expect("should read") + .expect("the table is present"); + assert_eq!(read.endpoint, "https://api.example.com/v1"); + assert!( + settings + .shared_config::("another") + .expect("an absent table reads as nothing") + .is_none() + ); + } + + #[test] + fn a_shared_table_its_vendor_cannot_read_is_refused_naming_the_table() { + let settings = with( + r#" + [shared.example] + endpoint = "https://api.example.com/v1" + unknown = true + "#, + ) + .expect("core reads none of the table, so it loads"); + + let refused = format!( + "{:?}", + settings + .shared_config::("example") + .expect_err("the vendor does not know the key") + ); assert!( - Settings::ADMIN_ENDPOINTS.contains(route), - "Router has admin route \"{route}\" that is missing from \ - Settings::ADMIN_ENDPOINTS — add it to ensure auth coverage" + refused.contains("[shared.example] could not be read") + && refused.contains("unknown"), + "{refused}" + ); + } + + #[test] + fn a_shared_table_failing_its_validation_is_refused_naming_the_table() { + let settings = with( + r#" + [shared.example] + endpoint = "not-a-url" + "#, + ) + .expect("core reads none of the table, so it loads"); + + let refused = format!( + "{:?}", + settings + .shared_config::("example") + .expect_err("the endpoint is not a URL") + ); + assert!( + refused.contains("[shared.example] failed validation"), + "{refused}" + ); + } + + #[test] + fn a_shared_table_not_named_as_a_vendor_is_refused() { + for table in ["[shared.Example]", "[shared.\"an.example\"]"] { + let refused = with(&format!("{table}\nendpoint = \"https://api.example.com\"")) + .expect_err("a vendor's table is named by its folder"); + + assert!( + format!("{refused:?}").contains("which is not a vendor's name"), + "{table}: {refused:?}" + ); + } + } + + /// The tables hold secrets, so a settings dump shows the vendors alone, + /// and the serialized settings carry the tables whole. + #[test] + fn a_shared_table_is_shown_by_its_vendor_and_serialized_whole() { + let settings = with( + r#" + [shared.example] + endpoint = "https://api.example.com/v1" + license_key = "a-secret-value" + "#, + ) + .expect("should load"); + + let shown = format!("{:?}", settings.shared); + assert!(shown.contains("example"), "{shown}"); + assert!(!shown.contains("a-secret-value"), "{shown}"); + + let serialized = serde_json::to_value(&settings).expect("should serialize"); + assert_eq!( + serialized.pointer("/shared/example/endpoint"), + Some(&serde_json::json!("https://api.example.com/v1")) ); } + + #[test] + fn settings_without_a_shared_section_are_written_without_one() { + let settings = with("").expect("should load"); + + let serialized = serde_json::to_value(&settings).expect("should serialize"); + assert!(serialized.get("shared").is_none()); + } } } diff --git a/crates/trusted-server-core/src/settings_data.rs b/crates/trusted-server-core/src/settings_data.rs index 07108fac64..b753f658f7 100644 --- a/crates/trusted-server-core/src/settings_data.rs +++ b/crates/trusted-server-core/src/settings_data.rs @@ -4,8 +4,9 @@ use serde::Deserialize; use sha2::{Digest as _, Sha256}; pub use crate::config_payload::DEFAULT_CONFIG_STORE_ID; -use crate::config_payload::{DEFAULT_SECRET_STORE_ID, settings_from_config_blob}; +use crate::config_payload::{DEFAULT_SECRET_STORE_ID, settings_from_config_blob_with}; use crate::error::TrustedServerError; +use crate::integrations::IntegrationBuilder; use crate::platform::{PlatformConfigStore, PlatformSecretStore, StoreName}; use crate::settings::Settings; @@ -68,7 +69,13 @@ pub fn default_secret_store_name() -> StoreName { StoreName::from(EnvConfig::from_env().store_name("secrets", DEFAULT_SECRET_STORE_ID)) } -/// Loads [`Settings`] from a platform config store and key. +/// Loads [`Settings`] from a platform config store and key, validating +/// against the built-in integrations only. +/// +/// A deployment that composes builders of its own calls +/// [`get_settings_from_config_store_with`] instead, because validation runs +/// during the load and refuses a `[demand]` or `[ad-server]` name it cannot +/// see a builder for. /// /// # Errors /// @@ -81,10 +88,41 @@ pub fn get_settings_from_config_store( store_name: &StoreName, key: &str, default_secret_store_name: &StoreName, +) -> Result> { + get_settings_from_config_store_with( + config_store, + secret_store, + store_name, + key, + default_secret_store_name, + &[], + ) +} + +/// Loads [`Settings`] from a platform config store and key, validating +/// against the built-in integrations followed by `extra_integrations`. +/// +/// # Errors +/// +/// Returns [`TrustedServerError::Configuration`] when the config blob is +/// missing, cannot be read, fails envelope verification, secret resolution, +/// or Trusted Server settings validation. +pub fn get_settings_from_config_store_with( + config_store: &dyn PlatformConfigStore, + secret_store: &dyn PlatformSecretStore, + store_name: &StoreName, + key: &str, + default_secret_store_name: &StoreName, + extra_integrations: &[IntegrationBuilder], ) -> Result> { let raw_value = read_config_entry(config_store, store_name, key)?; let envelope_json = resolve_fastly_chunk_pointer(config_store, store_name, &raw_value)?; - settings_from_config_blob(&envelope_json, secret_store, default_secret_store_name) + settings_from_config_blob_with( + &envelope_json, + secret_store, + default_secret_store_name, + extra_integrations, + ) } fn read_config_entry( diff --git a/crates/trusted-server-core/src/streaming_processor.rs b/crates/trusted-server-core/src/streaming_processor.rs index d95e6ebf85..e3d9272e6b 100644 --- a/crates/trusted-server-core/src/streaming_processor.rs +++ b/crates/trusted-server-core/src/streaming_processor.rs @@ -310,8 +310,8 @@ impl lol_html::OutputSink for RcVecSink { /// Adapter to use `lol_html` [`HtmlRewriter`](lol_html::HtmlRewriter) as a [`StreamProcessor`]. /// /// Output is emitted incrementally on every [`process_chunk`](StreamProcessor::process_chunk) -/// call. Script rewriters that receive text from `lol_html` must be fragment-safe — -/// they accumulate text fragments internally until `is_last_in_text_node` is true. +/// call. A text handler that receives text from `lol_html` must be fragment-safe, +/// so it accumulates text fragments until the last chunk of the text node arrives. /// /// The adapter is single-use: one adapter per request. Calling [`StreamProcessor::reset`] /// is a no-op because the rewriter consumes its settings on construction. @@ -1731,8 +1731,8 @@ mod tests { } /// Verify that `lol_html` fragments text nodes when input chunks split - /// mid-text-node. Script rewriters must be fragment-safe — they accumulate - /// text fragments internally until `is_last_in_text_node` is true. + /// mid-text-node. A text handler must be fragment-safe, so it accumulates + /// text fragments until the last chunk of the text node arrives. #[test] fn lol_html_fragments_text_across_chunk_boundaries() { use std::cell::RefCell; diff --git a/crates/trusted-server-core/src/tdl.rs b/crates/trusted-server-core/src/tdl.rs new file mode 100644 index 0000000000..8adcddb65d --- /dev/null +++ b/crates/trusted-server-core/src/tdl.rs @@ -0,0 +1,191 @@ +//! Terms Document Locators, the labels saying what terms the data offered for +//! a request is available under. +//! +//! A locator is the address of a published document a person can read, stating +//! the basis on which the data at hand may be collected, shared and used. A +//! recipient reads the locators alongside the data, decides whether those terms +//! are ones it accepts, and decides on the same basis whether it may pass the +//! data on. No locator means no terms are declared, which a recipient must not +//! read as permission. +//! +//! Nothing here interprets a document. Core carries the locators a +//! [`PermissionSignalModule`](crate::permission_signal::PermissionSignalModule) +//! declares for the request and makes them visible to what consumes the data, +//! and the module for a terms scheme decides which document applies. Model +//! Terms for Marketing (MTM) is the first such scheme to arrive and one of many +//! rather than the only one, because a publisher, a trade body or a regulator +//! can each publish terms and each set becomes a module. +//! +//! # The document must not change +//! +//! A locator has to point at a document that is never edited once published, +//! which is why a version belongs in its address. A document that can be +//! rewritten tomorrow means a recipient can never prove what it agreed to, and +//! one edit silently rewrites the basis of every transaction already sent under +//! it. That is a property of how the document is published, so no code here can +//! check it, and it is the reason this type refuses nothing but an address that +//! could not be fetched at all. + +use core::str::FromStr; + +use error_stack::Report; +use url::Url; + +use crate::error::TrustedServerError; + +/// The address of a published terms document. +/// +/// Absolute, and `http` or `https`, because a recipient has to be able to +/// fetch and read the document. See the module documentation for why the +/// document itself must be immutable and versioned. +#[derive(Debug, Clone, PartialEq, Eq, Hash, derive_more::Display)] +pub struct Tdl(String); + +impl Tdl { + /// Builds a locator from an address. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::Configuration`] when the address is not an + /// absolute `http` or `https` URL with a host, because a recipient given + /// one of those has nothing it can fetch. + /// + /// # Examples + /// + /// ``` + /// use trusted_server_core::tdl::Tdl; + /// + /// let tdl = Tdl::new("https://terms.example.com/marketing/2.txt") + /// .expect("should accept an absolute https address"); + /// assert_eq!(tdl.as_str(), "https://terms.example.com/marketing/2.txt"); + /// + /// assert!(Tdl::new("/marketing/2.txt").is_err()); + /// ``` + pub fn new(locator: &str) -> Result> { + let parsed = Url::parse(locator).map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: format!( + "Terms document locator `{locator}` is not an absolute URL: {error}" + ), + }) + })?; + if !matches!(parsed.scheme(), "http" | "https") { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "Terms document locator `{locator}` must be http or https, so a \ + recipient can read the document, and not `{}`", + parsed.scheme() + ), + })); + } + if parsed.host().is_none() { + return Err(Report::new(TrustedServerError::Configuration { + message: format!("Terms document locator `{locator}` names no host"), + })); + } + Ok(Self(locator.to_owned())) + } + + /// The address, as it is carried to a recipient. + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl FromStr for Tdl { + type Err = Report; + + fn from_str(locator: &str) -> Result { + Self::new(locator) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn an_absolute_https_address_is_accepted() { + let tdl = Tdl::new("https://terms.example.com/marketing/2.txt") + .expect("should accept an absolute https address"); + assert_eq!( + tdl.as_str(), + "https://terms.example.com/marketing/2.txt", + "should carry the address unchanged" + ); + } + + #[test] + fn an_absolute_http_address_is_accepted() { + assert!( + Tdl::new("http://terms.example.com/marketing/2.txt").is_ok(), + "should accept http, because a document served over http is still readable" + ); + } + + #[test] + fn a_relative_address_is_refused() { + let error = Tdl::new("/marketing/2.txt") + .expect_err("should refuse an address a recipient cannot resolve"); + assert!( + format!("{error:?}").contains("absolute"), + "should say the address is not absolute" + ); + } + + #[test] + fn another_scheme_is_refused() { + let error = + Tdl::new("mailto:terms@example.com").expect_err("should refuse a scheme with no page"); + assert!( + format!("{error:?}").contains("http or https"), + "should name the schemes a recipient can read" + ); + } + + #[test] + fn an_empty_address_is_refused() { + assert!( + Tdl::new("").is_err(), + "should refuse an empty address rather than carry it to a recipient" + ); + } + + #[test] + fn parsing_from_a_string_gives_the_same_answer() { + let parsed: Tdl = "https://terms.example.com/marketing/2.txt" + .parse() + .expect("should parse an absolute https address"); + assert_eq!( + parsed, + Tdl::new("https://terms.example.com/marketing/2.txt") + .expect("should accept an absolute https address"), + "should match the constructor" + ); + } + + #[test] + fn two_locators_for_the_same_document_compare_equal() { + let one = Tdl::new("https://terms.example.com/marketing/2.txt") + .expect("should accept an absolute https address"); + let two = Tdl::new("https://terms.example.com/marketing/2.txt") + .expect("should accept an absolute https address"); + assert_eq!( + one, two, + "should compare equal so the same document is carried once" + ); + } + + #[test] + fn versions_of_one_document_are_different_locators() { + let two = Tdl::new("https://terms.example.com/marketing/2.txt") + .expect("should accept an absolute https address"); + let three = Tdl::new("https://terms.example.com/marketing/3.txt") + .expect("should accept an absolute https address"); + assert_ne!( + two, three, + "should distinguish versions, because a recipient agreed to one of them" + ); + } +} diff --git a/crates/trusted-server-core/src/test_support.rs b/crates/trusted-server-core/src/test_support.rs index 37b8a9e657..9fea391024 100644 --- a/crates/trusted-server-core/src/test_support.rs +++ b/crates/trusted-server-core/src/test_support.rs @@ -1,48 +1,80 @@ -#[cfg(test)] +#[cfg(any(test, feature = "test-utils"))] pub mod tests { use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::{ConsentContext, ConsentSource, TcfConsent}; - use crate::settings::Settings; + use crate::ec::module::{EcModuleSelection, HMAC_MODULE_KEY, HOST_SIGNALS_MODULE_KEY}; + use crate::redacted::Redacted; + use crate::settings::{Ec, EcModuleBlock, HmacModuleConfig, HostSignalsModuleConfig, Settings}; #[must_use] pub fn crate_test_settings_str() -> String { r#" - [[handlers]] - path = "^/secure" - username = "user" - password = "pass" - - [[handlers]] - path = "^/_ts/admin" - username = "admin" - password = "admin-pass" - [publisher] domain = "test-publisher.com" cookie_domain = ".test-publisher.com" origin_url = "https://origin.test-publisher.com" proxy_secret = "unit-test-proxy-secret" - [integrations.prebid] - enabled = true - external_bundle_url = "https://assets.example/prebid/trusted-prebid.js" + [geo] + # A gdpr-eu country, where every permission requires a signal. This + # reproduces the prior no-default floor, so existing tests are + # unaffected by the now-required default. + # Tests run with no geo module, so single-jurisdiction operation + # is acknowledged the same way a deployment would. + assume_single_jurisdiction = true - [integrations.prebid.bundle.modules] - bidder = ["exampleBidderBidAdapter"] - - [integrations.nextjs] - enabled = false - rewrite_attributes = ["href", "link", "url"] + [auction] [ec] + module = "hmac" + + [ec.hmac] passphrase = "test-secret-key-32-bytes-minimum" + [request_signing] - config_store_id = "test-config-store-id" - secret_store_id = "test-secret-store-id" + enabled = false "# .to_owned() } + /// The shared fixture TOML with the module `name` selected in `section` + /// as well, written with the section's type folder left off, for a test + /// that appends that module's own table. + #[must_use] + pub fn crate_test_settings_str_running(section: &str, name: &str) -> String { + let written = crate::module_name::short_form(section, name); + if section == "auction" { + return crate_test_settings_str().replace( + "[auction]\n", + &format!("[auction]\nmodules = [\"{written}\"]\n"), + ); + } + format!( + "{}\n[{section}]\nmodules = [\"{written}\"]\n", + crate_test_settings_str() + ) + } + + /// The crate test configuration with its whole `[ec]` section replaced by + /// `ec_section`, which carries its own `[ec]` header and any module + /// blocks. + /// + /// # Panics + /// + /// Panics if the embedded TOML configuration no longer has an `[ec]` + /// section followed by a `[request_signing]` section. + #[must_use] + pub fn crate_test_settings_str_with_ec_section(ec_section: &str) -> String { + let base = crate_test_settings_str(); + let (before, rest) = base + .split_once("[ec]") + .expect("should find the [ec] section in the test settings"); + let (_, after) = rest + .split_once("[request_signing]") + .expect("should find the [request_signing] section in the test settings"); + format!("{before}{ec_section}\n\n[request_signing]{after}") + } + #[must_use] /// Creates test settings from embedded TOML configuration. /// @@ -56,10 +88,73 @@ pub mod tests { settings } + /// Selects the built-in HMAC module under `name` with `passphrase`, + /// replacing whatever Edge Cookie module the settings carried. + /// + /// A `name` other than `hmac` is a label, so the block names the + /// implementation it configures. + pub fn select_hmac_module(ec: &mut Ec, name: &str, passphrase: &str) { + let mut block = EcModuleBlock::from(HmacModuleConfig { + passphrase: Redacted::new(passphrase.to_owned()), + }); + if name != HMAC_MODULE_KEY { + block.implementation = Some(HMAC_MODULE_KEY.to_owned()); + } + ec.module = Some(EcModuleSelection::from(name)); + ec.module_blocks.clear(); + ec.module_blocks.insert(name.to_owned(), block); + } + + /// Selects the built-in host-signal module under its own name with + /// `passphrase`, replacing whatever Edge Cookie module the settings + /// carried. + pub fn select_host_signals_module(ec: &mut Ec, passphrase: &str) { + ec.module = Some(EcModuleSelection::from(HOST_SIGNALS_MODULE_KEY)); + ec.module_blocks.clear(); + ec.module_blocks.insert( + HOST_SIGNALS_MODULE_KEY.to_owned(), + EcModuleBlock::from(HostSignalsModuleConfig { + passphrase: Redacted::new(passphrase.to_owned()), + }), + ); + } + + /// The passphrase the block `name` holds. + /// + /// # Panics + /// + /// Panics if `name` has no block, or if its block configures another + /// module. + #[must_use] + #[allow( + clippy::panic, + reason = "a fixture names the block it could not find, which `expect` cannot" + )] + pub fn hmac_passphrase<'a>(ec: &'a Ec, name: &str) -> &'a str { + ec.module_blocks + .get(name) + .and_then(EcModuleBlock::hmac_settings) + .unwrap_or_else(|| panic!("settings should configure the hmac module under `{name}`")) + .passphrase + .expose() + } + /// A valid EC ID in `{64-hex}.{6-alnum}` format for use in tests. pub const VALID_SYNTHETIC_ID: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.Ab1234"; + /// The permissions of a reader who allowed storage and withheld + /// personalised-ad selection: the Edge Cookie stands, partner sharing does + /// not. + #[must_use] + pub fn storage_only_permissions() -> crate::permissions::PermissionState { + crate::permissions::PermissionState::new( + [crate::permissions::Permission::StoreOnDevice] + .into_iter() + .collect(), + ) + } + /// GDPR consent granting TCF Purpose 1 (storage/EC) but denying Purpose 4 /// (personalized ads), so EC is allowed while EID persistence is not. #[must_use] @@ -94,6 +189,71 @@ pub mod tests { } } +/// Page fixtures shared by core's tests and the tests of a module crate. +#[cfg(any(test, feature = "test-utils"))] +pub mod fixtures { + /// A publisher page carrying the tags of several vendors, as an origin + /// would serve it. + pub const PUBLISHER_PAGE_HTML: &str = include_str!("html_processor.test.html"); +} + +/// The operator-facing settings template, for tests that check what it +/// documents. +#[cfg(any(test, feature = "test-utils"))] +pub mod template { + /// The source-controlled template an operator starts from. + pub const EXAMPLE_TEMPLATE: &str = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../trusted-server.example.toml" + )); + + /// The template with its required secret-store key references replaced by + /// resolved test values, so parsing it as settings can exercise the + /// optional blocks a test uncomments. + #[must_use] + pub fn template_with_resolved_required_secrets() -> String { + EXAMPLE_TEMPLATE + .replace( + "proxy_secret = \"publisher_proxy_secret\"", + "proxy_secret = \"unit-test-resolved-publisher-proxy-secret-0001\"", + ) + .replace( + "passphrase = \"ec_passphrase\"", + "passphrase = \"unit-test-resolved-ec-passphrase-secret-0001\"", + ) + } + + /// Uncomments the contiguous `#`-prefixed block that begins at the line + /// `# {header}`, leaving the rest of the template untouched. Stops at the + /// first line that is not a comment, so a blank line ends the block. + #[must_use] + pub fn uncomment_block(template: &str, header: &str) -> String { + let header_line = format!("# {header}"); + let mut out = Vec::new(); + let mut uncommenting = false; + + for line in template.lines() { + if line == header_line { + uncommenting = true; + } else if uncommenting && !line.trim_start().starts_with('#') { + uncommenting = false; + } + + if uncommenting { + let bare = line + .strip_prefix("# ") + .or_else(|| line.strip_prefix('#')) + .unwrap_or(line); + out.push(bare.to_owned()); + } else { + out.push(line.to_owned()); + } + } + + out.join("\n") + } +} + /// Shared Next.js + auction origin fixture. /// /// Adapters exercise the buffered publisher path against this fixture in their @@ -132,11 +292,6 @@ pub mod nextjs_auction { pub fn settings() -> Settings { let mut settings = Settings::from_toml( r#" - [[handlers]] - path = "^/_ts/admin" - username = "admin" - password = "admin-pass" - [publisher] domain = "test-publisher.example.com" cookie_domain = ".test-publisher.example.com" @@ -145,30 +300,53 @@ pub mod nextjs_auction { [ec] passphrase = "test-secret-key-32-bytes-minimum" + + # The fixture supplies its own geo module through the runtime + # services, so it selects the host lookup to reach it. Without a + # selection the jurisdiction is unknown, and a server-side auction + # fails closed on an unknown jurisdiction. + [geo] + module = "platform" + + # What Next.js changes in a page is a middleware, which runs on + # the pages an entry names it for. + [[fetch]] + media_type = "text/html" + middleware = ["framework.nextjs"] "#, ) .expect("should parse Next.js auction fixture settings"); settings - .integrations - .insert_config( - "nextjs", + .insert_module_config( + "framework", + "framework.nextjs", &serde_json::json!({ - "enabled": true, "rewrite_attributes": ["href", "link", "url"], }), ) - .expect("should enable the fixture Next.js integration"); + .expect("should select the fixture Next.js integration"); settings.auction.enabled = true; - settings.auction.mediator = None; - settings.auction.providers = serde_json::from_value(serde_json::json!({ - "fixture": { - "protocol": "openrtb-2.6", - "endpoint": "https://auction.example.com/bid", - "routing": "all_eligible", - "timeout_ms": 5000 - } - })) - .expect("should configure the fixture auction provider"); + // One `[demand.fixture]` source, which is where an auction provider is + // configured. `implementation` states the wire format, so there is no + // separate `protocol`. + settings.demand = crate::provider_table::ProviderTable::new( + vec!["fixture".to_owned()], + std::collections::BTreeMap::from([( + "fixture".to_owned(), + serde_json::Map::from_iter([ + ( + "implementation".to_owned(), + serde_json::json!("auction-protocol.openrtb"), + ), + ( + "endpoint".to_owned(), + serde_json::json!("https://auction.example.com/bid"), + ), + ("routing".to_owned(), serde_json::json!("all_eligible")), + ("timeout_ms".to_owned(), serde_json::json!(5000)), + ]), + )]), + ); settings.creative_opportunities = Some( toml::from_str( r#" @@ -312,10 +490,12 @@ pub mod nextjs_auction { struct FixtureGeo; + #[async_trait::async_trait(?Send)] impl PlatformGeo for FixtureGeo { - fn lookup( + async fn lookup( &self, _client_ip: Option, + _services: &RuntimeServices, ) -> Result, Report> { Ok(Some(GeoInfo { country: "AU".to_owned(), diff --git a/crates/trusted-server-core/src/tester_cookie.rs b/crates/trusted-server-core/src/tester_cookie.rs index 0eeda9fcb3..8dc59647ab 100644 --- a/crates/trusted-server-core/src/tester_cookie.rs +++ b/crates/trusted-server-core/src/tester_cookie.rs @@ -1,7 +1,8 @@ //! Tester-cookie endpoint helpers. //! //! The tester routes are intentionally disabled unless configured. When enabled, -//! they set or clear a first-party `ts-tester` cookie scoped to the configured +//! they set or clear a first-party `-tester` cookie, such as +//! `ts-tester`, scoped to the configured //! publisher cookie domain. They are self-service routes rather than admin //! routes; the cookie only affects tester routing and must not gate sensitive //! behavior. @@ -10,31 +11,36 @@ use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt}; use http::{HeaderValue, Response, StatusCode, header}; -use crate::constants::COOKIE_TS_TESTER; +use crate::constants::COOKIE_TESTER; use crate::error::TrustedServerError; +use crate::prefix::PublicPrefix; use crate::settings::Settings; /// Formats the tester cookie `Set-Cookie` header value. -fn format_tester_cookie(domain: &str, max_age_seconds: u32) -> String { +fn format_tester_cookie(public: &PublicPrefix, domain: &str, max_age_seconds: u32) -> String { format!( "{}=true; Domain={}; Path=/; Secure; SameSite=Lax; Max-Age={}", - COOKIE_TS_TESTER, domain, max_age_seconds, + public.cookie_name(COOKIE_TESTER), + domain, + max_age_seconds, ) } /// Formats the tester cookie clearing `Set-Cookie` header value. -fn format_clear_tester_cookie(domain: &str) -> String { +fn format_clear_tester_cookie(public: &PublicPrefix, domain: &str) -> String { format!( "{}=; Domain={}; Path=/; Secure; SameSite=Lax; Max-Age=0", - COOKIE_TS_TESTER, domain, + public.cookie_name(COOKIE_TESTER), + domain, ) } /// Handles `GET /_ts/set-tester`. /// /// Returns `404 Not Found` while `[tester_cookie].enabled` is false. When the -/// feature is enabled, returns `204 No Content` with `Set-Cookie: ts-tester=true` -/// scoped to `publisher.cookie_domain` and persisted for +/// feature is enabled, returns `204 No Content` with `Set-Cookie` writing +/// `-tester=true`, such as `ts-tester=true`, scoped to +/// `publisher.cookie_domain` and persisted for /// `[tester_cookie].max_age_seconds` (30 days by default). /// /// # Errors @@ -43,6 +49,7 @@ fn format_clear_tester_cookie(domain: &str) -> String { /// domain cannot be rendered as an HTTP header value. pub fn handle_set_tester( settings: &Settings, + public: &PublicPrefix, ) -> Result, Report> { if !settings.tester_cookie.enabled { let mut response = Response::new(EdgeBody::empty()); @@ -51,6 +58,7 @@ pub fn handle_set_tester( } let set_cookie = HeaderValue::from_str(&format_tester_cookie( + public, &settings.publisher.cookie_domain, settings.tester_cookie.max_age_seconds, )) @@ -74,7 +82,7 @@ pub fn handle_set_tester( /// /// Returns `404 Not Found` while `[tester_cookie].enabled` is false. When the /// feature is enabled, returns `204 No Content` with `Set-Cookie` expiring the -/// `ts-tester` cookie scoped to `publisher.cookie_domain`. +/// tester cookie scoped to `publisher.cookie_domain`. /// /// # Errors /// @@ -82,6 +90,7 @@ pub fn handle_set_tester( /// domain cannot be rendered as an HTTP header value. pub fn handle_clear_tester( settings: &Settings, + public: &PublicPrefix, ) -> Result, Report> { if !settings.tester_cookie.enabled { let mut response = Response::new(EdgeBody::empty()); @@ -90,6 +99,7 @@ pub fn handle_clear_tester( } let set_cookie = HeaderValue::from_str(&format_clear_tester_cookie( + public, &settings.publisher.cookie_domain, )) .change_context(TrustedServerError::InvalidHeaderValue { @@ -119,7 +129,8 @@ mod tests { settings.tester_cookie.enabled = true; settings.publisher.cookie_domain = ".tester.example".to_string(); - let response = handle_set_tester(&settings).expect("should build tester response"); + let response = handle_set_tester(&settings, &PublicPrefix::default()) + .expect("should build tester response"); assert_eq!( response.status(), @@ -154,7 +165,8 @@ mod tests { settings.tester_cookie.max_age_seconds = 3_600; settings.publisher.cookie_domain = ".tester.example".to_string(); - let response = handle_set_tester(&settings).expect("should build tester response"); + let response = handle_set_tester(&settings, &PublicPrefix::default()) + .expect("should build tester response"); let set_cookie = response .headers() @@ -169,11 +181,32 @@ mod tests { ); } + #[test] + fn tester_cookie_is_named_after_the_public_prefix() { + let mut settings = create_test_settings(); + settings.tester_cookie.enabled = true; + let public = PublicPrefix::new(["_ex"]).expect("should accept the prefix"); + + for response in [ + handle_set_tester(&settings, &public).expect("should set the tester cookie"), + handle_clear_tester(&settings, &public).expect("should clear the tester cookie"), + ] { + let set_cookie = response + .headers() + .get(header::SET_COOKIE) + .expect("should write the tester cookie") + .to_str() + .expect("should render set-cookie as utf-8"); + assert!(set_cookie.starts_with("_ex-tester="), "{set_cookie}"); + } + } + #[test] fn tester_cookie_route_is_disabled_by_default() { let settings = create_test_settings(); - let response = handle_set_tester(&settings).expect("should build disabled tester response"); + let response = handle_set_tester(&settings, &PublicPrefix::default()) + .expect("should build disabled tester response"); assert_eq!( response.status(), @@ -192,7 +225,8 @@ mod tests { settings.tester_cookie.enabled = true; settings.publisher.cookie_domain = ".tester.example".to_string(); - let response = handle_clear_tester(&settings).expect("should build clear tester response"); + let response = handle_clear_tester(&settings, &PublicPrefix::default()) + .expect("should build clear tester response"); assert_eq!( response.status(), @@ -224,8 +258,8 @@ mod tests { fn clear_tester_cookie_route_is_disabled_by_default() { let settings = create_test_settings(); - let response = - handle_clear_tester(&settings).expect("should build disabled clear tester response"); + let response = handle_clear_tester(&settings, &PublicPrefix::default()) + .expect("should build disabled clear tester response"); assert_eq!( response.status(), diff --git a/crates/trusted-server-core/src/tsjs.rs b/crates/trusted-server-core/src/tsjs.rs index e8275b8144..0ad3b765f2 100644 --- a/crates/trusted-server-core/src/tsjs.rs +++ b/crates/trusted-server-core/src/tsjs.rs @@ -1,24 +1,30 @@ -use trusted_server_js::{concatenated_hash, single_module_hash}; +//! URLs and `", - tsjs_script_src(module_ids), + tsjs_script_src(parts), ) } -/// `/static` URL for the unified bundle when exact module IDs are unavailable. +/// `/static` URL for the unified bundle when the exact parts are unavailable. /// /// This intentionally omits `?v=` because the serving path can only mark a URL /// immutable when the hash matches the exact enabled module set. Use -/// [`tsjs_script_src`] with exact module IDs when [`IntegrationRegistry`] is -/// available. +/// [`tsjs_script_src`] with the registry's parts when [`IntegrationRegistry`] +/// is available. /// /// [`IntegrationRegistry`]: crate::integrations::IntegrationRegistry #[must_use] @@ -59,7 +65,7 @@ pub fn tsjs_unified_script_src() -> String { "/static/tsjs=tsjs-unified.min.js".to_string() } -/// `", - tsjs_deferred_script_src(module_id) + tsjs_deferred_script_src(part) ) } -/// Generate all deferred `"), "should generate exactly one trusted server script tag" ); @@ -204,18 +251,18 @@ mod tests { #[test] fn publisher_tsjs_script_tag_renders_static_attributes() { - let module_ids = ["gpt"]; - let src = tsjs_script_src(&module_ids); + let parts = compile_time_parts(&["gpt"]); + let src = tsjs_script_src(&parts); assert_eq!( - tsjs_script_tag_with_attributes(&module_ids, &[("data-ts-gam-attribution", "true")]), + tsjs_script_tag_with_attributes(&parts, &[("data-ts-gam-attribution", "true")]), format!( "" ), "should render trusted static attributes on the publisher bundle tag" ); assert_eq!( - tsjs_script_tag(&module_ids), + tsjs_script_tag(&parts), format!(""), "should keep the generic tag byte-for-byte unmarked" ); @@ -226,7 +273,10 @@ mod tests { expected = "attribute name should contain only lowercase ASCII letters, digits, and hyphens" )] fn publisher_tsjs_script_tag_rejects_invalid_attribute_name() { - let _ = tsjs_script_tag_with_attributes(&["gpt"], &[("data-bad_name", "true")]); + let _ = tsjs_script_tag_with_attributes( + &compile_time_parts(&["gpt"]), + &[("data-bad_name", "true")], + ); } #[test] @@ -234,31 +284,43 @@ mod tests { expected = "attribute name should contain only lowercase ASCII letters, digits, and hyphens" )] fn publisher_tsjs_script_tag_rejects_empty_attribute_name() { - let _ = tsjs_script_tag_with_attributes(&["gpt"], &[("", "true")]); + let _ = tsjs_script_tag_with_attributes(&compile_time_parts(&["gpt"]), &[("", "true")]); } #[test] #[should_panic(expected = "attribute value should not contain HTML-sensitive characters")] fn publisher_tsjs_script_tag_rejects_double_quote_in_attribute_value() { - let _ = tsjs_script_tag_with_attributes(&["gpt"], &[("data-safe-name", "bad\"value")]); + let _ = tsjs_script_tag_with_attributes( + &compile_time_parts(&["gpt"]), + &[("data-safe-name", "bad\"value")], + ); } #[test] #[should_panic(expected = "attribute value should not contain HTML-sensitive characters")] fn publisher_tsjs_script_tag_rejects_ampersand_in_attribute_value() { - let _ = tsjs_script_tag_with_attributes(&["gpt"], &[("data-safe-name", "bad&value")]); + let _ = tsjs_script_tag_with_attributes( + &compile_time_parts(&["gpt"]), + &[("data-safe-name", "bad&value")], + ); } #[test] #[should_panic(expected = "attribute value should not contain HTML-sensitive characters")] fn publisher_tsjs_script_tag_rejects_less_than_in_attribute_value() { - let _ = tsjs_script_tag_with_attributes(&["gpt"], &[("data-safe-name", "badvalue")]); + let _ = tsjs_script_tag_with_attributes( + &compile_time_parts(&["gpt"]), + &[("data-safe-name", "bad>value")], + ); } #[test] @@ -278,36 +340,58 @@ mod tests { #[test] fn tsjs_single_module_script_src_formats_known_module_url_with_hash() { - let src = tsjs_single_module_script_src("creative"); + let parts = compile_time_parts(&["creative"]); + let src = tsjs_single_module_script_src(&parts[0]); assert!( src.starts_with("/static/tsjs=tsjs-creative.min.js?v="), "should use per-module static bundle path" ); assert_sha256_hex_hash(hash_query_value(&src)); + assert_eq!( + src, + format!( + "/static/tsjs=tsjs-creative.min.js?v={}", + trusted_server_js::single_module_hash("creative") + .expect("should have compiled creative in") + ), + "should keep today's URL for a compile-time module" + ); } #[test] - fn tsjs_deferred_script_src_hashes_prebid_shim_and_empties_unknown_module() { - let prebid_src = tsjs_deferred_script_src("prebid"); + fn tsjs_deferred_script_src_hashes_prebid_shim() { + let parts = compile_time_parts(&["prebid"]); + let prebid_src = tsjs_deferred_script_src(&parts[0]); + assert!( prebid_src.starts_with("/static/tsjs=tsjs-prebid.min.js?v="), "prebid shim should be served from the deferred tsjs route" ); assert_sha256_hex_hash(hash_query_value(&prebid_src)); + } + + #[test] + fn tsjs_deferred_script_tag_carries_a_carried_parts_own_hash() { + let part = carried_part("probe", "(() => { window.probe = 1; })()"); + assert_eq!( - tsjs_deferred_script_src("unknown-module"), - "/static/tsjs=tsjs-unknown-module.min.js?v=", - "should document current unknown-module hash behavior" + tsjs_deferred_script_tag(&part), + format!( + "", + part.sha256 + ), + "should version a carried deferred module by its own content hash" ); } #[test] fn tsjs_deferred_script_tag_marks_script_defer() { - let src = tsjs_deferred_script_src("prebid"); + let parts = compile_time_parts(&["prebid"]); + let src = tsjs_deferred_script_src(&parts[0]); assert_eq!( - tsjs_deferred_script_tag("prebid"), + tsjs_deferred_script_tag(&parts[0]), format!(""), "should generate a deferred script tag" ); @@ -324,12 +408,14 @@ mod tests { #[test] fn tsjs_deferred_script_tags_preserves_input_order() { + let parts = compile_time_parts(&["prebid", "creative"]); + assert_eq!( - tsjs_deferred_script_tags(&["prebid", "creative"]), + tsjs_deferred_script_tags(&parts), format!( "{}{}", - tsjs_deferred_script_tag("prebid"), - tsjs_deferred_script_tag("creative") + tsjs_deferred_script_tag(&parts[0]), + tsjs_deferred_script_tag(&parts[1]) ), "should preserve caller-provided deferred module order" ); @@ -353,18 +439,9 @@ mod tests { #[test] fn tsjs_script_src_differs_for_different_module_sets() { assert_ne!( - tsjs_script_src(&["lockr"]), - tsjs_script_src(&["lockr", "permutive"]), + tsjs_script_src(&compile_time_parts(&["lockr"])), + tsjs_script_src(&compile_time_parts(&["lockr", "permutive"])), "should bust the cache when the module set content changes" ); } - - #[test] - fn tsjs_deferred_script_src_has_empty_hash_for_unknown_module() { - assert_eq!( - tsjs_deferred_script_src("does-not-exist"), - "/static/tsjs=tsjs-does-not-exist.min.js?v=", - "should fall back to an empty cache-busting hash for an unknown module" - ); - } } diff --git a/crates/trusted-server-core/src/tsjs_bundle.rs b/crates/trusted-server-core/src/tsjs_bundle.rs new file mode 100644 index 0000000000..4e98e491c8 --- /dev/null +++ b/crates/trusted-server-core/src/tsjs_bundle.rs @@ -0,0 +1,413 @@ +//! Composition of the served tsjs script from module parts. +//! +//! `trusted-server-js` knows only the modules compiled into it. A module a +//! vendor crate carries on its registration is not in that map, so the +//! composition of the served script and its cache-busting hash live here, +//! keyed on content rather than on ids. The byte rule is unchanged from +//! `trusted_server_js::concatenate_modules`: core first, then each part in +//! order, joined by `;\n`, so every existing `?v=` hash is preserved. + +use std::collections::HashMap; +use std::sync::{Mutex, MutexGuard, OnceLock}; + +use sha2::{Digest as _, Sha256}; + +/// One module of the served script. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct JsModulePart { + /// Module id, for example `core` or `lockr`. + pub id: &'static str, + /// The built IIFE. + pub source: &'static str, + /// SHA-256 of `source`, hex encoded. Identifies the content in the memo. + /// + /// The memo in [`compose_hash`] trusts this value rather than hashing + /// `source` again, so a part that declares the wrong hash serves a stale + /// `?v=` under a valid-looking URL. Debug builds and tests check the + /// value against `source`; release builds do not. + pub sha256: &'static str, +} + +impl JsModulePart { + /// Looks up a compile-time module of `trusted-server-js` by id. + /// + /// Returns `None` when no module with that id was compiled in. + /// + /// # Examples + /// + /// ``` + /// use trusted_server_core::tsjs_bundle::JsModulePart; + /// + /// assert!(JsModulePart::compile_time("core").is_some()); + /// assert!(JsModulePart::compile_time("not-a-module").is_none()); + /// ``` + #[must_use] + pub fn compile_time(id: &'static str) -> Option { + let source = trusted_server_js::module_bundle(id)?; + let sha256 = trusted_server_js::single_module_hash(id)?; + Some(Self { id, source, sha256 }) + } +} + +/// Resolves compile-time parts for a list of ids, dropping unknown ids, as +/// `trusted_server_js` does today. +/// +/// # Examples +/// +/// ``` +/// use trusted_server_core::tsjs_bundle::compile_time_parts; +/// +/// let parts = compile_time_parts(&["core", "not-a-module"]); +/// +/// assert_eq!(parts.len(), 1); +/// assert_eq!(parts[0].id, "core"); +/// ``` +#[must_use] +pub fn compile_time_parts(ids: &[&'static str]) -> Vec { + ids.iter() + .filter_map(|id| JsModulePart::compile_time(id)) + .collect() +} + +/// Concatenates the parts into the served script. +/// +/// Core comes first (the given `core` part, or the compile-time core when +/// none is given), then the remaining parts in input order. Each id appears +/// once, keeping its first occurrence, and parts are joined by `;\n` so one +/// IIFE cannot run into the next. +/// +/// # Examples +/// +/// ``` +/// use sha2::{Digest as _, Sha256}; +/// use trusted_server_core::tsjs_bundle::{JsModulePart, compose}; +/// +/// fn part(id: &'static str, source: &'static str) -> JsModulePart { +/// let sha256 = Box::leak(hex::encode(Sha256::digest(source)).into_boxed_str()); +/// JsModulePart { id, source, sha256 } +/// } +/// +/// let parts = [ +/// part("example", "(() => { window.example = true; })()"), +/// part("core", "(() => { window.tsjs = {}; })()"), +/// ]; +/// +/// assert_eq!( +/// compose(&parts), +/// "(() => { window.tsjs = {}; })();\n(() => { window.example = true; })()" +/// ); +/// ``` +#[must_use] +pub fn compose(parts: &[JsModulePart]) -> String { + let ordered = ordered(parts); + // Every piece the visit yields has a known length before the walk, so + // reserve the exact byte count once rather than letting the pushes grow + // the buffer. A bundle of a dozen parts is hundreds of kilobytes, and + // growing to that size copies roughly twice the bundle on every request + // that serves it. + let mut size = 0; + visit_parts(&ordered, |part| size += part.len()); + let mut body = String::with_capacity(size); + visit_parts(&ordered, |part| body.push_str(part)); + body +} + +/// SHA-256 of [`compose`]'s output, hex encoded, without materializing it. +/// +/// The result is memoized per ordered set of `(id, sha256)` pairs. Because the +/// key carries each part's content hash, a carried module that keeps its id +/// but changes its source gets a new hash. The memo never evicts, so feed it +/// only sets derived from configuration, never sets derived from request +/// input. +/// +/// The memo can only hit where the process outlives the request, which of the +/// four adapters means the Axum dev server alone, because its `main` builds +/// the router once before serving. Fastly starts a fresh Wasm instance per +/// request, and `edgezero_adapter_cloudflare::run_app` and +/// `edgezero_adapter_spin::run_app` both call `build_app` inside the +/// per-request entry point, so on those three every call is a miss. A miss +/// costs a key vector, two mutex locks and a stored copy of the hash, all +/// beside a SHA-256 over the whole bundle, so the memo is close to free where +/// it cannot hit and removes the hash entirely where it can. +/// +/// # Panics +/// +/// In debug builds, panics when a part's `sha256` is not the SHA-256 of its +/// `source`. Release builds trust the declared hash. +/// +/// # Examples +/// +/// ``` +/// use sha2::{Digest as _, Sha256}; +/// use trusted_server_core::tsjs_bundle::{JsModulePart, compose_hash}; +/// +/// fn part(id: &'static str, source: &'static str) -> JsModulePart { +/// let sha256 = Box::leak(hex::encode(Sha256::digest(source)).into_boxed_str()); +/// JsModulePart { id, source, sha256 } +/// } +/// +/// let core = JsModulePart::compile_time("core").expect("should have compiled core in"); +/// let before = [core, part("example", "(() => { window.example = 1; })()")]; +/// let after = [core, part("example", "(() => { window.example = 2; })()")]; +/// +/// assert_eq!(compose_hash(&before).len(), 64); +/// assert_eq!(compose_hash(&before), compose_hash(&before)); +/// assert_ne!(compose_hash(&before), compose_hash(&after)); +/// ``` +#[must_use] +pub fn compose_hash(parts: &[JsModulePart]) -> String { + for part in parts { + debug_assert_eq!( + part.sha256, + hex::encode(Sha256::digest(part.source)), + "should declare the SHA-256 of its source for part `{}`", + part.id + ); + } + + let ordered = ordered(parts); + let key = ordered + .iter() + .map(|part| (part.id, part.sha256)) + .collect::>(); + if let Some(hash) = lock_cache().get(&key).cloned() { + return hash; + } + + let mut hasher = Sha256::new(); + visit_parts(&ordered, |part| hasher.update(part.as_bytes())); + let hash = hex::encode(hasher.finalize()); + lock_cache().insert(key, hash.clone()); + hash +} + +/// Orders the parts for the served script: core first, then every non-core +/// part in input order, keeping the first occurrence of each id. +fn ordered(parts: &[JsModulePart]) -> Vec { + let mut result = Vec::with_capacity(parts.len() + 1); + + let core = parts + .iter() + .find(|part| part.id == "core") + .copied() + .or_else(|| JsModulePart::compile_time("core")); + if let Some(core) = core { + result.push(core); + } + + for part in parts { + if part.id == "core" { + continue; + } + if result.iter().any(|taken| taken.id == part.id) { + continue; + } + result.push(*part); + } + + result +} + +/// Visits the byte pieces of the served script in order, with `;\n` between +/// consecutive parts. +fn visit_parts(parts: &[JsModulePart], mut visit: F) { + let mut first = true; + for part in parts { + if first { + first = false; + } else { + visit(";\n"); + } + visit(part.source); + } +} + +type HashCache = HashMap, String>; + +fn lock_cache() -> MutexGuard<'static, HashCache> { + static CACHE: OnceLock> = OnceLock::new(); + match CACHE.get_or_init(|| Mutex::new(HashMap::new())).lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sha256_hex(bytes: &[u8]) -> String { + hex::encode(Sha256::digest(bytes)) + } + + /// Builds a part whose `sha256` really is the hash of `source`, so no two + /// parts with different content ever share a memo key. + fn part(id: &'static str, source: &'static str) -> JsModulePart { + let sha256 = Box::leak(sha256_hex(source.as_bytes()).into_boxed_str()); + JsModulePart { id, source, sha256 } + } + + #[test] + fn compose_puts_core_first_and_joins_with_a_semicolon_and_newline() { + let parts = [ + part("lockr", "L"), + part("core", "C"), + part("permutive", "P"), + ]; + + assert_eq!( + compose(&parts), + "C;\nL;\nP", + "should order core first and join parts" + ); + } + + #[test] + fn compose_hash_matches_the_compile_time_hash_for_built_in_modules() { + let ids = ["lockr", "permutive"]; + let parts = compile_time_parts(&ids); + + assert_eq!( + compose_hash(&parts), + trusted_server_js::concatenated_hash(&ids), + "should reproduce today's hash for a built-in module set" + ); + assert_eq!( + compose(&parts), + trusted_server_js::concatenate_modules(&ids), + "should reproduce today's bytes for a built-in module set" + ); + } + + #[test] + fn compose_of_no_parts_is_the_compile_time_core_alone() { + assert_eq!( + compose(&[]), + trusted_server_js::concatenate_modules(&[]), + "should serve core alone when no parts are given" + ); + assert_eq!( + compose_hash(&[]), + trusted_server_js::concatenated_hash(&[]), + "should hash core alone when no parts are given" + ); + } + + #[test] + fn compose_matches_every_compile_time_module_set_the_old_api_produces() { + let all = trusted_server_js::all_module_ids(); + let non_core = all + .iter() + .copied() + .filter(|id| *id != "core") + .collect::>(); + let mut cases = vec![all.to_vec(), non_core.clone()]; + cases.push(non_core.iter().rev().copied().collect()); + cases.push(vec!["core"]); + + for ids in cases { + let parts = compile_time_parts(&ids); + assert_eq!( + compose(&parts), + trusted_server_js::concatenate_modules(&ids), + "should reproduce today's bytes for {ids:?}" + ); + assert_eq!( + compose_hash(&parts), + trusted_server_js::concatenated_hash(&ids), + "should reproduce today's hash for {ids:?}" + ); + } + } + + #[test] + fn compose_hash_changes_when_a_carried_module_changes() { + let before = [part("core", "C"), part("probe", "A")]; + let after = [part("core", "C"), part("probe", "B")]; + + assert_ne!( + compose_hash(&before), + compose_hash(&after), + "should hash carried content" + ); + } + + #[test] + fn compose_keeps_a_duplicate_id_once_using_its_first_occurrence() { + let parts = [ + part("core", "C"), + part("lockr", "L1"), + part("permutive", "P"), + part("lockr", "L2"), + part("core", "C2"), + ]; + + assert_eq!( + compose(&parts), + "C;\nL1;\nP", + "should keep the first occurrence of each id and drop later ones" + ); + } + + #[test] + fn compose_reserves_the_exact_bundle_size_before_writing_it() { + let parts = compile_time_parts(trusted_server_js::all_module_ids()); + let body = compose(&parts); + + assert_eq!( + body.capacity(), + body.len(), + "should allocate the bundle once at its exact size" + ); + } + + #[test] + fn compose_hash_is_the_hex_sha256_of_compose() { + let parts = [ + part("core", "(() => {})()"), + part("carried", "(() => { window.carried = true; })()"), + part("lockr", "L"), + ]; + + assert_eq!( + compose_hash(&parts), + sha256_hex(compose(&parts).as_bytes()), + "should hash the exact bytes compose produces" + ); + } + + #[test] + #[should_panic(expected = "should declare the SHA-256 of its source for part `lying`")] + fn compose_hash_rejects_a_part_whose_declared_hash_is_wrong() { + let parts = [ + part("core", "C"), + JsModulePart { + id: "lying", + source: "A", + sha256: "not-the-hash-of-a", + }, + ]; + + let _ = compose_hash(&parts); + } + + #[test] + fn compile_time_parts_drops_unknown_ids() { + let parts = compile_time_parts(&["lockr", "not-a-module", "permutive"]); + let ids = parts.iter().map(|part| part.id).collect::>(); + + assert_eq!( + ids, + ["lockr", "permutive"], + "should keep known ids in order and drop unknown ids" + ); + for part in parts { + assert_eq!( + part.sha256, + sha256_hex(part.source.as_bytes()), + "should carry the compile-time hash of {}", + part.id + ); + } + } +} diff --git a/crates/trusted-server-core/src/version_header.rs b/crates/trusted-server-core/src/version_header.rs deleted file mode 100644 index 822e04b26c..0000000000 --- a/crates/trusted-server-core/src/version_header.rs +++ /dev/null @@ -1,176 +0,0 @@ -//! `x-ts-version`: the deployed git version compiled in by `build.rs`. - -use edgezero_core::http::{HeaderValue, Response}; - -use crate::constants::{HEADER_X_TS_VERSION, TS_GIT_VERSION}; - -/// Returns the compiled-in git version as a header value. -/// -/// `None` when no version is known or the value is not a valid header value. -#[must_use] -pub fn git_version_header_value() -> Option { - header_value_from(TS_GIT_VERSION) -} - -/// Converts `version` to a header value, logging and returning `None` when it -/// is not a valid header value. -#[must_use] -fn header_value_from(version: Option<&str>) -> Option { - let version = version?; - match HeaderValue::from_str(version) { - Ok(value) => Some(value), - Err(_) => { - log::warn!("Skipping invalid TRUSTED_SERVER__GIT_VERSION response header value"); - None - } - } -} - -/// Sets `x-ts-version` to the compiled-in git version, if one is known. -pub fn apply_git_version_header(response: &mut Response) { - apply_git_version_header_from(TS_GIT_VERSION, response); -} - -/// Sets `x-ts-version` to `version`, or removes it when `version` is unknown -/// or invalid, so an origin's `x-ts-version` is never reported as ours. -fn apply_git_version_header_from(version: Option<&str>, response: &mut Response) { - if let Some(value) = header_value_from(version) { - response.headers_mut().insert(HEADER_X_TS_VERSION, value); - } else { - response.headers_mut().remove(HEADER_X_TS_VERSION); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use edgezero_core::body::Body; - use edgezero_core::http::response_builder; - - fn empty_response() -> Response { - response_builder() - .body(Body::empty()) - .expect("should build empty test response") - } - - fn response_with_upstream_version() -> Response { - response_builder() - .header(HEADER_X_TS_VERSION, "upstream-v1") - .body(Body::empty()) - .expect("should build test response with an upstream x-ts-version") - } - - fn version_of(response: &Response) -> Option<&str> { - response - .headers() - .get(HEADER_X_TS_VERSION) - .and_then(|v| v.to_str().ok()) - } - - #[test] - fn header_value_from_valid_version() { - assert_eq!( - header_value_from(Some("v1.2.3")), - Some(HeaderValue::from_static("v1.2.3")), - "should convert a valid version" - ); - } - - #[test] - fn header_value_from_unknown_or_invalid_version() { - assert_eq!(header_value_from(None), None, "should be None when unknown"); - assert_eq!( - header_value_from(Some("bad\nvalue")), - None, - "should be None for a non-header-safe version" - ); - } - - #[test] - fn git_version_header_value_matches_compiled_in_version() { - assert_eq!( - git_version_header_value() - .as_ref() - .and_then(|v| v.to_str().ok()), - TS_GIT_VERSION, - "should convert the compiled-in TS_GIT_VERSION" - ); - } - - #[test] - fn sets_header_from_version() { - let mut response = empty_response(); - apply_git_version_header_from(Some("v1.2.3"), &mut response); - assert_eq!( - version_of(&response), - Some("v1.2.3"), - "should set x-ts-version" - ); - } - - #[test] - fn omits_header_when_version_unknown() { - let mut response = empty_response(); - apply_git_version_header_from(None, &mut response); - assert_eq!( - version_of(&response), - None, - "should omit x-ts-version when unknown" - ); - } - - #[test] - fn skips_invalid_header_value() { - let mut response = empty_response(); - apply_git_version_header_from(Some("bad\nvalue"), &mut response); - assert_eq!( - version_of(&response), - None, - "should skip a non-header-safe version" - ); - } - - #[test] - fn replaces_upstream_header_with_version() { - let mut response = response_with_upstream_version(); - apply_git_version_header_from(Some("v2.0.0"), &mut response); - assert_eq!( - version_of(&response), - Some("v2.0.0"), - "should replace an upstream x-ts-version with ours" - ); - } - - #[test] - fn removes_upstream_header_when_version_unknown() { - let mut response = response_with_upstream_version(); - apply_git_version_header_from(None, &mut response); - assert_eq!( - version_of(&response), - None, - "should remove an upstream x-ts-version when ours is unknown" - ); - } - - #[test] - fn removes_upstream_header_when_version_invalid() { - let mut response = response_with_upstream_version(); - apply_git_version_header_from(Some("bad\nvalue"), &mut response); - assert_eq!( - version_of(&response), - None, - "should remove an upstream x-ts-version when ours is invalid" - ); - } - - #[test] - fn default_uses_compiled_in_version() { - let mut response = empty_response(); - apply_git_version_header(&mut response); - assert_eq!( - version_of(&response), - TS_GIT_VERSION, - "should report the compiled-in TS_GIT_VERSION" - ); - } -} diff --git a/crates/trusted-server-integration-tests/Cargo.toml b/crates/trusted-server-integration-tests/Cargo.toml index 39f985860a..c83377fca4 100644 --- a/crates/trusted-server-integration-tests/Cargo.toml +++ b/crates/trusted-server-integration-tests/Cargo.toml @@ -25,6 +25,7 @@ edgezero-core = { workspace = true } serde_json = { workspace = true } toml = { workspace = true } trusted-server-core = { workspace = true, features = ["test-utils"] } +trusted-server-modules = { workspace = true } [dev-dependencies] async-trait = { workspace = true } diff --git a/crates/trusted-server-integration-tests/README.md b/crates/trusted-server-integration-tests/README.md index 2d3981721e..8ca3655a32 100644 --- a/crates/trusted-server-integration-tests/README.md +++ b/crates/trusted-server-integration-tests/README.md @@ -7,7 +7,7 @@ end-to-end publisher behavior. It is not linked into an adapter artifact. - `tests/parity.rs` calls Axum, Cloudflare, and Spin routers in process and compares their shared route behavior. -- `tests/documentation_snippets.rs` extracts the checked integration-guide +- `tests/documentation_snippets.rs` extracts each checked integration-guide fixture and compiles it in an isolated offline crate. - `tests/integration.rs` exercises the Fastly/Viceroy and Axum paths against the WordPress and Next.js fixture containers. diff --git a/crates/trusted-server-integration-tests/browser/initial-render/run.cjs b/crates/trusted-server-integration-tests/browser/initial-render/run.cjs index 681ba584a1..d69652bda6 100644 --- a/crates/trusted-server-integration-tests/browser/initial-render/run.cjs +++ b/crates/trusted-server-integration-tests/browser/initial-render/run.cjs @@ -67,7 +67,7 @@ const scriptContents = new Map( [ path.join( repo, - "crates/trusted-server-core/src/integrations/gpt_bootstrap.js", + "crates/ad-tag/google/src/gpt_bootstrap.js", ), path.join(dist, "tsjs-core.js"), path.join(dist, "tsjs-gpt.js"), @@ -163,7 +163,7 @@ const scenarios = [ scripts.push( path.join( repo, - "crates/trusted-server-core/src/integrations/gpt_bootstrap.js", + "crates/ad-tag/google/src/gpt_bootstrap.js", ), ); scripts.push( @@ -474,7 +474,7 @@ const scenarios = [ files: [ path.join( repo, - "crates/trusted-server-core/src/integrations/gpt_bootstrap.js", + "crates/ad-tag/google/src/gpt_bootstrap.js", ), path.join(dist, "tsjs-core.js"), path.join(dist, "tsjs-gpt.js"), diff --git a/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml b/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml index c3d9ff53d8..e49c170a2a 100644 --- a/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml +++ b/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml @@ -1,19 +1,23 @@ -[[handlers]] -path = "^/_ts/admin" -username = "admin" -password = "integration_admin_password" - [publisher] domain = "localhost" cookie_domain = "localhost" origin_url = "http://127.0.0.1:8888" proxy_secret = "integration_proxy_secret" +[geo] +# Viceroy maps the loopback client to US/CA (see viceroy-template.toml), so +# the platform geo module resolves a real place and the EC lifecycle +# scenarios exercise the US state opt-out machinery. +module = "platform" + [ec] -passphrase = "integration_ec_passphrase" +module = "hmac" ec_store = "ec_identity_store" pull_sync_concurrency = 3 +[ec.hmac] +passphrase = "integration_ec_passphrase" + [[ec.partners]] name = "Integration Test Partner" source_domain = "inttest.example.com" @@ -28,70 +32,18 @@ api_token = "integration_partner_token_bravo" [request_signing] enabled = false -config_store_id = "app_config" -secret_store_id = "secrets" - -[integrations.prebid] -enabled = false -timeout_ms = 1000 -debug = false -client_side_bidders = [] - -[integrations.nextjs] -enabled = false -rewrite_attributes = ["href", "link", "siteBaseUrl", "siteProductionDomain", "url"] -max_combined_payload_bytes = 10485760 - -[integrations.testlight] -enabled = false -endpoint = "https://testlight.example.com/openrtb2/auction" -timeout_ms = 1200 -rewrite_scripts = true -[integrations.didomi] -enabled = false -geo_query_parameters = false -sdk_origin = "https://sdk.example.com" -api_origin = "https://api.example.com" - -[integrations.sourcepoint] -enabled = false -rewrite_sdk = true -cdn_origin = "https://cdn.example.com" -cache_ttl_seconds = 3600 +# Only the GPT diagnostics overlay runs, which is what the browser tests need. +# A settings table its section does not select refuses startup, so no other +# integration has a table here. +[ad-tag] +modules = ["google.diagnostics"] -[integrations.permutive] -enabled = false -organization_id = "" -workspace_id = "" -project_id = "" -api_endpoint = "https://api.example.com" -secure_signals_endpoint = "https://secure-signals.example.com" - -[integrations.lockr] -enabled = false -app_id = "" -api_endpoint = "https://identity.example.com" -sdk_url = "https://identity.example.com/trusted-server.js" -cache_ttl_seconds = 3600 -rewrite_sdk = true - -[integrations.datadome] -enabled = false -sdk_origin = "https://sdk.example.com" -api_origin = "https://api.example.com" -cache_ttl_seconds = 3600 -rewrite_sdk = true - -[integrations.gpt] -enabled = false -gam_attribution_enabled = false -script_url = "https://ads.example.com/gpt.js" -cache_ttl_seconds = 3600 -rewrite_script = true - -[integrations.gpt_diagnostics] -enabled = true +# What the overlay writes into a page is a serve middleware, which runs on +# each reader's copy of the pages an entry names it for. +[[serve]] +media_type = "text/html" +middleware = ["ad-tag.google.diagnostics"] [proxy] certificate_check = false @@ -101,47 +53,31 @@ enabled = false timeout_ms = 2000 allowed_context_keys = [] -[auction.providers.pbs-main] -protocol = "openrtb-2.6" -profile = "prebid-server" +[demand] +modules = ["pbs_main", "aps_main"] + +[demand.pbs_main] +implementation = "auction.prebid-server" endpoint = "https://prebid.example.com/openrtb2/auction" routing = "explicit" - -[auction.providers.pbs-main.profile_config] debug = false test_mode = false consent_forwarding = "both" -[auction.providers.pbs-main.notifications] +[demand.pbs_main.notifications] suppress_all = false suppress_seats = [] -[auction.providers.aps-main] -protocol = "openrtb-2.6" -profile = "aps" +[demand.aps_main] +implementation = "auction.aps" endpoint = "https://aps.example.com/e/pb/bid" routing = "all_eligible" - -[auction.providers.aps-main.profile_config] account_id = "example-aps-account-id" debug = false allow_script_creatives = false [auction.bidders.example-bidder] -provider = "pbs-main" - -[integrations.google_tag_manager] -enabled = false -container_id = "GTM-EXAMPLE" -upstream_url = "https://tags.example.com" - -[integrations.adserver_mock] -enabled = false -endpoint = "https://adserver.example.com/mediate" -timeout_ms = 1000 - -[integrations.adserver_mock.context_query_params] -example_segments = "segments" +module = "pbs_main" [debug] ja4_endpoint_enabled = false diff --git a/crates/trusted-server-integration-tests/fixtures/configs/viceroy-template.toml b/crates/trusted-server-integration-tests/fixtures/configs/viceroy-template.toml index eae7220a84..8967f9bfdc 100644 --- a/crates/trusted-server-integration-tests/fixtures/configs/viceroy-template.toml +++ b/crates/trusted-server-integration-tests/fixtures/configs/viceroy-template.toml @@ -4,6 +4,52 @@ [local_server] + # Maps the loopback client to California, United States, so the platform + # geo module resolves a real place and the permission model applies the + # US/CA rules from the permissions.yaml tree. + [local_server.geolocation] + format = "inline-toml" + + [local_server.geolocation.addresses."127.0.0.1"] + as_name = "Integration Test" + as_number = 64496 + area_code = 0 + city = "Test City" + conn_speed = "broadband" + conn_type = "wired" + continent = "NA" + country_code = "US" + country_code3 = "USA" + country_name = "United States" + latitude = 0.0 + longitude = 0.0 + metro_code = 0 + postal_code = "00000" + proxy_description = "?" + proxy_type = "?" + region = "CA" + utc_offset = -800 + + [local_server.geolocation.addresses."::1"] + as_name = "Integration Test" + as_number = 64496 + area_code = 0 + city = "Test City" + conn_speed = "broadband" + conn_type = "wired" + continent = "NA" + country_code = "US" + country_code3 = "USA" + country_name = "United States" + latitude = 0.0 + longitude = 0.0 + metro_code = 0 + postal_code = "00000" + proxy_description = "?" + proxy_type = "?" + region = "CA" + utc_offset = -800 + [local_server.backends] [local_server.kv_stores] @@ -62,13 +108,6 @@ key = "ts-2025-10-A" data = "NVnTYrw5xoyTJDOwoUWoPJO3A6UCCXOJJUzgGTxxx7k=" - [[local_server.secret_stores.api-keys]] - key = "api_key" - data = "test-api-key" - - [[local_server.secret_stores.ts_secrets]] - key = "integration_admin_password" - data = "integration-admin-password-32-bytes-ok" [[local_server.secret_stores.ts_secrets]] key = "integration_proxy_secret" data = "integration-test-proxy-secret-32-bytes-ok" diff --git a/crates/trusted-server-integration-tests/src/bin/generate-viceroy-config.rs b/crates/trusted-server-integration-tests/src/bin/generate-viceroy-config.rs index aada5dcf54..f5bb7e80f5 100644 --- a/crates/trusted-server-integration-tests/src/bin/generate-viceroy-config.rs +++ b/crates/trusted-server-integration-tests/src/bin/generate-viceroy-config.rs @@ -121,6 +121,9 @@ fn build_app_config_envelope( if let Some(origin_url) = origin_url { settings.publisher.origin_url = origin_url.to_string(); } + // The fixture selects modules a stock build ships, so validation needs + // the stock list as the adapters and the `ts` tool have it. + trusted_server_core::config::register_deploy_integrations(trusted_server_modules::builders()); let app_config = TrustedServerAppConfig::new(settings) .map_err(|report| error_box(format!("invalid Trusted Server app config: {report:?}")))?; @@ -165,7 +168,7 @@ mod tests { use super::*; use error_stack::Report; use std::collections::HashMap; - use trusted_server_core::config_payload::settings_from_config_blob; + use trusted_server_core::config_payload::settings_from_config_blob_with; use trusted_server_core::platform::{PlatformError, PlatformSecretStore, StoreId, StoreName}; const TEMPLATE: &str = include_str!("../../fixtures/configs/viceroy-template.toml"); @@ -204,10 +207,6 @@ mod tests { fn integration_secret_store() -> IntegrationSecretStore { IntegrationSecretStore { values: HashMap::from([ - ( - "integration_admin_password".to_owned(), - b"integration-admin-password-32-bytes-ok".to_vec(), - ), ( "integration_proxy_secret".to_owned(), b"integration-test-proxy-secret-32-bytes-ok".to_vec(), @@ -317,10 +316,11 @@ mod tests { fn generated_blob_verifies_and_applies_origin_override() { let envelope = build_app_config_envelope(APP_CONFIG, Some("http://127.0.0.1:9999")) .expect("should build envelope"); - let settings = settings_from_config_blob( + let settings = settings_from_config_blob_with( &envelope, &integration_secret_store(), &StoreName::from("trusted_server_secrets"), + &trusted_server_modules::builders(), ) .expect("should verify blob"); diff --git a/crates/trusted-server-integration-tests/tests/common/config.rs b/crates/trusted-server-integration-tests/tests/common/config.rs index 0b793833f5..0ee777c35e 100644 --- a/crates/trusted-server-integration-tests/tests/common/config.rs +++ b/crates/trusted-server-integration-tests/tests/common/config.rs @@ -33,6 +33,9 @@ fn app_config_envelope( })?; let mut settings = app_config.into_settings(); settings.publisher.origin_url = origin_url; + // The fixture selects modules a stock build ships, so validation needs + // the stock list as the adapters and the `ts` tool have it. + trusted_server_core::config::register_deploy_integrations(trusted_server_modules::builders()); let app_config = TrustedServerAppConfig::new(settings).map_err(|report| { Report::new(TestError::ConfigGeneration) .attach(format!("invalid generated integration config: {report:?}")) @@ -57,14 +60,18 @@ pub fn cloudflare_config_json(origin_port: u16) -> TestResult { /// Builds a Cloudflare config binding with server-side auctions enabled. /// -/// The fixture's providers are removed so the enabled auction completes -/// through the orchestrator and telemetry without any outbound provider call. +/// The fixture's demand modules and bidder routes are removed so the enabled +/// auction completes through the orchestrator and telemetry without any +/// outbound call, and so Cloudflare, which runs one demand source at a time, +/// accepts it at startup. pub fn cloudflare_auction_config_json(origin_port: u16) -> TestResult { let envelope = app_config_envelope(origin_port, |table| { let auction = table_mut(table, "auction")?; auction.insert("enabled".to_string(), toml::Value::Boolean(true)); - auction.remove("providers"); auction.remove("bidders"); + let mut demand = toml::Table::new(); + demand.insert("modules".to_string(), toml::Value::Array(Vec::new())); + table.insert("demand".to_string(), toml::Value::Table(demand)); Ok(()) })?; cloudflare_binding_json(&serde_json::json!({ CONFIG_BLOB_KEY: envelope })) @@ -153,11 +160,7 @@ mod tests { .as_array() .expect("fastly.toml should define ts_secrets"); - for key in [ - "publisher_proxy_secret", - "ec_passphrase", - "handler_password", - ] { + for key in ["publisher_proxy_secret", "ec_passphrase"] { assert!( entries .iter() diff --git a/crates/trusted-server-integration-tests/tests/documentation_snippets.rs b/crates/trusted-server-integration-tests/tests/documentation_snippets.rs index d9c36d2338..b215d47388 100644 --- a/crates/trusted-server-integration-tests/tests/documentation_snippets.rs +++ b/crates/trusted-server-integration-tests/tests/documentation_snippets.rs @@ -3,15 +3,30 @@ use std::process::Command; use std::time::{SystemTime, UNIX_EPOCH}; const GUIDE: &str = include_str!("../../../docs/guide/integration-guide.md"); -const START: &str = ""; -const END: &str = ""; #[test] fn integration_guide_runtime_services_fixture_compiles() { + assert_documented_fixture_compiles("runtime-services"); +} + +#[test] +fn integration_guide_middleware_fixture_compiles() { + assert_documented_fixture_compiles("middleware"); +} + +/// Compiles the fence the guide marks as the snippet `name`, exactly as +/// written, as an isolated crate that depends on core alone. +#[allow( + clippy::panic, + reason = "a helper of the tests above, which fails the test that called it" +)] +fn assert_documented_fixture_compiles(name: &str) { + let start = format!(""); + let end = format!(""); let marked = GUIDE - .split_once(START) - .and_then(|(_, tail)| tail.split_once(END).map(|(body, _)| body)) - .expect("should contain one bounded runtime-services snippet"); + .split_once(&start) + .and_then(|(_, tail)| tail.split_once(&end).map(|(body, _)| body)) + .unwrap_or_else(|| panic!("should contain one bounded {name} snippet")); let source = marked .trim() .strip_prefix("```rust\n") @@ -23,7 +38,7 @@ fn integration_guide_runtime_services_fixture_compiles() { .expect("should have a current system clock") .as_nanos(); let fixture_root = std::env::temp_dir().join(format!( - "trusted-server-documentation-snippet-{}-{nonce}", + "trusted-server-documentation-snippet-{name}-{}-{nonce}", std::process::id() )); let source_dir = fixture_root.join("src"); @@ -38,18 +53,24 @@ fn integration_guide_runtime_services_fixture_compiles() { let core_path = repository_root.join("crates/trusted-server-core"); let workspace_manifest = fs::read_to_string(repository_root.join("Cargo.toml")) .expect("should read the workspace manifest"); - let error_stack_requirement = workspace_manifest - .lines() - .find_map(|line| { - let assignment = line.trim().strip_prefix("error-stack")?.trim_start(); - let value = assignment.strip_prefix('=')?; - let start = value.find('"')? + 1; - let end = start + value[start..].find('"')?; - Some(value[start..end].to_owned()) - }) - .expect("should find the workspace error-stack requirement"); + let requirement_of = |crate_name: &str| { + workspace_manifest + .lines() + .find_map(|line| { + let assignment = line.trim().strip_prefix(crate_name)?.trim_start(); + let value = assignment.strip_prefix('=')?; + let start = value.find('"')? + 1; + let end = start + value[start..].find('"')?; + Some(value[start..end].to_owned()) + }) + .unwrap_or_else(|| panic!("should find the workspace {crate_name} requirement")) + }; + let error_stack_requirement = requirement_of("error-stack"); + // The documented fixture implements `PlatformGeo`, whose `lookup` is + // asynchronous, so it carries the same attribute the trait does. + let async_trait_requirement = requirement_of("async-trait"); let manifest = format!( - "[package]\nname = \"documentation-snippet\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n[workspace]\n\n[dependencies]\nerror-stack = \"{error_stack_requirement}\"\ntrusted-server-core = {{ path = {:?} }}\n", + "[package]\nname = \"documentation-snippet\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n[workspace]\n\n[dependencies]\nasync-trait = \"{async_trait_requirement}\"\nerror-stack = \"{error_stack_requirement}\"\ntrusted-server-core = {{ path = {:?} }}\n", core_path ); fs::write(fixture_root.join("Cargo.toml"), manifest) @@ -76,7 +97,7 @@ fn integration_guide_runtime_services_fixture_compiles() { assert!( output.status.success(), - "documented integration fixture must compile:\nstdout:\n{}\nstderr:\n{}", + "the documented {name} fixture must compile:\nstdout:\n{}\nstderr:\n{}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr) ); diff --git a/crates/trusted-server-integration-tests/tests/environments/axum.rs b/crates/trusted-server-integration-tests/tests/environments/axum.rs index 6e1388cea3..0afdfbacc3 100644 --- a/crates/trusted-server-integration-tests/tests/environments/axum.rs +++ b/crates/trusted-server-integration-tests/tests/environments/axum.rs @@ -14,10 +14,6 @@ const AXUM_DEFAULT_PORT: u16 = 8787; /// Secret-store entries referenced by the integration app-config fixture. const INTEGRATION_SECRET_ENV: &[(&str, &str)] = &[ - ( - "TRUSTED_SERVER_SECRET_TRUSTED_SERVER_SECRETS_INTEGRATION_ADMIN_PASSWORD", - "integration-admin-password-32-bytes-ok", - ), ( "TRUSTED_SERVER_SECRET_TRUSTED_SERVER_SECRETS_INTEGRATION_PROXY_SECRET", "integration-test-proxy-secret-32-bytes-ok", diff --git a/crates/trusted-server-integration-tests/tests/frameworks/scenarios.rs b/crates/trusted-server-integration-tests/tests/frameworks/scenarios.rs index c1bbafe8df..a7a45f2f63 100644 --- a/crates/trusted-server-integration-tests/tests/frameworks/scenarios.rs +++ b/crates/trusted-server-integration-tests/tests/frameworks/scenarios.rs @@ -445,9 +445,10 @@ pub enum EcScenario { /// returns the scoped UID. FullLifecycle, - /// Consent withdrawal: GPC header triggers EC cookie deletion for a - /// seeded EC in the default US-state test geo. - ConsentWithdrawal, + /// Opt-out suppression: a GPC header suppresses use of a seeded EC + /// (identify answers 403) without expiring the cookie, in the default + /// US-state test geo. Opt-outs are never destructive. + OptOutSuppression, /// Identify without EC cookie returns 204. IdentifyWithoutEc, @@ -470,7 +471,7 @@ impl EcScenario { pub fn all() -> Vec { vec![ Self::FullLifecycle, - Self::ConsentWithdrawal, + Self::OptOutSuppression, Self::IdentifyWithoutEc, Self::IdentifyConsentDenied, Self::ConcurrentPartnerSyncs, @@ -489,7 +490,7 @@ impl EcScenario { pub fn run(&self, base_url: &str) -> TestResult<()> { match self { Self::FullLifecycle => ec_full_lifecycle(base_url), - Self::ConsentWithdrawal => ec_consent_withdrawal(base_url), + Self::OptOutSuppression => ec_opt_out_suppression(base_url), Self::IdentifyWithoutEc => ec_identify_without_ec(base_url), Self::IdentifyConsentDenied => ec_identify_consent_denied(base_url), Self::ConcurrentPartnerSyncs => ec_concurrent_partner_syncs(base_url), @@ -577,37 +578,48 @@ fn ec_full_lifecycle(base_url: &str) -> TestResult<()> { } /// Consent withdrawal: GPC header clears EC cookie. -fn ec_consent_withdrawal(base_url: &str) -> TestResult<()> { +fn ec_opt_out_suppression(base_url: &str) -> TestResult<()> { let client = EcTestClient::new(base_url); allow_ec_generation(&client); let seeded_ec_id = seeded_ec_id('b', "test02"); let ec_id = use_seeded_ec(&client, &seeded_ec_id); - log::info!("EC consent withdrawal: using seeded EC = {ec_id}"); + log::info!("EC opt-out suppression: using seeded EC = {ec_id}"); - // GPC overrides the allow cookie in US-CA, so this is an explicit - // withdrawal and must expire the EC cookie. + // GPC is a US-style opt-out. It suppresses use of the identifier for the + // request but is never destructive: the cookie is not expired and no + // tombstone is written, so lifting the opt-out restores the identity. let resp = client.get_with_headers("/", &[("sec-gpc", "1")])?; - - if !is_ec_cookie_expired(&resp) { + if is_ec_cookie_expired(&resp) { return Err(Report::new(TestError::UnexpectedContent) - .attach("consent withdrawal should expire ts-ec cookie (expected Max-Age=0)")); + .attach("an opt-out must suppress without expiring the ts-ec cookie")); } - if client.ec_cookie_value().is_some() { + if client.ec_cookie_value().is_none() { return Err(Report::new(TestError::UnexpectedContent) - .attach("client should stop tracking ts-ec after explicit withdrawal")); + .attach("the client should keep tracking ts-ec through an opt-out")); } - // 3. With consent still granted and the EC cookie revoked, identify should - // now report no EC present. - let resp = identify(&client, INTTEST_API_TOKEN)?; - assert_status(&resp, 204).attach("identify should return 204 after cookie revocation")?; - - // 4. With GPC still asserted, identify should reflect consent denial. + // With GPC asserted, identify reflects the suppressed permissions. let resp = identify_with_headers(&client, INTTEST_API_TOKEN, &[("sec-gpc", "1")])?; assert_status(&resp, 403) - .attach("identify with GPC should return 403 after consent withdrawal")?; + .attach("identify with GPC should return 403 while the opt-out is asserted")?; + + // Without GPC the identity is usable again: the cookie still carries the + // identifier, so identify answers 200 with consent ok. No row was seeded + // for this identifier, so there is no enrichment, and the absence of a + // 403 proves the opt-out destroyed nothing. + let resp = identify(&client, INTTEST_API_TOKEN)?; + let body = assert_json_response(resp, 200)?; + if body.get("consent").and_then(|v| v.as_str()) != Some("ok") { + return Err(Report::new(TestError::UnexpectedContent).attach(format!( + "identify without GPC should report consent ok, got {body}" + ))); + } + if body.get("uid").is_some() { + return Err(Report::new(TestError::UnexpectedContent) + .attach("no partner UID was seeded, so identify should carry no enrichment")); + } - log::info!("EC consent withdrawal: PASSED"); + log::info!("EC opt-out suppression: PASSED"); Ok(()) } diff --git a/crates/trusted-server-integration-tests/tests/parity.rs b/crates/trusted-server-integration-tests/tests/parity.rs index 2428d7799b..65b8f9fabd 100644 --- a/crates/trusted-server-integration-tests/tests/parity.rs +++ b/crates/trusted-server-integration-tests/tests/parity.rs @@ -2,7 +2,7 @@ //! //! Sends identical requests to all host-runnable adapters and asserts that: //! - Response status codes match -//! - Critical headers (X-Geo-Info-Available, WWW-Authenticate on 401) match +//! - Critical headers (X-Geo-Info-Available) match //! //! Fastly parity is verified via cargo test-fastly + Viceroy in CI. @@ -28,17 +28,9 @@ use trusted_server_core::test_support::nextjs_auction; /// The settings baked into the binaries contain placeholder secrets that /// `get_settings()` rejects by design, so all routers are built through /// their `routes_with_settings` testing seams from this known-good config. -/// The handler regex is the production-shaped `^/_ts/admin`, matching -/// `Settings::ADMIN_ENDPOINTS` and the default config, so the canonical -/// `/_ts/admin/keys/*` routes are auth-gated exactly as in production. fn test_settings() -> Settings { Settings::from_toml( r#" - [[handlers]] - path = "^/_ts/admin" - username = "admin" - password = "admin-pass" - [publisher] domain = "test-publisher.example.com" cookie_domain = ".test-publisher.example.com" @@ -46,7 +38,13 @@ fn test_settings() -> Settings { proxy_secret = "parity-test-proxy-secret" [ec] + module = "hmac" + + [ec.hmac] passphrase = "test-secret-key-32-bytes-minimum" + + [geo] + assume_single_jurisdiction = true "#, ) .expect("should parse parity test settings") @@ -434,184 +432,231 @@ async fn discovery_route_body_is_json_parity() { } } +/// The configuration view is built by core from the settings alone, so +/// every adapter shows the same one. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn verify_signature_route_parity() { - // known divergence: without real signing-key configuration the handler may - // return 5xx. The parity assertion is that both adapters agree on the status - // (routing and middleware are wired identically). - let (axum_status, _) = axum_post_headers("/verify-signature", "{}").await; - let (cf_status, _) = cf_post_headers("/verify-signature", "{}").await; - let (spin_status, _) = spin_post_headers("/verify-signature", "{}").await; +async fn config_endpoint_parity() { + use http_body_util::BodyExt as _; + use serde_json::Value; - assert_ne!(axum_status, 404, "Axum /verify-signature must be routed"); - assert_ne!( - cf_status, 404, - "Cloudflare /verify-signature must be routed" - ); - assert_ne!(spin_status, 404, "Spin /verify-signature must be routed"); + const PATH: &str = "/_ts/config.json"; + + let (axum_status, axum_body) = { + let mut svc = EdgeZeroAxumService::new(axum_router()); + let req = AxumRequest::builder() + .method("GET") + .uri(PATH) + .body(AxumBody::empty()) + .expect("should build GET request"); + let resp = svc + .ready() + .await + .expect("should be ready") + .call(req) + .await + .expect("should respond"); + let status = resp.status().as_u16(); + let body = resp + .into_body() + .collect() + .await + .expect("should collect body") + .to_bytes(); + (status, body) + }; + let (cf_status, cf_body) = { + let req = request_builder() + .method("GET") + .uri(PATH) + .body(edgezero_core::body::Body::empty()) + .expect("should build GET request"); + let resp = cf_router().oneshot(req).await.expect("should respond"); + let status = resp.status().as_u16(); + (status, resp.into_body().into_bytes().unwrap_or_default()) + }; + let (spin_status, _, spin_body) = spin_get_body(PATH).await; + + assert_eq!(axum_status, 200, "Axum should answer {PATH}"); + assert_eq!(cf_status, 200, "Cloudflare should answer {PATH}"); + assert_eq!(spin_status, 200, "Spin should answer {PATH}"); + + let axum: Value = serde_json::from_slice(&axum_body).expect("Axum should answer JSON"); + let cf: Value = serde_json::from_slice(&cf_body).expect("Cloudflare should answer JSON"); + let spin: Value = serde_json::from_slice(&spin_body).expect("Spin should answer JSON"); + assert_eq!(axum, cf, "{PATH} must match across adapters (axum, cf)"); + assert_eq!(cf, spin, "{PATH} must match across adapters (cf, spin)"); assert_eq!( - axum_status, cf_status, - "/verify-signature must return same status: axum={axum_status} cf={cf_status}" + axum["settings"]["publisher"]["proxy_secret"], "XXXX", + "a secret should be masked" ); - assert_eq!( - cf_status, spin_status, - "/verify-signature must return same status: cf={cf_status} spin={spin_status}" + assert!( + !String::from_utf8_lossy(&axum_body).contains("parity-test-proxy-secret"), + "the view should not carry the secret anywhere" ); } +/// Whether a request is a browser opening the page is core's decision, and +/// none of these three adapters keeps an identity graph, so each refuses and +/// answers alike. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn admin_rotate_unauthenticated_parity() { - // Both adapters must return 401 for unauthenticated admin requests on the - // canonical `/_ts/admin/keys/*` path that production config auth-gates. - // The authenticated-path divergence (Axum→501 no-KV, CF→4xx no-KV) - // is separate and not covered here. - let (axum_status, axum_headers) = axum_post_headers("/_ts/admin/keys/rotate", "{}").await; - let (cf_status, cf_headers) = cf_post_headers("/_ts/admin/keys/rotate", "{}").await; - let (spin_status, spin_headers) = spin_post_headers("/_ts/admin/keys/rotate", "{}").await; +async fn data_endpoint_parity() { + use http_body_util::BodyExt as _; - assert_eq!( - axum_status, 401, - "Axum must return 401 for unauthenticated admin route" - ); - assert_eq!( - cf_status, 401, - "Cloudflare must return 401 for unauthenticated admin route" - ); - assert_eq!( - spin_status, 401, - "Spin must return 401 for unauthenticated admin route" - ); - assert_eq!( - axum_status, cf_status, - "Axum and Cloudflare must return the same status for unauthenticated admin route" - ); - assert_eq!( - cf_status, spin_status, - "Cloudflare and Spin must return the same status for unauthenticated admin route" - ); + const PATH: &str = "/_ts/data"; + const OPENED_AS_A_PAGE: [(&str, &str); 2] = [ + ("sec-fetch-mode", "navigate"), + ("sec-fetch-dest", "document"), + ]; - let axum_www_auth = axum_headers - .get("www-authenticate") - .expect("Axum 401 must include WWW-Authenticate") - .to_str() - .expect("should be valid UTF-8"); - let cf_www_auth = cf_headers - .get("www-authenticate") - .expect("Cloudflare 401 must include WWW-Authenticate") - .to_str() - .expect("should be valid UTF-8"); - assert_eq!( - axum_www_auth, cf_www_auth, - "WWW-Authenticate header value must match across adapters for /admin/keys/rotate" - ); - assert!( - axum_www_auth.starts_with("Basic"), - "WWW-Authenticate must use Basic scheme: {axum_www_auth:?}" - ); - let spin_www_auth = spin_headers - .get("www-authenticate") - .expect("Spin 401 must include WWW-Authenticate") - .to_str() - .expect("should be valid UTF-8"); - assert_eq!( - cf_www_auth, spin_www_auth, - "WWW-Authenticate value must match: cf={cf_www_auth:?} spin={spin_www_auth:?}" - ); + async fn edge( + router: RouterService, + headers: &[(&str, &str)], + ) -> (u16, Option, bytes::Bytes) { + let mut builder = request_builder().method("GET").uri(PATH); + for (name, value) in headers { + builder = builder.header(*name, *value); + } + let req = builder + .body(edgezero_core::body::Body::empty()) + .expect("should build GET request"); + let resp = router.oneshot(req).await.expect("should respond"); + ( + resp.status().as_u16(), + resp.headers().get("content-security-policy").cloned(), + resp.into_body().into_bytes().unwrap_or_default(), + ) + } + + for (headers, expected) in [(&[][..], 403), (&OPENED_AS_A_PAGE[..], 200)] { + let axum = { + let mut svc = EdgeZeroAxumService::new(axum_router()); + let mut builder = AxumRequest::builder().method("GET").uri(PATH); + for (name, value) in headers { + builder = builder.header(*name, *value); + } + let req = builder + .body(AxumBody::empty()) + .expect("should build GET request"); + let resp = svc + .ready() + .await + .expect("should be ready") + .call(req) + .await + .expect("should respond"); + let status = resp.status().as_u16(); + let policy = resp.headers().get("content-security-policy").cloned(); + let body = resp + .into_body() + .collect() + .await + .expect("should collect body") + .to_bytes(); + (status, policy, body) + }; + let cf = edge(cf_router(), headers).await; + let spin = edge(spin_router(), headers).await; + + assert_eq!( + axum.0, expected, + "Axum should answer {PATH} with {expected}" + ); + assert_eq!(axum, cf, "{PATH} must match across adapters (axum, cf)"); + assert_eq!(cf, spin, "{PATH} must match across adapters (cf, spin)"); + assert!( + axum.1 + .as_ref() + .and_then(|policy| policy.to_str().ok()) + .is_some_and(|policy| policy.starts_with("sandbox;")), + "every answer should be in an origin of its own" + ); + } +} + +/// The permissions a request resolves to are core's decision, so the +/// endpoint that shows them answers the same on every adapter. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn permissions_endpoint_parity() { + use http_body_util::BodyExt as _; + use serde_json::Value; + + const PATH: &str = "/_ts/permissions.json"; + + let (axum_status, axum_body) = { + let mut svc = EdgeZeroAxumService::new(axum_router()); + let req = AxumRequest::builder() + .method("GET") + .uri(PATH) + .body(AxumBody::empty()) + .expect("should build GET request"); + let resp = svc + .ready() + .await + .expect("should be ready") + .call(req) + .await + .expect("should respond"); + let status = resp.status().as_u16(); + let body = resp + .into_body() + .collect() + .await + .expect("should collect body") + .to_bytes(); + (status, body) + }; + let (cf_status, cf_body) = { + let req = request_builder() + .method("GET") + .uri(PATH) + .body(edgezero_core::body::Body::empty()) + .expect("should build GET request"); + let resp = cf_router().oneshot(req).await.expect("should respond"); + let status = resp.status().as_u16(); + (status, resp.into_body().into_bytes().unwrap_or_default()) + }; + let (spin_status, _, spin_body) = spin_get_body(PATH).await; + + assert_eq!(axum_status, 200, "Axum should answer {PATH}"); + assert_eq!(cf_status, 200, "Cloudflare should answer {PATH}"); + assert_eq!(spin_status, 200, "Spin should answer {PATH}"); + + let axum: Value = serde_json::from_slice(&axum_body).expect("Axum should answer JSON"); + let cf: Value = serde_json::from_slice(&cf_body).expect("Cloudflare should answer JSON"); + let spin: Value = serde_json::from_slice(&spin_body).expect("Spin should answer JSON"); + assert_eq!(axum, cf, "{PATH} must match across adapters (axum, cf)"); + assert_eq!(cf, spin, "{PATH} must match across adapters (cf, spin)"); assert!( - spin_www_auth.starts_with("Basic"), - "Spin WWW-Authenticate must use Basic scheme: {spin_www_auth:?}" + axum["set"].is_array() && axum["awaiting"].is_array(), + "the resolution should say what is set and what is awaited" ); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn admin_deactivate_unauthenticated_parity() { - // Mirror of admin_rotate_unauthenticated_parity for the deactivate endpoint. - let (axum_status, axum_headers) = axum_post_headers("/_ts/admin/keys/deactivate", "{}").await; - let (cf_status, cf_headers) = cf_post_headers("/_ts/admin/keys/deactivate", "{}").await; - let (spin_status, spin_headers) = spin_post_headers("/_ts/admin/keys/deactivate", "{}").await; +async fn verify_signature_route_parity() { + // known divergence: without real signing-key configuration the handler may + // return 5xx. The parity assertion is that both adapters agree on the status + // (routing and middleware are wired identically). + let (axum_status, _) = axum_post_headers("/verify-signature", "{}").await; + let (cf_status, _) = cf_post_headers("/verify-signature", "{}").await; + let (spin_status, _) = spin_post_headers("/verify-signature", "{}").await; - assert_eq!( - axum_status, 401, - "Axum must return 401 for unauthenticated admin/keys/deactivate" - ); - assert_eq!( - cf_status, 401, - "Cloudflare must return 401 for unauthenticated admin/keys/deactivate" - ); - assert_eq!( - spin_status, 401, - "Spin must return 401 for unauthenticated admin/keys/deactivate" + assert_ne!(axum_status, 404, "Axum /verify-signature must be routed"); + assert_ne!( + cf_status, 404, + "Cloudflare /verify-signature must be routed" ); + assert_ne!(spin_status, 404, "Spin /verify-signature must be routed"); assert_eq!( axum_status, cf_status, - "Axum and Cloudflare must return the same status for unauthenticated admin/keys/deactivate" + "/verify-signature must return same status: axum={axum_status} cf={cf_status}" ); assert_eq!( cf_status, spin_status, - "Cloudflare and Spin must return the same status for unauthenticated admin/keys/deactivate" - ); - - let axum_www_auth = axum_headers - .get("www-authenticate") - .expect("Axum 401 on admin/keys/deactivate must include WWW-Authenticate") - .to_str() - .expect("should be valid UTF-8"); - let cf_www_auth = cf_headers - .get("www-authenticate") - .expect("Cloudflare 401 on admin/keys/deactivate must include WWW-Authenticate") - .to_str() - .expect("should be valid UTF-8"); - assert_eq!( - axum_www_auth, cf_www_auth, - "WWW-Authenticate header value must match across adapters for /admin/keys/deactivate" - ); - assert!( - axum_www_auth.starts_with("Basic"), - "WWW-Authenticate must use Basic scheme: {axum_www_auth:?}" - ); - let spin_www_auth = spin_headers - .get("www-authenticate") - .expect("Spin 401 on admin/keys/deactivate must include WWW-Authenticate") - .to_str() - .expect("should be valid UTF-8"); - assert_eq!( - cf_www_auth, spin_www_auth, - "WWW-Authenticate value must match: cf={cf_www_auth:?} spin={spin_www_auth:?}" - ); - assert!( - spin_www_auth.starts_with("Basic"), - "Spin WWW-Authenticate must use Basic scheme: {spin_www_auth:?}" + "/verify-signature must return same status: cf={cf_status} spin={spin_status}" ); } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn spin_legacy_admin_aliases_are_denied_locally_not_proxied() { - // The production handler regex `^/_ts/admin` only matches the canonical - // `/_ts/admin/keys/*` paths. Legacy aliases must therefore fail closed with a - // local 404 instead of reaching either the admin handlers or the publisher - // fallback. - for alias in ["/admin/keys/rotate", "/admin/keys/deactivate"] { - let (canonical_status, _) = spin_post_headers(&format!("/_ts{alias}"), "{}").await; - assert_eq!( - canonical_status, 401, - "canonical /_ts{alias} must challenge unauthenticated callers" - ); - - for method in ["POST", "GET"] { - let (alias_status, alias_headers) = - spin_authorized_json(method, alias, r#"{"key_id":"leak-me"}"#).await; - assert_eq!( - alias_status, 404, - "Spin legacy {method} {alias} must be denied locally with 404" - ); - assert!( - !alias_headers.contains_key("www-authenticate"), - "Spin legacy {method} {alias} must not issue an admin auth challenge" - ); - } - } -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn geo_header_parity_on_all_responses() { let routes_to_check: &[(&str, &str, &str)] = &[ @@ -671,14 +716,11 @@ async fn geo_header_parity_on_all_responses() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn auction_not_challenged_by_auth_parity() { +async fn auction_is_routed_parity() { let (axum_status, _) = axum_post_headers("/auction", r#"{"adUnits":[]}"#).await; let (cf_status, _) = cf_post_headers("/auction", r#"{"adUnits":[]}"#).await; let (spin_status, _) = spin_post_headers("/auction", r#"{"adUnits":[]}"#).await; - assert_ne!(axum_status, 401, "Axum /auction must not 401"); - assert_ne!(cf_status, 401, "Cloudflare /auction must not 401"); - assert_ne!(spin_status, 401, "Spin /auction must not 401"); assert_ne!(axum_status, 404, "Axum /auction must be routed (not 404)"); assert_ne!( cf_status, 404, @@ -749,7 +791,6 @@ async fn spin_auction_ignores_spoofed_forwarded_headers() { ) .await; - assert_ne!(spoofed_status, 401, "Spin /auction must not 401"); assert_ne!( spoofed_status, 404, "Spin /auction must be routed (not 404)" @@ -814,177 +855,40 @@ async fn unknown_route_returns_same_status_parity() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn legacy_admin_aliases_are_denied_locally_not_proxied() { - // The production handler regex `^/_ts/admin` only matches the canonical - // `/_ts/admin/keys/*` paths, so the legacy `/admin/keys/*` aliases are not - // auth-gated. They must be denied locally with 404 — never routed to the key - // handlers (which would execute admin operations), and never left unrouted to - // fall through to the publisher fallback, which forwards the request - // (including any `Authorization` header and key-management body) to the origin - // and leaks admin credentials. - // - // Primary guard: assert the route tables directly. The legacy aliases must - // be registered (to the local deny) so they can never reach the publisher - // fallback as an unrouted path. - let cf_registered: Vec<(String, String)> = cf_router() - .routes() - .iter() - .map(|route| (route.method().to_string(), route.path().to_string())) - .collect(); - let spin_registered: Vec<(String, String)> = spin_router() - .routes() - .iter() - .map(|route| (route.method().to_string(), route.path().to_string())) - .collect(); - let cf_is_registered = - |method: &str, path: &str| cf_registered.iter().any(|(m, p)| m == method && p == path); - let spin_is_registered = |method: &str, path: &str| { - spin_registered - .iter() - .any(|(m, p)| m == method && p == path) - }; - - for path in ["/_ts/admin/keys/rotate", "/_ts/admin/keys/deactivate"] { - assert!( - cf_is_registered("POST", path), - "Cloudflare POST {path} must be a registered route" - ); - assert!( - spin_is_registered("POST", path), - "Spin POST {path} must be a registered route" - ); - } - for path in ["/admin/keys/rotate", "/admin/keys/deactivate"] { - for method in ["GET", "POST", "HEAD", "OPTIONS", "PUT", "PATCH", "DELETE"] { - assert!( - cf_is_registered(method, path), - "Cloudflare {method} {path} must be a registered route" - ); - assert!( - spin_is_registered(method, path), - "Spin {method} {path} must be a registered route" - ); - } - } - - // Secondary guard: confirm runtime behavior. Canonical paths challenge - // unauthenticated callers (401). Legacy aliases are denied locally with 404 - // and carry no auth challenge — a reintroduced key handler at these ungated - // paths would not return 404, and the publisher fallback would not either, so - // 404 proves the local deny ran. - for alias in ["/admin/keys/rotate", "/admin/keys/deactivate"] { - let (canonical_status, _) = cf_post_headers(&format!("/_ts{alias}"), "{}").await; - assert_eq!( - canonical_status, 401, - "Cloudflare canonical /_ts{alias} must challenge unauthenticated callers" - ); - let (spin_canonical_status, _) = spin_post_headers(&format!("/_ts{alias}"), "{}").await; - assert_eq!( - spin_canonical_status, 401, - "Spin canonical /_ts{alias} must challenge unauthenticated callers" - ); - +async fn key_administration_paths_are_answered_here_and_never_proxied() { + // A request to a key administration path can carry an operator's + // credentials and a key payload. Every adapter must answer it with 404 and + // never reach the publisher fallback, which forwards the request, + // `Authorization` header and body included, to the origin. A key handler + // at one of these paths would not return 404, and the publisher fallback + // would not either, so 404 proves the request was answered here. + for path in [ + "/_ts/admin/keys/rotate", + "/_ts/admin/keys/deactivate", + "/admin/keys/rotate", + "/admin/keys/deactivate", + ] { for method in ["POST", "GET"] { let (axum_status, axum_headers) = - axum_authorized_json(method, alias, r#"{"key_id":"leak-me"}"#).await; + axum_authorized_json(method, path, r#"{"key_id":"leak-me"}"#).await; let (cf_status, cf_headers) = - cf_authorized_json(method, alias, r#"{"key_id":"leak-me"}"#).await; + cf_authorized_json(method, path, r#"{"key_id":"leak-me"}"#).await; let (spin_status, spin_headers) = - spin_authorized_json(method, alias, r#"{"key_id":"leak-me"}"#).await; - assert_eq!( - axum_status, 404, - "Axum legacy {method} {alias} must be denied locally with 404" - ); - assert_eq!( - cf_status, 404, - "Cloudflare legacy {method} {alias} must be denied locally with 404" - ); - assert_eq!( - axum_status, cf_status, - "legacy {method} {alias} status must match across adapters" - ); - assert_eq!( - cf_status, spin_status, - "legacy {method} {alias} status must match across adapters" - ); - assert!( - !axum_headers.contains_key("www-authenticate"), - "Axum legacy {method} {alias} must not issue an admin auth challenge" - ); - assert!( - !cf_headers.contains_key("www-authenticate"), - "Cloudflare legacy {method} {alias} must not issue an admin auth challenge" - ); - assert!( - !spin_headers.contains_key("www-authenticate"), - "Spin legacy {method} {alias} must not issue an admin auth challenge" - ); - } - } -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn admin_cache_purge_not_implemented_parity() { - // The template cache is Fastly-backed, so the other three adapters answer 501 rather - // than letting the path fall through to the publisher origin and 404 — a CMS purge - // webhook needs to tell "not supported here" from "no such endpoint". - let body = r#"{"scope":"all"}"#; - - let (axum_status, _) = axum_authorized_json("POST", "/_ts/admin/cache/purge", body).await; - let (cf_status, _) = cf_authorized_json("POST", "/_ts/admin/cache/purge", body).await; - let (spin_status, _) = spin_authorized_json("POST", "/_ts/admin/cache/purge", body).await; - - assert_eq!(axum_status, 501, "Axum must answer cache purge with 501"); - assert_eq!( - cf_status, 501, - "Cloudflare must answer cache purge with 501" - ); - assert_eq!(spin_status, 501, "Spin must answer cache purge with 501"); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn admin_cache_purge_unauthenticated_parity() { - // Guards the test above from passing for the wrong reason. Without credentials the - // path must 401, which proves the 501s were reached through auth rather than being - // the 401s of a probe that never arrived at a handler. - let body = r#"{"scope":"all"}"#; - - let (axum_status, _) = axum_post_headers("/_ts/admin/cache/purge", body).await; - let (cf_status, _) = cf_post_headers("/_ts/admin/cache/purge", body).await; - let (spin_status, _) = spin_post_headers("/_ts/admin/cache/purge", body).await; - - for (adapter, status) in [ - ("Axum", axum_status), - ("Cloudflare", cf_status), - ("Spin", spin_status), - ] { - assert_eq!( - status, 401, - "{adapter} must require auth on the cache purge path" - ); - } -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn admin_cache_purge_rejects_credential_forwarding_methods() { - // The guard the Fastly route exists for, asserted cross-adapter: a method the route - // does not claim falls through to the publisher with the Authorization header still - // attached. Every method must be answered locally, never forwarded. - for method in ["GET", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD"] { - let (axum_status, _) = axum_authorized_json(method, "/_ts/admin/cache/purge", "").await; - let (cf_status, _) = cf_authorized_json(method, "/_ts/admin/cache/purge", "").await; - let (spin_status, _) = spin_authorized_json(method, "/_ts/admin/cache/purge", "").await; - - for (adapter, status) in [ - ("Axum", axum_status), - ("Cloudflare", cf_status), - ("Spin", spin_status), - ] { - assert_eq!( - status, 501, - "{adapter} must answer {method} locally; a fallthrough would ship the \ - admin credential to the origin" - ); + spin_authorized_json(method, path, r#"{"key_id":"leak-me"}"#).await; + for (adapter, status, headers) in [ + ("Axum", axum_status, &axum_headers), + ("Cloudflare", cf_status, &cf_headers), + ("Spin", spin_status, &spin_headers), + ] { + assert_eq!( + status, 404, + "{adapter} {method} {path} must be answered here with 404" + ); + assert!( + !headers.contains_key("www-authenticate"), + "{adapter} {method} {path} must not ask for credentials" + ); + } } } } @@ -1119,3 +1023,176 @@ async fn adapter_buffers_nextjs_auction_output() { } } } + +/// The shared settings with an `[attestation]` section whose endpoint sits +/// under a prefix of its own, so the deployment owns the prefix. +fn attestation_settings() -> Settings { + let mut settings = test_settings(); + settings.attestation = Some(trusted_server_core::attestation::AttestationConfig { + endpoint: "/_ex/attestation".to_owned(), + operator: "Example Operator".to_owned(), + context: "example-attestation:v1".to_owned(), + verify_url: "https://verifier.example/verify".to_owned(), + }); + settings +} + +/// The value of header `name`, or an empty string when it is absent. +fn header_text(headers: &HeaderMap, name: &str) -> String { + headers + .get(name) + .and_then(|value| value.to_str().ok()) + .unwrap_or_default() + .to_owned() +} + +/// One adapter's answer to a request. +struct Answer { + adapter: &'static str, + status: u16, + headers: HeaderMap, +} + +/// What each host-runnable adapter answers to `method` at `uri` under +/// `settings`. +async fn answers(settings: &Settings, method: &str, uri: &str) -> Vec { + let mut axum = EdgeZeroAxumService::new( + AxumApp::routes_with_settings(settings.clone()).expect("should build Axum router"), + ); + let request = AxumRequest::builder() + .method(method) + .uri(uri) + .body(AxumBody::empty()) + .expect("should build request"); + let response = axum + .ready() + .await + .expect("should be ready") + .call(request) + .await + .expect("should respond"); + let mut answers = vec![Answer { + adapter: "axum", + status: response.status().as_u16(), + headers: response.headers().clone(), + }]; + + for (adapter, router) in [ + ( + "cloudflare", + CloudflareApp::routes_with_settings(settings.clone()) + .expect("should build Cloudflare router"), + ), + ( + "spin", + SpinApp::routes_with_settings(settings.clone()).expect("should build Spin router"), + ), + ] { + let request = request_builder() + .method(method) + .uri(uri) + .body(edgezero_core::body::Body::empty()) + .expect("should build request"); + let response = router.oneshot(request).await.expect("should respond"); + answers.push(Answer { + adapter, + status: response.status().as_u16(), + headers: response.headers().clone(), + }); + } + answers +} + +/// A deployment whose endpoint sits under a prefix of its own answers +/// everything beneath that prefix the same way on every adapter. Reads of the +/// two attestation addresses reach the handler, the prefix lists its pages, +/// and anything else is not found rather than sent to the publisher's origin. +/// Test builds carry no signing key, so the handler answers 503 in JSON. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn attestation_prefix_parity() { + let cases = [ + ("GET", "/_ex", 200, "text/html"), + ("GET", "/_ex/", 200, "text/html"), + ("GET", "/_ex/attestation.json", 503, "application/json"), + ("HEAD", "/_ex/attestation.json", 503, "application/json"), + ("GET", "/_ex/attestation", 503, "application/json"), + ("POST", "/_ex/attestation", 404, "text/html"), + ("PUT", "/_ex/attestation.json", 404, "text/html"), + ("DELETE", "/_ex", 404, "text/html"), + ("GET", "/_ex/unknown/page", 404, "text/html"), + ]; + let settings = attestation_settings(); + for (method, uri, status, content_type) in cases { + for answer in answers(&settings, method, uri).await { + let case = format!("{} {method} {uri}", answer.adapter); + let got_type = header_text(&answer.headers, "content-type"); + assert_eq!(answer.status, status, "should answer {case} alike"); + assert!( + got_type.starts_with(content_type), + "should answer {case} as {content_type}, got {got_type}" + ); + assert_eq!( + header_text(&answer.headers, "cache-control"), + "no-store", + "should keep the answer to {case} out of every cache" + ); + } + } +} + +/// When the endpoint is not the default, every adapter sends a read of the +/// default address to the endpoint, page to page and JSON to JSON, keeping +/// the nonce. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn attestation_default_address_redirect_parity() { + let cases = [ + ("GET", "/_ts/attestation", "/_ex/attestation"), + ("HEAD", "/_ts/attestation", "/_ex/attestation"), + ( + "GET", + "/_ts/attestation.json?nonce=abc123", + "/_ex/attestation.json?nonce=abc123", + ), + ]; + let settings = attestation_settings(); + for (method, uri, location) in cases { + for answer in answers(&settings, method, uri).await { + let case = format!("{} {method} {uri}", answer.adapter); + assert_eq!(answer.status, 301, "should redirect {case}"); + assert_eq!( + header_text(&answer.headers, "location"), + location, + "should send {case} to the endpoint with its query kept" + ); + assert_eq!( + header_text(&answer.headers, "cache-control"), + "no-store", + "should keep the redirect of {case} out of every cache" + ); + } + } +} + +/// Without the section no adapter takes the default address, so a read of it +/// reaches the publisher's origin as any other path does. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn attestation_is_not_served_without_its_section_parity() { + let settings = test_settings(); + let unknown = answers(&settings, "GET", "/a-path-no-route-names").await; + let attestation = answers(&settings, "GET", "/_ts/attestation.json").await; + + for (unknown, attestation) in unknown.iter().zip(&attestation) { + assert_eq!( + attestation.status, unknown.status, + "{} should answer the attestation address as it answers any path of the origin", + attestation.adapter + ); + // The handler answers 404 where there is no section, so a 404 here + // would say a route had been registered for it. + assert_ne!( + attestation.status, 404, + "{} should not answer from the attestation handler", + attestation.adapter + ); + } +} diff --git a/crates/trusted-server-js/lib/build-all.mjs b/crates/trusted-server-js/lib/build-all.mjs index 24f842e764..2a48d7071b 100644 --- a/crates/trusted-server-js/lib/build-all.mjs +++ b/crates/trusted-server-js/lib/build-all.mjs @@ -11,7 +11,7 @@ * The prebid integration builds here as the tsjs shim only — Prebid.js itself * is never bundled into tsjs. Use build-prebid-external.mjs to generate the * pure Prebid.js external bundle (core + adapters + user ID modules) that the - * shim requires at runtime via integrations.prebid.external_bundle_url. + * shim requires at runtime via auction.prebid.external_bundle_url. * * The Rust build script passes `--out-dir` with a private directory under * Cargo's OUT_DIR, so concurrent cargo builds never share (or clean) the same diff --git a/crates/trusted-server-js/lib/build-prebid-external.mjs b/crates/trusted-server-js/lib/build-prebid-external.mjs index a02c5bf60a..7cce1f103b 100644 --- a/crates/trusted-server-js/lib/build-prebid-external.mjs +++ b/crates/trusted-server-js/lib/build-prebid-external.mjs @@ -5,7 +5,7 @@ * * Unlike build-all.mjs, this script is intended to run outside the Cargo build. * It produces an immutable bundle and manifest that can be hosted on an asset - * CDN, then referenced by integrations.prebid.external_bundle_url. + * CDN, then referenced by auction.prebid.external_bundle_url. */ import crypto from 'node:crypto'; @@ -43,19 +43,19 @@ const MODULE_KIND_DEFINITIONS = Object.freeze([ { requestKey: 'bidder', tomlKey: 'bidder', - fieldPath: 'integrations.prebid.bundle.modules.bidder', + fieldPath: 'auction.prebid.bundle.modules.bidder', metadataType: 'bidder', }, { requestKey: 'userId', tomlKey: 'user_id', - fieldPath: 'integrations.prebid.bundle.modules.user_id', + fieldPath: 'auction.prebid.bundle.modules.user_id', metadataType: 'userId', }, { requestKey: 'analytics', tomlKey: 'analytics', - fieldPath: 'integrations.prebid.bundle.modules.analytics', + fieldPath: 'auction.prebid.bundle.modules.analytics', metadataType: 'analytics', }, ]); diff --git a/crates/trusted-server-js/lib/src/core/index.ts b/crates/trusted-server-js/lib/src/core/index.ts index b2c4e41e15..2678a879f1 100644 --- a/crates/trusted-server-js/lib/src/core/index.ts +++ b/crates/trusted-server-js/lib/src/core/index.ts @@ -13,6 +13,7 @@ import { log } from './log'; import { setConfig, getConfig } from './config'; import { requestAds } from './request'; import { installQueue } from './queue'; +import { installPermissions } from './permissions'; const VERSION = '0.1.0'; @@ -42,6 +43,10 @@ api.requestAds = requestAds; // instead of throwing. Injected scripts overwrite these wholesale. api.adSlots ??= []; api.bids ??= {}; +// The edge also injects the request's resolved permission state, either at head +// open (inline mode) or at the seam (shared-template mode). An accessor +// observes the seam's plain assignment so page code can await it either way. +installPermissions(api); // Point global tsjs w.tsjs = api; @@ -68,5 +73,6 @@ log.info('tsjs initialized', { 'addAdUnits', 'renderAdUnit', 'renderAllAdUnits', + 'whenPermissions', ], }); diff --git a/crates/trusted-server-js/lib/src/core/permissions.ts b/crates/trusted-server-js/lib/src/core/permissions.ts new file mode 100644 index 0000000000..6f037b0215 --- /dev/null +++ b/crates/trusted-server-js/lib/src/core/permissions.ts @@ -0,0 +1,89 @@ +// Permission state the edge injects into the page, and the promise page code +// awaits so it can read that state whichever order the injection arrives in. +import { log } from './log'; +import type { PermissionsSnapshot, TsjsApi } from './types'; + +function isSnapshot(value: unknown): value is PermissionsSnapshot { + return typeof value === 'object' && value !== null; +} + +// Page code reads the four lists without checking any exists, so each is an +// array whatever arrived. The edge always sends all four, and a page assigning +// a snapshot by hand, or an older edge, may not. +function normalize(snapshot: PermissionsSnapshot): PermissionsSnapshot { + return { + ...snapshot, + set: Array.isArray(snapshot.set) ? snapshot.set : [], + awaiting: Array.isArray(snapshot.awaiting) ? snapshot.awaiting : [], + signals: Array.isArray(snapshot.signals) ? snapshot.signals : [], + tdls: Array.isArray(snapshot.tdls) ? snapshot.tdls : [], + }; +} + +/** + * Install the `permissions` accessor and `whenPermissions()` on the API object. + * + * The edge injects `window.tsjs.permissions` either before this bundle runs + * (inline mode, at head open) or after it has initialized (shared-template + * mode, as a plain assignment at the `` seam), and on a page with no + * seam it never arrives at all. An accessor observes the plain assignment, so + * the promise resolves in every one of those orders. + */ +export function installPermissions(api: TsjsApi): void { + // A value already on the API object came from the head-open injection, so it + // is the current value; otherwise page code must still read a defined value. + const injected = api.permissions; + let current: PermissionsSnapshot = isSnapshot(injected) + ? normalize(injected) + : { set: [], awaiting: [], signals: [], tdls: [] }; + let settled = false; + let resolvePending: (snapshot: PermissionsSnapshot) => void = () => {}; + const pending = new Promise((resolve) => { + resolvePending = resolve; + }); + + function settle(snapshot: PermissionsSnapshot): void { + if (settled) return; + settled = true; + resolvePending(snapshot); + } + + Object.defineProperty(api, 'permissions', { + get(): PermissionsSnapshot { + return current; + }, + set(value: PermissionsSnapshot) { + current = isSnapshot(value) + ? normalize(value) + : { set: [], awaiting: [], signals: [], tdls: [] }; + log.debug('permissions: received', current); + settle(current); + }, + enumerable: true, + configurable: true, + }); + + if (isSnapshot(injected)) { + log.debug('permissions: present at initialization', injected); + settle(current); + } else if (typeof document !== 'undefined') { + // The seam sits at ``, so anything it was going to assign has run by + // the time the document is parsed. Resolve with whatever the current value + // is rather than leaving page code waiting on a page that has no seam. A + // bundle that initializes after parsing has already missed any seam, so it + // resolves at once instead of waiting for an event that will never fire. + if (document.readyState === 'loading') { + document.addEventListener( + 'DOMContentLoaded', + () => { + settle(current); + }, + { once: true } + ); + } else { + settle(current); + } + } + + api.whenPermissions = () => pending; +} diff --git a/crates/trusted-server-js/lib/src/core/types.ts b/crates/trusted-server-js/lib/src/core/types.ts index 3407d487a0..742e8355af 100644 --- a/crates/trusted-server-js/lib/src/core/types.ts +++ b/crates/trusted-server-js/lib/src/core/types.ts @@ -365,6 +365,36 @@ export interface GptSlotHandoff { suppressPublisherRefresh: boolean; } +/** + * Permission state the server resolved for this request. + * + * The names in `set` are IAB Privacy Taxonomy Data Use keys, as resolved by the + * server for this request. + * + * `tdls` are the terms documents the data for this request is available under, + * as declared by the permission signal modules the deployment runs, in the + * order they were asked. Each entry addresses a published document a person can + * read. An empty list says no terms were declared, which is not the same as + * terms that permit anything, so page code that needs a basis and finds none + * has none. + */ +export interface PermissionsSnapshot { + set: string[]; + /** + * Permissions still waiting for a signal. Not set, and not refused + * either, which page code must tell apart from `set` being empty. + */ + awaiting: string[]; + /** + * The signals the server's modules read and found valid, each as it + * was received. A signal absent from this list was missing, could not be + * read, or is one the deployment does not act on, and page code relies + * on exactly these and no other. + */ + signals: ValidSignal[]; + tdls: string[]; +} + export type FirstImpressionOwner = 'publisher' | 'trusted_server'; export type FirstImpressionPhase = 'auctioning' | 'delivery_pending' | 'requested' | 'rendered'; @@ -403,6 +433,16 @@ export interface FirstImpressionState { fallbackSlots: Record; } +/** One signal a server-side module read and found valid. */ +export interface ValidSignal { + /** The module that read it, by its configured id. */ + module: string; + /** The scheme it belongs to, for example `tcf` or `gpp`. */ + scheme: string; + /** The signal as received. */ + value: string; +} + export interface TsjsApi { version: string; que: Array<() => void>; @@ -430,6 +470,20 @@ export interface TsjsApi { adSlots?: AuctionSlot[]; /** Winning bid targeting data injected before . */ bids?: Record; + /** + * Permission state resolved by the server for this request, injected at head + * open in inline mode or at the body seam in shared-template mode. + */ + permissions?: PermissionsSnapshot; + /** + * Resolves with the permission state once it arrives. + * + * It resolves straight away when the state was already present at + * initialization, on the first assignment when the body seam makes one, and + * on `DOMContentLoaded` with whatever the current value is when no assignment + * arrives at all. Every later call returns the same resolved promise. + */ + whenPermissions?(): Promise; /** * Bounded client-side Prebid APS renderer capabilities keyed by Prebid's generated * `hb_adid`. The Universal Creative bridge consumes each entry at most once. diff --git a/crates/trusted-server-js/lib/src/integrations/ec_client_fixed/index.ts b/crates/trusted-server-js/lib/src/integrations/ec_client_fixed/index.ts new file mode 100644 index 0000000000..c8dee2ecbc --- /dev/null +++ b/crates/trusted-server-js/lib/src/integrations/ec_client_fixed/index.ts @@ -0,0 +1,102 @@ +// Demonstration client for the client-cycle Edge Cookie module (`client_fixed`). +// +// Client and server share one fixed, known word. When the resolved marker is +// absent, this posts that word to the resolve endpoint. With the `client_fixed` +// module selected, the server verifies the word and, on a match, persists the +// identity-graph row and sets the coded form of the word (cfix~an-ec) as an +// HttpOnly Edge Cookie on the +// response, together with a non-HttpOnly marker cookie. The Edge Cookie itself +// is HttpOnly, so this script can never see it; the marker is what tells it a +// resolve already succeeded, so it does not post again on every page view. +// +// The value is verifiable precisely because it is a known constant, which is the +// point of the demo. It is useless in production, because a fixed value is not an +// identity and every client posts the same word. For demonstration and testing +// only. A real client-cycle module posts and verifies a real payload (for +// example an OWID signature) instead of a shared constant. +import { log } from '../../core/log'; +import { cookieName } from '../../shared/public_prefix'; + +const RESOLVE_ENDPOINT = '/_ts/api/v1/ec/resolve'; + +// The non-HttpOnly companion the server sets alongside the Edge Cookie, named +// after the public prefix the page's bundle tag carries, such as `ts-ecr`. +// Must match COOKIE_EC_RESOLVED in crates/trusted-server-core/src/constants.rs; +// a Rust test asserts the two stay in sync. +const MARKER_COOKIE = 'ecr'; + +// The fixed, known word shared with the server. Must match EXPECTED_VALUE in +// crates/trusted-server-core/src/ec/module.rs; a Rust test asserts the two +// stay in sync. +const FIXED_WORD = 'an-ec'; + +// The permission this module requires, the same declaration the server-side +// module makes in `required_permissions` (crates/trusted-server-core/src/ec/ +// module.rs). A page module is treated like any other module: it declares +// what it requires and checks that against the resolved state the server +// hands the page before it does anything. The server enforces the same gate on +// the resolve endpoint, so this check is the page's half of one decision, not +// a substitute for the server's. A Rust test asserts the two stay in sync. +const REQUIRED_PERMISSION = 'necessary.operations.storage'; + +// Waits for the resolved permission state the edge injects into the page +// (`window.tsjs.permissions`, via `tsjs.whenPermissions()`) and returns +// whether the permission this module requires is set. With no permission state +// on the page there is nothing to check against, so the answer is no. +export async function requiredPermissionIsSet(): Promise { + const whenPermissions = window.tsjs?.whenPermissions; + if (typeof whenPermissions !== 'function') { + log.warn('ec client_fixed: no permission state on the page, not posting'); + return false; + } + const snapshot = await whenPermissions(); + return Array.isArray(snapshot?.set) && snapshot.set.includes(REQUIRED_PERMISSION); +} + +// Returns true when the resolved marker is present in `cookieString`. The Edge +// Cookie itself is HttpOnly and never appears in `document.cookie`, so the +// marker is the only signal the page has. +export function hasResolvedMarker(cookieString: string, marker: string): boolean { + return cookieString.split(';').some((part) => part.trim().startsWith(`${marker}=`)); +} + +// Posts the fixed known word to the resolve endpoint when no resolved marker is +// present and the required permission is set. Returns the word posted, or null +// when nothing was sent or the post failed (a resolve already succeeded, the +// page names no public prefix, the required permission is not set, the +// environment lacks `document`/`fetch`, or the request threw). +export async function resolveEdgeCookie(): Promise { + if (typeof document === 'undefined' || typeof fetch !== 'function') { + return null; + } + const marker = cookieName(MARKER_COOKIE); + if (!marker) { + log.info('ec client_fixed: the page names no public prefix, not posting'); + return null; + } + if (hasResolvedMarker(document.cookie, marker)) { + return null; + } + if (!(await requiredPermissionIsSet())) { + log.info('ec client_fixed: required permission not set, not posting'); + return null; + } + + try { + await fetch(RESOLVE_ENDPOINT, { + method: 'POST', + credentials: 'same-origin', + headers: { 'Content-Type': 'text/plain' }, + body: FIXED_WORD, + }); + log.info('ec client_fixed: posted the known word to the resolve endpoint'); + return FIXED_WORD; + } catch (err) { + log.warn('ec client_fixed: resolve request failed', err); + return null; + } +} + +if (typeof window !== 'undefined') { + void resolveEdgeCookie(); +} diff --git a/crates/trusted-server-js/lib/src/integrations/google_tag_manager/script_guard.ts b/crates/trusted-server-js/lib/src/integrations/google_tag_manager/script_guard.ts index 2072c04032..874748bec8 100644 --- a/crates/trusted-server-js/lib/src/integrations/google_tag_manager/script_guard.ts +++ b/crates/trusted-server-js/lib/src/integrations/google_tag_manager/script_guard.ts @@ -21,7 +21,7 @@ const GTM_URL_PATTERN = /** * Supported paths that the server can proxy. * Must match the route patterns defined in the GoogleTagManagerIntegration handler - * in crates/trusted-server-core/src/integrations/google_tag_manager.rs + * in crates/tag/google-tag-manager/src/lib.rs */ const SUPPORTED_PATHS = ['/gtm.js', '/gtag/js', '/gtag.js', '/collect', '/g/collect']; diff --git a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts index fc18beb5e9..ed07fae993 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts @@ -1499,10 +1499,10 @@ const PAGE_BIDS_PATH = '/_ts/page-bids'; * Deprecated alias of {@link PAGE_BIDS_PATH}, kept registered server-side so * pre-rename bundles keep working. This bundle falls back to it when the * canonical path does not serve page-bids: a server rolled back to before the - * rename does not register the canonical path, and an operator `[[handlers]]` - * auth regex broad enough to cover `/_ts` answers it with `401` that no - * anonymous browser fetch can satisfy. Without the fallback either case - * silently drops ads on every SPA navigation. + * rename does not register the canonical path, and a gate in front of the + * service that covers `/_ts` answers it with `401` that no anonymous browser + * fetch can satisfy. Without the fallback either case silently drops ads on + * every SPA navigation. * * Removed together with the server-side alias in IABTechLab/trusted-server#970. */ @@ -1547,7 +1547,7 @@ async function fetchPageBids( signal, }); if (!res.ok) { - // 401: an operator auth handler regex covers this path. 404: this server + // 401: a gate in front of the service covers this path. 404: this server // does not know the route. Either way the other path may still answer. // 403 (cross-site gate) and 5xx would repeat on both, so they do not. return { data: null, wrongEndpoint: res.status === 401 || res.status === 404 }; diff --git a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts index 82c0e8c6b7..4f3444dad5 100644 --- a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts @@ -23,6 +23,7 @@ import { resolveFirstImpressionElement, } from '../../core/first_impression'; import { log } from '../../core/log'; +import { cookieName } from '../../shared/public_prefix'; import { buildAdRequest, parseAuctionResponse } from '../../core/auction'; import { registerApsPrebidRenderer, validateApsRenderer } from '../aps/render'; import type { AuctionBid, AuctionEid } from '../../core/auction'; @@ -45,9 +46,9 @@ type PbjsGlobal = typeof _pbjsDefault & { }; // Prebid.js itself is NOT bundled into this module. It is served as the -// external bundle configured via `integrations.prebid.external_bundle_url` -// (required whenever the prebid integration is enabled) and owns the -// `window.pbjs` global. The Rust head injector emits a stub +// external bundle configured via `auction.prebid.external_bundle_url` +// (required whenever the prebid integration runs) and owns the +// `window.pbjs` global. The Rust module's middleware emits a stub // (`window.pbjs = window.pbjs || {que:[],cmd:[]}`) before any script runs and // Prebid.js installs its API onto that same object, so capturing the reference // at module scope is safe regardless of evaluation order. @@ -179,7 +180,7 @@ export interface PrebidNpmConfig { /** * Shape of the server-injected config at `window.__tsjs_prebid`. - * Set by the Rust IntegrationHeadInjector from trusted-server.toml values. + * Set by the Rust module's middleware from trusted-server.toml values. */ interface InjectedPrebidConfig { accountId?: string; @@ -1988,7 +1989,7 @@ function installApsBidResponseRegistry(): void { export function installPrebidNpm(config?: Partial): typeof pbjs { // The prebid integration requires the external Prebid.js bundle - // (integrations.prebid.external_bundle_url). When it failed to load (network + // (auction.prebid.external_bundle_url). When it failed to load (network // error, SRI mismatch) window.pbjs is still the head-injected stub with no // API — installing the adapter is impossible, so bail out loudly. if (!hasPrebidJsApi()) { @@ -2359,8 +2360,9 @@ export function installPrebidNpm(config?: Partial): typeof pbjs const requestScopedBidRequests = [...validBidRequests]; const hasUserIdApi = typeof pbjs.getUserIdsAsEids === 'function'; const auctionEids = collectAuctionEids(); - if (hasUserIdApi && !auctionEids) { - clearPrebidEidsCookie(); + const eidsCookie = cookieName(EID_COOKIE); + if (hasUserIdApi && !auctionEids && eidsCookie) { + clearPrebidEidsCookie(eidsCookie); } const payload = buildAdRequest(validBidRequests, { eids: auctionEids }); return { @@ -2609,7 +2611,7 @@ export function installPrebidNpm(config?: Partial): typeof pbjs log.error( `[tsjs-prebid] client-side bidder "${bidder}" has no adapter in the external ` + 'Prebid bundle. Add its exact Prebid module stem to ' + - '[integrations.prebid.bundle.modules].bidder in trusted-server.toml and ' + + '[auction.prebid.bundle.modules].bidder in trusted-server.toml and ' + 'rebuild it with `ts prebid client`.' ); } @@ -2904,15 +2906,16 @@ export function installUserIdModules(): void { /** Maximum cookie payload size in bytes (leave room for other cookies). */ const MAX_EID_COOKIE_BYTES = 3072; -/** Cookie name for persisted Prebid EIDs. */ -const EID_COOKIE_NAME = 'ts-eids'; +/** The cookie persisted Prebid EIDs are written in, named after the page's + * public prefix, such as `ts-eids`. */ +const EID_COOKIE = 'eids'; /** Cookie max-age in seconds (1 day). */ const EID_COOKIE_MAX_AGE = 86400; -/** Clears any previously persisted Prebid EIDs cookie. */ -function clearPrebidEidsCookie(): void { - document.cookie = `${EID_COOKIE_NAME}=; Path=/; Secure; SameSite=Lax; Max-Age=0`; +/** Clears any previously persisted Prebid EIDs cookie, named `name`. */ +function clearPrebidEidsCookie(name: string): void { + document.cookie = `${name}=; Path=/; Secure; SameSite=Lax; Max-Age=0`; } /** @@ -2976,31 +2979,37 @@ function logTrimmedEidSources(droppedSources: Set, trimmedSources: Set-eids`, such as + * `ts-eids`) for backend ingestion and auction fallback on later requests. A + * page naming no public prefix writes no cookie. */ function syncPrebidEidsCookie(): void { + const name = cookieName(EID_COOKIE); + if (!name) { + log.debug('the page names no public prefix, so the EIDs cookie is not written'); + return; + } try { if (typeof pbjs.getUserIdsAsEids !== 'function') { // Without Prebid EIDs to forward, stale auction fallback IDs must not persist. - clearPrebidEidsCookie(); + clearPrebidEidsCookie(name); return; } const eids = collectAuctionEids(); if (!eids) { - clearPrebidEidsCookie(); + clearPrebidEidsCookie(name); return; } const payload = fitAuctionEidsToCookie(eids); if (!payload) { - clearPrebidEidsCookie(); + clearPrebidEidsCookie(name); return; } const encoded = btoa(JSON.stringify(payload)); - document.cookie = `${EID_COOKIE_NAME}=${encoded}; Path=/; Secure; SameSite=Lax; Max-Age=${EID_COOKIE_MAX_AGE}`; + document.cookie = `${name}=${encoded}; Path=/; Secure; SameSite=Lax; Max-Age=${EID_COOKIE_MAX_AGE}`; log.debug(`[tsjs-prebid] synced ${payload.length} EID sources to cookie`); } catch (err) { diff --git a/crates/trusted-server-js/lib/src/integrations/sourcepoint/script_guard.ts b/crates/trusted-server-js/lib/src/integrations/sourcepoint/script_guard.ts index 3e1493bdbb..dd4624aa77 100644 --- a/crates/trusted-server-js/lib/src/integrations/sourcepoint/script_guard.ts +++ b/crates/trusted-server-js/lib/src/integrations/sourcepoint/script_guard.ts @@ -12,7 +12,7 @@ function normalizeSourcepointUrl(url: string): string | null { if (trimmed.startsWith('http://') || trimmed.startsWith('https://')) return trimmed; // Keep in sync with Rust `parse_sourcepoint_url()` in: - // crates/trusted-server-core/src/integrations/sourcepoint.rs + // crates/cmp/sourcepoint/src/lib.rs // (protocol-relative + bare-host handling + hostname-only host checks). // Bare domain or path — attempt to parse as https URL. // The hostname === check in isSourcepointUrl rejects non-matching domains. diff --git a/crates/trusted-server-js/lib/src/shared/public_prefix.ts b/crates/trusted-server-js/lib/src/shared/public_prefix.ts new file mode 100644 index 0000000000..2dba302c3d --- /dev/null +++ b/crates/trusted-server-js/lib/src/shared/public_prefix.ts @@ -0,0 +1,36 @@ +// The deployment's public prefix, which every cookie name this script writes +// starts with, such as `ts` for `ts-eids`. +// +// The server writes it on the bundle's own tag, so the script holds no copy of +// its own. A page whose bundle tag carries none has no names to write, and a +// module asking for one is told `null` and writes nothing. + +/** The `id` the server gives the bundle's `"#, - ctx.request_host - )] + fn create(&self, context: &MiddlewareContext<'_>) -> MiddlewareAction { + let origin = context.origin_host.to_owned(); + let host = context.request_host.to_owned(); + MiddlewareAction { + element_handlers: vec![Box::new(AttributeRewrite::new( + "data-link", + Rc::new(move |matched| { + if matched.value.contains(&origin) { + AttributeRewriteAction::replace(matched.value.replace(&origin, &host)) + } else { + AttributeRewriteAction::keep() + } + }), + ))], + ..MiddlewareAction::pass() + } } } ``` -**Behavior**: - -- Snippets are prepended into `` after the TSJS bundle tag -- Called once per HTML response -- Multiple integrations can each contribute snippets -- If no snippets are returned, no extra markup is added - -See [Integration Guide](/guide/integration-guide) for creating custom rewriters. +See [Changing a page](/guide/integration-guide#changing-a-page) for what a +middleware can rely on, and +[Placing page changes](/guide/configuration#placing-page-changes) for the +entries that place one. ## TSJS Injection @@ -800,7 +781,8 @@ The Trusted Server JavaScript (TSJS) library is automatically injected: ### Integration bundles -The integration registry selects TSJS modules by enabled integration ID. A +The integration registry selects TSJS modules by the integrations the +settings select, each known to the browser by its integration ID. A registered ID with a compiled module is included in the immediate unified bundle unless its builder calls `.with_deferred_js()`; deferred modules are served separately as `/static/tsjs=tsjs-.min.js`. Builders can call @@ -819,7 +801,7 @@ Each integration is built as a separate IIFE at compile time (into Cargo's `OUT_ - Prebid is built externally with `build-prebid-external.mjs` and served through `/integrations/prebid/bundle.js` - `tsjs-lockr.js`, `tsjs-permutive.js`, `tsjs-didomi.js`, `tsjs-datadome.js`, `tsjs-testlight.js` — Other integrations -At runtime, the server concatenates `tsjs-core.js` + the modules for enabled integrations. The URL stays `/static/tsjs=tsjs-unified.min.js?v=` for backward compatibility. +At runtime, the server concatenates `tsjs-core.js` + the modules for the integrations that run. The URL stays `/static/tsjs=tsjs-unified.min.js?v=` for backward compatibility. ## Performance Optimization diff --git a/docs/guide/ec-setup-guide.md b/docs/guide/ec-setup-guide.md index cccb04daee..87122b4fac 100644 --- a/docs/guide/ec-setup-guide.md +++ b/docs/guide/ec-setup-guide.md @@ -25,9 +25,12 @@ The `replace-with-*` values below are intentionally rejected placeholders: ```toml [ec] -passphrase = "ec_passphrase" +module = "hmac" ec_store = "ec_identity_store" +[ec.hmac] +passphrase = "ec_passphrase" + [[ec.partners]] name = "Mocktioneer SSP" source_domain = "formally-vital-lion.edgecompute.app" @@ -40,10 +43,11 @@ secret store, not the credential values. This workflow calls the inbound identify and batch-sync APIs, so its partner needs `api_token`. Partners that do not call either API may omit it. Provision high-entropy values under `ec_passphrase` and `partner_api_token`; see -[Configuration](/guide/configuration#secret-store-migration). +[Configuration](/guide/configuration#static-secret-references). Required behavior assumptions: +- `module = "hmac"` selects the built-in HMAC module, and its `passphrase` lives under `[ec.hmac]` - The value stored under `ec_passphrase` is long-lived HMAC-SHA256 keying material for EC ID derivation; use a high-entropy random value of at least 32 characters - `ec_store` is linked to the active Fastly service version - `ec_store` is the only KV-backed EC lifecycle store; it contains identity graph state, minimal consent metadata, source-domain keyed partner UIDs, and withdrawal tombstones @@ -99,13 +103,13 @@ curl -si "${TS_BASE_URL}/" \ Look for: -- `Set-Cookie: ts-ec=<64hex.6chars>` +- `Set-Cookie: ts-ec=hmac~<64hex.6chars>` (the `hmac~` prefix is the module code, and a bare value with no prefix, as a browser may already hold, still resolves) ## 5) Batch Sync (S2S) Endpoint: `POST /_ts/api/v1/batch-sync` -Important: request field is `ec_id` (full `{64hex}.{6alnum}` value). The `timestamp` field remains required for API compatibility, but it no longer orders writes because EC identity entries do not store per-partner sync timestamps. Within one request, valid mappings are grouped by normalized EC ID and the last valid UID for each group is applied once; unchanged UIDs are accepted without a write. Group outcomes are reported for every original mapping, and infrastructure failures abort the remaining groups. See the [API Reference](/guide/api-reference) for the complete accounting and failure contract. +Important: request field is `ec_id` (the full value as issued, `hmac~{64hex}.{6alnum}`, with the bare form that has no prefix also accepted). The `timestamp` field remains required for API compatibility, but it no longer orders writes because EC identity entries do not store per-partner sync timestamps. Within one request, valid mappings are grouped by normalized EC ID and the last valid UID for each group is applied once; unchanged UIDs are accepted without a write. Group outcomes are reported for every original mapping, and infrastructure failures abort the remaining groups. See the [API Reference](/guide/api-reference) for the complete accounting and failure contract. ```bash BATCH_UID="${PARTNER_UID}-batch" diff --git a/docs/guide/edge-cookies.md b/docs/guide/edge-cookies.md index 113066cdaf..8204b793a4 100644 --- a/docs/guide/edge-cookies.md +++ b/docs/guide/edge-cookies.md @@ -8,12 +8,9 @@ permit EC use. ## Policy Posture -Trusted Server is technology. It is neutral on policy. The Edge Cookie -gives the deployer a cookie slot and configuration over the surrounding -attributes. The deployer determines the policy posture based on the -laws and contractual arrangements that apply to their deployment. -Privacy outcomes follow from that configuration, not from the cookie -mechanism itself. +Trusted Server is technology. It is neutral on policy. The Edge Cookie gives the deployer a cookie slot and configuration over the surrounding attributes. The deployer determines the policy posture based on the laws and contractual arrangements that apply to their deployment. Privacy outcomes follow from that configuration, not from the cookie mechanism itself. + +An Edge Cookie (EC) is a first-party identifier that the built-in module derives on a first site visit with an HMAC of the client IP address plus a short random suffix, created only when the permission model allows it. It is passed in requests on subsequent visits and activity. Trusted Server surfaces the current EC ID via response headers and a first-party cookie. For the exact header and cookie names, see the [API Reference](/guide/api-reference). For full operational onboarding (partner configuration, batch sync, identify, and auction verification), use the [EC Setup Guide](/guide/ec-setup-guide). @@ -42,7 +39,7 @@ maps to a stable base. ### Request Lifecycle -Every request passes through four phases. EC generation only happens on organic routes (publisher proxy, integration proxy, auction) — read-only endpoints like `/identify` and `/batch-sync` skip generation entirely. During pre-routing, Trusted Server builds consent from request-local cookies, headers, geolocation, and policy defaults; it does not load consent from a separate KV store. +Every request passes through four phases. EC generation only happens on organic routes (publisher proxy, integration proxy, auction). Read-only endpoints like `/identify` and `/batch-sync` skip generation entirely. During pre-routing, Trusted Server builds the consent context from request-local cookies, headers, geolocation, and policy defaults. It does not load consent from a separate KV store. ```mermaid sequenceDiagram @@ -60,7 +57,7 @@ sequenceDiagram Note over TS: Phase 3: Finalize
Ingest Prebid EID cookies TS-->>B: Response + Set-Cookie: ts-ec=... else Return Visit (EC cookie present) - Note over TS: Phase 2: Routing
EC exists — skip generation + Note over TS: Phase 2: Routing
EC exists, skip generation Note over TS: Phase 3: Finalize
Ingest Prebid EID cookies TS-->>B: Response
(no cookie refresh) end @@ -70,11 +67,11 @@ sequenceDiagram ### Response Finalization -After routing completes, the server evaluates consent state and cookie presence to decide what to do with the EC cookie on the response. +After routing completes, the server evaluates the permission state and cookie presence to decide what to do with the EC cookie on the response. ```mermaid flowchart TD - Start[ec_finalize_response] --> ConsentCheck{Consent
allows EC?} + Start[ec_finalize_response] --> ConsentCheck{Permissions
allow EC?} ConsentCheck -- "No" --> ExplicitWithdrawal{Explicit
withdrawal?} ExplicitWithdrawal -- "Yes" --> CookiePresent{Cookie was
present?} @@ -87,44 +84,71 @@ flowchart TD WasPresent -- "No, just generated" --> NewEc["Ingest Prebid EID cookies
Set ts-ec cookie"] ``` -When consent cannot be verified for the current request — for example, unknown jurisdiction or missing/undecodable consent signals in a regulated region — Trusted Server fails closed for EC use by stripping EC headers, but it does **not** treat that as authoritative revocation of an already-issued EC. +When the required permissions are not set for the current request (for example a location lookup that failed, or a baseline that requires a signal no module has answered), Trusted Server fails closed for EC use by stripping EC headers, but it does **not** treat that as authoritative revocation of an already-issued EC. + +## Permission Gating + +EC creation is gated through the [permission model](/guide/permission-model), not by a jurisdiction rule baked into the core. The Edge Cookie module advertises the permissions its data use requires, and Trusted Server creates an Edge Cookie only when every required permission is set. The built-in HMAC module requires `necessary.operations.storage` (TCF Purpose 1), because the `Set-Cookie` operation stores information on the device. + +The Edge Cookie code never reads consent. It checks only whether the required **permission** is set. Consent is one of the sources that _set_ a permission, not something the gate reads directly, so the Edge Cookie logic does not change when a consent framework changes. Two sources combine for each request: + +- **A country and region baseline.** The country, and an optional region such as a US state, that the geo module returns. A region rule takes precedence over its country, and when no country is identified, or neither the region nor the country has a rule, the baseline on the top node of the rules tree applies. +- **Consent and privacy signals.** TCF, GPP, a US Privacy string, GPC and the Model Terms for Marketing preference (`euconsent-v2`, `__gpp` / `__gpp_sid`, `us_privacy`, `Sec-GPC`, `__mtm_pref`), each read by its own [permission signal module](/guide/permission-signals) and mapped onto permissions as a **grant or a revoke** on top of that baseline, in the configured order. There is no separate consent KV fallback. + +Every permission is resolved this way, and the Edge Cookie is created only when each permission its module requires is set. A location lookup that fails resolves at the requires-signal floor, so the cookie is not created unless a signal grants the permission. The core encodes no jurisdiction's law. The deployer brings the policy, and the per-country and per-region rules live in the policy file rather than in core logic. See the [permission model](/guide/permission-model) for the full list of permission sources and the resolution order. Writing browser-supplied Prebid EIDs into the identity graph, and server-side pull sync, additionally need the storage and personalised-ad permissions both set, the pair that also gates EIDs in the bid request. With storage alone the Edge Cookie stays first-party and partner sharing is withheld, so the EC ID is not sent to pull-sync partners. See [Prebid EID Flow](#prebid-eid-flow). + +```mermaid +flowchart TD + Start[Resolve country and region] --> Baseline[Country or region rule,
else the top of the rules tree] + Baseline --> Signals[Apply consent/privacy signals
as a grant or revoke] + Signals --> Check{Module's required
permissions all set?} + Check -- "Yes" --> Allow([Create EC]) + Check -- "No" --> Deny([No EC]) +``` + +The `ec_identity_store` KV store is the only EC lifecycle store. It holds identity graph state, source-domain keyed partner UIDs, a minimal consent snapshot used for EC entry metadata, withdrawal tombstones, and completion markers that prevent stale point-read misses from rewriting completed tombstones. A marker key records the original tombstone's validity bound and is ignored at or after that time, even if its KV row has not expired yet. If the clock is unusable, the marker is ignored and withdrawal falls back to a strong root-existence check; only a confirmed existing root can be written. This prevents a stale marker from suppressing withdrawal when an expired EC key is created again. With a healthy store, repeated withdrawal of an already tombstoned EC ID leaves the row unchanged instead of refreshing its 24-hour TTL. Permission resolution for each request is based on the live request signals listed above. + +## Module Types: Server-Side and Client-Side -## Consent Model +The Edge Cookie identifier comes from a configurable module, selected by `[ec] module`. A module is one of two types, and the permission gate above applies to both. The two reach the **same outcome** (a `ts-ec` cookie set and carried on every later request) by **different routes**. -EC creation is gated by jurisdiction. The server detects jurisdiction from geolocation data attached to the request and applies the corresponding consent rules. Live consent comes from request-local signals (`euconsent-v2`, `__gpp`, `__gpp_sid`, `us_privacy`, `Sec-GPC`) plus geolocation and policy defaults; there is no separate consent KV fallback. +- **Server-side** (for example the built-in HMAC module, or the built-in `host_signals` module that derives an identifier from the host's TLS JA4 and HTTP/2 signals on a host that supplies them). The module derives the identifier at the edge from request data in `generate()`, and the **page response** sets the cookie. Nothing client-side is involved. +- **Client-side** (for example the `client_fixed` demonstration module). The module cannot derive the identifier at the edge, so `generate()` defers and returns no identifier. The page then runs the module's own JavaScript in the browser, which does its work and posts the result to the resolve endpoint. The module derives an identifier from that value in `resolve_from_client()`, and the **resolve response** sets the cookie. ```mermaid flowchart TD - Start[Detect Jurisdiction] --> J{Jurisdiction?} - - J -- "GDPR
(EU/UK)" --> TCF{TCF string
present?} - TCF -- "Yes" --> P1{Purpose 1
granted?} - P1 -- "Yes" --> Allow([Allow EC]) - P1 -- "No" --> Deny([Deny EC]) - TCF -- "No" --> Deny - - J -- "US State" --> GPC{GPC header
set?} - GPC -- "Yes" --> Deny - GPC -- "No" --> USTCF{TCF from CMP
e.g. Didomi?} - USTCF -- "Yes" --> USP1{Purpose 1
granted?} - USP1 -- "Yes" --> Allow - USP1 -- "No" --> Deny - USTCF -- "No" --> USP{US Privacy
string?} - USP -- "Yes" --> OptOut{Opt-out
sale?} - OptOut -- "No" --> Allow - OptOut -- "Yes" --> Deny - USP -- "No" --> Deny - - J -- "Non-regulated" --> Allow - J -- "Unknown
(no geo data)" --> Deny + Start(["Page request, no Edge Cookie"]) --> Type{"Module type"} + + Type -->|"Server-side (e.g. HMAC)"| SGen["generate() derives at the edge from request data"] + SGen --> SSet["Page response sets ts-ec"] + + Type -->|"Client-side (e.g. client_fixed)"| CDefer["generate() defers, returns no identifier"] + CDefer --> CPage["Page response, no cookie, delivers the module JS"] + CPage --> CBox[["Module JS (black box): runs in the browser and does its work"]] + CBox --> CPost["JS posts the result to POST /_ts/api/v1/ec/resolve"] + CPost --> CResolve["resolve_from_client() verifies and derives"] + CResolve --> CSet["Resolve response sets ts-ec"] + + SSet --> Same(["Same outcome: ts-ec set, carried on every later request"]) + CSet --> Same ``` -- **GDPR**: Opt-in required. TCF Purpose 1 (store/access device) must be explicitly consented. Writing browser-supplied Prebid EIDs into the identity graph additionally requires TCF Purpose 4 (personalized ads); see [Prebid EID Flow](#prebid-eid-flow). Without Purpose 4, server-side pull sync is also skipped, so the EC ID is not sent to pull-sync partners. -- **US State**: Opt-out model with three-tier fallback — GPC always blocks, then TCF if a CMP uses it, then US Privacy string, then fail-closed. -- **Non-regulated**: EC always allowed. -- **Unknown**: Fail-closed when jurisdiction cannot be determined. +The two types differ only in route and in the methods they use: + +| Feature | Server-side | Client-side | +| -------------------- | ------------------------- | ---------------------------------------------- | +| Example | HMAC (`hmac`) | `client_fixed` (demonstration) | +| Created in | `generate()`, at the edge | `resolve_from_client()`, from the posted value | +| `generate()` returns | the identifier | no identifier (defers) | +| Client JavaScript | none | the module JS (black box), which posts back | +| Endpoint | none | `POST /_ts/api/v1/ec/resolve` | +| Cookie set on | the page response | the resolve response | + +The resolve endpoint requires an `Origin` on the publisher's domain and a `text/plain` or `application/json` body, and it answers `409` rather than silently replacing an identity the request already carries. A created identifier is persisted to the identity graph before the cookie is set, so withdrawal reaches a client-set identity the same way it reaches an edge-created one. On success the cookie is set on the endpoint's own first-party `200` response, so the value is live for every subsequent request without a second navigation. The cookie is `HttpOnly`, so the page script never reads it back, and a non-`HttpOnly` marker cookie (`ts-ecr=1`, carrying no identity) tells the script a resolve succeeded so it does not post again on every page view. Every resolve response carries `Cache-Control: no-store`. The `client_fixed` demonstration module is compiled only behind the `client-fixed-demo` cargo feature, so a production build rejects selecting it at startup. + +Because the posted value comes from the browser, **verification is the module's responsibility**. A client-side module must verify the payload (for example a signature) before creating an identifier, or a client could forge an Edge Cookie. The endpoint itself is module-agnostic. It bounds the body, applies the same permission gate as organic generation, calls the module, and writes the cookie. -The `ec_identity_store` KV store is the only EC lifecycle store. It holds identity graph state, source-domain keyed partner UIDs, a minimal consent snapshot used for EC entry metadata, withdrawal tombstones, and completion markers that prevent stale point-read misses from rewriting completed tombstones. A marker key records the original tombstone's validity bound and is ignored at or after that time, even if its KV row has not expired yet. If the clock is unusable, the marker is ignored and withdrawal falls back to a strong root-existence check; only a confirmed existing root can be written. This prevents a stale marker from suppressing withdrawal when an expired EC key is created again. With a healthy store, repeated withdrawal of an already tombstoned EC ID leaves the row unchanged instead of refreshing its 24-hour TTL. Consent interpretation for each request remains based on the live request signals listed above. +A built-in `client_fixed` module demonstrates the client-side type end to end with no vendor coupling. Client and server share one fixed, known word. When no Edge Cookie is present, the page script (shipped in the tsjs bundle when that module is selected) posts that word, the server verifies it matches, and on a match sets it as the Edge Cookie. The value is verifiable because it is a known constant, which is the point of the demo. It is useless in production, because a fixed value is not an identity, so it is for demonstration and testing only. ## Partner Sync Channels @@ -156,7 +180,7 @@ Browser-resolved Prebid EIDs reach the identity graph when Trusted Server finali The `/auction` body carries the full EID set Prebid.js resolved, so `/auction` ingestion is not limited by the size cap TSJS applies to the `ts-eids` cookie. The cookie still bridges Prebid user ID modules with the identity graph on requests that carry no EID body, such as `GET /_ts/page-bids` and page navigations. -These Prebid EID writes follow the same consent rule as bidstream EID forwarding: when a TCF signal is present, Purpose 1 (store/access device) and Purpose 4 (personalized ads) must both be consented, and when GDPR applies without a TCF signal the writes are withheld. A user who consents to Purpose 1 but not Purpose 4 keeps their EC, but no EIDs from the request are persisted. +These Prebid EID writes follow the same permission rule as bidstream EID forwarding: the storage and personalised-ad permissions must both be set. A reader who allows storage but not personalised ads keeps their EC, but no EIDs from the request are persisted. ```mermaid sequenceDiagram @@ -176,7 +200,7 @@ sequenceDiagram Note over B,TS: Next eligible EID-sync request B->>TS: Document navigation, POST /auction,
or admitted GET /_ts/page-bids TS->>TS: Base64 decode → parse OpenRTB-style EIDs
match source domains to partners - TS->>KV: Add missing partner IDs in one conditional write
skip when UIDs already match
(requires TCF Purpose 1 + 4 under GDPR) + TS->>KV: Add missing partner IDs in one conditional write
skip when UIDs already match
(requires the storage and personalised-ad permissions) ``` Current TSJS writers preserve the full OpenRTB-style `{source, uids:[...]}` shape in `ts-eids`. The server remains backward-compatible with earlier flattened `{source, id, atype}` cookies during rollout, but new cookies use the structured `uids[]` form. @@ -225,7 +249,7 @@ sequenceDiagram DSP-->>PS: OpenRTB bid response PS-->>TS: OpenRTB seatbid response TS-->>B: Auction response + x-ts-eids header when available - TS->>KV: Add missing partner UIDs from request eids[]
+ ts-eids/sharedId cookies (TCF Purpose 1 + 4 under GDPR) + TS->>KV: Add missing partner UIDs from request eids[]
+ ts-eids/sharedId cookies (storage and personalised-ad permissions) ``` The relevant OpenRTB structure forwarded to Prebid Server and downstream partners is: @@ -310,7 +334,7 @@ sets `Path=/`, `Secure`, `HttpOnly`, `SameSite=Lax`, and a `Max-Age`. - Returning requests with consent and an existing `ts-ec` do not refresh the EC cookie or KV TTL. - Newly generated ECs receive `Set-Cookie: ts-ec=...`. -- When consent is blocked but not explicitly withdrawn, Trusted Server strips EC response headers for that request but leaves any existing `ts-ec` cookie intact; cookie expiry and tombstones happen only on explicit withdrawal. +- When the permission is not set but nothing was explicitly withdrawn, Trusted Server strips EC response headers for that request but leaves any existing `ts-ec` cookie intact; cookie expiry and tombstones happen only on explicit withdrawal. Withdrawal is deliberately narrow: a TCF record refusing storage in a jurisdiction whose baseline did not grant it. US-style opt-outs (GPC, a GPP sale opt-out, or a US Privacy opt-out) suppress use for the request but never expire the cookie or write a tombstone, so lifting the opt-out restores the identity. - `/_ts/api/v1/identify` is read-oriented and returns identity enrichment for the authenticated partner. It computes `cluster_size` only when the EC entry does not already store one. - `/_ts/api/v1/batch-sync` validates every input, groups valid mappings by normalized EC ID, and applies the last valid UID for each group once. Group outcomes still account for every original input; infrastructure failures reject the failing and remaining groups. Mapping timestamps remain required for API compatibility but do not order writes. See the [API Reference](/guide/api-reference) for the complete contract. - Pull sync fills missing partner UIDs only. Existing partner UIDs are not periodically refreshed because EC entries no longer store per-partner sync timestamps. diff --git a/docs/guide/edgezero.md b/docs/guide/edgezero.md index ed9c4f2f2c..7c7e4bc2eb 100644 --- a/docs/guide/edgezero.md +++ b/docs/guide/edgezero.md @@ -69,8 +69,9 @@ store before deletion, especially when using `--store` or `--no-env`. 4. Run `ts config push --adapter ADAPTER`. 5. Start or deploy the adapter and verify a non-health publisher route. A healthy endpoint alone may not prove that application configuration loaded. - Confirm that its `x-ts-version` response header reports the release you - deployed. + Confirm which build is serving, from the attestation document's `build` + fields, or from the `x-ts-version` header on a page where a `[[serve]]` + entry names `service_version`. See [Configuration](/guide/configuration) for the field contract, the [CLI guide](/guide/cli) for exact command help, and diff --git a/docs/guide/error-reference.md b/docs/guide/error-reference.md index c8d48be43d..67c25e0e57 100644 --- a/docs/guide/error-reference.md +++ b/docs/guide/error-reference.md @@ -75,7 +75,7 @@ TOML file. - `publisher.domain` - `publisher.origin_url` - `publisher.proxy_secret` -- `ec.passphrase` +- `ec.hmac.passphrase` (when `ec.module = "hmac"`) --- @@ -94,15 +94,13 @@ credentials or fragment: ```toml # ❌ Wrong -[auction.providers.pbs-main] -protocol = "openrtb-2.6" -profile = "prebid-server" +[demand.pbs_main] +implementation = "auction.prebid-server" endpoint = "prebid.example.com/openrtb2/auction" # ✅ Correct -[auction.providers.pbs-main] -protocol = "openrtb-2.6" -profile = "prebid-server" +[demand.pbs_main] +implementation = "auction.prebid-server" endpoint = "https://prebid.example.com/openrtb2/auction" ``` @@ -118,13 +116,12 @@ Failed to parse environment variable: TRUSTED_SERVER__PUBLISHER__DOMAIN **Cause:** Environment variable format doesn't match expected type -**Solution:** Override an existing scalar leaf with the expected type. Provider -map keys preserve hyphens, so shell users must invoke the CLI through `env`: +**Solution:** Override an existing scalar leaf with the expected type. Demand +source names are snake_case, so each maps straight onto a path segment: ```bash env 'TRUSTED_SERVER__PUBLISHER__DOMAIN=example.com' \ - 'TRUSTED_SERVER__AUCTION__PROVIDERS__PBS-MAIN__TIMEOUT_MS=1000' \ - 'TRUSTED_SERVER__INTEGRATIONS__PREBID__ENABLED=true' \ + 'TRUSTED_SERVER__DEMAND__PBS_MAIN__TIMEOUT_MS=1000' \ ts config validate ``` @@ -149,10 +146,13 @@ Failed to generate EC ID: HMAC error **Solution:** -1. Ensure `passphrase` names a secret-store entry in `trusted-server.toml`: +1. Ensure the `hmac` module is selected and its `passphrase` names a secret-store entry in `trusted-server.toml`: ```toml [ec] +module = "hmac" + +[ec.hmac] passphrase = "ec_passphrase" ``` @@ -160,7 +160,8 @@ passphrase = "ec_passphrase" passphrase value: ```bash -TRUSTED_SERVER__EC__PASSPHRASE=ec_passphrase +TRUSTED_SERVER__EC__MODULE=hmac +TRUSTED_SERVER__EC__HMAC__PASSPHRASE=ec_passphrase ``` 3. Provision a high-entropy value of at least 32 characters under @@ -176,9 +177,9 @@ TRUSTED_SERVER__EC__PASSPHRASE=ec_passphrase Backend not found: prebid-server ``` -**Cause:** Dynamic backend creation for a configured provider endpoint failed. -Provider backends are derived from `[auction.providers.]`; they are not -manually named static Fastly backends. +**Cause:** Dynamic backend creation for a configured demand endpoint failed. +Demand backends are derived from `[demand.]`; they are not manually +named static Fastly backends. **Solution:** @@ -227,14 +228,14 @@ Upstream request timeout after 1000ms **Solution:** -1. Increase the affected server provider timeout: +1. Increase the affected demand source's timeout: ```toml -[auction.providers.pbs-main] +[demand.pbs_main] timeout_ms = 2000 ``` -Browser `[integrations.prebid].timeout_ms` is independent and does not control +Browser `[auction.prebid].timeout_ms` is independent and does not control Prebid Server transport. 2. Verify upstream service is responsive: @@ -289,14 +290,14 @@ Prebid Server returned 400: Invalid OpenRTB request **Solution:** -1. Enable debug mode on the Prebid Server profile: +1. Enable debug mode on the Prebid Server demand source: ```toml -[auction.providers.pbs-main] -profile_config = { debug = true } +[demand.pbs_main] +debug = true ``` -`[integrations.prebid].debug` controls browser Prebid.js only. +`[auction.prebid].debug` controls browser Prebid.js only. 2. Check logs for request/response details 3. Verify bidders are supported by your Prebid Server @@ -331,8 +332,10 @@ Next.js links still pointing to origin domain 2. Update `rewrite_attributes` to match actual keys: ```toml -[integrations.nextjs] -enabled = true +[framework] +module = "nextjs" + +[framework.nextjs] rewrite_attributes = ["href", "link", "url", "src"] # Add keys you find ``` @@ -360,7 +363,10 @@ Failed to fetch Permutive SDK: 404 Not Found 2. Update configuration: ```toml -[integrations.permutive] +[audience] +module = "permutive" + +[audience.permutive] organization_id = "myorg" workspace_id = "workspace-123" ``` @@ -381,15 +387,15 @@ curl https://myorg.edge.permutive.app/workspace-123-web.js No route matched for /integrations/custom/endpoint ``` -**Cause:** Integration not enabled or route not registered +**Cause:** No section selects the integration's module, or the route is not registered **Solution:** -1. Enable integration: +1. Select the module in the section of its type so it runs: ```toml -[integrations.custom] -enabled = true +[testing] +modules = ["custom"] ``` 2. Verify integration is compiled in (check build logs) @@ -413,12 +419,8 @@ Signing key not found: ts-2025-01-A 1. Check Config Store `current-kid` value 2. Verify corresponding key exists in Secret Store -3. Run key rotation to generate new key: - -```bash -curl -X POST https://edge.example.com/_ts/admin/keys/rotate \ - -u admin:password -``` +3. Rotate to a new key with `ts keys rotate`, as + [Key Rotation](./key-rotation.md) describes. --- @@ -436,13 +438,8 @@ curl -X POST https://edge.example.com/_ts/admin/keys/rotate \ **Solution:** -1. Initialize keys using rotation endpoint: - -```bash -curl -X POST https://edge.example.com/_ts/admin/keys/rotate \ - -u admin:password -``` - +1. Create the first key with `ts keys rotate`, as + [Key Rotation](./key-rotation.md) describes. 2. Verify Config Store has `active-kids` entry 3. Check Secret Store contains the key @@ -502,13 +499,11 @@ Config store not found: jwks_store - Service → Configuration → Config Stores - Add store with exact name from configuration -3. Update `trusted-server.toml`: +3. Switch request signing on in `trusted-server.toml`: ```toml [request_signing] enabled = true -config_store_id = "your-config-store-id" # From Fastly dashboard -secret_store_id = "your-secret-store-id" ``` --- @@ -668,18 +663,21 @@ cargo install viceroy --version 0.17.0 --locked --force ### Enable Debug Logging -Browser Prebid.js debug remains under `[integrations.prebid]`: +Browser Prebid.js debug remains under `[auction.prebid]`: ```toml -[integrations.prebid] +[auction] +modules = ["prebid"] + +[auction.prebid] debug = true ``` -For Prebid Server diagnostics, enable debug in that provider's profile: +For Prebid Server diagnostics, enable debug on that demand source: ```toml -[auction.providers.pbs-main] -profile_config = { debug = true } +[demand.pbs_main] +debug = true ``` **Check Fastly logs:** diff --git a/docs/guide/fastly.md b/docs/guide/fastly.md index fa64280542..aefc396190 100644 --- a/docs/guide/fastly.md +++ b/docs/guide/fastly.md @@ -309,6 +309,10 @@ for the initial provisioning and linking sequence, subsequent CLI pushes, and precautions when changing a live mapping. A process-environment override alone does not configure the Fastly runtime. +One service can serve several publishers, each from an app-config blob of its +own that the request's host chooses. See +[One blob for each host](/guide/configuration#one-blob-for-each-host). + Create the separate request-signing store when that feature is enabled: ```bash @@ -336,8 +340,11 @@ Configure the secret-store key name in `trusted-server.toml`: ```toml [ec] -passphrase = "ec_passphrase" +module = "hmac" ec_store = "ec_identity_store" + +[ec.hmac] +passphrase = "ec_passphrase" ``` Store the high-entropy passphrase under that key in `ts_secrets`. The resolved @@ -377,10 +384,10 @@ Run the repository smoke from a clean shell: The script creates an isolated application config, applies its publisher-origin overrides, and runs strict validation. It then executes `ts config push ---adapter fastly --local`, adds all three required entries to +--adapter fastly --local`, adds both required entries to `[local_server.secret_stores.ts_secrets]`, and starts `fastly compute serve` -through Viceroy. The required keys are `handler_password`, -`publisher_proxy_secret`, and `ec_passphrase`. +through Viceroy. The required keys are `publisher_proxy_secret` and +`ec_passphrase`. The check deliberately proves both halves of startup. With no config entry, it requires `/health` to return 200 while the publisher route returns 500 with the diff --git a/docs/guide/gdpr-compliance.md b/docs/guide/gdpr-compliance.md index 556a7abfd2..235825265f 100644 --- a/docs/guide/gdpr-compliance.md +++ b/docs/guide/gdpr-compliance.md @@ -7,9 +7,11 @@ Consent signal handling in Trusted Server. Trusted Server reads consent signals from each request, decodes them, and applies built-in enforcement rules to consent-gated activities such as EC creation and EID forwarding. The publisher configures how -signals are interpreted: which countries and US states map to each -jurisdiction's rules, how Global Privacy Control is read, how -conflicting signals are resolved, and when stored signals expire. +signals are interpreted, meaning how Global Privacy Control is read, +how conflicting signals are resolved, and when stored signals expire. +Which countries and US states fall under which jurisdiction's rules is +not set here, because the `permissions.yaml` rules tree states it. See +the [Permission Model](/guide/permission-model). The per-activity gates and their fail-closed defaults are built in. ## Policy Posture @@ -96,12 +98,6 @@ Configure consent handling in the `[consent]` section of mode = "interpreter" # or "proxy" (forward raw strings without decoding) max_consent_age_days = 365 # expiration check for dated signals -[consent.gdpr] -applies_in = ["DE", "FR"] # countries mapped to the GDPR rules - -[consent.us_states] -privacy_states = ["CA", "CO"] # US states mapped to the US state rules - [consent.us_privacy_defaults] gpc_implies_optout = true # how the Sec-GPC header is interpreted @@ -112,6 +108,17 @@ mode = "restrictive" # or "newest" / "permissive" Each field tunes how signals are interpreted. The per-jurisdiction gates and their fail-closed defaults are built in. +Which jurisdiction applies to a visitor is not configured here. The +`[consent.gdpr] applies_in` and `[consent.us_states] privacy_states` +lists are retired, and a `jurisdiction` attribute on the +`permissions.yaml` rules tree does their job. Every node of that tree +may name the jurisdiction for the places it covers, a node that names +none inherits the nearest one above it, and the top of the tree answers +a visitor whose country cannot be resolved. One file therefore carries +the permission baselines and the jurisdiction assignment together. The +[Permission Model](/guide/permission-model) documents the tree, so this +page does not repeat it. + ## Operational Behavior - Consent checks run before consent-gated activities (EC creation, @@ -120,7 +127,8 @@ gates and their fail-closed defaults are built in. Resolution of conflicting signals is configurable (restrictive, newest, or permissive). - Audit logging records the consent decision per gated activity. -- Regional rules are applied per detected jurisdiction. +- Regional rules are applied per detected jurisdiction, which the rules + tree assigns from the visitor's country and region. ## Best Practices @@ -136,6 +144,7 @@ gates and their fail-closed defaults are built in. ## Next Steps +- [Permission Model](/guide/permission-model) - [Configuration Reference](/guide/configuration) - [Edge Cookies](/guide/edge-cookies) - [Architecture](/guide/architecture) diff --git a/docs/guide/getting-started.md b/docs/guide/getting-started.md index d83b0cc863..60a7e25229 100644 --- a/docs/guide/getting-started.md +++ b/docs/guide/getting-started.md @@ -90,11 +90,10 @@ export TRUSTED_SERVER_CONFIG_TRUSTED_SERVER_CONFIG_TRUSTED_SERVER_CONFIG="$( jq -r '.trusted_server_config' .edgezero/local-config-trusted_server_config.json )" -# Populate the three secret references from the starter config for this shell. +# Populate the two secret references from the starter config for this shell. # Use stable values only if you need existing proxy URLs or EC IDs to remain valid. export TRUSTED_SERVER_SECRET_TRUSTED_SERVER_SECRETS_PUBLISHER_PROXY_SECRET="$(openssl rand -base64 32)" export TRUSTED_SERVER_SECRET_TRUSTED_SERVER_SECRETS_EC_PASSPHRASE="$(openssl rand -base64 32)" -export TRUSTED_SERVER_SECRET_TRUSTED_SERVER_SECRETS_HANDLER_PASSWORD="$(openssl rand -base64 32)" # Build and start the dev server in the same shell. cargo run -p trusted-server-adapter-axum @@ -129,8 +128,8 @@ ephemeral secret-store values only into the current shell; do not put secret values in the TOML config, config-store blob, or a source-controlled environment file. -> **Dev server limitations:** The Axum adapter does not support KV store, -> geo lookup, config/secret-store writes, or admin key-management routes. +> **Dev server limitations:** The Axum adapter does not support KV store or +> geo lookup. > See [Architecture](/guide/architecture) for the full list. ### Build the Project @@ -174,18 +173,20 @@ only the asset proxy entries you want to serve or block. Then validate it. Edit `trusted-server.toml` to configure: -- browser integrations under `[integrations.*]`; -- server auction providers under map-shaped `[auction.providers.]`; -- server bidder routes under `[auction.bidders.]`; -- KV store mappings; -- EC configuration; -- consent settings (`[gdpr]`); and -- stable key names for `trusted_server_secrets`. +- the page modules that run, each selected in the section of its type (`[cmp]`, `[tag]`, `[ad-tag]`, `[bot-protection]`, `[identity]`, `[audience]`, `[framework]`, and `[auction]` for Prebid), with their settings under `[
.]` +- the `[[fetch]]` and `[[serve]]` entries that say which pages each selected module's page changes run on, which the audit's draft writes for the modules it selects +- the demand sources, in `[demand] modules`, each with its settings under `[demand.]` +- the ad server, if one runs, in `[ad-server] module`, with its settings under `[ad-server.]` +- server bidder routes under `[auction.bidders.]` +- KV store mappings +- Edge Cookie configuration under `[ec]` +- stable key names for `trusted_server_secrets` Do not put a Prebid Server URL or server bidder list under -`[integrations.prebid]`, and do not put APS account/endpoint/timeout fields under -`[integrations.aps]`. Those server values belong to auction provider common -fields and `profile_config`. +`[auction.prebid]`. Those server values belong to a `[demand.]` table +whose `implementation` is `auction.prebid-server`, and APS has no integration table at +all, because it is selected in `[demand]` too. The rules every one of these +tables follows are in [Configuration Rules](/guide/configuration-rules). Before the first push, provision the physical store mapped from logical `trusted_server_secrets` with the credential values referenced by the config. @@ -211,13 +212,15 @@ Restart or redeploy instances after secret rotation. See TRUSTED_SERVER__GIT_VERSION=v1.3.0 fastly compute publish ``` -The build compiles `TRUSTED_SERVER__GIT_VERSION` into the binary and reports it -in the `x-ts-version` response header. The Fastly service version is in -`x-ts-fastly-version`. Without the variable, a build reports the checked-out -tag, else the branch, else the first 6 characters of the commit, and a build -outside a git checkout omits the header. The value is public on every -response, so pass a neutral value such as the commit hash when a branch name -is sensitive. +The build compiles `TRUSTED_SERVER__GIT_VERSION` into the binary. Without the +variable, a build compiles in the checked-out tag, else the branch, else the +first 6 characters of the commit, and a build outside a git checkout knows no +version. No adapter writes it on a response. A deployment that names core's +`service_version` middleware in a `[[serve]]` entry writes it as `x-ts-version` +on the pages the entry covers, with the Fastly service version as +`x-ts-service-version`, so pass a neutral value such as the commit hash when a +branch name is sensitive. See +[Headers that describe the deployment](/guide/configuration#headers-that-describe-the-deployment). ## Next Steps diff --git a/docs/guide/integration-guide.md b/docs/guide/integration-guide.md index f127d4207c..664a514a92 100644 --- a/docs/guide/integration-guide.md +++ b/docs/guide/integration-guide.md @@ -7,14 +7,14 @@ Axum, or Spin SDK types. ## Choose the narrowest hook -- `IntegrationProxy` owns explicit method/path endpoints and receives - `Settings`, `RuntimeServices`, and an EdgeZero-neutral request. -- `IntegrationAttributeRewriter` inspects selected HTML attributes. -- `IntegrationScriptRewriter` handles one declared selector. -- `IntegrationHeadInjector` inserts deterministic head markup. -- `IntegrationHtmlPostProcessor` is for bounded whole-document work that - cannot be performed during streaming. +- `IntegrationProxy` owns explicit method/path endpoints and receives an + EdgeZero-neutral request and its call into the request's module context, + from which it names the `Settings`, the `RuntimeServices` and whatever + else it needs. - `IntegrationRequestFilter` makes an early request decision. +- `Middleware` changes a page, on the pages where the operator's settings + place it, and is the one way a module changes a page. See + [Changing a page](#changing-a-page). Build one `IntegrationRegistration` with only the hooks the feature needs. Use `with_deferred_js()` only for a separately loaded integration module and @@ -23,7 +23,7 @@ namespaced and bounded; do not introduce a general outbound proxy. ## Compiling core-neutral fixture -The fixture below registers an attribute rewriter and constructs every +The fixture below registers a middleware and constructs every required `RuntimeServices` service without an adapter dependency. The documentation test extracts this exact fence and compiles it as an isolated crate. @@ -32,12 +32,16 @@ crate. ```rust use std::net::IpAddr; +use std::rc::Rc; use std::sync::Arc; use error_stack::Report; use trusted_server_core::integrations::{ - AttributeRewriteAction, IntegrationAttributeContext, - IntegrationAttributeRewriter, IntegrationRegistration, + AttributeRewriteAction, IntegrationRegistration, +}; +use trusted_server_core::middleware::{ + AttributeRewrite, AttributeRewriteFn, Middleware, MiddlewareAction, + MiddlewareContext, MiddlewarePhase, }; use trusted_server_core::platform::{ BackendNamingPolicy, ClientInfo, GeoInfo, PlatformBackend, @@ -130,42 +134,46 @@ impl PlatformBackend for FixtureBackend { struct FixtureGeo; +#[async_trait::async_trait(?Send)] impl PlatformGeo for FixtureGeo { - fn lookup( + async fn lookup( &self, _client_ip: Option, + _services: &RuntimeServices, ) -> Result, Report> { Ok(None) } } -struct AssetRewriter; +struct AssetAddress; -impl IntegrationAttributeRewriter for AssetRewriter { - fn integration_id(&self) -> &'static str { - "example" +impl Middleware for AssetAddress { + fn middleware_id(&self) -> &'static str { + "testing.example" } - fn handles_attribute(&self, attribute: &str) -> bool { - matches!(attribute, "src" | "href") + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Fetch] } - fn rewrite( - &self, - _attribute: &str, - value: &str, - _context: &IntegrationAttributeContext<'_>, - ) -> AttributeRewriteAction { - value - .strip_prefix("https://assets.example/") - .map(|path| AttributeRewriteAction::replace(format!("/assets/{path}"))) - .unwrap_or_else(AttributeRewriteAction::keep) + fn create(&self, _context: &MiddlewareContext<'_>) -> MiddlewareAction { + let decide: Rc = Rc::new(|matched| { + matched + .value + .strip_prefix("https://assets.example/") + .map(|path| AttributeRewriteAction::replace(format!("/assets/{path}"))) + .unwrap_or_else(AttributeRewriteAction::keep) + }); + MiddlewareAction { + element_handlers: AttributeRewrite::each(&["src", "href"], &decide), + ..MiddlewareAction::pass() + } } } pub fn registration() -> IntegrationRegistration { IntegrationRegistration::builder("example") - .with_attribute_rewriter(Arc::new(AssetRewriter)) + .with_middleware(Arc::new(AssetAddress)) .build() } @@ -189,14 +197,121 @@ Production adapters replace every unavailable or fixture service with the target implementation. The builder deliberately panics when a required service is omitted, so adapter startup must construct the complete service graph. +### A middleware for a media type other than HTML + +A middleware works on `text/html` unless it says otherwise, with +`handles_media_type`, and an entry naming it under a media type it does not +handle is refused when the registry is built. On a media type other than +HTML an action may carry a stream processor alone, which is handed the body +after core has moved the origin's addresses in it, chunk by chunk, with the +last chunk marked. Head markup and the element and text handlers are +decisions about HTML, and an action that makes one on another media type is +refused by name when the chain is planned. + +A middleware named in an entry with a path, for a media type other than +HTML, also runs where the origin answers `404 Not Found` for a path under +it, on an empty document of the entry's media type, and what it writes is +the file. A middleware that adds to a file therefore writes the same on an +empty one, with nothing before its first line. + +```rust +struct Sellers; + +impl Middleware for Sellers { + fn middleware_id(&self) -> &'static str { + "example.sellers" + } + + fn handles_media_type(&self, media_type: &str) -> bool { + media_type == "text/plain" + } + + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Fetch] + } + + fn create(&self, _context: &MiddlewareContext<'_>) -> MiddlewareAction { + MiddlewareAction { + stream: Some(Box::new(AppendLines::new(["example-ssp.com, 1, DIRECT"]))), + ..MiddlewareAction::pass() + } + } +} +``` + +The entry that places it names the media type and, where the change is for +one file, its path. See "Entries for a media type other than HTML" in the +[configuration guide](/guide/configuration#entries-for-a-media-type-other-than-html). + +### A middleware that prepares, and one that refuses + +A middleware whose decision needs the request's services, or has to wait on +the key-value store or the network, answers `prepare` with a future. Core +runs it in the request's handler, before the document is processed, as the +module the middleware belongs to and with the permissions it declares in +`required_permissions`. What the future answers is kept under the +middleware's name in the context's `prepared`, and `create`, which stays +synchronous, takes it from there as the type it prepared. A middleware that +prepares nothing leaves `prepare` at its default and is asked `create` as +before. + +```rust +impl Middleware for Sellers { + fn prepare<'a>( + &'a self, + call: ModuleCall<'a>, + context: &'a MiddlewareContext<'a>, + ) -> Option> { + Some(Box::pin(async move { + let lines = self.lines_for(call, context.request_host).await?; + Ok(Box::new(lines) as Box) + })) + } + + fn create(&self, context: &MiddlewareContext<'_>) -> MiddlewareAction { + let lines: Vec = context.take_prepared(MODULE).unwrap_or_default(); + MiddlewareAction { + stream: Some(Box::new(AppendLines::new(lines))), + ..MiddlewareAction::pass() + } + } +} +``` + +The future may answer a `Refusal` instead, with a status, an optional +`Retry-After` and a message. The request is then answered with that status +and the message as plain text, no document is processed, and nothing is +stored or served from the store, so a middleware that cannot make the +document it is responsible for fails closed rather than letting an empty or +unchanged one through. `Refusal::service_unavailable` is the one for a +source that cannot answer now. The preparations of one document run in the +order the entry names the middleware, and the first refusal ends them. + +On a document that may be stored the fetch phase's preparations are handed +the request's services and nothing of the reader, because what they answer +is stored with the document. The serve phase is prepared with the reader's +state on every document, the one fetched for this reader as much as a copy +read from the store, so what a serve middleware finds prepared never depends +on whether its reader was the first. On a document that is not stored both +phases are prepared with the reader's state. + ## Proxy implementation rules -An `IntegrationProxy::handle` implementation receives the complete runtime -service graph. Register or predict backends through `services.backend()`, -send through `services.http_client()`, bound request and response bodies, and -return `Report` with integration context. The registry -strips internal identity headers before dispatch; an integration must opt into -any explicit forwarding behavior. +An `IntegrationProxy::handle` implementation receives the request and its +`ModuleCall`. It hands a function of its own to `call.inject_with`, with the +request as that function's own argument, and names what else it needs as +parameters, such as `&Settings` and `&RuntimeServices`, which is the complete +runtime service graph. Register or predict backends through +`services.backend()`, send through `services.http_client()`, bound request and +response bodies, and return `Report` with the route's +context. The registry strips internal identity headers before dispatch, and a +route must opt into any explicit forwarding behavior. + +A route that names a value whose use needs a permission, such as the Edge +Cookie identifier, declares that permission in `required_permissions` and is +passed the value only on a request that grants it. Nothing gates the route +itself, so a route that must not run without a permission reads the +`&PermissionState` and refuses. Streaming support is an adapter capability. Request `PlatformHttpRequest::with_stream_response()` only when the caller has @@ -212,12 +327,598 @@ DOM insertion dispatcher, remain idempotent, and leave unmatched elements untouched. See [Trusted Server JavaScript](/guide/tsjs) and [GPT's guarded handoff](/guide/integrations/gpt#server-slot-handoff). +## Integrations that ship outside core + +Every integration ships in a crate of its own, which a deployment composes in +at startup, and the hooks described above are what that crate implements. +The vendor owns the code, the release cycle and the integration's own rules, +and core never names the vendor. The crates in this repository sit under +`crates///`, and a vendor's crate can equally live in a +repository of its own. + +`crates/testing/seam-probe` is the worked example. It is a test fixture rather +than something to deploy, and it exercises every part of the seam from a +vendor crate's position. The round-trip tests in +`crates/trusted-server-adapter-axum/tests/seam_probe.rs` drive each part +through the Axum adapter, and the Fastly adapter carries the same round trip. + +### What the crate provides + +The crate hands out an `IntegrationBuilder`, which names the integration and +points at the functions that do the work. + +| Part | Purpose | +| --------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Id | Names the integration in its routes under `/integrations//`, in its browser module and in diagnostics | +| Source | The crate or package name, reported when two builders claim one id, so an operator can tell which crates collided | +| Build function | Reads `Settings` and returns the registration. It is called only when a section selects the module | +| Validate function | The integration's own deploy rules. It runs for every builder, selected or not | +| `.with_module_name(".")` | The name a section selects the module by, which is the crate's path under `crates/` with `.` between the parts. `module_name!()` derives it from the crate's folder | +| `.with_request_preparer(...)` | Optional. Runs before routing, once for a request, whether or not the module is selected | +| `.with_response_finalizer(...)` | Optional. Finishes the response the page path returns, from what the module's request hooks left for the request | +| `.with_auction_token()` | Optional. Declares that the module's browser script reads the token an auction publishes with its winning bids, which is made only when a selected module reads one | +| `.with_secret_settings(...)` | Optional. Lists the settings in the module's own table that hold the name of a secret, so each is looked up as the settings load | +| `.with_plan_registration(...)` | Optional. Registers the module from the compiled auction plan, for a module whose page support follows what the plan selects. It runs whether or not a section selects the module, and its hooks go first | +| `.with_plan_validator(...)` | Optional. Checks the module's settings against the compiled auction plan, when a deployment is validated and as the settings load | +| `.with_demand(...)` and `.with_adserver(...)` | Optional. Registers an auction implementation that `[demand]` or `[ad-server]` can name. A crate that supplies nothing else starts from `IntegrationBuilder::implementations(...)` | +| `.with_loading_cache(...)` | Optional. Offers the loading cache the crate supplies, which a feature selects by the module's name in its `[cache.]` table. An adapter takes the selections with `loading_caches_from` | +| `.with_shared_settings(...)` | Optional. Runs the builder when its vendor's `[shared.]` table is present, for a vendor whose modules all read that one table. A builder the table runs is selected by no section | + +```rust +pub fn module_name() -> &'static str { + trusted_server_core::module_name!() +} + +pub fn builder() -> IntegrationBuilder { + IntegrationBuilder::new(EXAMPLE_ID, EXAMPLE_SOURCE, register, validate) + .with_module_name(module_name()) + .with_request_preparer(prepare_request) +} +``` + +A crate at `crates/cmp/example` is the module `cmp.example`, so a deployment +runs it with `[cmp] module = "example"` and gives it settings in +`[cmp.example]`. + +### Who maintains the crate + +Every crate outside core names its maintainers in its manifest, the way +Prebid.js requires a named maintainer of every adapter. + +```toml +[package.metadata.maintainers] +owner = "Example Vendor" +status = "vendor owned" +``` + +`status` is `vendor owned` once the vendor has adopted the crate, +`seeking vendor owner` while the Trusted Server maintainers hold it for a +vendor, and `project owned` for a crate that is the project's own, such as a +test module. A test in `crates/trusted-server-modules` fails for a crate in +this repository that leaves the declaration out. + +### What a registration can declare + +The build function returns an `IntegrationRegistration`, built with the +builder every integration uses. + +| Declaration | What it does | +| ----------------------------------------------------- | ----------------------------------------------------------------------------------------------- | +| `.with_proxy(...)` | Routes the paths the proxy declares | +| `.with_request_filter(...)` | Inspects a request and can turn it back before it reaches the origin | +| `.with_middleware(...)` | Offers a page change that an entry may place by the middleware's name | +| `.with_bundle_tag_attribute(name, value)` | Puts an attribute on the script bundle's tag, for a browser module that reads a setting from it | +| `.with_js_module(CarriedJsModule { source, sha256 })` | Carries the integration's own browser script, built outside `trusted-server-js` | +| `.with_deferred_js()` | Serves the script as its own ` ``` -If `rewrite_sdk` is enabled, Trusted Server rewrites existing DataDome script tags in HTML: +If `rewrite_sdk` is enabled, Trusted Server rewrites existing DataDome script tags in HTML, on the pages a `[[fetch]]` entry names the middleware `bot-protection.datadome` for: ```html @@ -161,7 +167,7 @@ When `enable_protection = true`, Trusted Server calls DataDome before normal rou A request is protected when all of the following are true: -1. The DataDome integration is enabled. +1. `[bot-protection] module` selects DataDome. 2. `enable_protection = true`. 3. The method is not listed in `protection_excluded_methods`. 4. The path is not one of Trusted Server's internal routes. @@ -170,7 +176,7 @@ A request is protected when all of the following are true: 7. No `protection_exclusion_rules` match. 8. The request does not contain a matching enabled `protection_test_bypass` credential while `FASTLY_IS_STAGING=1`. -Static assets are excluded by default using a case-insensitive file-extension regex. Trusted Server internal routes such as `/static/tsjs=`, `/integrations/`, `/first-party/`, admin routes, discovery routes, and signature-verification routes are also excluded by default. +Static assets are excluded by default using a case-insensitive file-extension regex. Trusted Server internal routes such as `/static/tsjs=`, `/integrations/`, `/first-party/`, discovery routes, and signature-verification routes are also excluded by default. Auction traffic at `/auction` is protected by default. @@ -182,7 +188,10 @@ Protection API: ```toml # Runtime activation also requires FASTLY_IS_STAGING=1. -[integrations.datadome.protection_test_bypass] +[bot-protection] +module = "datadome" + +[bot-protection.datadome.protection_test_bypass] enabled = true credential_secret_name = "datadome_test_bypass" ``` @@ -262,7 +271,10 @@ fields. For example: Use structured rules for all DataDome protection exclusions. Each rule has an `id`, optional `methods`, and a typed matcher. The default configuration includes a `path_regex` rule for common static assets. ```toml -[[integrations.datadome.protection_exclusion_rules]] +[bot-protection] +module = "datadome" + +[[bot-protection.datadome.protection_exclusion_rules]] id = "legacy-static-get-head" methods = ["GET", "HEAD"] type = "path_regex" @@ -272,7 +284,7 @@ patterns = [ "^/robots\\.txt$", ] -[[integrations.datadome.protection_exclusion_rules]] +[[bot-protection.datadome.protection_exclusion_rules]] id = "next-rsc" methods = ["GET", "HEAD"] type = "query_param_non_empty" @@ -292,7 +304,10 @@ Supported rule types are: Config Store-backed CIDR sources accept newline-, comma-, whitespace-, or JSON-array encoded CIDR lists. They are useful for large or frequently updated vendor crawler lists. ```toml -[[integrations.datadome.protection_excluded_ip_cidr_sources]] +[bot-protection] +module = "datadome" + +[[bot-protection.datadome.protection_excluded_ip_cidr_sources]] config_store = "datadome-ip-bypass" key = "googlebot_ips" ``` @@ -367,14 +382,13 @@ sequenceDiagram Override configuration via environment variables: ```bash -TRUSTED_SERVER__INTEGRATIONS__DATADOME__ENABLED=true -TRUSTED_SERVER__INTEGRATIONS__DATADOME__SDK_ORIGIN=https://js.datadome.co -TRUSTED_SERVER__INTEGRATIONS__DATADOME__API_ORIGIN=https://api-js.datadome.co -TRUSTED_SERVER__INTEGRATIONS__DATADOME__CACHE_TTL_SECONDS=3600 -TRUSTED_SERVER__INTEGRATIONS__DATADOME__REWRITE_SDK=true -TRUSTED_SERVER__INTEGRATIONS__DATADOME__ENABLE_PROTECTION=true -TRUSTED_SERVER__INTEGRATIONS__DATADOME__SERVER_SIDE_KEY_SECRET_NAME=datadome_server_side_key -TRUSTED_SERVER__INTEGRATIONS__DATADOME__CLIENT_SIDE_KEY=your-client-side-key +TRUSTED_SERVER__BOT-PROTECTION__DATADOME__SDK_ORIGIN=https://js.datadome.co +TRUSTED_SERVER__BOT-PROTECTION__DATADOME__API_ORIGIN=https://api-js.datadome.co +TRUSTED_SERVER__BOT-PROTECTION__DATADOME__CACHE_TTL_SECONDS=3600 +TRUSTED_SERVER__BOT-PROTECTION__DATADOME__REWRITE_SDK=true +TRUSTED_SERVER__BOT-PROTECTION__DATADOME__ENABLE_PROTECTION=true +TRUSTED_SERVER__BOT-PROTECTION__DATADOME__SERVER_SIDE_KEY_SECRET_NAME=datadome_server_side_key +TRUSTED_SERVER__BOT-PROTECTION__DATADOME__CLIENT_SIDE_KEY=your-client-side-key ``` ## Client-side script guard @@ -393,11 +407,11 @@ This keeps DataDome scripts routed through first-party context, even when insert ### Script not loading -Check that the integration is enabled: +Check that `[bot-protection]` selects the module: ```toml -[integrations.datadome] -enabled = true +[bot-protection] +module = "datadome" ``` If you rely on auto-injection, verify `client_side_key` is non-empty and `inject_client_side_tag = true`. @@ -415,8 +429,10 @@ curl -X POST https://www.example.com/integrations/datadome/js/check Check that both fields are configured: ```toml -[integrations.datadome] -enabled = true +[bot-protection] +module = "datadome" + +[bot-protection.datadome] enable_protection = true server_side_key_secret_name = "datadome_server_side_key" ``` @@ -425,7 +441,7 @@ Also verify the request is not excluded by the default internal/static route exc ### HTML rewriting not working -Ensure `rewrite_sdk = true` and that your pages are being proxied through Trusted Server's HTML processing pipeline. +Ensure `rewrite_sdk = true`, that a `[[fetch]]` entry names `bot-protection.datadome` for the page, and that your pages are being proxied through Trusted Server's HTML processing pipeline. ## See also diff --git a/docs/guide/integrations/didomi.md b/docs/guide/integrations/didomi.md index a2a64e434b..e4669de5bf 100644 --- a/docs/guide/integrations/didomi.md +++ b/docs/guide/integrations/didomi.md @@ -26,8 +26,10 @@ for the upstream contract. Add the integration to the operator-owned `trusted-server.toml`: ```toml -[integrations.didomi] -enabled = true +[cmp] +module = "didomi" + +[cmp.didomi] geo_query_parameters = true # proxy_path = "my-custom-consent" # sdk_origin = "https://sdk.privacy-center.org" @@ -40,13 +42,12 @@ Publish application configuration with: ts config push --adapter fastly ``` -| Field | Type | Required | Default | Description | -| ---------------------- | ------- | -------- | ------------------------------------- | ------------------------------------------------------- | -| `enabled` | boolean | No | `true` in a present integration block | Enables the integration | -| `geo_query_parameters` | boolean | No | `false` | Enables trusted geo canonicalization for notice loaders | -| `proxy_path` | string | No | `integrations/didomi/consent` | Changes the first-party path prefix | -| `sdk_origin` | string | No | `https://sdk.privacy-center.org` | Changes the SDK origin, primarily for testing | -| `api_origin` | string | No | `https://api.privacy-center.org` | Changes the API origin, primarily for testing | +| Field | Type | Required | Default | Description | +| ---------------------- | ------- | -------- | -------------------------------- | ------------------------------------------------------- | +| `geo_query_parameters` | boolean | No | `false` | Enables trusted geo canonicalization for notice loaders | +| `proxy_path` | string | No | `integrations/didomi/consent` | Changes the first-party path prefix | +| `sdk_origin` | string | No | `https://sdk.privacy-center.org` | Changes the SDK origin, primarily for testing | +| `api_origin` | string | No | `https://api.privacy-center.org` | Changes the API origin, primarily for testing | `geo_query_parameters` is disabled by default for compatibility. It currently supports Fastly only because Cloudflare does not expose a trusted region through @@ -62,8 +63,10 @@ already exists in the TOML input. `proxy_path` helps avoid a predictable integration path: ```toml -[integrations.didomi] -enabled = true +[cmp] +module = "didomi" + +[cmp.didomi] geo_query_parameters = true proxy_path = "my-custom-consent" ``` @@ -76,7 +79,10 @@ This serves Didomi at `/my-custom-consent/*`. The path: - may start with `/`; Trusted Server normalizes the leading slash. Trusted Server passes the resolved path to its browser bundle through -`window.__tsjs_didomi.proxyPath`. +`window.__tsjs_didomi.proxyPath`. The script that sets it is the middleware +`cmp.didomi`, which runs on the pages a `[[fetch]]` entry names it for, so +name it for every page that loads Didomi. See +[Placing page changes](/guide/configuration#placing-page-changes). ## Notice-loader geo flow diff --git a/docs/guide/integrations/gam.md b/docs/guide/integrations/gam.md index 304619ede9..5560c7d752 100644 --- a/docs/guide/integrations/gam.md +++ b/docs/guide/integrations/gam.md @@ -1,7 +1,7 @@ # Google Ad Manager integration status -Trusted Server does not ship a direct Google Ad Manager integration or an -`[integrations.gam]` configuration section. +Trusted Server does not ship a direct Google Ad Manager integration, so there +is no `gam` module to select or configure. The TSJS GPT module can place server-side auction results into publisher-owned GPT slots, including APS renderer winners. That browser handoff is not a GAM @@ -10,4 +10,4 @@ GAM demand. Use the [Integrations overview](/guide/integrations-overview) for the current integration inventory and [Auction orchestration](/guide/auction-orchestration) -for shipped provider profiles. +for the demand implementations it ships. diff --git a/docs/guide/integrations/google_tag_manager.md b/docs/guide/integrations/google_tag_manager.md index c60efac227..4e89dafecd 100644 --- a/docs/guide/integrations/google_tag_manager.md +++ b/docs/guide/integrations/google_tag_manager.md @@ -34,8 +34,10 @@ This is a deliberate design choice to limit data forwarded to Google. If your us Add the GTM configuration to `trusted-server.toml`: ```toml -[integrations.google_tag_manager] -enabled = true +[tag] +modules = ["google-tag-manager"] + +[tag.google-tag-manager] container_id = "GTM-XXXXXX" # upstream_url = "https://www.googletagmanager.com" # Optional override (must be https) # allowed_tag_ids = ["G-XXXXXXXX"] # Tag ids besides container_id that gtag/js may serve first-party @@ -45,14 +47,13 @@ container_id = "GTM-XXXXXX" ### Configuration Options -| Field | Type | Required | Description | -| ---------------------- | ------- | -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `enabled` | boolean | No | Enable/disable integration (default: `false`) | -| `container_id` | string | Yes | Your GTM Container ID (e.g., `GTM-A1B2C3`) | -| `upstream_url` | string | No | Custom upstream, as a credential-free `https` bare origin with a literal host — no path, query, fragment, userinfo or wildcard (default: `https://www.googletagmanager.com`) | -| `allowed_tag_ids` | array | No | Extra tag ids servable on `gtag/js` besides `container_id` (default: none) | -| `cache_max_age` | number | No | Cache duration in seconds (default: `900`, range: `60`-`86400`) | -| `max_beacon_body_size` | number | No | Max POST body size in bytes (default: `65536`, range: `1024`-`1048576`) | +| Field | Type | Required | Description | +| ---------------------- | ------ | -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `container_id` | string | Yes | Your GTM Container ID (e.g., `GTM-A1B2C3`) | +| `upstream_url` | string | No | Custom upstream, as a credential-free `https` bare origin with a literal host, with no path, query, fragment, userinfo or wildcard (default: `https://www.googletagmanager.com`) | +| `allowed_tag_ids` | array | No | Extra tag ids servable on `gtag/js` besides `container_id` (default: none) | +| `cache_max_age` | number | No | Cache duration in seconds (default: `900`, range: `60`-`86400`) | +| `max_beacon_body_size` | number | No | Max POST body size in bytes (default: `65536`, range: `1024`-`1048576`) | ### Upgrading: check `allowed_tag_ids` @@ -101,7 +102,7 @@ A trailing slash is still accepted; it is trimmed before targets are built. **This is a breaking change.** A path-based value such as `https://tags.example.com/gateway` passed validation before this release and -produced targets below that path, so an enabled deployment using one now fails +produced targets below that path, so a deployment using one now fails config validation. `ts config push` rejects it, which is where you should expect to see this. Validation is fail-closed rather than skip-the-integration: a config that reaches the runtime without having been pushed through that check @@ -129,7 +130,7 @@ flowchart TD ### 1. Script Rewriting -When Trusted Server processes an HTML response, it automatically rewrites GTM script tags to point to the local proxy: +On the pages a `[[fetch]]` entry names the middleware `tag.google-tag-manager` for, Trusted Server rewrites GTM script tags to point to the local proxy. See [Placing page changes](/guide/configuration#placing-page-changes). **Before:** @@ -295,7 +296,7 @@ _Expected_: `200 OK` (or 204). ## Implementation Details -See [crates/trusted-server-core/src/integrations/google_tag_manager.rs](https://github.com/IABTechLab/trusted-server/blob/main/crates/trusted-server-core/src/integrations/google_tag_manager.rs). +See [crates/tag/google-tag-manager/src/lib.rs](https://github.com/IABTechLab/trusted-server/blob/main/crates/tag/google-tag-manager/src/lib.rs). ## Next Steps diff --git a/docs/guide/integrations/gpt-diagnostics.md b/docs/guide/integrations/gpt-diagnostics.md index af6f2e3248..d2a7dad051 100644 --- a/docs/guide/integrations/gpt-diagnostics.md +++ b/docs/guide/integrations/gpt-diagnostics.md @@ -23,19 +23,30 @@ server integration configuration plus `?ts_console=1`; it does not require new publisher JavaScript, React, Next.js, DOM, or GAM configuration. The diagnostics integration is independent of the -[GPT first-party script integration](./gpt.md). Either integration can be enabled +[GPT first-party script integration](./gpt.md). Either integration can run without the other, although Trusted Server creative-progress evidence is available only for slots served through the existing GPT integration. ## Deployment Configuration -The module is unavailable unless explicitly enabled for the deployment: +The module is unavailable unless the deployment names it: ```toml -[integrations.gpt_diagnostics] -enabled = true +[ad-tag] +modules = ["google.diagnostics"] ``` +What the module writes into a page is the serve middleware `ad-tag.google.diagnostics`, +which runs on each reader's copy of the pages a `[[serve]]` entry names it for: + +```toml +[[serve]] +media_type = "text/html" +middleware = ["ad-tag.google.diagnostics"] +``` + +See [Placing page changes](/guide/configuration#placing-page-changes). + Deployment configuration makes the module available; it does not activate any browser session. Inactive browser sessions receive no diagnostics module. When activated, the standalone content-hashed module loads synchronously after the core bundle so it can @@ -477,7 +488,7 @@ trigger GPT or Prebid work, gate an auction, or delay delivery. ### The API or panel is absent -1. Confirm `[integrations.gpt_diagnostics]` is enabled in the deployed configuration. +1. Confirm `[ad-tag] modules` selects `google.diagnostics` in the deployed configuration. 2. Activate the browser session with an exact recognized `ts_console` value. 3. Confirm the Trusted Server script bundle loaded successfully. 4. Use `ts_console=false` and then `ts_console=true` on a new document to reset diff --git a/docs/guide/integrations/gpt.md b/docs/guide/integrations/gpt.md index ae070950f8..2196c3da49 100644 --- a/docs/guide/integrations/gpt.md +++ b/docs/guide/integrations/gpt.md @@ -25,7 +25,7 @@ All of these are served from `securepubads.g.doubleclick.net`. Publisher HTML │ ├─ + + diff --git a/tools/permissions-inspector/wasm/.gitignore b/tools/permissions-inspector/wasm/.gitignore new file mode 100644 index 0000000000..2f7896d1d1 --- /dev/null +++ b/tools/permissions-inspector/wasm/.gitignore @@ -0,0 +1 @@ +target/ diff --git a/tools/permissions-inspector/wasm/Cargo.lock b/tools/permissions-inspector/wasm/Cargo.lock new file mode 100644 index 0000000000..8391023d81 --- /dev/null +++ b/tools/permissions-inspector/wasm/Cargo.lock @@ -0,0 +1,2552 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common", + "generic-array", +] + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "async-compression" +version = "0.4.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3976abdc8fe7d1133d43d304afd42abdf5bc3e1319d263d223bde07b5efc4be8" +dependencies = [ + "compression-codecs", + "compression-core", + "futures-io", + "pin-project-lite", +] + +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base16ct" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "bitstream-io" +version = "4.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eff00be299a18769011411c9def0d827e8f2d7bf0c3dbf53633147a8867fd1f" +dependencies = [ + "no_std_io2", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "brotli" +version = "8.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "build-print" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8e6738dfb11354886f890621b4a34c0b177f75538023f7100b608ab9adbd66b" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chacha20" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + +[[package]] +name = "chacha20poly1305" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" +dependencies = [ + "aead", + "chacha20", + "cipher", + "poly1305", + "zeroize", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", + "zeroize", +] + +[[package]] +name = "compression-codecs" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +dependencies = [ + "brotli", + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "convert_case" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "cookie" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a373e3602691c3cdea496d2f0ee5935151e6168fe87739483c463db1b2f2f87" +dependencies = [ + "time", + "version_check", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "generic-array", + "rand_core", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "rand_core", + "typenum", +] + +[[package]] +name = "cssparser" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dae61cf9c0abb83bd659dab65b7e4e38d8236824c85f0f804f173567bda257d2" +dependencies = [ + "cssparser-macros", + "dtoa-short", + "itoa", + "phf", + "smallvec", +] + +[[package]] +name = "cssparser-macros" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" +dependencies = [ + "quote", + "syn 2.0.119", +] + +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures", + "curve25519-dalek-derive", + "digest", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.119", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "zeroize", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.119", + "unicode-xid", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", + "subtle", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "dtoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590" + +[[package]] +name = "dtoa-short" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87" +dependencies = [ + "dtoa", +] + +[[package]] +name = "ecdsa" +version = "0.16.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" +dependencies = [ + "der", + "digest", + "elliptic-curve", + "rfc6979", + "signature", + "spki", +] + +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek", + "ed25519", + "rand_core", + "serde", + "sha2", + "subtle", + "zeroize", +] + +[[package]] +name = "edgezero-core" +version = "0.1.0" +source = "git+https://github.com/stackpop/edgezero?rev=683202c66948146ec360f490126f1d60f920288b#683202c66948146ec360f490126f1d60f920288b" +dependencies = [ + "anyhow", + "async-compression", + "async-stream", + "async-trait", + "bytes", + "edgezero-macros", + "futures", + "futures-util", + "http", + "http-body", + "log", + "matchit", + "ryu", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sha2", + "thiserror", + "toml", + "tower-service", + "tracing", + "validator", + "web-time", +] + +[[package]] +name = "edgezero-macros" +version = "0.1.0" +source = "git+https://github.com/stackpop/edgezero?rev=683202c66948146ec360f490126f1d60f920288b#683202c66948146ec360f490126f1d60f920288b" +dependencies = [ + "log", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn 3.0.4", + "toml", + "validator", +] + +[[package]] +name = "elliptic-curve" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" +dependencies = [ + "base16ct", + "crypto-bigint", + "digest", + "ff", + "generic-array", + "group", + "pkcs8", + "rand_core", + "sec1", + "subtle", + "zeroize", +] + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "error-stack" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b878b3fac9613c3c7f22eb70bc8a3c6ebdc03cc11479ee60fde1692d747fd45f" +dependencies = [ + "anyhow", + "rustc_version", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "ff" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +dependencies = [ + "rand_core", + "subtle", +] + +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + +[[package]] +name = "find-msvc-tools" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" + +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", + "zeroize", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff", + "rand_core", + "subtle", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "iab_gpp" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3be2d0191a3376e0176bb3df53b2754c644ead6edd50d9494ee8fa376a70e02" +dependencies = [ + "bitstream-io", + "fnv", + "iab_gpp_derive", + "num-derive", + "num-iter", + "num-traits", + "prettyplease", + "proc-macro2", + "quote", + "strum_macros", + "syn 2.0.119", + "thiserror", + "walkdir", +] + +[[package]] +name = "iab_gpp_derive" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5acda598b043c6386d20fffe86c600b63c7ca4980ee9a28f7e9aaa15d749747" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" + +[[package]] +name = "icu_properties" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" + +[[package]] +name = "icu_provider" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07aa2048142242915a31d35844fb311e0e53fcca590c3a0a40dcf1b841fa09eb" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jose-b64" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bec69375368709666b21c76965ce67549f2d2db7605f1f8707d17c9656801b56" +dependencies = [ + "base64ct", + "serde", + "subtle", + "zeroize", +] + +[[package]] +name = "jose-jwa" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ab78e053fe886a351d67cf0d194c000f9d0dcb92906eb34d853d7e758a4b3a7" +dependencies = [ + "serde", +] + +[[package]] +name = "jose-jwk" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "280fa263807fe0782ecb6f2baadc28dffc04e00558a58e33bfdb801d11fd58e7" +dependencies = [ + "jose-b64", + "jose-jwa", + "p256", + "p384", + "rsa", + "serde", + "zeroize", +] + +[[package]] +name = "js-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "lol_html" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00aad58f6ec3990e795943872f13651e7a5fa59dca2c8f31a74faf8a0e0fb652" +dependencies = [ + "bitflags", + "cfg-if", + "cssparser", + "encoding_rs", + "foldhash", + "hashbrown", + "memchr", + "mime", + "precomputed-hash", + "selectors", + "thiserror", +] + +[[package]] +name = "matchit" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8863b587001c1b9a8a4e36008cebc6b3612cb1226fe2de94858e06092687b608" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + +[[package]] +name = "no_std_io2" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418abd1b6d34fbf6cae440dc874771b0525a604428704c76e48b29a5e67b8003" +dependencies = [ + "memchr", +] + +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand", + "smallvec", + "zeroize", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + +[[package]] +name = "p256" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2", +] + +[[package]] +name = "p384" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe42f1670a52a47d448f14b6a5c61dd78fce51856e68edaa38f7ae3a46b8d6b6" +dependencies = [ + "elliptic-curve", + "primeorder", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "permissions-inspector-wasm" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "trusted-server-core", + "trusted-server-permission-signal-gpc", + "trusted-server-permission-signal-gpp", + "trusted-server-permission-signal-tcf", + "trusted-server-permission-signal-us-privacy", +] + +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_macros", + "phf_shared", + "serde", +] + +[[package]] +name = "phf_codegen" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" +dependencies = [ + "fastrand", + "phf_shared", +] + +[[package]] +name = "phf_macros" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "poly1305" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" +dependencies = [ + "cpufeatures", + "opaque-debug", + "universal-hash", +] + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "primeorder" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve", +] + +[[package]] +name = "proc-macro-error-attr3" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e564d14133360e1ae169ffde5da25881b5fa47261665b8e5713c212c27799da" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error3" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f0d4471b3436c22106b21913b1dda531558918ae9b7ec55d58aa84b43552233" +dependencies = [ + "proc-macro-error-attr3", + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rfc6979" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +dependencies = [ + "hmac", + "subtle", +] + +[[package]] +name = "rsa" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" +dependencies = [ + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core", + "signature", + "spki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "sec1" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + +[[package]] +name = "selectors" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cfaaa6035167f0e604e42723c7650d59ee269ef220d7bbe0565602c8a0173b9" +dependencies = [ + "bitflags", + "cssparser", + "derive_more", + "log", + "new_debug_unreachable", + "phf", + "phf_codegen", + "precomputed-hash", + "rustc-hash", + "servo_arc", + "smallvec", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_yaml_ng" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4db627b98b36d4203a7b458cf3573730f2bb591b28871d916dfa9efabfd41f" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "servo_arc" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "170fb83ab34de17dc69aa7c67482b22218ddb85da56546f9bd6b929e32a05930" +dependencies = [ + "stable_deref_trait", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core", +] + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9be42f50aa861c555654aa3a37f52f4b1074bacf4e48fe0ef7fa584e80f1f0f" + +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "toml" +version = "1.1.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12c0ba9680044b4ce98d391a62094047eada0d64860b80166c39f4a6b5640785" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow", +] + +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_parser" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +dependencies = [ + "winnow", +] + +[[package]] +name = "toml_writer" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "trusted-server-core" +version = "0.1.0" +dependencies = [ + "async-stream", + "async-trait", + "base64", + "brotli", + "bytes", + "chacha20poly1305", + "chrono", + "cookie", + "cssparser", + "derive_more", + "ed25519-dalek", + "edgezero-core", + "error-stack", + "flate2", + "futures", + "getrandom 0.2.17", + "glob", + "hex", + "hmac", + "http", + "httpdate", + "iab_gpp", + "jose-jwk", + "log", + "lol_html", + "matchit", + "mime", + "p256", + "rand", + "regex", + "serde", + "serde_json", + "serde_yaml_ng", + "sha2", + "subtle", + "toml", + "trusted-server-js", + "trusted-server-openrtb", + "url", + "urlencoding", + "uuid", + "validator", + "web-time", +] + +[[package]] +name = "trusted-server-js" +version = "0.1.0" +dependencies = [ + "build-print", + "hex", + "sha2", + "which", +] + +[[package]] +name = "trusted-server-openrtb" +version = "0.1.0" +dependencies = [ + "log", + "serde", + "serde_json", +] + +[[package]] +name = "trusted-server-permission-signal-gpc" +version = "0.1.0" +dependencies = [ + "trusted-server-core", +] + +[[package]] +name = "trusted-server-permission-signal-gpp" +version = "0.1.0" +dependencies = [ + "trusted-server-core", +] + +[[package]] +name = "trusted-server-permission-signal-tcf" +version = "0.1.0" +dependencies = [ + "trusted-server-core", +] + +[[package]] +name = "trusted-server-permission-signal-us-privacy" +version = "0.1.0" +dependencies = [ + "trusted-server-core", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common", + "subtle", +] + +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid" +version = "1.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5772d71c9be8a8a6ac2117d949c5b224c1b72241bb611d9a3012edcf8af7812" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "validator" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43fb22e1a008ece370ce08a3e9e4447a910e92621bb49b85d6e48a45397e7cfa" +dependencies = [ + "idna", + "once_cell", + "regex", + "serde", + "serde_derive", + "serde_json", + "url", + "validator_derive", +] + +[[package]] +name = "validator_derive" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240e4b81c20a1d6d50d1d7265c658dfbd204e8b9ac4d80f3c931f39462196335" +dependencies = [ + "darling", + "proc-macro-error3", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "which" +version = "8.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bae2f2b2b816647a1cab1acc91f5bd20812d53cb344382635ec2181940c8034f" +dependencies = [ + "libc", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +dependencies = [ + "serde", +] + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "zlib-rs" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/tools/permissions-inspector/wasm/Cargo.toml b/tools/permissions-inspector/wasm/Cargo.toml new file mode 100644 index 0000000000..2871aad40c --- /dev/null +++ b/tools/permissions-inspector/wasm/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "permissions-inspector-wasm" +version = "0.1.0" +edition = "2024" +publish = false + +[lib] +crate-type = ["cdylib"] + +[dependencies] +trusted-server-core = { path = "../../../crates/trusted-server-core" } +# The inspector runs the same permission signal modules an adapter ships, +# in the same default order, so the page answers as a deployment would. +trusted-server-permission-signal-gpc = { path = "../../../crates/permission-signal/gpc" } +trusted-server-permission-signal-gpp = { path = "../../../crates/permission-signal/gpp" } +trusted-server-permission-signal-tcf = { path = "../../../crates/permission-signal/tcf" } +trusted-server-permission-signal-us-privacy = { path = "../../../crates/permission-signal/us-privacy" } +serde = { version = "1", features = ["derive"] } +serde_json = "1" + +[profile.release] +opt-level = "z" +lto = true +strip = true +codegen-units = 1 + +# Deliberately its own workspace, like the integration-tests crate, so the +# main workspace's wasm32-wasip1 default target does not apply to it. +[workspace] diff --git a/tools/permissions-inspector/wasm/build.rs b/tools/permissions-inspector/wasm/build.rs new file mode 100644 index 0000000000..5e8f11cb32 --- /dev/null +++ b/tools/permissions-inspector/wasm/build.rs @@ -0,0 +1,46 @@ +use std::process::Command; + +fn git(args: &[&str]) -> String { + Command::new("git") + .args(args) + .output() + .ok() + .and_then(|o| String::from_utf8(o.stdout).ok()) + .map(|s| s.trim().to_string()) + .unwrap_or_default() +} + +/// Reads the workspace version from the repository root manifest, so the page +/// reports the same version as the trusted-server crates it runs. +fn workspace_version() -> String { + let root = std::fs::read_to_string("../../../Cargo.toml").unwrap_or_default(); + let mut in_package = false; + for line in root.lines() { + let line = line.trim(); + if line.starts_with('[') { + in_package = line == "[workspace.package]"; + } else if in_package && line.starts_with("version") { + if let Some(version) = line.split('"').nth(1) { + return version.to_string(); + } + } + } + String::from("unknown") +} + +fn main() { + println!("cargo:rustc-env=TS_CORE_VERSION={}", workspace_version()); + println!( + "cargo:rustc-env=TS_CORE_COMMIT={}", + git(&["rev-parse", "--short=9", "HEAD"]) + ); + println!( + "cargo:rustc-env=TS_CORE_DATE={}", + git(&["show", "-s", "--format=%cs", "HEAD"]) + ); + println!( + "cargo:rustc-env=TS_CORE_BRANCH={}", + git(&["rev-parse", "--abbrev-ref", "HEAD"]) + ); + println!("cargo:rerun-if-changed=../../../Cargo.toml"); +} diff --git a/tools/permissions-inspector/wasm/src/lib.rs b/tools/permissions-inspector/wasm/src/lib.rs new file mode 100644 index 0000000000..a826d8388d --- /dev/null +++ b/tools/permissions-inspector/wasm/src/lib.rs @@ -0,0 +1,182 @@ +//! The permissions bit of Trusted Server, compiled to WebAssembly for the +//! inspector page. Inputs in, resulting permissions out, through the same +//! functions the server runs: `build_context_from_signals` decodes the raw +//! consent signals and `assemble_permissions` resolves the policy. + +use std::sync::Arc; + +use serde::Deserialize; +use serde_json::json; +use trusted_server_core::consent::build_context_from_signals; +use trusted_server_core::consent::types::RawConsentSignals; +use trusted_server_core::ec::consent::{GeoStatus, assemble_permissions}; +use trusted_server_core::evidence::OwnedRequestInfo; +use trusted_server_core::module_context::ModuleContext; +use trusted_server_core::permission_signal::PermissionSignalModule; +use trusted_server_core::permissions::{Permission, PermissionMaps}; +use trusted_server_core::platform::GeoInfo; + +/// The signal modules the inspector asks, in the order every adapter offers +/// them when `[permission-signal] modules` names none. +fn modules() -> Vec> { + vec![ + Arc::new(trusted_server_permission_signal_gpc::GpcModule::new()), + Arc::new(trusted_server_permission_signal_gpp::GppSaleOptOutModule::new()), + Arc::new(trusted_server_permission_signal_us_privacy::UsPrivacyModule::new()), + Arc::new(trusted_server_permission_signal_tcf::TcfModule::new()), + ] +} + +/// The inspector's evaluation request. +#[derive(Deserialize)] +struct EvalInput { + /// `located`, `none`, or `failed`. + geo: String, + country: Option, + region: Option, + tc: Option, + gpp: Option, + us_privacy: Option, + #[serde(default)] + gpc: bool, +} + +fn eval_json(input: &str) -> String { + let input: EvalInput = match serde_json::from_str(input) { + Ok(input) => input, + Err(e) => return json!({"ok": false, "error": e.to_string()}).to_string(), + }; + let signals = RawConsentSignals { + raw_tc_string: input.tc.filter(|s| !s.is_empty()), + raw_gpp_string: input.gpp.filter(|s| !s.is_empty()), + raw_gpp_sid: None, + raw_us_privacy: input.us_privacy.filter(|s| !s.is_empty()), + gpc: input.gpc, + }; + let ctx = build_context_from_signals(&signals); + // The page carries no request, only the consent signals above, and each of + // the four modules answers from the consent record rather than from + // request evidence, so empty evidence changes none of their answers. A + // module that read a header or a cookie would need real evidence here. + let evidence = OwnedRequestInfo::default(); + // The page stands for no request, so the modules are asked in the empty + // context with the signals entered. + let context = ModuleContext::empty() + .with_consent(&ctx) + .with_evidence(&evidence); + let modules = modules(); + let maps = PermissionMaps::standard(); + let (state, jurisdiction) = match input.geo.as_str() { + "failed" => { + let state = assemble_permissions(&context, GeoStatus::Failed, &modules); + (state, "unknown".to_string()) + } + "none" => { + let state = assemble_permissions(&context, GeoStatus::NoLocation, &modules); + (state, jurisdiction_name(maps.default_jurisdiction())) + } + _ => { + let info = GeoInfo { + city: String::new(), + country: input.country.clone().unwrap_or_default(), + continent: String::new(), + latitude: 0.0, + longitude: 0.0, + metro_code: 0, + region: input.region.clone().filter(|r| !r.is_empty()), + asn: None, + }; + let state = assemble_permissions(&context, GeoStatus::Located(&info), &modules); + let jurisdiction = jurisdiction_name( + maps.jurisdiction_for(input.country.as_deref(), input.region.as_deref()), + ); + (state, jurisdiction) + } + }; + let set: Vec<&'static str> = Permission::all() + .filter(|p| state.is_set(*p)) + .map(Permission::as_str) + .collect(); + json!({ + "ok": true, + "jurisdiction": jurisdiction, + "set": set, + "tcf_decoded": ctx.tcf.is_some(), + // No `malformed_record`. Whether an absent, unreadable or expired + // signal changes the permissions is each module's own decision, + // taken in its answer, so nothing at this level can report one and + // `false` would read as "none was found". + }) + .to_string() +} + +fn jurisdiction_name(j: trusted_server_core::consent::jurisdiction::Jurisdiction) -> String { + let name = format!("{j:?}").to_lowercase(); + let name = name.split('(').next().unwrap_or(&name).to_string(); + name.replace("usstate", "us-state") + .replace("nonregulated", "non-regulated") +} + +fn validate_json(yaml: &str) -> String { + match PermissionMaps::from_yaml(yaml) { + Ok(_) => json!({"ok": true}).to_string(), + Err(e) => json!({"ok": false, "error": e.to_string()}).to_string(), + } +} + +fn meta_json() -> String { + json!({ + "version": env!("TS_CORE_VERSION"), + "commit": env!("TS_CORE_COMMIT"), + "date": env!("TS_CORE_DATE"), + "branch": env!("TS_CORE_BRANCH"), + }) + .to_string() +} + +/// Leaks a length-prefixed buffer the host reads and then frees. +fn out(s: String) -> *mut u8 { + let bytes = s.into_bytes(); + let mut buf = Vec::with_capacity(4 + bytes.len()); + buf.extend_from_slice(&(bytes.len() as u32).to_le_bytes()); + buf.extend_from_slice(&bytes); + let ptr = buf.as_mut_ptr(); + core::mem::forget(buf); + ptr +} + +#[unsafe(no_mangle)] +pub extern "C" fn ts_alloc(len: usize) -> *mut u8 { + let mut buf = vec![0u8; len]; + let ptr = buf.as_mut_ptr(); + core::mem::forget(buf); + ptr +} + +/// # Safety +/// `ptr` must come from `ts_alloc` or an `out` buffer with capacity `len`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ts_free(ptr: *mut u8, len: usize) { + unsafe { drop(Vec::from_raw_parts(ptr, len, len)) }; +} + +/// # Safety +/// `ptr`/`len` must describe a valid UTF-8 JSON buffer from `ts_alloc`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ts_eval(ptr: *const u8, len: usize) -> *mut u8 { + let input = unsafe { core::slice::from_raw_parts(ptr, len) }; + out(eval_json(core::str::from_utf8(input).unwrap_or("{}"))) +} + +/// # Safety +/// `ptr`/`len` must describe a valid UTF-8 YAML buffer from `ts_alloc`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ts_validate(ptr: *const u8, len: usize) -> *mut u8 { + let input = unsafe { core::slice::from_raw_parts(ptr, len) }; + out(validate_json(core::str::from_utf8(input).unwrap_or(""))) +} + +#[unsafe(no_mangle)] +pub extern "C" fn ts_meta() -> *mut u8 { + out(meta_json()) +} diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 066ad659e3..04af689ad5 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -1,21 +1,43 @@ # ============================================================================= -# Trusted Server — application configuration template +# Trusted Server application configuration template # ============================================================================= # # This is the source-controlled starting point for an operator-owned # `trusted-server.toml`. Copy it (`ts config init`), fill in the required # values, and push it (`ts config push`) as an EdgeZero app-config blob. # -# Only three sections are REQUIRED for the server to start and pass validation: -# 1. [[handlers]] covering /_ts/admin (admin authentication) -# 2. [publisher] (domain + origin) -# 3. [ec] passphrase (Edge Cookie identity secret) +# Only one section is REQUIRED for the server to start and pass validation, +# which is [publisher] (domain + origin). # -# Everything below those is OPTIONAL. Most optional blocks are commented out — -# uncomment and edit one to enable it — but a few integrations are kept as active -# `enabled = false` stubs (see the Integrations section for why). All example -# hosts use `example.com`; replace them with your real endpoints in your private -# config, not in this template. +# Edge Cookie identity is optional and stays off until an [ec] module is +# selected, so no identity secret is needed to start. See the Edge Cookie +# section below. +# +# Everything below it is OPTIONAL. Most optional blocks are commented out, so +# uncomment and edit one to switch it on. All example hosts use `example.com`, +# so replace them with your real endpoints in your private config, not in this +# template. +# +# Everything this deployment can switch on is a module, selected the same way, +# with `module` where one runs and `modules` where several run: +# +# [] +# module = "" # modules = [...] where several run +# +# [.] # only when that name has settings +# setting = "value" +# +# The types are ec, geo, device, permission-signal, demand, ad-server, +# robots-txt, analytics and integration. A settings table its type's selector +# does not name refuses startup, so there is no stale block to misread. The +# complete rules, and what is checked when, are in +# docs/guide/configuration-rules.md. +# +# [shared.] holds what one vendor's modules share, such as an endpoint +# and a license key. That vendor's crates alone read it, its presence runs the +# builder they share, and a table no module reads is refused. [prefix] selects +# the module that names the browser's cookies and the shared store keys, and +# core's own `ts` runs when it selects nothing. # # `TRUSTED_SERVER__` env vars can override values when the `ts` CLI builds and # validates this config for push (e.g. TRUSTED_SERVER__PUBLISHER__DOMAIN=...; @@ -28,29 +50,7 @@ # ----------------------------------------------------------------------------- -# REQUIRED — Admin authentication -# ----------------------------------------------------------------------------- -# HTTP Basic-auth handler(s). At least one handler whose `path` regex covers -# the /_ts/admin endpoints is mandatory; startup fails without it. Each handler -# needs a non-placeholder username/password (deploy validation rejects the -# sample password below). -[[handlers]] -# Regex matched against the request path. This one guards the admin surface. -path = "^/_ts/admin" -username = "admin" -password = "handler_password" - -# You can add more handlers to basic-auth-protect other path prefixes. The -# sample password below is a known placeholder that deploy validation rejects, -# so it forces a real secret before push: -# [[handlers]] -# path = "^/secure" -# username = "user" -# password = "replace-with-admin-password" - - -# ----------------------------------------------------------------------------- -# REQUIRED — Publisher / origin +# REQUIRED - Publisher / origin # ----------------------------------------------------------------------------- [publisher] # Public domain Trusted Server is fronting. @@ -70,21 +70,67 @@ proxy_secret = "publisher_proxy_secret" # ----------------------------------------------------------------------------- -# REQUIRED — Edge Cookie (EC) identity +# OPTIONAL - Edge Cookie (EC) identity # ----------------------------------------------------------------------------- [ec] -# Secret Store key for the secret used to derive EC identifiers. -passphrase = "ec_passphrase" +# Edge Cookie identity is OFF by default. With no module selected, Trusted +# Server runs statelessly and generates no Edge Cookie. Activate one by +# uncommenting the selector below, together with that module's own +# [ec.] table when it has settings (a table the selector does not name +# is rejected at startup). Deployment tooling can merge a +# TRUSTED_SERVER__EC__MODULE environment value into the published +# configuration before it is loaded, and the running server itself reads its +# settings from the platform config store rather than the environment. The +# built-in hmac module is host-neutral, and a vendor module ships in its +# own crate that the adapter composes in. +# module = "hmac" # KV store that persists EC identity state. This is the physical store name # bound per adapter (e.g. `ec_identity_store` in fastly.toml); edgezero.toml's # logical KV id is `trusted_server_kv`. ec_store = "ec_identity_store" +# Extra origins allowed to POST the client resolve endpoint. The endpoint +# always accepts https:// and nothing else by default. List +# further origins here if the pages that resolve identity are served from +# another origin, www being the common case. Each entry is a bare origin +# (RFC 6454), being http:// or https:// and a host with an optional port, with +# no path, trailing slash, query or fragment, and any other entry is refused +# when the configuration is pushed or loaded. Entries are compared by the +# same-origin test, so the scheme, host and port all have to match, and a +# subdomain is never accepted just for being a subdomain. +# resolve_allowed_origins = ["https://www.example.com"] # Max concurrent partner pull-sync requests. pull_sync_concurrency = 3 # Optional cluster-heuristic tuning (defaults shown): # cluster_trust_threshold = 10 # entries with cluster_size <= this are individual users # cluster_recheck_secs = 3600 # re-evaluate cluster_size after this many seconds +# Built-in HMAC module table. Uncomment it together with the +# `module = "hmac"` selector above. `passphrase` names the Secret Store key +# holding the secret used to derive EC identifiers. That secret must be +# >= 32 bytes and non-placeholder in production. Provision referenced keys in +# trusted_server_secrets before validating/pushing. +# [ec.hmac] +# passphrase = "ec_passphrase" + +# A table can name the implementation it configures, which makes the table's +# own name a label of your choosing. The same module could therefore be +# selected as `module = "primary"` and configured in the table below +# instead, which is how a deployment gives a module a name that means +# something to the people running it. +# A module from a crate is named by its folder below crates/, and may be +# written in full, as edgecookie., or with that type folder left off. +# [ec.primary] +# implementation = "hmac" +# passphrase = "ec_passphrase" + +# Built-in host-signal module table. Creates the identifier from the host's +# TLS JA4 and HTTP/2 signals together with the client IP, so it needs a host +# that supplies those signals (Fastly does). Uncomment it together with +# `module = "host_signals"` above, and give it its own secret on the same +# terms as the hmac table. +# [ec.host_signals] +# passphrase = "ec_host_signals_passphrase" + # Example partner configuration. Provision referenced keys in # trusted_server_secrets before validating/pushing. # [[ec.partners]] @@ -102,26 +148,99 @@ pull_sync_concurrency = 3 # batch_rate_limit = 60 # max batch-sync requests/min (default 60) # pull_sync_enabled = false # default false +# Which module resolves location. Leave the whole section out to keep the +# host's own lookup, which is what every adapter ships with today. +# [geo] +# `none` resolves no location at all, or name a registered module that declares +# a geo module to have that module resolve location instead of the host. +# module = "none" + # ============================================================================= -# OPTIONAL — Core features (disabled/omitted by default) +# OPTIONAL - Core features (disabled/omitted by default) # ============================================================================= +# Device-detection module. Selects how a request is classified into device +# signals (browser/bot gating). Default "builtin" classifies from the User-Agent +# alone and makes no host call, so a default deployment stays host-neutral. The +# opt-in "fastly" module adds the host's TLS/H2 signals to the gate. +# Override at deployment with TRUSTED_SERVER__device__module. +# [device] +# module = "builtin" # or "fastly" to add TLS/H2 signal evidence + +[geo] +# Geo / IP intelligence module. No module is the default: Trusted Server +# resolves no geolocation and makes no host geo call, so a default deployment is +# not tied to any host geo service. Opt into the host lookup with +# module = "platform". Override with TRUSTED_SERVER__geo__module. +# module = "platform" +# +# The permission baseline for a request the geo module leaves unmatched +# (and, with no geo module, for every request) is the top of the rules tree +# in the permissions.yaml compiled into the build (the repository sample is +# config/permissions/sample.yaml). Edit that file to +# change it. +# +# With no geo module, every request is treated as that top node, so a +# visitor from another jurisdiction receives its permission rules. A +# deployment that runs an Edge Cookie module without a geo module must +# acknowledge that by uncommenting the line below, or select a geo module +# instead. +# assume_single_jurisdiction = true + # Custom headers added to every response (e.g. X-Robots-Tag: noindex). # [response_headers] # X-Robots-Tag = "noindex" -# Sign outbound OpenRTB/API requests. These are platform management-API store -# IDs used for key-rotation WRITES; the runtime reads keys from fixed store names -# (`jwks_store` / `signing_keys`), not these. To keep signing OFF, leave this -# whole block commented out. If you uncomment it - even with `enabled = false` - -# both store IDs are required and must be REAL values: the key rotate/deactivate -# admin routes read them regardless of `enabled`, so deploy validation rejects -# the placeholders below (replace them before pushing). +# What /robots.txt answers. With this section absent the publisher keeps +# their own file and /robots.txt reaches their origin like any other path. +# +# `modules` names what makes the file, in order, as one name or a list. +# refuse_all refuses every crawler and puts X-Robots-Tag: noindex, nofollow on +# the responses it sends, allow_all allows every crawler everywhere, and any +# other name is a module that supplies rules. Modules contribute rules and +# Trusted Server writes the file. A module that cannot answer has its last +# answer served past its age, and with nothing held the answer is 503, never +# an empty file, which a crawler would read as permission. +# +# A module's own settings go in its own block, [robots-txt.]. +# A publisher who sells advertising wants /ads.txt in always_allow, because +# a crawler may ignore an ads.txt that robots.txt disallows. The file is made +# by core's robots_txt middleware where a [[fetch]] entry for text/plain on +# the file's path names it, see the entry below the ads.txt one. +# [robots-txt] +# modules = ["refuse_all"] +# always_allow = ["/ads.txt"] +# sitemap = "https://publisher.example.com/sitemap.xml" + +# The sellers added to the publisher's ads.txt. The origin's file stays +# authoritative and the lines are written at its end, where a [[fetch]] entry +# for text/plain on the file's path names the middleware. Each line is a data +# record or a variable of the ads.txt grammar, checked as the settings load. +# [ads-txt] +# modules = ["sellers"] + +# [ads-txt.sellers] +# lines = ["ssp.example, 12345, DIRECT, f08c47fec0942fa0"] + +# [[fetch]] +# media_type = "text/plain" +# path = "/ads.txt" +# middleware = ["ads-txt.sellers"] + +# The robots.txt the [robots-txt] section describes, made by core's own +# middleware where this entry names it. The file replaces the origin's own +# and is made where the origin has none. +# [[fetch]] +# media_type = "text/plain" +# path = "/robots.txt" +# middleware = ["robots_txt"] + +# Sign outbound OpenRTB/API requests. The runtime reads its keys from fixed +# store names (`jwks_store` / `signing_keys`), and `ts keys` writes them. To keep +# signing OFF, leave this whole block commented out. # [request_signing] # enabled = false -# config_store_id = "" -# secret_store_id = "" # First-party HTML/CSS rewriting controls. # [rewrite] @@ -145,17 +264,17 @@ pull_sync_concurrency = 3 # enabled = false # max_age_seconds = 2592000 -# Consent forwarding. All values below are the defaults — uncomment to override. +# Consent forwarding. All values below are the defaults, so uncomment one to +# override it. # [consent] # mode = "interpreter" # "interpreter" (decode + forward) or "proxy" (raw passthrough) # check_expiration = true # check TCF consent freshness # max_consent_age_days = 395 # max age before consent is treated as expired (~13 months) # -# [consent.gdpr] -# applies_in = ["AT","BE","BG","HR","CY","CZ","DK","EE","FI","FR","DE","GR","HU","IE","IT","LV","LT","LU","MT","NL","PL","PT","RO","SK","SI","ES","SE","IS","LI","NO","GB"] -# -# [consent.us_states] -# privacy_states = ["CA","VA","CO","CT","UT","MT","OR","TX","FL","DE","IA","NE","NH","NJ","TN","MN","MD","IN","KY","RI"] +# Which places fall under GDPR, and which US states have a comprehensive +# privacy law, are no longer listed here. They are the `jurisdiction` values in +# the permissions.yaml rules tree, so one file states the policy for both the +# permission baseline and the consent handling. # # [consent.us_privacy_defaults] # notice_given = true # has the publisher shown CCPA notice? @@ -166,22 +285,115 @@ pull_sync_concurrency = 3 # mode = "restrictive" # "restrictive" | "newest" | "permissive" # freshness_threshold_days = 30 -[integrations.js_asset_proxy] -enabled = false +# Which permission signal modules run, and in what order. +# +# Signals compose rather than select. A request can carry a TCF string and a +# Global Privacy Control header at once, and both have something to say, so +# modules here is a list, whereas in [ec], [geo] and [device] module names +# one module. The order is the policy, because the last module with an +# opinion decides. +# +# Listed below is every module the shipped adapters link, which is also +# exactly what runs when modules is left out. Each is a crate under +# crates/permission-signal, outside the core, and is named by its folder +# there. Remove the ones this +# publisher does not want to act on. A module that is not on the list does +# not run, and there is no separate switch to turn one off. An unknown or +# repeated name is refused at startup rather than quietly ignored. None of the +# four has settings, so none needs a [permission-signal.] table. +# +# [permission-signal] +# modules = [ +# "gpc", # the Sec-GPC request header, Global Privacy Control +# "gpp", # a GPP US sale opt-out +# "us-privacy", # a US Privacy string sale opt-out +# "tcf", # TCF v2, with its purpose mapping in the crate +# ] +# +# Each name is the module's crate folder below crates/permission-signal, and +# may be written in full, as permission-signal.gpp. +# +# A module that gains settings will take them in a [permission-signal.] +# block named for it. None of these four has settings, so such a block is +# refused rather than ignored. +# +# The three opt-outs are separate entries so that a publisher who does not act +# on Global Privacy Control can remove "gpc" and keep the GPP and US Privacy +# opt-outs working. +# +# A consent record that arrives and cannot be read revokes, whichever modules +# are configured. That is error handling rather than a module to choose, so +# it is not in this list and cannot be removed. +# +# The default order asks Global Privacy Control first, being a browser setting +# with no interface of its own, and the three that carry a choice someone made +# through an interface after, so an answer given at a prompt amends the header +# the visitor arrived with. Reorder the list to change that. +# +# The layering, what a module may consult, and how to add a scheme are +# documented in crates/trusted-server-core/src/permission_signal/README.md +# and in the Permission Signals guide. + +# Page integrations are modules, each selected in the section of its type, +# which is [cmp] for a consent prompt, [tag] and [ad-tag] for tags, +# [bot-protection], [identity], [audience], [framework], [auction] for the +# modules the auction runs and [proxy] for the script proxy. A section names +# what runs with `module` where one runs or `modules` where several run, a +# module with settings gets its own [
.] table, and a section +# that selects nothing or a table its section does not select refuses +# startup. Nothing is selected by default. The sections and tables are +# further down, commented out. + +# A module changes a page through a middleware, and a middleware runs only on +# the pages where a [[fetch]] or [[serve]] entry names it. A page takes the +# first entry that covers it and runs that entry's middleware in the order +# written. The two entries below name every middleware the modules further +# down supply, in the order `ts audit` writes them, and core's own +# `service_version` last. Keep the names of the modules that are selected and +# remove the rest, because an entry that names the middleware of a module no +# section selects refuses startup. See "Placing page changes" in the +# configuration guide. +# [[fetch]] +# media_type = "text/html" +# middleware = [ +# "auction.prebid", +# "js_asset_proxy", +# "testing.testlight", +# "framework.nextjs", +# "audience.permutive", +# "identity.lockr", +# "cmp.didomi", +# "cmp.inmobi", +# "cmp.sourcepoint", +# "tag.google-tag-manager", +# "bot-protection.datadome", +# "ad-tag.google", +# ] + +# [[serve]] +# media_type = "text/html" +# middleware = ["bot-protection.datadome.tag", "ad-tag.google.diagnostics", "service_version"] + +# First-party proxying of named third-party scripts. Uncomment the +# `modules` line under [proxy] below to switch it on. +# [proxy.js_asset_proxy] # Uncomment to override upstream cache headers for every asset below. # This replaces upstream directives, including private and no-store. # Use only when each asset's bytes are identical for every visitor. # cache_ttl_seconds = 3600 # Asset fetches use a fixed TrustedServer/1.0 User-Agent. Do not proxy assets # that vary by browser User-Agent or use integrity hashes for UA-specific bytes. - -[[integrations.js_asset_proxy.assets]] -path = "/assets/example-vendor-loader.js" -origin_url = "https://cdn.example.com/vendor-loader.js" -proxy = "enabled" +# +# [[proxy.js_asset_proxy.assets]] +# path = "/assets/example-vendor-loader.js" +# origin_url = "https://cdn.example.com/vendor-loader.js" +# proxy = "enabled" # Proxy behavior and first-party asset routing. Kept active with defaults. [proxy] +# The modules [proxy] selects. Uncomment to run the first-party script proxy +# configured in [proxy.js_asset_proxy] above. +# modules = ["js_asset_proxy"] # Verify TLS certs when proxying to HTTPS origins (default true; false only for # local self-signed dev). # certificate_check = true @@ -197,7 +409,7 @@ proxy = "enabled" # prefix = "/.image/" # origin_url = "https://assets.example.com" # Optional path rewrite before sending upstream (the prefix must match what the -# pattern expects — here both use `/.image/`): +# pattern expects, and here both use `/.image/`): # path_pattern = "^/\\.image/(.*)/[^/]+\\.([^/.]+)$" # target_path = "/image/upload/$1.$2" # @@ -230,6 +442,19 @@ proxy = "enabled" # thumbnail = "width=150&crop=1:1,smart" # medium = "format=auto&width=828" +# The loading cache that holds each robots.txt module's answer, named by the +# module that offers it, and how long the copies last, in seconds: the brief +# copy in process memory, the longest the copy in the shared store is used, and +# how long that copy lasts unused. A module's own refresh ends its answer +# sooner. Memory may be 0, the other two must be at least 1, and none may be +# more than a year. Without `module` the one loading cache module the build +# links is used, and with none or several linked no copy is kept. +# [cache.robots-txt] +# module = "51degrees" +# memory_seconds = 60 +# store_seconds = 604800 +# store_idle_seconds = 2592000 + # Operator-controlled cache header policies for static/rehosted assets. Disabled # rules never match, and matcher/policy validation is deferred until a rule is # enabled; IDs must be non-empty and unique. Keep rules disabled unless the @@ -256,9 +481,11 @@ proxy = "enabled" # edge_ttl_seconds = 31536000 # immutable = true -# Server-side auction. Providers are declared below; mediator names must match -# enabled integrations. Kept active with the creative-processing leaves present -# so the EdgeZero environment override can apply to them. +# Server-side auction. [auction] holds the settings that belong to the auction +# itself, [demand] selects the demand sources, [ad-server] selects the ad server +# that picks the winner, and [auction.bidders.] says which demand source a +# browser bidder code is routed to. Kept active with the creative-processing +# leaves present so the EdgeZero environment override can apply to them. [auction] # Keep disabled until provider endpoints, routes, and profile values below are # replaced with deployment-specific settings. @@ -280,47 +507,69 @@ rewrite_creatives = true # blank slots on a script-heavy demand stack. sanitize_creatives = false timeout_ms = 2000 -# mediator = "adserver_mock" # optional mediator integration # Context keys the JS client may forward into auction requests (allowlist; # empty blocks all). allowed_context_keys = [] - -# Example server-side provider declarations. Remove or customize before enabling auctions. -[auction.providers.pbs-main] -protocol = "openrtb-2.6" -profile = "prebid-server" +# The modules [auction] selects, with their tables under OPTIONAL below. Add +# "testing.testlight" for the Testlight test module. +# modules = ["prebid"] + +# Example demand sources. Remove or customize before enabling auctions. The +# table name is this source's identity, and its `implementation` line names +# the implementation by module path, which is how two Prebid Servers run side +# by side under names of their own. The built-in implementations are +# auction-protocol.openrtb, auction.prebid-server and auction.aps. Every +# endpoint must be HTTPS, or HTTP to 127.0.0.1, ::1 or localhost. +[demand] +modules = ["pbs_main"] + +[demand.pbs_main] +implementation = "auction.prebid-server" endpoint = "https://prebid.example.com/openrtb2/auction" -# `prebid-server` defaults to 1000 ms when omitted. +# `auction.prebid-server` defaults to 1000 ms when omitted. routing = "explicit" - -[auction.providers.pbs-main.profile_config] debug = false test_mode = false consent_forwarding = "both" -[auction.providers.pbs-main.notifications] +[demand.pbs_main.notifications] suppress_all = false suppress_seats = [] [auction.bidders.example-bidder] -provider = "pbs-main" - -# APS server behavior is also provider/profile-owned. Use routing = "all_eligible" -# when every banner-compatible slot should be eligible. The APS profile timeout -# defaults to 800 ms; debug and script creatives default to false. -# [auction.providers.aps-main] -# protocol = "openrtb-2.6" -# profile = "aps" +module = "pbs_main" + +# APS is a demand implementation too, not a page integration, so everything it +# owns lives in its own table. Use routing = "all_eligible" when every +# banner-compatible slot should be eligible. The APS timeout defaults to +# 800 ms, and debug, script creatives and publisher-native rendering all +# default to off. Add "aps_main" to the [demand] modules list above when +# uncommenting this. +# [demand.aps_main] +# implementation = "auction.aps" # endpoint = "https://aps.example.com/e/pb/bid" # routing = "all_eligible" -# -# [auction.providers.aps-main.profile_config] # account_id = "example-aps-account-id" # debug = false # allow_script_creatives = false +# rendering_mode = "trusted_server" # inventory_domain = "publisher.example" # inventory_page_origin = "https://www.publisher.example" +# The ad server that picks the auction winner. With no [ad-server] selection the +# highest bid wins. The only implementation this repository ships is +# ad-server.mock, for development and testing, written here as mock because +# its type is this section's. +# [ad-server] +# module = "mock" +# +# [ad-server.mock] +# endpoint = "https://adserver.example.com/decide" +# timeout_ms = 1000 +# Map auction context keys to decision-URL query params: +# [ad-server.mock.context_query_params] +# example_segments = "segments" + # Server-side ad slot templates + creative-opportunity auction. Kept active. [creative_opportunities] # Set false to disable server-side ad templates while keeping slot definitions @@ -333,7 +582,7 @@ provider = "pbs-main" enabled = true gam_network_id = "123456789" price_granularity = "dense" -# FCP is not affected by this value — body content above has already +# FCP is not affected by this value, because body content above has already # streamed and painted before the hold begins. What this caps is the slip on # DOMContentLoaded and window.load. 500 ms is the recommended default; raise # only if your SSPs need more headroom and analytics confirm the DCL slip is OK. @@ -347,7 +596,7 @@ auction_timeout_ms = 500 # assembly_mode = "inline" # Request headers (besides Accept-Encoding) the origin may name in Vary. Every # emitted Vary name must be covered here or template storage is refused. Never -# include Cookie — shared templates must be reader-neutral. +# include Cookie, because shared templates must be reader-neutral. # template_cache_vary = [ # "rsc", # "next-router-state-tree", @@ -434,28 +683,30 @@ auction_timeout_ms = 500 # "/news/x" -> /123456789/example/news # "/reviews/y" -> /123456789/example/reviews -# Direct Tinybird auction telemetry (off by default). When enabled, `api_host` -# is required and must be a bare host (no scheme or path). -# [tinybird] -# enabled = true -# api_host = "api.us-east.tinybird.example" # required when enabled; host only +# Direct Tinybird auction telemetry, sent only when [analytics] selects it. +# `api_host` is required and must be a bare host (no scheme or path). +# [analytics] +# module = "tinybird" +# +# [analytics.tinybird] +# api_host = "api.us-east.tinybird.example" # required; host only # auction_dataset = "auction_events" # Events API datasource name # auction_token_secret = "tinybird_auction_append_token" # Key in trusted_server_secrets -# Debug endpoints (all default false — never enable in production). +# Debug endpoints (all default false, and never enable one in production). # [debug] # Exposes GET /_ts/debug/ja4 returning TLS/JA4 fingerprint details. Disable # after investigation; it reflects data browser JS cannot normally read. # ja4_endpoint_enabled = false # Injects a `` auction dump before ``. NEVER enable # in production: even the default "redacted" mode still includes bid-level fields -# and creative previews — it is not a fully anonymized dump. +# and creative previews, so it is not a fully anonymized dump. # auction_html_comment = false # # Fine-grained control over the dump (all optional; defaults shown). # [debug.auction_html_comment_options] # include_provider_responses = true -# include_mediator_response = true +# include_adserver_response = true # include_bids = true # Subset of the fixed validated metadata keys shown in "redacted" (and, for these # three keys, "upstream") mode. Any other key fails config load. @@ -471,35 +722,29 @@ auction_timeout_ms = 500 # ============================================================================= -# OPTIONAL — Integrations +# OPTIONAL - Integration settings # ============================================================================= -# Every integration is off by default. Most are fully commented out; uncomment a -# block and set `enabled = true` to activate it. Four (gpt, didomi, datadome, -# google_tag_manager) are kept as active `enabled = false` stubs so the `ts audit` -# CLI can flip them in place — leave those sections present. Integrations whose -# `enabled` defaults to true (prebid, permutive, lockr, ...) still stay OFF while -# their section is commented out. Required fields are noted per block. +# Every module is off by default, because no section below is uncommented. To +# switch one on, uncomment its section's selection and its settings table. A +# module with nothing to set needs no table at all. A table its section does +# not select refuses startup, so a table left behind after a module is +# switched off is caught rather than sitting unread. Required fields are +# noted per table. # ============================================================================= - -# Browser-side Prebid.js integration. Server-side bidder routing belongs under -# [auction.providers] and [auction.bidders]. When enabled, -# `external_bundle_url` is required and its host must be listed in -# [proxy].allowed_domains. Kept active but disabled. -[integrations.prebid] -enabled = false -timeout_ms = 1000 -debug = false -client_side_bidders = [] # bidders running via native Prebid.js adapters +# [auction.prebid] +# timeout_ms = 1000 +# debug = false +# client_side_bidders = [] # bidders running via native Prebid.js adapters # Keep selected GAM inventory out of Trusted Server's Prebid refresh auctions. # Matching slots still refresh through GAM. # excluded_gam_ad_unit_path_suffixes = ["/trackingonly"] -# Runtime bundle metadata — set after running `ts prebid client` and uploading: +# Runtime bundle metadata, set after running `ts prebid client` and uploading: # external_bundle_url = "https://assets.example.com/prebid/trusted-prebid-.js" # external_bundle_sha256 = "" # external_bundle_sri = "" # Bundle build inputs consumed by `ts prebid client`, not by the edge runtime. # Values are exact upstream module stems without `.js`. -# [integrations.prebid.bundle.modules] +# [auction.prebid.bundle.modules] # bidder = ["rubiconBidAdapter"] # Omit user_id for the curated default preset; use [] to select none. # user_id = ["sharedIdSystem"] @@ -514,7 +759,7 @@ client_side_bidders = [] # bidders running via native Prebid.js adapter # the checked-in User ID registry. Each `name` must appear only once. # # The module must be present in the built bundle: name it under -# [integrations.prebid.bundle.modules].user_id, or omit that list to take the +# [auction.prebid.bundle.modules].user_id, or omit that list to take the # generator's default preset. `ts prebid client` resolves every managed name # through the checked-in User ID registry and fails if the generated manifest # omits its required module without updating the configured hash or SRI. @@ -524,63 +769,78 @@ client_side_bidders = [] # bidders running via native Prebid.js adapter # OpenRTB EID source; without one the ID still reaches the auction but is never # written to KV. # -# [[integrations.prebid.managed_user_ids]] +# [[auction.prebid.managed_user_ids]] # name = "sharedId" # -# [integrations.prebid.managed_user_ids.storage] +# [auction.prebid.managed_user_ids.storage] # type = "cookie" # or "html5" # name = "_sharedid" # expires = 15 # days; omit to keep Prebid's default # refresh_in_seconds = 1800 # omit to keep Prebid's default # Next.js first-party rewriting for App Router / RSC payloads. -# [integrations.nextjs] -# enabled = true +# [framework] +# module = "nextjs" + +# [framework.nextjs] # rewrite_attributes = ["href", "link", "siteBaseUrl", "siteProductionDomain", "url"] # max_combined_payload_bytes = 10485760 # 10 MiB -# Testlight OpenRTB test integration. `endpoint` required when enabled. -# [integrations.testlight] -# enabled = true +# Testlight OpenRTB test integration, selected as "testing.testlight" in +# [auction] modules. `endpoint` is required. +# [auction.testing.testlight] # endpoint = "https://testlight.example.com/openrtb2/auction" # timeout_ms = 1200 # rewrite_scripts = true -# Didomi CMP SDK/API first-party proxy. Kept active but disabled so `ts audit` -# can flip `enabled` to true when Didomi is detected on the audited page. -[integrations.didomi] -enabled = false -geo_query_parameters = false +# Didomi CMP SDK/API first-party proxy. `ts audit` selects "didomi" in [cmp] +# when it detects Didomi on the audited page; the table below stays yours to +# uncomment and fill. +# [cmp] +# module = "didomi" + +# [cmp.didomi] +# geo_query_parameters = false # sdk_origin = "https://sdk.example.com" # api_origin = "https://api.example.com" # Sourcepoint CMP first-party proxy. # `cdn_origin` is intentionally omitted: the upstream CDN origin is pinned to # Sourcepoint's own host and is not operator-selectable. Leave it unset so the -# validated default applies — overriding it with any other host fails config -# validation. -# [integrations.sourcepoint] -# enabled = true +# validated default applies, because overriding it with any other host fails +# config validation. Selected with [cmp] module = "sourcepoint". +# [cmp.sourcepoint] # rewrite_sdk = true # cache_ttl_seconds = 3600 # auth_cookie_name = "sp_auth" # optional: forward a custom authCookie upstream -# Osano consent management (proxy toggle only). -# [integrations.osano] -# enabled = true +# Osano consent management. It has nothing to set, so [cmp] module = "osano" +# is the whole configuration and it needs no table of its own. -# Permutive DMP. `organization_id` and `workspace_id` required when enabled. -# [integrations.permutive] -# enabled = true +# InMobi Choice consent prompt. `script_url` is required, https on cmp.inmobi.com +# or cdn.inmobi.com with no user name or password. Name `cmp.inmobi` in the +# [[fetch]] entry before any middleware that writes a vendor's tag. Selected +# with [cmp] module = "inmobi". +# [cmp.inmobi] +# script_url = "https://cmp.inmobi.com/choice///choice.js?tag_version=V3" +# cmp_id = 10 + +# Permutive DMP. `organization_id` and `workspace_id` are required. +# [audience] +# module = "permutive" + +# [audience.permutive] # organization_id = "your-permutive-organization-id" # required (non-empty) # workspace_id = "your-permutive-workspace-id" # required (non-empty) # project_id = "your-permutive-project-id" # api_endpoint = "https://api.example.com" # secure_signals_endpoint = "https://secure-signals.example.com" -# lockr identity SDK. `app_id` required when enabled. -# [integrations.lockr] -# enabled = true +# lockr identity SDK. `app_id` is required. +# [identity] +# module = "lockr" + +# [identity.lockr] # app_id = "your-lockr-app-id" # required (non-empty) # api_endpoint = "https://identity.example.com" # sdk_url = "https://identity.example.com/trusted-server.js" @@ -588,9 +848,12 @@ geo_query_parameters = false # rewrite_sdk = true # DataDome bot protection. Proxies tags.js + signal-collection API first-party. -# Kept active but disabled so `ts audit` can flip `enabled` when detected. -[integrations.datadome] -enabled = false +# `ts audit` selects "datadome" in [bot-protection] when it detects DataDome; +# the table below stays yours to uncomment and fill. +# [bot-protection] +# module = "datadome" + +# [bot-protection.datadome] # sdk_origin = "https://sdk.example.com" # api_origin = "https://api.example.com" # cache_ttl_seconds = 3600 @@ -612,45 +875,43 @@ enabled = false # API validation and is stripped before the origin sees it. The credential is # loaded from the Secret Store at runtime (>= 32 bytes of high-entropy material). # Never enable in production. -# [integrations.datadome.protection_test_bypass] +# [bot-protection.datadome.protection_test_bypass] # enabled = true # credential_secret_store = "ts_secrets" # credential_secret_name = "datadome_protection_test_bypass" -# Google Publisher Tag (GPT) first-party proxy. Kept active but disabled so -# `ts audit` can flip `enabled` to true when GPT is detected. -[integrations.gpt] -enabled = false -# Kept as an active leaf so the environment override can apply; the overlay -# cannot create a missing configuration leaf. GAM attribution stays off until -# this is set true. -gam_attribution_enabled = false +# Google Publisher Tag (GPT) first-party proxy. `ts audit` selects "google" in +# [ad-tag] when it detects GPT; the table below stays yours. GAM attribution +# stays off until `gam_attribution_enabled` is set true. The environment +# overlay can only replace a leaf the document already has, and it names a +# section as written, so the variable for this leaf is +# TRUSTED_SERVER__AD-TAG__GOOGLE__GAM_ATTRIBUTION_ENABLED, which a shell's +# `export` cannot set. +# [ad-tag] +# modules = ["google"] + +# [ad-tag.google] +# gam_attribution_enabled = false # script_url = "https://ads.example.com/gpt.js" # cache_ttl_seconds = 3600 # rewrite_script = true -# GPT runtime diagnostics browser overlay. Optional and enabled manually (not -# flipped by `ts audit`); serves a diagnostics module gated behind an activation -# query param + session cookie. -# [integrations.gpt_diagnostics] -# enabled = true - -# APS browser renderer ownership. Server-side APS behavior belongs under an -# [auction.providers] entry with profile = "aps". -# [integrations.aps] -# enabled = true -# Default `trusted_server` uses TS's opaque static renderer route. Set -# `publisher_native` only for a controlled publisher-origin friendly-frame cohort. -# rendering_mode = "trusted_server" - -# Google Tag Manager first-party proxy. Kept active but disabled so `ts audit` -# can fill container_id and flip `enabled` when GTM is detected. `container_id` -# is required when this integration is actually enabled. -[integrations.google_tag_manager] -enabled = false -# Invalid placeholder on purpose: enabling GTM without a real GTM-XXXXXX id -# fails validation. `ts audit` overwrites this when it detects a real container. -container_id = "GTM-REPLACE-ME" +# GPT runtime diagnostics browser overlay. Selected manually rather than by +# `ts audit`, it serves a diagnostics module gated behind an activation query +# param and a session cookie. It has nothing to set, so adding +# "google.diagnostics" to [ad-tag] modules is the whole configuration. + +# Google Tag Manager first-party proxy. `ts audit` selects "google-tag-manager" +# in [tag] and writes this table with the container_id it finds when it +# detects GTM. +# `container_id` is required. +# [tag] +# modules = ["google-tag-manager"] + +# [tag.google-tag-manager] +# Invalid placeholder on purpose, because selecting GTM without a real +# GTM-XXXXXX id fails validation. +# container_id = "GTM-REPLACE-ME" # Must be a credential-free https bare origin with a literal host: no path, # query, fragment, `user:pass@` or wildcard. Targets are built by appending a # rooted path to this value, so anything beyond an origin corrupts them. @@ -667,11 +928,17 @@ container_id = "GTM-REPLACE-ME" # provides. Find the ids by grepping your rendered HTML for `gtag/js?id=`. # allowed_tag_ids = ["G-EXAMPLE1", "AW-000000", "MC-EXAMPLE1"] -# Mock ad server used for auction mediation in dev/testing. -# [integrations.adserver_mock] -# enabled = true -# endpoint = "https://adserver.example.com/mediate" -# timeout_ms = 1000 -# Map auction context keys to mediation URL query params: -# [integrations.adserver_mock.context_query_params] -# example_segments = "segments" +# Signed evidence of who operates this deployment and which build it runs, +# served at `endpoint` and at the same address with `.json`. Without this +# table the address belongs to the publisher's origin. The signing keys are +# compiled into the build from the file TRUSTED_SERVER_ATTESTATION_KEYS names +# and are never configured, so a build given none answers 503 there. +# See docs/guide/attestation.md. +# [attestation] +# Defaults to "/_ts/attestation". +# endpoint = "/_ts/attestation" +# operator = "Example Operator" +# The text signed ahead of the evidence. A verifier must use the same text. +# context = "example-attestation:v1" +# Where the page sends a reader to check the claim. Must be https. +# verify_url = "https://verifier.example/verify?host=example.com"